android + ui: update target sdk, add camera support, camera control from native code, camera view, qr scan modal

android/app/build.gradle

@@ -7,13 +7,13 @@ def keystoreProperties = new Properties()
 keystoreProperties.load(new FileInputStream(keystorePropertiesFile))
 
 android {
-    compileSdk 33
+    compileSdk 34
     ndkVersion '26.0.10792818'
 
     defaultConfig {
         applicationId "mw.gri.android"
         minSdk 24
-        targetSdk 33
+        targetSdk 34
         versionCode 1
         versionName "0.1.0"
     }
@@ -48,19 +48,12 @@ android {
 dependencies {
     implementation 'androidx.appcompat:appcompat:1.6.1'
 
-    // To use the Android Frame Pacing library
-    //implementation "androidx.games:games-frame-pacing:1.9.1"
-
-    // To use the Android Performance Tuner
-    //implementation "androidx.games:games-performance-tuner:1.5.0"
-
     // To use the Games Activity library
     implementation "androidx.games:games-activity:2.0.2"
 
-    // To use the Games Controller Library
-    //implementation "androidx.games:games-controller:1.1.0"
-
-    // To use the Games Text Input Library
-    //implementation "androidx.games:games-text-input:1.1.0"
+    // Android Camera
+    implementation 'androidx.camera:camera-core:1.2.3'
+    implementation 'androidx.camera:camera-camera2:1.2.3'
+    implementation 'androidx.camera:camera-lifecycle:1.2.3'
 }
 

android/app/src/main/AndroidManifest.xml

@@ -2,10 +2,12 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
 >
 
+    <uses-feature android:name="android.hardware.camera" android:required="false"/>
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
     <uses-permission android:name="android.permission.INTERNET"/>
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.CAMERA"/>
 
     <application
             android:hardwareAccelerated="true"
@@ -20,7 +22,7 @@
         <activity
                 android:launchMode="singleTask"
                 android:name=".MainActivity"
-                android:configChanges="orientation|screenSize|screenLayout|keyboardHidden|uiMode|keyboard"
+                android:configChanges="orientation|screenSize"
                 android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />

android/app/src/main/java/mw/gri/android/MainActivity.java

@@ -3,21 +3,29 @@ package mw.gri.android;
 import android.Manifest;
 import android.content.*;
 import android.content.pm.PackageManager;
+import android.content.res.Configuration;
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Process;
 import android.system.ErrnoException;
 import android.system.Os;
+import android.util.Size;
 import android.view.KeyEvent;
 import android.view.View;
 import android.view.inputmethod.InputMethodManager;
 import androidx.annotation.NonNull;
+import androidx.camera.core.*;
+import androidx.camera.lifecycle.ProcessCameraProvider;
+import androidx.core.content.ContextCompat;
 import androidx.core.graphics.Insets;
 import androidx.core.view.DisplayCutoutCompat;
 import androidx.core.view.ViewCompat;
 import androidx.core.view.WindowInsetsCompat;
 import com.google.androidgamesdk.GameActivity;
-import org.jetbrains.annotations.NotNull;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import static android.content.ClipDescription.MIMETYPE_TEXT_HTML;
 import static android.content.ClipDescription.MIMETYPE_TEXT_PLAIN;
@@ -26,6 +34,7 @@ public class MainActivity extends GameActivity {
     public static String STOP_APP_ACTION = "STOP_APP";
 
     private static final int NOTIFICATIONS_PERMISSION_CODE = 1;
+    private static final int CAMERA_PERMISSION_CODE = 2;
 
     static {
         System.loadLibrary("grim");
@@ -41,9 +50,19 @@ public class MainActivity extends GameActivity {
         }
     };
 
+    private final ImageAnalysis mImageAnalysis = new ImageAnalysis.Builder()
+            .setTargetResolution(new Size(640, 480))
+            .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+            .build();
+
+    private ListenableFuture<ProcessCameraProvider> mCameraProviderFuture = null;
+    private ProcessCameraProvider mCameraProvider = null;
+    private ExecutorService mCameraExecutor = null;
+    private boolean mUseBackCamera = true;
+
     @Override
     protected void onCreate(Bundle savedInstanceState) {
-        // Setup HOME environment variable for native code configurations.
+        // Setup environment variables for native code.
         try {
             Os.setenv("HOME", getExternalFilesDir("").getPath(), true);
             Os.setenv("XDG_CACHE_HOME", getExternalCacheDir().getPath(), true);
@@ -87,7 +106,7 @@ public class MainActivity extends GameActivity {
         });
 
         findViewById(android.R.id.content).post(() -> {
-            // Request notifications permissions.
+            // Request notifications permissions if needed.
             if (Build.VERSION.SDK_INT >= 33) {
                 String notificationsPermission = Manifest.permission.POST_NOTIFICATIONS;
                 if (checkSelfPermission(notificationsPermission) != PackageManager.PERMISSION_GRANTED) {
@@ -104,12 +123,30 @@ public class MainActivity extends GameActivity {
     }
 
     @Override
-    public void onRequestPermissionsResult(int requestCode, @NonNull @NotNull String[] permissions, @NonNull @NotNull int[] grantResults) {
-        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
-        if (requestCode == NOTIFICATIONS_PERMISSION_CODE && grantResults.length != 0 &&
-                grantResults[0] == PackageManager.PERMISSION_GRANTED) {
-            // Start notification service.
-            BackgroundService.start(this);
+    public void onConfigurationChanged(Configuration newConfig) {
+        super.onConfigurationChanged(newConfig);
+        // Called on screen orientation change to restart camera.
+        if (mCameraProvider != null) {
+            stopCamera();
+            startCamera();
+        }
+    }
+
+    @Override
+    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] results) {
+        super.onRequestPermissionsResult(requestCode, permissions, results);
+        if (results.length != 0 && results[0] == PackageManager.PERMISSION_GRANTED) {
+            switch (requestCode) {
+                case NOTIFICATIONS_PERMISSION_CODE: {
+                    // Start notification service.
+                    BackgroundService.start(this);
+                    return;
+                }
+                case CAMERA_PERMISSION_CODE: {
+                    // Start camera.
+                    startCamera();
+                }
+            }
         }
     }
 
@@ -121,7 +158,7 @@ public class MainActivity extends GameActivity {
                 onInput(event.getCharacters());
                 return false;
             }
-        // Pass any other input values into native code.
+            // Pass any other input values into native code.
         } else if (event.getAction() == KeyEvent.ACTION_UP &&
                 event.getKeyCode() != KeyEvent.KEYCODE_ENTER &&
                 event.getKeyCode() != KeyEvent.KEYCODE_BACK) {
@@ -201,13 +238,98 @@ public class MainActivity extends GameActivity {
         return text;
     }
 
+    // Called from native code to show keyboard.
     public void showKeyboard() {
-        InputMethodManager imm = (InputMethodManager )getSystemService(Context.INPUT_METHOD_SERVICE);
+        InputMethodManager imm = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE);
         imm.showSoftInput(getWindow().getDecorView(), InputMethodManager.SHOW_IMPLICIT);
     }
 
+    // Called from native code to hide keyboard.
     public void hideKeyboard() {
-        InputMethodManager imm = (InputMethodManager )getSystemService(Context.INPUT_METHOD_SERVICE);
+        InputMethodManager imm = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE);
         imm.hideSoftInputFromWindow(getWindow().getDecorView().getWindowToken(), 0);
     }
+
+    // Called from native code to start camera.
+    public void startCamera() {
+        // Check permissions.
+        String notificationsPermission = Manifest.permission.CAMERA;
+        if (checkSelfPermission(notificationsPermission) != PackageManager.PERMISSION_GRANTED) {
+            requestPermissions(new String[] { notificationsPermission }, CAMERA_PERMISSION_CODE);
+        } else {
+            // Start .
+            if (mCameraProviderFuture == null) {
+                mCameraProviderFuture = ProcessCameraProvider.getInstance(this);
+                mCameraProviderFuture.addListener(() -> {
+                    try {
+                        mCameraProvider = mCameraProviderFuture.get();
+                        // Launch camera.
+                        openCamera();
+                    } catch (Exception e) {
+                        View content = findViewById(android.R.id.content);
+                        if (content != null) {
+                            content.post(this::stopCamera);
+                        }
+                    }
+                }, ContextCompat.getMainExecutor(this));
+            } else {
+                View content = findViewById(android.R.id.content);
+                if (content != null) {
+                    content.post(this::openCamera);
+                }
+            }
+        }
+    }
+
+    // Open camera after initialization or start after stop.
+    private void openCamera() {
+        // Set up the image analysis use case which will process frames in real time.
+        if (mCameraExecutor == null) {
+            mCameraExecutor = Executors.newSingleThreadExecutor();
+            mImageAnalysis.setAnalyzer(mCameraExecutor, image -> {
+                // Convert image to JPEG.
+                byte[] data = Utils.convertCameraImage(image);
+                // Send image to native code.
+                onCameraImage(data, image.getImageInfo().getRotationDegrees());
+                image.close();
+            });
+        }
+
+        // Select back camera initially.
+        CameraSelector cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA;
+        if (!mUseBackCamera) {
+            cameraSelector = CameraSelector.DEFAULT_FRONT_CAMERA;
+        }
+        // Apply declared configs to CameraX using the same lifecycle owner
+        mCameraProvider.unbindAll();
+        mCameraProvider.bindToLifecycle(this, cameraSelector, mImageAnalysis);
+    }
+
+    // Called from native code to stop camera.
+    public void stopCamera() {
+        View content = findViewById(android.R.id.content);
+        if (content != null) {
+            content.post(() -> {
+                mCameraProvider.unbindAll();
+            });
+        }
+    }
+
+    // Called from native code to get number of cameras.
+    public int camerasAmount() {
+        if (mCameraProvider == null) {
+            return 0;
+        }
+        return mCameraProvider.getAvailableCameraInfos().size();
+    }
+
+    // Called from native code to switch camera.
+    public void switchCamera() {
+        mUseBackCamera = true;
+        stopCamera();
+        startCamera();
+    }
+
+    // Pass image from camera into native code.
+    public native void onCameraImage(byte[] buff, int rotation);
 }

android/app/src/main/java/mw/gri/android/Utils.java

@@ -1,10 +1,138 @@
 package mw.gri.android;
 
 import android.content.Context;
+import android.graphics.ImageFormat;
+import android.graphics.Rect;
+import android.graphics.YuvImage;
+import android.media.Image;
+import androidx.camera.core.ImageProxy;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
 
 public class Utils {
     // Convert Pixels to DensityPixels
     public static int pxToDp(int px, Context context) {
         return (int) (px / context.getResources().getDisplayMetrics().density);
     }
+
+    /** Converts a YUV_420_888 image from CameraX API to a bitmap. */
+    public static byte[] convertCameraImage(ImageProxy image) {
+        // Convert image to nv21 and get buffer.
+        ByteBuffer nv21Buffer =
+                yuv420ThreePlanesToNV21(image.getPlanes(), image.getWidth(), image.getHeight());
+        nv21Buffer.rewind();
+        byte[] nv21 = new byte[nv21Buffer.limit()];
+        nv21Buffer.get(nv21);
+        // Convert to JPEG.
+        YuvImage yuvImage = new YuvImage(nv21, ImageFormat.NV21, image.getWidth(), image.getHeight(), null);
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        yuvImage.compressToJpeg(new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()), 100, out);
+        return out.toByteArray();
+    }
+
+    /**
+     * Converts YUV_420_888 to NV21 bytebuffer.
+     *
+     * <p>The NV21 format consists of a single byte array containing the Y, U and V values. For an
+     * image of size S, the first S positions of the array contain all the Y values. The remaining
+     * positions contain interleaved V and U values. U and V are subsampled by a factor of 2 in both
+     * dimensions, so there are S/4 U values and S/4 V values. In summary, the NV21 array will contain
+     * S Y values followed by S/4 VU values: YYYYYYYYYYYYYY(...)YVUVUVUVU(...)VU
+     *
+     * <p>YUV_420_888 is a generic format that can describe any YUV image where U and V are subsampled
+     * by a factor of 2 in both dimensions. {@link Image#getPlanes} returns an array with the Y, U and
+     * V planes. The Y plane is guaranteed not to be interleaved, so we can just copy its values into
+     * the first part of the NV21 array. The U and V planes may already have the representation in the
+     * NV21 format. This happens if the planes share the same buffer, the V buffer is one position
+     * before the U buffer and the planes have a pixelStride of 2. If this is case, we can just copy
+     * them to the NV21 array.
+     */
+    private static ByteBuffer yuv420ThreePlanesToNV21(ImageProxy.PlaneProxy[] yuv420888planes, int width, int height) {
+        int imageSize = width * height;
+        byte[] out = new byte[imageSize + 2 * (imageSize / 4)];
+
+        if (areUVPlanesNV21(yuv420888planes, width, height)) {
+            // Copy the Y values.
+            yuv420888planes[0].getBuffer().get(out, 0, imageSize);
+
+            ByteBuffer uBuffer = yuv420888planes[1].getBuffer();
+            ByteBuffer vBuffer = yuv420888planes[2].getBuffer();
+            // Get the first V value from the V buffer, since the U buffer does not contain it.
+            vBuffer.get(out, imageSize, 1);
+            // Copy the first U value and the remaining VU values from the U buffer.
+            uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1);
+        } else {
+            // Fallback to copying the UV values one by one, which is slower but also works.
+            // Unpack Y.
+            unpackPlane(yuv420888planes[0], width, height, out, 0, 1);
+            // Unpack U.
+            unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2);
+            // Unpack V.
+            unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2);
+        }
+
+        return ByteBuffer.wrap(out);
+    }
+
+    /** Checks if the UV plane buffers of a YUV_420_888 image are in the NV21 format. */
+    private static boolean areUVPlanesNV21(ImageProxy.PlaneProxy[] planes, int width, int height) {
+        int imageSize = width * height;
+
+        ByteBuffer uBuffer = planes[1].getBuffer();
+        ByteBuffer vBuffer = planes[2].getBuffer();
+
+        // Backup buffer properties.
+        int vBufferPosition = vBuffer.position();
+        int uBufferLimit = uBuffer.limit();
+
+        // Advance the V buffer by 1 byte, since the U buffer will not contain the first V value.
+        vBuffer.position(vBufferPosition + 1);
+        // Chop off the last byte of the U buffer, since the V buffer will not contain the last U value.
+        uBuffer.limit(uBufferLimit - 1);
+
+        // Check that the buffers are equal and have the expected number of elements.
+        boolean areNV21 =
+                (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0);
+
+        // Restore buffers to their initial state.
+        vBuffer.position(vBufferPosition);
+        uBuffer.limit(uBufferLimit);
+
+        return areNV21;
+    }
+
+    /**
+     * Unpack an image plane into a byte array.
+     *
+     * <p>The input plane data will be copied in 'out', starting at 'offset' and every pixel will be
+     * spaced by 'pixelStride'. Note that there is no row padding on the output.
+     */
+    private static void unpackPlane(
+            ImageProxy.PlaneProxy plane, int width, int height, byte[] out, int offset, int pixelStride) {
+        ByteBuffer buffer = plane.getBuffer();
+        buffer.rewind();
+
+        // Compute the size of the current plane.
+        // We assume that it has the aspect ratio as the original image.
+        int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride();
+        if (numRow == 0) {
+            return;
+        }
+        int scaleFactor = height / numRow;
+        int numCol = width / scaleFactor;
+
+        // Extract the data in the output buffer.
+        int outputPos = offset;
+        int rowStart = 0;
+        for (int row = 0; row < numRow; row++) {
+            int inputPos = rowStart;
+            for (int col = 0; col < numCol; col++) {
+                out[outputPos] = buffer.get(inputPos);
+                outputPos += pixelStride;
+                inputPos += plane.getPixelStride();
+            }
+            rowStart += plane.getRowStride();
+        }
+    }
 }

android/build.gradle

@@ -1,7 +1,7 @@
 // Top-level build file where you can add configuration options common to all sub-projects/modules.
 plugins {
-    id 'com.android.application' version '8.1.0' apply false
-    id 'com.android.library' version '8.1.0' apply false
+    id 'com.android.application' version '8.1.1' apply false
+    id 'com.android.library' version '8.1.1' apply false
 }
 
 task clean(type: Delete) {