From 57c806868005aa1de4d9be2e1d8e9f3aa6c1b4dd Mon Sep 17 00:00:00 2001 From: shubham0204 Date: Sat, 1 Oct 2022 18:58:10 +0530 Subject: [PATCH] Resolved issue #30 --- README.md | 7 ++ app/build.gradle | 10 +-- .../quaterion/facenetdetection/BitmapUtils.kt | 7 +- .../quaterion/facenetdetection/FileReader.kt | 82 ++++++++++--------- .../facenetdetection/FrameAnalyser.kt | 2 +- .../ml/quaterion/facenetdetection/Logger.kt | 2 +- .../facenetdetection/MainActivity.kt | 2 - .../facenetdetection/model/FaceNetModel.kt | 8 +- .../model/MaskDetectionModel.kt | 4 +- .../facenetdetection/model/ModelInfo.kt | 4 +- build.gradle | 24 +----- gradle/wrapper/gradle-wrapper.properties | 4 +- settings.gradle | 14 ++++ 13 files changed, 89 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index ffe4d39..dd43f6b 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,13 @@ We don't need to modify the app/retrain any ML model to add more people ( subjec ## What's New +### Updates - September 2022 + +- Modified `settings.gradle` to use the new plugin management system. +- The conversion of `Bitmap` to NV21-formatted `ByteArray` ( YUV420 ) is now transformed into a suspending function +to avoid blocking of the UI thread when a large number of images are being processed. +- + ### Updates - December 2021 - Users can now control the use of `GpuDelegate` and `XNNPack` using `useGpu` and `useXNNPack` in diff --git a/app/build.gradle b/app/build.gradle index 2fe3b7f..054232b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -4,11 +4,11 @@ plugins { } android { - compileSdk 32 + compileSdk 33 defaultConfig { - applicationId "com.edu.shubham0204.attendancesystem" + applicationId "com.ml.quaterion.facenetdetection" minSdk 25 - targetSdk 32 + targetSdk 33 versionCode 1 versionName "1.0" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" @@ -33,8 +33,8 @@ android { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.core:core-ktx:1.8.0' - implementation 'androidx.appcompat:appcompat:1.5.0' + implementation 'androidx.core:core-ktx:1.9.0' + implementation 'androidx.appcompat:appcompat:1.5.1' implementation 'com.google.android.material:material:1.6.1' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/BitmapUtils.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/BitmapUtils.kt index bd7d823..0c5e88d 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/BitmapUtils.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/BitmapUtils.kt @@ -20,6 +20,9 @@ import android.graphics.* import android.media.Image import android.net.Uri import android.os.ParcelFileDescriptor +import android.util.Log +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.io.ByteArrayOutputStream import java.io.File import java.io.FileDescriptor @@ -109,13 +112,13 @@ class BitmapUtils { // Convert the given Bitmap to NV21 ByteArray // See this comment -> https://github.com/firebase/quickstart-android/issues/932#issuecomment-531204396 - fun bitmapToNV21ByteArray(bitmap: Bitmap): ByteArray { + suspend fun bitmapToNV21ByteArray(bitmap: Bitmap): ByteArray = withContext(Dispatchers.Default) { val argb = IntArray(bitmap.width * bitmap.height ) bitmap.getPixels(argb, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) val yuv = ByteArray(bitmap.height * bitmap.width + 2 * Math.ceil(bitmap.height / 2.0).toInt() * Math.ceil(bitmap.width / 2.0).toInt()) encodeYUV420SP( yuv, argb, bitmap.width, bitmap.height) - return yuv + return@withContext yuv } private fun encodeYUV420SP(yuv420sp: ByteArray, argb: IntArray, width: Int, height: Int) { diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/FileReader.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/FileReader.kt index 92392f9..ed24590 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/FileReader.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/FileReader.kt @@ -32,7 +32,8 @@ class FileReader( private var faceNetModel: FaceNetModel ) { .setPerformanceMode( FaceDetectorOptions.PERFORMANCE_MODE_FAST ) .build() private val detector = FaceDetection.getClient( realTimeOpts ) - private val coroutineScope = CoroutineScope( Dispatchers.Main ) + private val defaultScope = CoroutineScope( Dispatchers.Default ) + private val mainScope = CoroutineScope( Dispatchers.Main ) private var numImagesWithNoFaces = 0 private var imageCounter = 0 private var numImages = 0 @@ -62,52 +63,55 @@ class FileReader( private var faceNetModel: FaceNetModel ) { // Crop faces and produce embeddings ( using FaceNet ) from given image. // Store the embedding in imageData private fun scanImage( name : String , image : Bitmap ) { - val inputImage = InputImage.fromByteArray( - BitmapUtils.bitmapToNV21ByteArray( image ) , - image.width, - image.height, - 0, - InputImage.IMAGE_FORMAT_NV21 - ) - detector.process( inputImage ) - .addOnSuccessListener { faces -> - if ( faces.size != 0 ) { - coroutineScope.launch{ - val embedding = getEmbedding( image , faces[ 0 ].boundingBox ) - imageData.add( Pair( name , embedding ) ) - // Embedding stored, now proceed to the next image. - if ( imageCounter + 1 != numImages ) { - imageCounter += 1 - scanImage( data[ imageCounter].first , data[ imageCounter ].second ) - } - else { - // Processing done, reset the file reader. - callback.onProcessCompleted( imageData , numImagesWithNoFaces ) - reset() + mainScope.launch { + val inputImage = InputImage.fromByteArray( + BitmapUtils.bitmapToNV21ByteArray(image), + image.width, + image.height, + 0, + InputImage.IMAGE_FORMAT_NV21 + ) + detector.process(inputImage) + .addOnSuccessListener { faces -> + if (faces.size != 0) { + mainScope.launch { + val embedding = getEmbedding(image, faces[0].boundingBox) + imageData.add(Pair(name, embedding)) + // Embedding stored, now proceed to the next image. + if (imageCounter + 1 != numImages) { + imageCounter += 1 + scanImage(data[imageCounter].first, data[imageCounter].second) + } else { + // Processing done, reset the file reader. + callback.onProcessCompleted(imageData, numImagesWithNoFaces) + reset() + } } } - } - else { - // The image contains no faces, proceed to the next one. - numImagesWithNoFaces += 1 - if ( imageCounter + 1 != numImages ) { - imageCounter += 1 - scanImage( data[ imageCounter].first , data[ imageCounter ].second ) - } else { - callback.onProcessCompleted( imageData , numImagesWithNoFaces ) - reset() + // The image contains no faces, proceed to the next one. + numImagesWithNoFaces += 1 + if (imageCounter + 1 != numImages) { + imageCounter += 1 + scanImage(data[imageCounter].first, data[imageCounter].second) + } else { + callback.onProcessCompleted(imageData, numImagesWithNoFaces) + reset() + } } } - - } + } } // Suspend function for running the FaceNet model - private suspend fun getEmbedding(image: Bitmap, bbox : Rect ) = - withContext( Dispatchers.Default ) { - return@withContext faceNetModel.getFaceEmbedding( BitmapUtils.cropRectFromBitmap( image , bbox ) ) - } + private suspend fun getEmbedding(image: Bitmap, bbox : Rect ) : FloatArray = withContext( Dispatchers.Default ) { + return@withContext faceNetModel.getFaceEmbedding( + BitmapUtils.cropRectFromBitmap( + image, + bbox + ) + ) + } private fun reset() { diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/FrameAnalyser.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/FrameAnalyser.kt index 2c3b8f4..0d9b7f1 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/FrameAnalyser.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/FrameAnalyser.kt @@ -92,7 +92,7 @@ class FrameAnalyser( private var context: Context , boundingBoxOverlay.frameWidth = frameBitmap.width } - val inputImage = InputImage.fromMediaImage( image.image , image.imageInfo.rotationDegrees ) + val inputImage = InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees ) detector.process(inputImage) .addOnSuccessListener { faces -> CoroutineScope( Dispatchers.Default ).launch { diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/Logger.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/Logger.kt index 871fb90..3d0a96f 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/Logger.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/Logger.kt @@ -24,7 +24,7 @@ class Logger { // To scroll to the last message // See this SO answer -> https://stackoverflow.com/a/37806544/10878733 while ( MainActivity.logTextView.canScrollVertically(1) ) { - MainActivity.logTextView.scrollBy(0, 10); + MainActivity.logTextView.scrollBy(0, 10) } } diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/MainActivity.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/MainActivity.kt index 6615265..7fb7d48 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/MainActivity.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/MainActivity.kt @@ -111,8 +111,6 @@ class MainActivity : AppCompatActivity() { activityMainBinding = ActivityMainBinding.inflate( layoutInflater ) setContentView( activityMainBinding.root ) - // Implementation of CameraX preview - previewView = activityMainBinding.previewView logTextView = activityMainBinding.logTextview logTextView.movementMethod = ScrollingMovementMethod() diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/model/FaceNetModel.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/model/FaceNetModel.kt index b9f6516..2ac0370 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/model/FaceNetModel.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/model/FaceNetModel.kt @@ -57,17 +57,17 @@ class FaceNetModel( context : Context , val interpreterOptions = Interpreter.Options().apply { // Add the GPU Delegate if supported. // See -> https://www.tensorflow.org/lite/performance/gpu#android - if ( CompatibilityList().isDelegateSupportedOnThisDevice ) { - if ( useGpu ) { + if ( useGpu ) { + if ( CompatibilityList().isDelegateSupportedOnThisDevice ) { addDelegate( GpuDelegate( CompatibilityList().bestOptionsForThisDevice )) } } else { // Number of threads for computation - setNumThreads( 4 ) + numThreads = 4 } setUseXNNPACK( useXNNPack ) - setUseNNAPI(true) + useNNAPI = true } interpreter = Interpreter(FileUtil.loadMappedFile(context, model.assetsFilename ) , interpreterOptions ) Logger.log("Using ${model.name} model.") diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/model/MaskDetectionModel.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/model/MaskDetectionModel.kt index ce6895d..8d33193 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/model/MaskDetectionModel.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/model/MaskDetectionModel.kt @@ -44,7 +44,7 @@ class MaskDetectionModel( context: Context ) { } else { // Number of threads for computation - setNumThreads( 4 ) + numThreads = 4 } setUseXNNPACK(true) } @@ -61,7 +61,7 @@ class MaskDetectionModel( context: Context ) { // Kotlin Extension function for arg max. // See https://kotlinlang.org/docs/extensions.html private fun FloatArray.argmax() : Int { - return this.indexOf( this.maxOrNull()!! ) + return this.indexOfFirst { it == this.maxOrNull()!! } } diff --git a/app/src/main/java/com/ml/quaterion/facenetdetection/model/ModelInfo.kt b/app/src/main/java/com/ml/quaterion/facenetdetection/model/ModelInfo.kt index a06475b..23caff8 100644 --- a/app/src/main/java/com/ml/quaterion/facenetdetection/model/ModelInfo.kt +++ b/app/src/main/java/com/ml/quaterion/facenetdetection/model/ModelInfo.kt @@ -6,6 +6,4 @@ class ModelInfo( val cosineThreshold : Float , val l2Threshold : Float , val outputDims : Int , - val inputDims : Int ) { - -} \ No newline at end of file + val inputDims : Int ) \ No newline at end of file diff --git a/build.gradle b/build.gradle index b492928..3251255 100644 --- a/build.gradle +++ b/build.gradle @@ -1,24 +1,8 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. - -buildscript { - ext.kotlin_version = '1.5.30' - repositories { - google() - mavenCentral() - } - dependencies { - classpath 'com.android.tools.build:gradle:7.2.0' - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - // NOTE: Do not place your application dependencies here; they belong - // in the individual module build.gradle files - } -} - -allprojects { - repositories { - google() - mavenCentral() - } +plugins { + id 'com.android.application' version '7.3.0' apply false + id 'com.android.library' version '7.3.0' apply false + id 'org.jetbrains.kotlin.android' version '1.6.10' apply false } task clean(type: Delete) { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 5bf6f53..4e3fea7 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Sun Sep 11 07:35:00 IST 2022 +#Sat Oct 01 09:16:18 IST 2022 distributionBase=GRADLE_USER_HOME -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip distributionPath=wrapper/dists zipStorePath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/settings.gradle b/settings.gradle index 4f92e3d..fa410fd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,2 +1,16 @@ +pluginManagement { + repositories { + gradlePluginPortal() + google() + mavenCentral() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} rootProject.name='FaceNet Detection' include ':app'