From 7ed66b827fc9672f51a4f49a679639f4b5193fbe Mon Sep 17 00:00:00 2001 From: tom5079 Date: Sat, 12 Sep 2020 09:48:30 +0900 Subject: [PATCH] Implemented eye recognition TODO: Move pages according to eye blinking --- app/build.gradle | 1 + app/src/main/AndroidManifest.xml | 11 +- .../xyz/quaver/pupil/ui/ReaderActivity.kt | 80 ++++++++++--- .../main/java/xyz/quaver/pupil/util/camera.kt | 113 ++++++++++++++++++ app/src/main/res/drawable/dot.xml | 30 +++++ app/src/main/res/drawable/eye.xml | 8 ++ app/src/main/res/drawable/eye_closed.xml | 44 +++++++ app/src/main/res/drawable/eye_off.xml | 8 ++ app/src/main/res/drawable/icon.xml | 30 +++++ app/src/main/res/drawable/icon_red.xml | 30 +++++ app/src/main/res/layout/activity_reader.xml | 12 ++ app/src/main/res/layout/reader_eye_card.xml | 67 +++++++++++ 12 files changed, 416 insertions(+), 18 deletions(-) create mode 100644 app/src/main/java/xyz/quaver/pupil/util/camera.kt create mode 100644 app/src/main/res/drawable/dot.xml create mode 100644 app/src/main/res/drawable/eye.xml create mode 100644 app/src/main/res/drawable/eye_closed.xml create mode 100644 app/src/main/res/drawable/eye_off.xml create mode 100644 app/src/main/res/drawable/icon.xml create mode 100644 app/src/main/res/drawable/icon_red.xml create mode 100644 app/src/main/res/layout/reader_eye_card.xml diff --git a/app/build.gradle b/app/build.gradle index a880fd7e..641cfab9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -78,6 +78,7 @@ dependencies { implementation 'com.google.firebase:firebase-crashlytics:17.2.1' implementation 'com.google.firebase:firebase-perf:19.0.8' implementation 'com.google.android.gms:play-services-oss-licenses:17.0.0' + implementation 'com.google.android.gms:play-services-mlkit-face-detection:16.1.1' implementation 'com.github.arimorty:floatingsearchview:2.1.1' implementation 'com.github.clans:fab:1.6.4' //implementation 'com.quiph.ui:recyclerviewfastscroller:0.2.1' diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 47df453c..d96312dc 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -6,10 +6,13 @@ - - + + + + + + + { + eyes.apply { + visibility = View.VISIBLE + TranslateAnimation(0F, 0F, -100F, 0F).apply { + duration = 500 + fillAfter = false + interpolator = OvershootInterpolator() + }.let { startAnimation(it) } + } + testCamera(context) { faces -> + eyes.dot.let { + it.visibility = View.VISIBLE + Timer().schedule(50) { + runOnUiThread { + it.visibility = View.GONE + } + } } + + if (faces.size != 1) + ResourcesCompat.getDrawable(resources, R.drawable.eye_off, context.theme).let { + eyes.left_eye.setImageDrawable(it) + eyes.right_eye.setImageDrawable(it) + + return@testCamera + } + + val left = ResourcesCompat.getDrawable(resources, + if (faces[0].rightEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed, + context.theme) + val right = ResourcesCompat.getDrawable(resources, + if (faces[0].leftEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed, + context.theme) + + eyes.left_eye.setImageDrawable(left) + eyes.right_eye.setImageDrawable(right) } } - setImageResource(R.drawable.clock_end) - } else { - autoTimer?.cancel() - autoTimer = null - setImageResource(R.drawable.clock_start) + else -> { + eyes.apply { + TranslateAnimation(0F, 0F, 0F, -100F).apply { + duration = 500 + fillAfter = false + interpolator = AnticipateInterpolator() + setAnimationListener(object: Animation.AnimationListener { + override fun onAnimationStart(p0: Animation?) {} + override fun onAnimationRepeat(p0: Animation?) {} + + override fun onAnimationEnd(p0: Animation?) { + eyes.visibility = View.GONE + } + }) + }.let { startAnimation(it) } + } + closeCamera() + } } } } diff --git a/app/src/main/java/xyz/quaver/pupil/util/camera.kt b/app/src/main/java/xyz/quaver/pupil/util/camera.kt new file mode 100644 index 00000000..f783a9e4 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/util/camera.kt @@ -0,0 +1,113 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2020 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +@file:Suppress("DEPRECATION") + +package xyz.quaver.pupil.util + +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.ImageFormat +import android.hardware.Camera +import android.view.Surface +import android.view.WindowManager +import com.google.android.gms.tasks.Task +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetectorOptions + +/** Check if this device has a camera */ +private fun Context.checkCameraHardware() = + this.packageManager.hasSystemFeature(PackageManager.FEATURE_CAMERA) + +private fun openFrontCamera() : Pair { + var camera: Camera? = null + var cameraID: Int = -1 + + val cameraInfo = Camera.CameraInfo() + + for (i in 0 until Camera.getNumberOfCameras()) { + Camera.getCameraInfo(i, cameraInfo) + if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) + runCatching { Camera.open(i) }.getOrNull()?.let { camera = it; cameraID = i } + + if (camera != null) break + } + + return Pair(camera, cameraID) +} + +val orientations = mapOf( + Surface.ROTATION_0 to 0, + Surface.ROTATION_90 to 90, + Surface.ROTATION_180 to 180, + Surface.ROTATION_270 to 270, +) + +private fun getRotation(context: Context, cameraID: Int): Int { + val cameraRotation = Camera.CameraInfo().also { Camera.getCameraInfo(cameraID, it) }.orientation + val rotation = orientations[(context.getSystemService(Context.WINDOW_SERVICE) as WindowManager).defaultDisplay.rotation] ?: error("") + + return (cameraRotation + rotation) % 360 +} + +var camera: Camera? = null +private val detector = FaceDetection.getClient( + FaceDetectorOptions.Builder() + .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL) + .build() +) +private var process: Task>? = null + +fun testCamera(context: Context, callback: (List) -> Unit) { + if (camera != null) closeCamera() + + val cameraID = openFrontCamera().let { (cam, cameraID) -> + cam ?: return + camera = cam + cameraID + } + + with (camera!!) { + parameters = parameters.apply { + setPreviewSize(640, 480) + previewFormat = ImageFormat.NV21 + flashMode = Camera.Parameters.FLASH_MODE_OFF + } + setPreviewCallback { bytes, camera -> + if (process?.isComplete == false) + return@setPreviewCallback + + val rotation = getRotation(context, cameraID) + + val image = InputImage.fromByteArray(bytes, 640, 480, rotation, InputImage.IMAGE_FORMAT_NV21) + process = detector.process(image) + .addOnSuccessListener(callback) + } + + startPreview() + } +} + +fun closeCamera() { + camera?.setPreviewCallback(null) + camera?.stopPreview() + camera?.release() + camera = null +} \ No newline at end of file diff --git a/app/src/main/res/drawable/dot.xml b/app/src/main/res/drawable/dot.xml new file mode 100644 index 00000000..f75e450c --- /dev/null +++ b/app/src/main/res/drawable/dot.xml @@ -0,0 +1,30 @@ + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/drawable/eye.xml b/app/src/main/res/drawable/eye.xml new file mode 100644 index 00000000..c5853f1e --- /dev/null +++ b/app/src/main/res/drawable/eye.xml @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/app/src/main/res/drawable/eye_closed.xml b/app/src/main/res/drawable/eye_closed.xml new file mode 100644 index 00000000..cb8e83fd --- /dev/null +++ b/app/src/main/res/drawable/eye_closed.xml @@ -0,0 +1,44 @@ + + + + + + + + + diff --git a/app/src/main/res/drawable/eye_off.xml b/app/src/main/res/drawable/eye_off.xml new file mode 100644 index 00000000..2f1a1e82 --- /dev/null +++ b/app/src/main/res/drawable/eye_off.xml @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/app/src/main/res/drawable/icon.xml b/app/src/main/res/drawable/icon.xml new file mode 100644 index 00000000..8bca8ba9 --- /dev/null +++ b/app/src/main/res/drawable/icon.xml @@ -0,0 +1,30 @@ + + + + + + diff --git a/app/src/main/res/drawable/icon_red.xml b/app/src/main/res/drawable/icon_red.xml new file mode 100644 index 00000000..00fdcd21 --- /dev/null +++ b/app/src/main/res/drawable/icon_red.xml @@ -0,0 +1,30 @@ + + + + + + diff --git a/app/src/main/res/layout/activity_reader.xml b/app/src/main/res/layout/activity_reader.xml index 2b9de763..f2b3fb7e 100644 --- a/app/src/main/res/layout/activity_reader.xml +++ b/app/src/main/res/layout/activity_reader.xml @@ -45,6 +45,14 @@ + + @@ -87,6 +96,7 @@ android:id="@+id/reader_fab_retry" android:layout_width="wrap_content" android:layout_height="wrap_content" + app:srcCompat="@drawable/refresh" app:fab_label="@string/reader_fab_retry" app:fab_size="mini"/> @@ -94,6 +104,7 @@ android:id="@+id/reader_fab_auto" android:layout_width="wrap_content" android:layout_height="wrap_content" + app:srcCompat="@drawable/clock_start" app:fab_label="@string/reader_fab_auto" app:fab_size="mini"/> @@ -101,6 +112,7 @@ android:id="@+id/reader_fab_fullscreen" android:layout_width="wrap_content" android:layout_height="wrap_content" + app:srcCompat="@drawable/ic_fullscreen" app:fab_label="@string/reader_fab_fullscreen" app:fab_size="mini"/> diff --git a/app/src/main/res/layout/reader_eye_card.xml b/app/src/main/res/layout/reader_eye_card.xml new file mode 100644 index 00000000..b199a922 --- /dev/null +++ b/app/src/main/res/layout/reader_eye_card.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + \ No newline at end of file