Blink Recognition

app/build.gradle

@@ -66,11 +66,12 @@ dependencies {
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.9"
     //implementation "org.jetbrains.kotlinx:kotlinx-serialization-core:1.0.0-RC"
     implementation 'androidx.appcompat:appcompat:1.2.0'
-    implementation 'androidx.constraintlayout:constraintlayout:2.0.1'
+    implementation "androidx.activity:activity-ktx:1.2.0-alpha08"
+    implementation 'androidx.fragment:fragment-ktx:1.3.0-alpha08'
     implementation 'androidx.preference:preference:1.1.1'
+    implementation 'androidx.constraintlayout:constraintlayout:2.0.1'
     implementation 'androidx.gridlayout:gridlayout:1.0.0'
     implementation "androidx.biometric:biometric:1.0.1"
-    implementation 'androidx.fragment:fragment-ktx:1.2.5'
     implementation "com.daimajia.swipelayout:library:1.2.0@aar"
     implementation 'com.google.android.material:material:1.3.0-alpha02'
     implementation 'com.google.firebase:firebase-core:17.5.0'

app/src/main/java/xyz/quaver/pupil/ui/ReaderActivity.kt

@@ -18,11 +18,14 @@
 
 package xyz.quaver.pupil.ui
 
+import android.Manifest
 import android.content.ComponentName
 import android.content.Intent
 import android.content.ServiceConnection
+import android.content.pm.PackageManager
 import android.graphics.drawable.Animatable
 import android.graphics.drawable.Drawable
+import android.os.Build
 import android.os.Bundle
 import android.os.IBinder
 import android.view.*
@@ -31,9 +34,9 @@ import android.view.animation.AnticipateInterpolator
 import android.view.animation.OvershootInterpolator
 import android.view.animation.TranslateAnimation
 import android.widget.Toast
+import androidx.activity.result.contract.ActivityResultContracts
 import androidx.appcompat.app.AlertDialog
 import androidx.core.content.ContextCompat
-import androidx.core.content.res.ResourcesCompat
 import androidx.preference.PreferenceManager
 import androidx.recyclerview.widget.LinearLayoutManager
 import androidx.recyclerview.widget.PagerSnapHelper
@@ -43,6 +46,7 @@ import androidx.vectordrawable.graphics.drawable.AnimatedVectorDrawableCompat
 import com.bumptech.glide.Glide
 import com.google.android.material.snackbar.Snackbar
 import com.google.firebase.crashlytics.FirebaseCrashlytics
+import com.google.mlkit.vision.face.Face
 import com.qtalk.recyclerviewfastscroller.RecyclerViewFastScroller
 import kotlinx.android.synthetic.main.activity_reader.*
 import kotlinx.android.synthetic.main.activity_reader.view.*
@@ -50,6 +54,7 @@ import kotlinx.android.synthetic.main.dialog_numberpicker.view.*
 import kotlinx.android.synthetic.main.reader_eye_card.view.*
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import xyz.quaver.Code
 import xyz.quaver.pupil.R
@@ -62,7 +67,7 @@ import xyz.quaver.pupil.util.camera
 import xyz.quaver.pupil.util.closeCamera
 import xyz.quaver.pupil.util.downloader.Cache
 import xyz.quaver.pupil.util.downloader.DownloadManager
-import xyz.quaver.pupil.util.testCamera
+import xyz.quaver.pupil.util.startCamera
 import java.util.*
 import kotlin.concurrent.schedule
 
@@ -97,11 +102,29 @@ class ReaderActivity : BaseActivity() {
     }
 
     private val timer = Timer()
-
     private val snapHelper = PagerSnapHelper()
-
     private var menu: Menu? = null
 
+    private val requestPermissionLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { isGranted ->
+        if (isGranted)
+            toggleCamera()
+        else
+            AlertDialog.Builder(this)
+                .setTitle(R.string.error)
+                .setMessage(R.string.camera_denied)
+                .setPositiveButton(android.R.string.ok) { _, _ ->}
+                .show()
+    }
+
+    enum class Eye {
+        LEFT,
+        RIGHT
+    }
+
+    private var cameraEnabled = false
+    private var eyeType: Eye? = null
+    private var eyeCount: Int = 0
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_reader)
@@ -226,6 +249,18 @@ class ReaderActivity : BaseActivity() {
         return true
     }
 
+    override fun onResume() {
+        super.onResume()
+
+        if (cameraEnabled)
+            startCamera(this, cameraCallback)
+    }
+
+    override fun onPause() {
+        super.onPause()
+        closeCamera()
+    }
+
     override fun onDestroy() {
         super.onDestroy()
 
@@ -390,66 +425,21 @@ class ReaderActivity : BaseActivity() {
         }
 
         with(reader_fab_auto) {
-            setImageResource(R.drawable.clock_start)
+            setImageResource(R.drawable.eye_white)
             setOnClickListener {
-                val eyes = this@ReaderActivity.eye_card
+                when {
-                when (camera) {
+                    ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED -> {
-                    null -> {
+                        toggleCamera()
-                        eyes.apply {
-                            visibility = View.VISIBLE
-                            TranslateAnimation(0F, 0F, -100F, 0F).apply {
-                                duration = 500
-                                fillAfter = false
-                                interpolator = OvershootInterpolator()
-                            }.let { startAnimation(it) }
-                        }
-                        testCamera(context) { faces ->
-                            eyes.dot.let {
-                                it.visibility = View.VISIBLE
-                                Timer().schedule(50) {
-                                    runOnUiThread {
-                                        it.visibility = View.GONE
-                                    }
-                                }
-                            }
-
-                            if (faces.size != 1)
-                                ResourcesCompat.getDrawable(resources, R.drawable.eye_off, context.theme).let {
-                                    eyes.left_eye.setImageDrawable(it)
-                                    eyes.right_eye.setImageDrawable(it)
-
-                                    return@testCamera
-                                }
-
-                            val left = ResourcesCompat.getDrawable(resources,
-                                if (faces[0].rightEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed,
-                            context.theme)
-                            val right = ResourcesCompat.getDrawable(resources,
-                                if (faces[0].leftEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed,
-                            context.theme)
-
-                            eyes.left_eye.setImageDrawable(left)
-                            eyes.right_eye.setImageDrawable(right)
-                        }
                     }
-                    else -> {
+                    Build.VERSION.SDK_INT >= 23 && shouldShowRequestPermissionRationale(Manifest.permission.CAMERA) -> {
-                        eyes.apply {
+                        AlertDialog.Builder(this@ReaderActivity)
-                            TranslateAnimation(0F, 0F, 0F, -100F).apply {
+                            .setTitle(R.string.warning)
-                                duration = 500
+                            .setMessage(R.string.camera_denied)
-                                fillAfter = false
+                            .setPositiveButton(android.R.string.ok) { _, _ ->}
-                                interpolator = AnticipateInterpolator()
+                            .show()
-                                setAnimationListener(object: Animation.AnimationListener {
-                                    override fun onAnimationStart(p0: Animation?) {}
-                                    override fun onAnimationRepeat(p0: Animation?) {}
-
-                                    override fun onAnimationEnd(p0: Animation?) {
-                                        eyes.visibility = View.GONE
-                                    }
-                                })
-                            }.let { startAnimation(it) }
-                        }
-                        closeCamera()
                     }
+                    else ->
+                        requestPermissionLauncher.launch(Manifest.permission.CAMERA)
                 }
             }
         }
@@ -502,6 +492,17 @@ class ReaderActivity : BaseActivity() {
         } else {
             snapHelper.attachToRecyclerView(reader_recyclerview)
             reader_recyclerview.layoutManager = LinearLayoutManager(this, LinearLayoutManager.HORIZONTAL, Preferences["rtl", false])
+
+            if (Preferences["rtl", false])
+                with(reader_progressbar) {
+                    scaleX = -1F
+                    translationX = 1F
+                }
+            else
+                with(reader_progressbar) {
+                    scaleX = 0F
+                    translationX = 0F
+                }
         }
 
         (reader_recyclerview.layoutManager as LinearLayoutManager).scrollToPositionWithOffset(currentPage-1, 0)
@@ -536,6 +537,125 @@ class ReaderActivity : BaseActivity() {
         }
     }
 
+    val cameraCallback: (List<Face>) -> Unit = callback@{ faces ->
+        eye_card.dot.let {
+            it.visibility = View.VISIBLE
+            CoroutineScope(Dispatchers.Main).launch {
+                delay(50)
+                it.visibility = View.INVISIBLE
+            }
+        }
+
+        if (faces.size != 1)
+            ContextCompat.getDrawable(this, R.drawable.eye_off).let {
+                with(eye_card) {
+                    left_eye.setImageDrawable(it)
+                    right_eye.setImageDrawable(it)
+                }
+
+                return@callback
+            }
+
+        val (left, right) = Pair(
+            faces[0].rightEyeOpenProbability?.let { it > 0.4 } == true,
+            faces[0].leftEyeOpenProbability?.let { it > 0.4 } == true
+        )
+
+        with(eye_card) {
+            left_eye.setImageDrawable(
+                ContextCompat.getDrawable(
+                    context,
+                    if (left) R.drawable.eye else R.drawable.eye_closed
+                )
+            )
+            right_eye.setImageDrawable(
+                ContextCompat.getDrawable(
+                    context,
+                    if (right) R.drawable.eye else R.drawable.eye_closed
+                )
+            )
+        }
+
+        when {
+            // Both closed / opened
+            !left.xor(right) -> {
+                eyeType = null
+                eyeCount = 0
+            }
+            !left -> {
+                if (eyeType != Eye.LEFT) {
+                    eyeType = Eye.LEFT
+                    eyeCount = 0
+                }
+                eyeCount++
+            }
+            !right -> {
+                if (eyeType != Eye.RIGHT) {
+                    eyeType = Eye.RIGHT
+                    eyeCount = 0
+                }
+                eyeCount++
+            }
+        }
+
+        if (eyeCount > 3) {
+            (this@ReaderActivity.reader_recyclerview.layoutManager as LinearLayoutManager).let {
+                it.scrollToPositionWithOffset(when(eyeType!!) {
+                    Eye.RIGHT -> {
+                        if (it.reverseLayout) currentPage - 2 else currentPage
+                    }
+                    Eye.LEFT -> {
+                        if (it.reverseLayout) currentPage else currentPage - 2
+                    }
+                }, 0)
+            }
+
+            eyeType = null
+            eyeCount = 0
+        }
+    }
+
+    private fun toggleCamera() {
+        val eyes = this@ReaderActivity.eye_card
+        when (camera) {
+            null -> {
+                reader_fab_auto.labelText = getString(R.string.reader_fab_auto_cancel)
+                reader_fab_auto.setImageResource(R.drawable.eye_off_white)
+                eyes.apply {
+                    visibility = View.VISIBLE
+                    TranslateAnimation(0F, 0F, -100F, 0F).apply {
+                        duration = 500
+                        fillAfter = false
+                        interpolator = OvershootInterpolator()
+                    }.let { startAnimation(it) }
+                }
+                startCamera(this, cameraCallback)
+                cameraEnabled = true
+            }
+            else -> {
+                reader_fab_auto.labelText = getString(R.string.reader_fab_auto)
+                reader_fab_auto.setImageResource(R.drawable.eye_white)
+                eyes.apply {
+                    TranslateAnimation(0F, 0F, 0F, -100F).apply {
+                        duration = 500
+                        fillAfter = false
+                        interpolator = AnticipateInterpolator()
+                        setAnimationListener(object: Animation.AnimationListener {
+                            override fun onAnimationStart(p0: Animation?) {}
+                            override fun onAnimationRepeat(p0: Animation?) {}
+
+                            override fun onAnimationEnd(p0: Animation?) {
+                                eyes.visibility = View.GONE
+                            }
+                        })
+                    }.let { startAnimation(it) }
+                }
+                closeCamera()
+                cameraEnabled = false
+            }
+        }
+    }
+
     override fun onLowMemory() {
         super.onLowMemory()
         Glide.get(this).onLowMemory()

app/src/main/java/xyz/quaver/pupil/util/camera.kt

@@ -16,13 +16,14 @@
  *     along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-@file:Suppress("DEPRECATION")
+@file:Suppress("DEPRECATION", "Recycle")
 
 package xyz.quaver.pupil.util
 
 import android.content.Context
 import android.content.pm.PackageManager
 import android.graphics.ImageFormat
+import android.graphics.SurfaceTexture
 import android.hardware.Camera
 import android.view.Surface
 import android.view.WindowManager
@@ -68,6 +69,7 @@ private fun getRotation(context: Context, cameraID: Int): Int {
 }
 
 var camera: Camera? = null
+var surfaceTexture: SurfaceTexture? = null
 private val detector = FaceDetection.getClient(
     FaceDetectorOptions.Builder()
         .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL)
@@ -75,7 +77,7 @@ private val detector = FaceDetection.getClient(
 )
 private var process: Task<List<Face>>? = null
 
-fun testCamera(context: Context, callback: (List<Face>) -> Unit) {
+fun startCamera(context: Context, callback: (List<Face>) -> Unit) {
     if (camera != null) closeCamera()
 
     val cameraID = openFrontCamera().let { (cam, cameraID) ->
@@ -88,9 +90,13 @@ fun testCamera(context: Context, callback: (List<Face>) -> Unit) {
         parameters = parameters.apply {
             setPreviewSize(640, 480)
             previewFormat = ImageFormat.NV21
-            flashMode = Camera.Parameters.FLASH_MODE_OFF
         }
-        setPreviewCallback { bytes, camera ->
+
+        setPreviewTexture(surfaceTexture ?: SurfaceTexture(0).also {
+            surfaceTexture = it
+        })
+        startPreview()
+        setPreviewCallback { bytes, _ ->
             if (process?.isComplete == false)
                 return@setPreviewCallback
 
@@ -100,14 +106,14 @@ fun testCamera(context: Context, callback: (List<Face>) -> Unit) {
             process = detector.process(image)
                 .addOnSuccessListener(callback)
         }
-
-        startPreview()
     }
 }
 
 fun closeCamera() {
     camera?.setPreviewCallback(null)
     camera?.stopPreview()
+    surfaceTexture?.release()
+    surfaceTexture = null
     camera?.release()
     camera = null
 }

app/src/main/res/drawable/eye_off_white.xml

@@ -0,0 +1,8 @@
+<!-- drawable/eye_off.xml -->
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:height="24dp"
+    android:width="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24">
+    <path android:fillColor="#fff" android:pathData="M11.83,9L15,12.16C15,12.11 15,12.05 15,12A3,3 0 0,0 12,9C11.94,9 11.89,9 11.83,9M7.53,9.8L9.08,11.35C9.03,11.56 9,11.77 9,12A3,3 0 0,0 12,15C12.22,15 12.44,14.97 12.65,14.92L14.2,16.47C13.53,16.8 12.79,17 12,17A5,5 0 0,1 7,12C7,11.21 7.2,10.47 7.53,9.8M2,4.27L4.28,6.55L4.73,7C3.08,8.3 1.78,10 1,12C2.73,16.39 7,19.5 12,19.5C13.55,19.5 15.03,19.2 16.38,18.66L16.81,19.08L19.73,22L21,20.73L3.27,3M12,7A5,5 0 0,1 17,12C17,12.64 16.87,13.26 16.64,13.82L19.57,16.75C21.07,15.5 22.27,13.86 23,12C21.27,7.61 17,4.5 12,4.5C10.6,4.5 9.26,4.75 8,5.2L10.17,7.35C10.74,7.13 11.35,7 12,7Z" />
+</vector>

app/src/main/res/drawable/eye_white.xml

@@ -0,0 +1,8 @@
+<!-- drawable/eye.xml -->
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:height="24dp"
+    android:width="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24">
+    <path android:fillColor="#fff" android:pathData="M12,9A3,3 0 0,0 9,12A3,3 0 0,0 12,15A3,3 0 0,0 15,12A3,3 0 0,0 12,9M12,17A5,5 0 0,1 7,12A5,5 0 0,1 12,7A5,5 0 0,1 17,12A5,5 0 0,1 12,17M12,4.5C7,4.5 2.73,7.61 1,12C2.73,16.39 7,19.5 12,19.5C17,19.5 21.27,16.39 23,12C21.27,7.61 17,4.5 12,4.5Z" />
+</vector>

app/src/main/res/layout/activity_reader.xml

@@ -106,7 +106,7 @@
             android:id="@+id/reader_fab_auto"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
-            app:srcCompat="@drawable/clock_start"
+            app:srcCompat="@drawable/eye_white"
             app:fab_label="@string/reader_fab_auto"
             app:fab_size="mini"/>
 

app/src/main/res/layout/item_mirrors.xml

@@ -42,7 +42,6 @@
         android:layout_width="wrap_content"
         android:layout_height="wrap_content"
         app:srcCompat="@drawable/menu"
-        app:tint="?attr/colorControlNormal"
         app:layout_constraintEnd_toEndOf="parent"
         app:layout_constraintTop_toTopOf="parent"
         app:layout_constraintBottom_toBottomOf="parent"

app/src/main/res/layout/reader_eye_card.xml

@@ -55,7 +55,7 @@
             android:id="@+id/dot"
             android:layout_width="4dp"
             android:layout_height="4dp"
-            android:visibility="gone"
+            android:visibility="invisible"
             app:srcCompat="@drawable/dot"
             app:layout_constraintLeft_toLeftOf="@id/left_eye"
             app:layout_constraintRight_toRightOf="@id/right_eye"

app/src/main/res/values-ja/strings.xml

@@ -132,7 +132,7 @@
     <string name="settings_user_id">ユーザーID</string>
     <string name="settings_user_id_toast">ユーザーIDをクリップボードにコピーしました</string>
     <string name="reader_fab_retry">リトライ</string>
-    <string name="reader_fab_auto">自動スクロール</string>
+    <string name="reader_fab_auto">まばたき検知スクロール</string>
     <string name="search_all">全てのギャラリーを対象に検索</string>
     <string name="settings_rtl">綴じ方向を左にする</string>
     <string name="settings_manage_favorites">ブックマーク管理</string>
@@ -148,4 +148,8 @@
     <string name="settings_oss">オープンソースライセンス</string>
     <string name="search_show_tags">お気に入りのタグを見る</string>
     <string name="search_show_histories">履歴を見る</string>
+    <string name="reader_fab_auto_cancel">まばたき検知を中止</string>
+    <string name="camera_denied">カメラ権限が拒否されているため、まばたき検知使用できません</string>
+    <string name="no_camera">この機器には前面カメラが装着されていません</string>
+    <string name="error">エラー</string>
 </resources>

app/src/main/res/values-ko/strings.xml

@@ -132,7 +132,7 @@
     <string name="settings_user_id">유저 ID</string>
     <string name="settings_user_id_toast">유저 ID를 클립보드에 복사했습니다</string>
     <string name="reader_fab_retry">재시도</string>
-    <string name="reader_fab_auto">자동 스크롤</string>
+    <string name="reader_fab_auto">눈 깜빡임 감지 스크롤</string>
     <string name="search_all">모든 갤러리 검색</string>
     <string name="settings_rtl">좌측으로 페이지 넘기기</string>
     <string name="settings_manage_favorites">즐겨찾기 관리</string>
@@ -148,4 +148,8 @@
     <string name="settings_oss">오픈 소스 라이선스</string>
     <string name="search_show_histories">검색 기록 보기</string>
     <string name="search_show_tags">즐겨찾기 태그 보기</string>
+    <string name="reader_fab_auto_cancel">눈 깜빡임 감지 중지</string>
+    <string name="camera_denied">카메라 권한이 거부되었기 때문에 눈 깜빡임 감지가 불가능합니다</string>
+    <string name="no_camera">이 장치에는 전면 카메라가 없습니다</string>
+    <string name="error">오류</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -20,6 +20,7 @@
     <!-- Translate needed down here -->
 
     <string name="warning">Warning</string>
+    <string name="error">Error</string>
 
     <string name="ignore_update">Ignore</string>
 
@@ -99,12 +100,16 @@
     <string name="reader_go_to_page">Go to page</string>
     <string name="reader_fab_fullscreen">Fullscreen</string>>
     <string name="reader_fab_retry">Retry</string>
-    <string name="reader_fab_auto">Automatic scroll</string>
+    <string name="reader_fab_auto">Scroll with eye blink</string>
+    <string name="reader_fab_auto_cancel">Stop scroll with eye blink</string>
     <string name="reader_fab_download">Background download</string>
     <string name="reader_fab_download_cancel">Cancel background download</string>
     <string name="reader_notification_text">Downloading&#8230;</string>
     <string name="reader_notification_complete">Download complete</string>
 
+    <string name="camera_denied">Eye blink detection cannot be used without a permission</string>
+    <string name="no_camera">There is no front facing camera in this device</string>
+
     <!-- DOWNLOADER -->
     <string name="downloader_running">Downloader running…</string>