Implemented eye recognition

TODO: Move pages according to eye blinking
This commit is contained in:
tom5079
2020-09-12 09:48:30 +09:00
parent df3a478ef3
commit 7ed66b827f
12 changed files with 416 additions and 18 deletions

View File

@@ -78,6 +78,7 @@ dependencies {
implementation 'com.google.firebase:firebase-crashlytics:17.2.1' implementation 'com.google.firebase:firebase-crashlytics:17.2.1'
implementation 'com.google.firebase:firebase-perf:19.0.8' implementation 'com.google.firebase:firebase-perf:19.0.8'
implementation 'com.google.android.gms:play-services-oss-licenses:17.0.0' implementation 'com.google.android.gms:play-services-oss-licenses:17.0.0'
implementation 'com.google.android.gms:play-services-mlkit-face-detection:16.1.1'
implementation 'com.github.arimorty:floatingsearchview:2.1.1' implementation 'com.github.arimorty:floatingsearchview:2.1.1'
implementation 'com.github.clans:fab:1.6.4' implementation 'com.github.clans:fab:1.6.4'
//implementation 'com.quiph.ui:recyclerviewfastscroller:0.2.1' //implementation 'com.quiph.ui:recyclerviewfastscroller:0.2.1'

View File

@@ -6,10 +6,13 @@
<uses-permission android:name="android.permission.INTERNET" /> <uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" /> <uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
<uses-permission android:name="android.permission.USE_BIOMETRIC" /> <uses-permission android:name="android.permission.USE_BIOMETRIC" />
<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" /> <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="21"/>
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="21" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" android:required="false" />
<uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
<application <application
android:name=".Pupil" android:name=".Pupil"
@@ -24,6 +27,10 @@
tools:replace="android:theme" tools:replace="android:theme"
tools:ignore="UnusedAttribute"> tools:ignore="UnusedAttribute">
<meta-data
android:name="com.google.mlkit.vision.DEPENDENCIES"
android:value="face" />
<provider <provider
android:authorities="${applicationId}.provider" android:authorities="${applicationId}.provider"
android:name="androidx.core.content.FileProvider" android:name="androidx.core.content.FileProvider"

View File

@@ -26,9 +26,14 @@ import android.graphics.drawable.Drawable
import android.os.Bundle import android.os.Bundle
import android.os.IBinder import android.os.IBinder
import android.view.* import android.view.*
import android.view.animation.Animation
import android.view.animation.AnticipateInterpolator
import android.view.animation.OvershootInterpolator
import android.view.animation.TranslateAnimation
import android.widget.Toast import android.widget.Toast
import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AlertDialog
import androidx.core.content.ContextCompat import androidx.core.content.ContextCompat
import androidx.core.content.res.ResourcesCompat
import androidx.preference.PreferenceManager import androidx.preference.PreferenceManager
import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.LinearLayoutManager
import androidx.recyclerview.widget.PagerSnapHelper import androidx.recyclerview.widget.PagerSnapHelper
@@ -42,6 +47,7 @@ import com.qtalk.recyclerviewfastscroller.RecyclerViewFastScroller
import kotlinx.android.synthetic.main.activity_reader.* import kotlinx.android.synthetic.main.activity_reader.*
import kotlinx.android.synthetic.main.activity_reader.view.* import kotlinx.android.synthetic.main.activity_reader.view.*
import kotlinx.android.synthetic.main.dialog_numberpicker.view.* import kotlinx.android.synthetic.main.dialog_numberpicker.view.*
import kotlinx.android.synthetic.main.reader_eye_card.view.*
import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
@@ -52,11 +58,13 @@ import xyz.quaver.pupil.favorites
import xyz.quaver.pupil.histories import xyz.quaver.pupil.histories
import xyz.quaver.pupil.services.DownloadService import xyz.quaver.pupil.services.DownloadService
import xyz.quaver.pupil.util.Preferences import xyz.quaver.pupil.util.Preferences
import xyz.quaver.pupil.util.camera
import xyz.quaver.pupil.util.closeCamera
import xyz.quaver.pupil.util.downloader.Cache import xyz.quaver.pupil.util.downloader.Cache
import xyz.quaver.pupil.util.downloader.DownloadManager import xyz.quaver.pupil.util.downloader.DownloadManager
import xyz.quaver.pupil.util.testCamera
import java.util.* import java.util.*
import kotlin.concurrent.schedule import kotlin.concurrent.schedule
import kotlin.concurrent.timer
class ReaderActivity : BaseActivity() { class ReaderActivity : BaseActivity() {
@@ -89,7 +97,6 @@ class ReaderActivity : BaseActivity() {
} }
private val timer = Timer() private val timer = Timer()
private var autoTimer: Timer? = null
private val snapHelper = PagerSnapHelper() private val snapHelper = PagerSnapHelper()
@@ -385,23 +392,64 @@ class ReaderActivity : BaseActivity() {
with(reader_fab_auto) { with(reader_fab_auto) {
setImageResource(R.drawable.clock_start) setImageResource(R.drawable.clock_start)
setOnClickListener { setOnClickListener {
if (autoTimer == null) { val eyes = this@ReaderActivity.eye_card
autoTimer = timer(initialDelay = 10000L, period = 10000L) { when (camera) {
CoroutineScope(Dispatchers.Main).launch { null -> {
with(this@ReaderActivity.reader_recyclerview) { eyes.apply {
val lastItem = visibility = View.VISIBLE
(layoutManager as LinearLayoutManager).findLastCompletelyVisibleItemPosition() TranslateAnimation(0F, 0F, -100F, 0F).apply {
duration = 500
fillAfter = false
interpolator = OvershootInterpolator()
}.let { startAnimation(it) }
}
testCamera(context) { faces ->
eyes.dot.let {
it.visibility = View.VISIBLE
Timer().schedule(50) {
runOnUiThread {
it.visibility = View.GONE
}
}
}
if (lastItem < adapter!!.itemCount - 1) if (faces.size != 1)
(layoutManager as LinearLayoutManager).scrollToPosition(lastItem + 1) ResourcesCompat.getDrawable(resources, R.drawable.eye_off, context.theme).let {
eyes.left_eye.setImageDrawable(it)
eyes.right_eye.setImageDrawable(it)
return@testCamera
}
val left = ResourcesCompat.getDrawable(resources,
if (faces[0].rightEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed,
context.theme)
val right = ResourcesCompat.getDrawable(resources,
if (faces[0].leftEyeOpenProbability?.let { it > 0.4 } == true) R.drawable.eye else R.drawable.eye_closed,
context.theme)
eyes.left_eye.setImageDrawable(left)
eyes.right_eye.setImageDrawable(right)
} }
} }
else -> {
eyes.apply {
TranslateAnimation(0F, 0F, 0F, -100F).apply {
duration = 500
fillAfter = false
interpolator = AnticipateInterpolator()
setAnimationListener(object: Animation.AnimationListener {
override fun onAnimationStart(p0: Animation?) {}
override fun onAnimationRepeat(p0: Animation?) {}
override fun onAnimationEnd(p0: Animation?) {
eyes.visibility = View.GONE
}
})
}.let { startAnimation(it) }
}
closeCamera()
} }
setImageResource(R.drawable.clock_end)
} else {
autoTimer?.cancel()
autoTimer = null
setImageResource(R.drawable.clock_start)
} }
} }
} }

View File

@@ -0,0 +1,113 @@
/*
* Pupil, Hitomi.la viewer for Android
* Copyright (C) 2020 tom5079
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
@file:Suppress("DEPRECATION")
package xyz.quaver.pupil.util
import android.content.Context
import android.content.pm.PackageManager
import android.graphics.ImageFormat
import android.hardware.Camera
import android.view.Surface
import android.view.WindowManager
import com.google.android.gms.tasks.Task
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.face.Face
import com.google.mlkit.vision.face.FaceDetection
import com.google.mlkit.vision.face.FaceDetectorOptions
/** Check if this device has a camera */
private fun Context.checkCameraHardware() =
this.packageManager.hasSystemFeature(PackageManager.FEATURE_CAMERA)
private fun openFrontCamera() : Pair<Camera?, Int> {
var camera: Camera? = null
var cameraID: Int = -1
val cameraInfo = Camera.CameraInfo()
for (i in 0 until Camera.getNumberOfCameras()) {
Camera.getCameraInfo(i, cameraInfo)
if (cameraInfo.facing == Camera.CameraInfo.CAMERA_FACING_FRONT)
runCatching { Camera.open(i) }.getOrNull()?.let { camera = it; cameraID = i }
if (camera != null) break
}
return Pair(camera, cameraID)
}
val orientations = mapOf(
Surface.ROTATION_0 to 0,
Surface.ROTATION_90 to 90,
Surface.ROTATION_180 to 180,
Surface.ROTATION_270 to 270,
)
private fun getRotation(context: Context, cameraID: Int): Int {
val cameraRotation = Camera.CameraInfo().also { Camera.getCameraInfo(cameraID, it) }.orientation
val rotation = orientations[(context.getSystemService(Context.WINDOW_SERVICE) as WindowManager).defaultDisplay.rotation] ?: error("")
return (cameraRotation + rotation) % 360
}
var camera: Camera? = null
private val detector = FaceDetection.getClient(
FaceDetectorOptions.Builder()
.setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_ALL)
.build()
)
private var process: Task<List<Face>>? = null
fun testCamera(context: Context, callback: (List<Face>) -> Unit) {
if (camera != null) closeCamera()
val cameraID = openFrontCamera().let { (cam, cameraID) ->
cam ?: return
camera = cam
cameraID
}
with (camera!!) {
parameters = parameters.apply {
setPreviewSize(640, 480)
previewFormat = ImageFormat.NV21
flashMode = Camera.Parameters.FLASH_MODE_OFF
}
setPreviewCallback { bytes, camera ->
if (process?.isComplete == false)
return@setPreviewCallback
val rotation = getRotation(context, cameraID)
val image = InputImage.fromByteArray(bytes, 640, 480, rotation, InputImage.IMAGE_FORMAT_NV21)
process = detector.process(image)
.addOnSuccessListener(callback)
}
startPreview()
}
}
fun closeCamera() {
camera?.setPreviewCallback(null)
camera?.stopPreview()
camera?.release()
camera = null
}

View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Pupil, Hitomi.la viewer for Android
~ Copyright (C) 2020 tom5079
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<shape
xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="oval">
<solid
android:color="@color/colorAccent"/>
<size
android:width="24dp"
android:height="24dp"/>
</shape>

View File

@@ -0,0 +1,8 @@
<!-- drawable/eye.xml -->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:height="24dp"
android:width="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path android:fillColor="?attr/colorControlNormal" android:pathData="M12,9A3,3 0 0,0 9,12A3,3 0 0,0 12,15A3,3 0 0,0 15,12A3,3 0 0,0 12,9M12,17A5,5 0 0,1 7,12A5,5 0 0,1 12,7A5,5 0 0,1 17,12A5,5 0 0,1 12,17M12,4.5C7,4.5 2.73,7.61 1,12C2.73,16.39 7,19.5 12,19.5C17,19.5 21.27,16.39 23,12C21.27,7.61 17,4.5 12,4.5Z" />
</vector>

View File

@@ -0,0 +1,44 @@
<!--
~ Pupil, Hitomi.la viewer for Android
~ Copyright (C) 2020 tom5079
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="22dp"
android:height="15dp"
android:viewportWidth="22"
android:viewportHeight="15">
<path
android:pathData="M21.61,5.4C14.21,13.39 7.16,13.37 0.43,5.32"
android:strokeWidth="1"
android:strokeColor="?attr/colorControlNormal"/>
<path
android:pathData="M1.32,9.8L3.03,7.8"
android:strokeWidth="1"
android:strokeColor="?attr/colorControlNormal"/>
<path
android:pathData="M5.14,12.37L6.16,10.37"
android:strokeWidth="1"
android:strokeColor="?attr/colorControlNormal"/>
<path
android:pathData="M16.27,12.37L15.25,10.37"
android:strokeWidth="1"
android:strokeColor="?attr/colorControlNormal"/>
<path
android:pathData="M18.78,7.8L20.49,9.8"
android:strokeWidth="1"
android:strokeColor="?attr/colorControlNormal"/>
</vector>

View File

@@ -0,0 +1,8 @@
<!-- drawable/eye_off.xml -->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:height="24dp"
android:width="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path android:fillColor="?attr/colorControlNormal" android:pathData="M11.83,9L15,12.16C15,12.11 15,12.05 15,12A3,3 0 0,0 12,9C11.94,9 11.89,9 11.83,9M7.53,9.8L9.08,11.35C9.03,11.56 9,11.77 9,12A3,3 0 0,0 12,15C12.22,15 12.44,14.97 12.65,14.92L14.2,16.47C13.53,16.8 12.79,17 12,17A5,5 0 0,1 7,12C7,11.21 7.2,10.47 7.53,9.8M2,4.27L4.28,6.55L4.73,7C3.08,8.3 1.78,10 1,12C2.73,16.39 7,19.5 12,19.5C13.55,19.5 15.03,19.2 16.38,18.66L16.81,19.08L19.73,22L21,20.73L3.27,3M12,7A5,5 0 0,1 17,12C17,12.64 16.87,13.26 16.64,13.82L19.57,16.75C21.07,15.5 22.27,13.86 23,12C21.27,7.61 17,4.5 12,4.5C10.6,4.5 9.26,4.75 8,5.2L10.17,7.35C10.74,7.13 11.35,7 12,7Z" />
</vector>

View File

@@ -0,0 +1,30 @@
<!--
~ Pupil, Hitomi.la viewer for Android
~ Copyright (C) 2020 tom5079
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="640"
android:viewportHeight="640">
<path
android:pathData="M640,320C640,496.61 496.61,640 320,640C143.39,640 0,496.61 0,320C0,143.38 143.39,0 320,0C496.61,0 640,143.38 640,320Z"
android:fillColor="#4ec1f5"/>
<path
android:pathData="M420,320C420,375.19 375.19,420 320,420C264.81,420 220,375.19 220,320C220,264.81 264.81,220 320,220C375.19,220 420,264.81 420,320Z"
android:fillColor="#1d1d1d"/>
</vector>

View File

@@ -0,0 +1,30 @@
<!--
~ Pupil, Hitomi.la viewer for Android
~ Copyright (C) 2020 tom5079
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="640"
android:viewportHeight="640">
<path
android:pathData="M640,320C640,496.61 496.61,640 320,640C143.39,640 0,496.61 0,320C0,143.38 143.39,0 320,0C496.61,0 640,143.38 640,320Z"
android:fillColor="@color/colorAccent"/>
<path
android:pathData="M420,320C420,375.19 375.19,420 320,420C264.81,420 220,375.19 220,320C220,264.81 264.81,220 320,220C375.19,220 420,264.81 420,320Z"
android:fillColor="#1d1d1d"/>
</vector>

View File

@@ -45,6 +45,14 @@
</com.qtalk.recyclerviewfastscroller.RecyclerViewFastScroller> </com.qtalk.recyclerviewfastscroller.RecyclerViewFastScroller>
<include layout="@layout/reader_eye_card"
android:id="@+id/eye_card"
android:visibility="gone"
android:layout_height="wrap_content"
android:layout_width="wrap_content"
android:layout_gravity="center_horizontal"
android:layout_margin="8dp"/>
<LinearLayout <LinearLayout
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="wrap_content" android:layout_height="wrap_content"
@@ -80,6 +88,7 @@
android:id="@+id/reader_fab_download" android:id="@+id/reader_fab_download"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
app:srcCompat="@drawable/ic_download"
app:fab_label="@string/reader_fab_download" app:fab_label="@string/reader_fab_download"
app:fab_size="mini"/> app:fab_size="mini"/>
@@ -87,6 +96,7 @@
android:id="@+id/reader_fab_retry" android:id="@+id/reader_fab_retry"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
app:srcCompat="@drawable/refresh"
app:fab_label="@string/reader_fab_retry" app:fab_label="@string/reader_fab_retry"
app:fab_size="mini"/> app:fab_size="mini"/>
@@ -94,6 +104,7 @@
android:id="@+id/reader_fab_auto" android:id="@+id/reader_fab_auto"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
app:srcCompat="@drawable/clock_start"
app:fab_label="@string/reader_fab_auto" app:fab_label="@string/reader_fab_auto"
app:fab_size="mini"/> app:fab_size="mini"/>
@@ -101,6 +112,7 @@
android:id="@+id/reader_fab_fullscreen" android:id="@+id/reader_fab_fullscreen"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
app:srcCompat="@drawable/ic_fullscreen"
app:fab_label="@string/reader_fab_fullscreen" app:fab_label="@string/reader_fab_fullscreen"
app:fab_size="mini"/> app:fab_size="mini"/>

View File

@@ -0,0 +1,67 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Pupil, Hitomi.la viewer for Android
~ Copyright (C) 2020 tom5079
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU General Public License for more details.
~
~ You should have received a copy of the GNU General Public License
~ along with this program. If not, see <http://www.gnu.org/licenses/>.
-->
<androidx.cardview.widget.CardView xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
app:cardCornerRadius="16dp">
<androidx.constraintlayout.widget.ConstraintLayout
android:layout_width="wrap_content"
android:layout_height="wrap_content">
<ImageView
android:id="@+id/left_eye"
android:layout_width="8dp"
android:layout_height="8dp"
app:srcCompat="@drawable/eye_off"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintLeft_toLeftOf="parent"
android:layout_margin="4dp"
tools:ignore="ContentDescription" />
<ImageView
android:id="@+id/right_eye"
android:layout_width="8dp"
android:layout_height="8dp"
app:srcCompat="@drawable/eye_off"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintLeft_toRightOf="@id/left_eye"
app:layout_constraintRight_toRightOf="parent"
android:layout_margin="4dp"
tools:ignore="ContentDescription" />
<ImageView
android:id="@+id/dot"
android:layout_width="4dp"
android:layout_height="4dp"
android:visibility="gone"
app:srcCompat="@drawable/dot"
app:layout_constraintLeft_toLeftOf="@id/left_eye"
app:layout_constraintRight_toRightOf="@id/right_eye"
app:layout_constraintTop_toTopOf="parent"
tools:ignore="ContentDescription" />
</androidx.constraintlayout.widget.ConstraintLayout>
</androidx.cardview.widget.CardView>