Rebuilding Downloader

2020-01-29 15:46:23 +09:00
parent 8a9ab6b36c
commit 9d80857a38
6 changed files with 237 additions and 89 deletions
--- a/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt
+++ b/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt
@@ -37,6 +37,7 @@ import xyz.quaver.hiyobi.createImgList
 import xyz.quaver.hiyobi.getReader
 import xyz.quaver.hiyobi.user_agent
 import xyz.quaver.pupil.ui.LockActivity
+import xyz.quaver.pupil.util.download.DownloadWorker
 import xyz.quaver.pupil.util.getDownloadDirectory
 import xyz.quaver.pupil.util.updateOldReaderGalleries
 import java.io.File
@@ -118,4 +119,24 @@ class ExampleInstrumentedTest {

        updateOldReaderGalleries(context)
    }
+
+    @Test
+    fun test_downloadWorker() {
+        val context = InstrumentationRegistry.getInstrumentation().targetContext
+
+        val galleryID = 515515
+
+        val worker = DownloadWorker.getInstance(context)
+
+        worker.queue.add(galleryID)
+
+        while(worker.progress.indexOfKey(galleryID) < 0 || worker.progress[galleryID] != null) {
+            Log.i("PUPILD", worker.progress[galleryID]?.joinToString(" ") ?: "null")
+
+            if (worker.progress[galleryID]?.all { !it.isFinite() } == true)
+                break
+        }
+
+        Log.i("PUPILD", "DONE!!")
+    }
 }
--- a/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt
+++ b/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt
@@ -19,7 +19,6 @@
 package xyz.quaver.pupil.adapters

 import android.graphics.drawable.Drawable
-import android.util.Log
 import android.util.SparseBooleanArray
 import android.view.LayoutInflater
 import android.view.View
@@ -130,8 +129,6 @@ class GalleryBlockAdapter(private val glide: RequestManager, private val galleri
                            with(view.galleryblock_progressbar) {
                                progress = imageCache.invoke().list()?.size ?: 0

-                                Log.i("PUPILD", progress.toString())
-
                                if (!readerCache.invoke().exists()) {
                                    visibility = View.GONE
                                    max = 0
--- a/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt
@@ -20,6 +20,7 @@ package xyz.quaver.pupil.util.download

 import android.content.Context
 import android.content.ContextWrapper
+import android.util.Base64
 import android.util.SparseArray
 import androidx.core.content.ContextCompat
 import androidx.preference.PreferenceManager
@@ -30,7 +31,10 @@ import kotlinx.serialization.parse
 import kotlinx.serialization.stringify
 import xyz.quaver.hitomi.GalleryBlock
 import xyz.quaver.hitomi.Reader
+import xyz.quaver.pupil.util.getDownloadDirectory
+import xyz.quaver.pupil.util.isParentOf
 import java.io.File
+import java.net.URL

 class Cache(context: Context) : ContextWrapper(context) {

@@ -86,6 +90,29 @@ class Cache(context: Context) : ContextWrapper(context) {
        }
    }

+    suspend fun getThumbnail(galleryID: Int): String? {
+        val metadata = Cache(this).getCachedMetadata(galleryID)
+
+        val thumbnail = if (metadata?.thumbnail == null)
+            withContext(Dispatchers.IO) {
+                val thumbnails = getGalleryBlock(galleryID)?.thumbnails
+                try {
+                    Base64.encodeToString(URL(thumbnails?.firstOrNull()).readBytes(), Base64.DEFAULT)
+                } catch (e: Exception) {
+                    null
+                }
+            }
+        else
+            metadata.thumbnail
+
+        setCachedMetadata(
+            galleryID,
+            Metadata(Cache(this).getCachedMetadata(galleryID), thumbnail = thumbnail)
+        )
+
+        return thumbnail
+    }
+
    suspend fun getGalleryBlock(galleryID: Int): GalleryBlock? {
        val metadata = Cache(this).getCachedMetadata(galleryID)

@@ -102,7 +129,7 @@ class Cache(context: Context) : ContextWrapper(context) {

        setCachedMetadata(
            galleryID,
-            Metadata(metadata, galleryBlock = galleryBlock)
+            Metadata(Cache(this).getCachedMetadata(galleryID), galleryBlock = galleryBlock)
        )

        return galleryBlock
@@ -129,7 +156,7 @@ class Cache(context: Context) : ContextWrapper(context) {
        if (readers.isNotEmpty())
            setCachedMetadata(
                galleryID,
-                Metadata(metadata, readers = readers)
+                Metadata(Cache(this).getCachedMetadata(galleryID), readers = readers)
            )

        val mirrors = preference.getString("mirrors", "")!!.split('>')
@@ -140,22 +167,44 @@ class Cache(context: Context) : ContextWrapper(context) {
    }

    fun getImages(galleryID: Int): SparseArray<File>? {
-        val regex = Regex("[0-9]+")
        val gallery = getCachedGallery(galleryID) ?: return null

        return SparseArray<File>().apply {
            gallery.listFiles { file ->
-                file.nameWithoutExtension.matches(regex)
+                file.nameWithoutExtension.toIntOrNull() != null
            }?.forEach {
                append(it.nameWithoutExtension.toInt(), it)
            }
        }
    }

-    fun putImage(galleryID: Int, index: Int, data: ByteArray) {
+    fun putImage(galleryID: Int, name: String, data: ByteArray) {
        val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID")

-        File(cache, index.toString()).writeBytes(data)
+        with(File(cache, name)) {
+
+            if (!parentFile!!.exists())
+                parentFile!!.mkdirs()
+
+            if (!exists())
+                createNewFile()
+
+            if (nameWithoutExtension.toIntOrNull() != null)
+                writeBytes(data)
+            else
+                IllegalArgumentException("File name is not a number")
+        }
+    }
+
+    fun moveToDownload(galleryID: Int) {
+        val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID")
+
+        val download = getDownloadDirectory(this)
+
+        if (!download.isParentOf(cache)) {
+            cache.copyRecursively(download)
+            cache.deleteRecursively()
+        }
    }

 }
--- a/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt
@@ -21,28 +21,39 @@ package xyz.quaver.pupil.util.download
 import android.content.Context
 import android.content.ContextWrapper
 import android.content.SharedPreferences
+import android.util.SparseArray
 import androidx.preference.PreferenceManager
 import com.crashlytics.android.Crashlytics
 import io.fabric.sdk.android.Fabric
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.ExperimentalCoroutinesApi
-import kotlinx.coroutines.channels.Channel
-import kotlinx.coroutines.launch
+import kotlinx.coroutines.*
 import okhttp3.*
 import okio.*
 import xyz.quaver.hitomi.Reader
+import xyz.quaver.hitomi.getReferer
 import xyz.quaver.hitomi.urlFromUrlFromHash
+import xyz.quaver.hiyobi.cookie
 import xyz.quaver.hiyobi.createImgList
-import java.io.FileInputStream
+import xyz.quaver.hiyobi.user_agent
 import java.io.IOException
+import java.util.concurrent.Executors
+import java.util.concurrent.LinkedBlockingQueue

@UseExperimental(ExperimentalCoroutinesApi::class)
 class DownloadWorker private constructor(context: Context) : ContextWrapper(context) {

-    val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)
+    private val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)

    //region ProgressListener
+    @Suppress("UNCHECKED_CAST")
+    private val progressListener = object: ProgressListener {
+        override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) {
+            val (galleryID, index) = (tag as? Pair<Int, Int>) ?: return
+
+            if (!done && progress[galleryID]!![index] != Float.POSITIVE_INFINITY)
+                progress[galleryID]!![index] = bytesRead * 100F / contentLength
+        }
+    }
+
    interface ProgressListener {
        fun update(tag: Any?, bytesRead : Long, contentLength: Long, done: Boolean)
    }
@@ -52,7 +63,7 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
        val responseBody: ResponseBody,
        val progressListener : ProgressListener
    ) : ResponseBody() {
-        var bufferedSource : BufferedSource? = null
+        private var bufferedSource : BufferedSource? = null

        override fun contentLength() = responseBody.contentLength()
        override fun contentType() = responseBody.contentType()
@@ -93,23 +104,46 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
    }
    //endregion

-    val queue = Channel<Int>()
-    /* VALUE
+    val queue = LinkedBlockingQueue<Int>()
+
+    /*
+    * KEY
+    *  primary galleryID
+    *  secondary index
+    * PRIMARY VALUE
+    *  MutableList -> Download in progress
+    *  null -> Loading / Gallery doesn't exist
+    * SECONDARY VALUE
    *  0 <= value < 100 -> Download in progress
    *  Float.POSITIVE_INFINITY -> Download completed
    *  Float.NaN -> Exception
    */
-    val progress = mutableMapOf<String, Float>()
-    val result = mutableMapOf<String, ByteArray>()
-    val exception = mutableMapOf<String, Throwable>()
+    val progress = SparseArray<MutableList<Float>?>()
+    /*
+    * KEY
+    *  primary galleryID
+    *  secondary index
+    * PRIMARY VALUE
+    *  MutableList -> Download in progress / Loading
+    *  null -> Gallery doesn't exist
+    * SECONDARY VALUE
+    *  Throwable -> Exception
+    *  null -> Download in progress / Loading
+    */
+    val exception = SparseArray<MutableList<Throwable?>?>()

-    val client = OkHttpClient.Builder()
-        .addNetworkInterceptor { chain ->
+    private val loop = loop()
+    private val worker = SparseArray<Job?>()
+    @Volatile var nRunners = 0
+
+    private val client = OkHttpClient.Builder()
+        .addInterceptor { chain ->
            val request = chain.request()
            var response = chain.proceed(request)

            var retry = preferences.getInt("retry", 3)
            while (!response.isSuccessful && retry > 0) {
+                response.close()
                response = chain.proceed(request)
                retry--
            }
@@ -117,72 +151,121 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
            response.newBuilder()
                .body(ProgressResponseBody(request.tag(), response.body!!, progressListener))
                .build()
-        }.build()
-
-
-    val progressListener = object: ProgressListener {
-        override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) {
-            if (tag !is String)
-                return
-
-            if (progress[tag] != Float.POSITIVE_INFINITY)
-                progress[tag] = bytesRead / contentLength.toFloat()
        }
+        .dispatcher(Dispatcher(Executors.newSingleThreadExecutor()))
+        .build()
+
+    fun stop() {
+        loop.cancel()
+        for (i in 0..worker.size())
+            worker[worker.keyAt(i)]?.cancel()
+
+        client.dispatcher.cancelAll()
    }
-    init {
-        CoroutineScope(Dispatchers.Unconfined).launch {
-            while (!(queue.isEmpty && queue.isClosedForReceive)) {
-                val lowQuality = preferences.getBoolean("low_quality", false)
-                val galleryID = queue.receive()

-                launch(Dispatchers.IO) io@{
-                    val reader = Cache(context).getReader(galleryID) ?: return@io
-                    val cache = Cache(context).getImages(galleryID)
+    fun cancel(galleryID: Int) {
+        worker[galleryID]?.cancel()

-                    reader.galleryInfo.forEachIndexed { index, galleryInfo ->
-                        val tag = "$galleryID-$index"
-                        val url = when(reader.code) {
-                            Reader.Code.HITOMI ->
-                                urlFromUrlFromHash(galleryID, galleryInfo, if (lowQuality) "webp" else null)
-                            Reader.Code.HIYOBI ->
-                                createImgList(galleryID, reader, lowQuality)[index].path
-                            else -> ""  //Shouldn't be called anyways
-                        }
+        client.dispatcher.queuedCalls()
+            .filter { it.request().tag(Pair::class.java)?.first == galleryID }
+            .forEach {
+                it.cancel()
+            }
+    }

-                        //Cache exists :P
-                        cache?.get(index)?.let {
-                            result[tag] = FileInputStream(it).readBytes()
-                            progress[tag] = Float.POSITIVE_INFINITY
+    private fun queueDownload(galleryID: Int, reader: Reader, index: Int, callback: Callback) {
+        val cache = Cache(this@DownloadWorker).getImages(galleryID)
+        val lowQuality = preferences.getBoolean("low_quality", false)

-                            return@io
-                        }
+        //Cache exists :P
+        cache?.get(index)?.let {
+            progress[galleryID]!![index] = Float.POSITIVE_INFINITY

-                        val request = Request.Builder()
-                            .url(url)
-                            .tag(tag)
-                            .build()
+            return
+        }

-                        client.newCall(request).enqueue(object: Callback {
-                            override fun onFailure(call: Call, e: IOException) {
-                                if (Fabric.isInitialized())
-                                    Crashlytics.logException(e)
-
-                                progress[tag] = Float.NaN
-                                exception[tag] = e
-                            }
-
-                            override fun onResponse(call: Call, response: Response) {
-                                response.use {
-                                    val res = it.body!!.bytes()
-                                    result[tag] = res
-                                    Cache(context).putImage(galleryID, index, res)
-                                    progress[tag] = Float.POSITIVE_INFINITY
-                                }
-                            }
-                        })
-                    }
+        val request = Request.Builder().apply {
+            when (reader.code) {
+                Reader.Code.HITOMI -> {
+                    url(
+                        urlFromUrlFromHash(
+                            galleryID,
+                            reader.galleryInfo[index],
+                            if (lowQuality) "webp" else null
+                        )
+                    )
+                    addHeader("Referer", getReferer(galleryID))
+                }
+                Reader.Code.HIYOBI -> {
+                    url(createImgList(galleryID, reader, lowQuality)[index].path)
+                    addHeader("User-Agent", user_agent)
+                    addHeader("Cookie", cookie)
+                }
+                else -> {
+                    //shouldn't be called anyway
                }
            }
+            tag(galleryID to index)
+        }.build()
+
+        client.newCall(request).enqueue(callback)
+    }
+
+    private fun download(galleryID: Int) = CoroutineScope(Dispatchers.IO).launch {
+        val reader = Cache(this@DownloadWorker).getReader(galleryID)
+
+        //gallery doesn't exist
+        if (reader == null) {
+            progress.put(galleryID, null)
+            exception.put(galleryID, null)
+            nRunners--
+            return@launch
+        }
+
+        progress.put(galleryID, reader.galleryInfo.map { 0F }.toMutableList())
+        exception.put(galleryID, reader.galleryInfo.map { null }.toMutableList())
+
+        for (i in reader.galleryInfo.indices) {
+            val callback = object : Callback {
+                override fun onFailure(call: Call, e: IOException) {
+                    if (Fabric.isInitialized())
+                        Crashlytics.logException(e)
+
+                    progress[galleryID]!![i] = Float.NaN
+                    exception[galleryID]!![i] = e
+
+                    if (progress[galleryID]!!.all { !it.isFinite() })
+                        nRunners--
+                }
+
+                override fun onResponse(call: Call, response: Response) {
+                    response.use {
+                        val res = it.body!!.bytes()
+                        val ext =
+                            call.request().url.encodedPath.split('.').last()
+
+                        Cache(this@DownloadWorker).putImage(galleryID, "$i.$ext", res)
+                        progress[galleryID]!![i] = Float.POSITIVE_INFINITY
+                    }
+
+                    if (progress[galleryID]!!.all { !it.isFinite() })
+                        nRunners--
+                }
+            }
+
+            queueDownload(galleryID, reader, i, callback)
+        }
+    }
+
+    private fun loop() = CoroutineScope(Dispatchers.Default).launch {
+        while (true) {
+            if (queue.isEmpty() || nRunners > preferences.getInt("max_download", 4))
+                continue
+
+            val galleryID = queue.poll() ?: continue
+
+            worker.put(galleryID, download(galleryID))
+            nRunners++
        }
    }

--- a/app/src/main/java/xyz/quaver/pupil/util/file.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/file.kt
@@ -62,4 +62,6 @@ fun URL.download(to: File, onDownloadProgress: ((Long, Long) -> Unit)? = null) {
        }

    }
-}
+}
+
+fun File.isParentOf(file: File) = file.absolutePath.startsWith(this.absolutePath)
--- a/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt
+++ b/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt
@@ -26,20 +26,16 @@ package xyz.quaver.pupil
 * See [testing documentation](http://d.android.com/tools/testing).
 */

+import android.util.SparseArray
 import org.junit.Test
-import xyz.quaver.pupil.util.download
-import java.io.File
-import java.net.URL

 class ExampleUnitTest {

    @Test
    fun test() {
-        URL("https://github.com/tom5079/Pupil/releases/download/4.2-beta2-hotfix2/Pupil-v4.2-beta2-hotfix2.apk").download(
-            File(System.getenv("USERPROFILE"), "Pupil.apk")
-        ) { downloaded, fileSize ->
-            println("%.1f%%".format(downloaded*100.0/fileSize))
-        }
+        val arr = SparseArray<Float>()
+
+        print(arr.indexOfKey(34))
    }

 }