WIP

2020-12-23 17:09:48 +09:00
parent 521f3ad809
commit 3051d800bd
20 changed files with 576 additions and 746 deletions
--- a/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt
@@ -20,79 +20,63 @@ package xyz.quaver.pupil.util.downloader

 import android.content.Context
 import android.content.ContextWrapper
-import android.net.Uri
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
-import kotlinx.coroutines.withContext
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import okhttp3.Request
-import xyz.quaver.hitomi.GalleryBlock
-import xyz.quaver.hitomi.GalleryInfo
 import xyz.quaver.io.FileX
-import xyz.quaver.io.util.*
-import xyz.quaver.pupil.client
-import xyz.quaver.pupil.util.Preferences
-import java.io.File
-import java.io.IOException
+import xyz.quaver.io.util.deleteRecursively
+import xyz.quaver.io.util.getChild
+import xyz.quaver.io.util.outputStream
+import xyz.quaver.io.util.writeText
+import xyz.quaver.pupil.sources.ItemInfo
+import xyz.quaver.pupil.sources.sources
+import java.io.InputStream
 import java.util.concurrent.ConcurrentHashMap

@Serializable
 data class Metadata(
-    var galleryBlock: GalleryBlock? = null,
-    var reader: GalleryInfo? = null,
+    var itemInfo: ItemInfo? = null,
    var imageList: MutableList<String?>? = null
 ) {
-    fun copy(): Metadata = Metadata(galleryBlock, reader, imageList?.let { MutableList(it.size) { i -> it[i] } })
+    fun copy(): Metadata = Metadata(itemInfo, imageList?.let { MutableList(it.size) { i -> it[i] } })
 }

-class Cache private constructor(context: Context, val galleryID: String) : ContextWrapper(context) {
+class Cache private constructor(context: Context, source: String, private val itemID: String) : ContextWrapper(context) {

    companion object {
        val instances = ConcurrentHashMap<String, Cache>()

-        fun getInstance(context: Context, galleryID: String) =
-            instances[galleryID] ?: synchronized(this) {
-                instances[galleryID] ?: Cache(context, galleryID).also { instances[galleryID] = it }
+        fun getInstance(context: Context, source: String, itemID: String): Cache {
+            val key = "$source/$itemID"
+            return instances[key] ?: synchronized(this) {
+                instances[key] ?: Cache(context, source, itemID).also { instances[key] = it }
            }
+        }

        @Synchronized
-        fun delete(context: Context, galleryID: String) {
-            File(context.cacheDir, "imageCache/$galleryID").deleteRecursively()
-            instances.remove(galleryID)
+        fun delete(source: String, itemID: String) {
+            val key = "$source/$itemID"
+
+            instances[key]?.cacheFolder?.deleteRecursively()
+            instances.remove("$source/$itemID")
        }
    }

-    init {
-        cacheFolder.mkdirs()
-    }
-
-    var metadata = kotlin.runCatching {
-        findFile(".metadata")?.readText()?.let {
-            Json.decodeFromString<Metadata>(it)
-        }
-    }.getOrNull() ?: Metadata()
+    val source = sources[source]!!

    val downloadFolder: FileX?
-        get() = DownloadManager.getInstance(this).getDownloadFolder(galleryID)
+        get() = DownloadManager.getInstance(this).getDownloadFolder(source.name, itemID)

    val cacheFolder: FileX
-        get() = FileX(this, cacheDir, "imageCache/$galleryID").also {
+        get() = FileX(this, cacheDir, "imageCache/$source/$itemID").also {
            if (!it.exists())
                it.mkdirs()
        }

-    fun findFile(fileName: String): FileX? =
-         downloadFolder?.let { downloadFolder -> downloadFolder.getChild(fileName).let {
-            if (it.exists()) it else null
-        } } ?: cacheFolder.getChild(fileName).let {
-            if (it.exists()) it else null
-        }
+    val metadata: Metadata = kotlin.runCatching {
+        Json.decodeFromString<Metadata>(findFile(".metadata")!!.readText())
+    }.getOrDefault(Metadata())

    @Suppress("BlockingMethodInNonBlockingContext")
    fun setMetadata(change: (Metadata) -> Unit) {
@@ -108,156 +92,26 @@ class Cache private constructor(context: Context, val galleryID: String) : Conte
        }
    }

-    suspend fun getGalleryBlock(): GalleryBlock? {
-        val sources = listOf(
-            { xyz.quaver.hitomi.getGalleryBlock(galleryID.toInt()) }
-           // { xyz.quaver.hiyobi.getGalleryBlock(galleryID) }
-        )
-
-        return metadata.galleryBlock
-            ?: withContext(Dispatchers.IO) {
-                var galleryBlock: GalleryBlock? = null
-
-                for (source in sources) {
-                    galleryBlock = try {
-                        source.invoke()
-                    } catch (e: Exception) { null }
-
-                    if (galleryBlock != null)
-                        break
-                }
-
-                galleryBlock?.also {
-                    setMetadata { metadata -> metadata.galleryBlock = it }
-                }
-            }
-    }
-
-    @Suppress("BlockingMethodInNonBlockingContext")
-    suspend fun getThumbnail(): Uri =
-        findFile(".thumbnail")?.uri
-            ?: getGalleryBlock()?.thumbnails?.firstOrNull()?.let { withContext(Dispatchers.IO) {
-                kotlin.runCatching {
-                    val request = Request.Builder()
-                        .url(it)
-                        .build()
-
-                    client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() }
-                }.getOrNull()?.let { thumbnail -> kotlin.runCatching {
-                    cacheFolder.getChild(".thumbnail").also {
-                        if (!it.exists())
-                            it.createNewFile()
-
-                        it.writeBytes(thumbnail)
-                    }
-                }.getOrNull()?.uri }
-            } } ?: Uri.EMPTY
-
-    suspend fun getReader(): GalleryInfo? {
-        val mirrors = Preferences.get<String>("mirrors").let { if (it.isEmpty()) emptyList() else it.split('>') }
-
-        val sources = mapOf(
-            "hitomi" to { xyz.quaver.hitomi.getGalleryInfo(galleryID.toInt()) },
-            //Code.HIYOBI to { xyz.quaver.hiyobi.getReader(galleryID) }
-        )
-
-        return metadata.reader
-            ?: withContext(Dispatchers.IO) {
-                var reader: GalleryInfo? = null
-
-                for (source in sources) {
-                    reader = try {
-                        source.value.invoke()
-                    } catch (e: Exception) {
-                        null
-                    }
-
-                   if (reader != null)
-                       break
-                }
-
-                reader?.also {
-                    setMetadata { metadata ->
-                        metadata.reader = it
-
-                        if (metadata.imageList == null)
-                            metadata.imageList = MutableList(reader.files.size) { null }
-                    }
-                }
-            }
-    }
-
-    fun getImage(index: Int): FileX? =
-        metadata.imageList?.getOrNull(index)?.let { findFile(it) }
-
-    @Suppress("BlockingMethodInNonBlockingContext")
-    fun putImage(index: Int, fileName: String, data: ByteArray) {
-        val file = cacheFolder.getChild(fileName)
-
-        if (!file.exists())
-            file.createNewFile()
-        file.writeBytes(data)
-        setMetadata { metadata -> metadata.imageList!![index] = fileName }
-    }
-
-    private val lock = ConcurrentHashMap<String, Mutex>()
-    @Suppress("BlockingMethodInNonBlockingContext")
-    fun moveToDownload() = CoroutineScope(Dispatchers.IO).launch {
-        val downloadFolder = downloadFolder ?: return@launch
-
-        if (lock[galleryID]?.isLocked == true)
-            return@launch
-
-        (lock[galleryID] ?: Mutex().also { lock[galleryID] = it }).withLock {
-            val cacheMetadata = cacheFolder.getChild(".metadata")
-            val downloadMetadata = downloadFolder.getChild(".metadata")
-
-            if (!cacheMetadata.exists())
-                return@launch
-
-            if (cacheMetadata.exists()) {
-                kotlin.runCatching {
-                    if (!downloadMetadata.exists())
-                        downloadMetadata.createNewFile()
-
-                    downloadMetadata.writeText(Json.encodeToString(metadata))
-                }
-            }
-
-            val cacheThumbnail = cacheFolder.getChild(".thumbnail")
-            val downloadThumbnail = downloadFolder.getChild(".thumbnail")
-
-            if (cacheThumbnail.exists()) {
-                kotlin.runCatching {
-                    if (!downloadThumbnail.exists())
-                        downloadThumbnail.createNewFile()
-
-                    downloadThumbnail.outputStream()?.use { target -> target.channel.truncate(0L); cacheThumbnail.inputStream()?.use { source ->
-                        source.copyTo(target)
-                    } }
-                    cacheThumbnail.delete()
-                }
-            }
-
-            metadata.imageList?.forEach { imageName ->
-                imageName ?: return@forEach
-                val target = downloadFolder.getChild(imageName)
-                val source = cacheFolder.getChild(imageName)
-
-                if (!source.exists())
-                    return@forEach
-
-                kotlin.runCatching {
-                    if (!target.exists())
-                        target.createNewFile()
-
-                    target.outputStream()?.use { target -> target.channel.truncate(0L); source.inputStream()?.use { source ->
-                        source.copyTo(target)
-                    } }
-                }
-            }
-
-            cacheFolder.deleteRecursively()
+    private fun findFile(fileName: String): FileX? =
+         downloadFolder?.let { downloadFolder -> downloadFolder.getChild(fileName).let {
+            if (it.exists()) it else null
+        } } ?: cacheFolder.getChild(fileName).let {
+            if (it.exists()) it else null
        }
+
+    fun putImage(index: Int, name: String, `is`: InputStream) {
+        cacheFolder.getChild(name).also {
+            if (!it.exists())
+                it.createNewFile()
+        }.outputStream()?.use {
+            it.channel.truncate(0L)
+            `is`.copyTo(it)
+        }
+
+        setMetadata { metadata -> metadata.imageList!![index] = name }
+    }
+
+    fun getImage(index: Int): FileX? {
+        return metadata.imageList?.get(index)?.let { findFile(it) }
    }
 }
--- a/app/src/main/java/xyz/quaver/pupil/util/downloader/DownloadManager.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/downloader/DownloadManager.kt
@@ -20,16 +20,15 @@ package xyz.quaver.pupil.util.downloader

 import android.content.Context
 import android.content.ContextWrapper
-import android.util.Log
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import okhttp3.Call
 import xyz.quaver.io.FileX
 import xyz.quaver.io.util.*
-import xyz.quaver.pupil.client
-import xyz.quaver.pupil.services.DownloadService
+import xyz.quaver.pupil.sources.sources
 import xyz.quaver.pupil.util.Preferences
 import xyz.quaver.pupil.util.formatDownloadFolder

@@ -83,44 +82,33 @@ class DownloadManager private constructor(context: Context) : ContextWrapper(con
            return downloadFolderMapInstance ?: mutableMapOf()
        }

+    @Synchronized
+    fun getDownloadFolder(source: String, itemID: String): FileX? =
+        downloadFolderMap["$source-$itemID"]?.let { downloadFolder.getChild(it) }

    @Synchronized
-    fun isDownloading(galleryID: String): Boolean {
-        val isThisGallery: (Call) -> Boolean = { (it.request().tag() as? DownloadService.Tag)?.galleryID == galleryID }
+    fun addDownloadFolder(source: String, itemID: String) = CoroutineScope(Dispatchers.IO).launch {
+        val name = "A" // TODO

-        return downloadFolderMap.containsKey(galleryID)
-                && client.dispatcher().let { it.queuedCalls().any(isThisGallery) || it.runningCalls().any(isThisGallery) }
-    }
-
-    @Synchronized
-    fun getDownloadFolder(galleryID: String): FileX? =
-        downloadFolderMap[galleryID]?.let { downloadFolder.getChild(it) }
-
-    @Synchronized
-    fun addDownloadFolder(galleryID: String) {
-        val name = runBlocking {
-            Cache.getInstance(this@DownloadManager, galleryID).getGalleryBlock()
-        }?.formatDownloadFolder() ?: return
-
-        val folder = downloadFolder.getChild(name)
+        val folder = downloadFolder.getChild("$source/$name")

        if (folder.exists())
-            return
+            return@launch

        folder.mkdir()

-        downloadFolderMap[galleryID] = folder.name
+        downloadFolderMap["$source/$itemID"] = folder.name

        downloadFolder.getChild(".download").let { if (!it.exists()) it.createNewFile() }
        downloadFolder.getChild(".download").writeText(Json.encodeToString(downloadFolderMap))
    }

    @Synchronized
-    fun deleteDownloadFolder(galleryID: String) {
-        downloadFolderMap[galleryID]?.let {
+    fun deleteDownloadFolder(source: String, itemID: String) {
+        downloadFolderMap["$source/$itemID"]?.let {
            kotlin.runCatching {
                downloadFolder.getChild(it).deleteRecursively()
-                downloadFolderMap.remove(galleryID)
+                downloadFolderMap.remove("$source/$itemID")

                downloadFolder.getChild(".download").let { if (!it.exists()) it.createNewFile() }
                downloadFolder.getChild(".download").writeText(Json.encodeToString(downloadFolderMap))
--- a/app/src/main/java/xyz/quaver/pupil/util/downloader/Downloader.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/downloader/Downloader.kt
@@ -0,0 +1,221 @@
+/*
+ *     Pupil, Hitomi.la viewer for Android
+ *     Copyright (C) 2020  tom5079
+ *
+ *     This program is free software: you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation, either version 3 of the License, or
+ *     (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package xyz.quaver.pupil.util.downloader
+
+import android.content.Context
+import com.google.firebase.crashlytics.FirebaseCrashlytics
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import okhttp3.*
+import okio.*
+import xyz.quaver.pupil.PupilInterceptor
+import xyz.quaver.pupil.client
+import xyz.quaver.pupil.interceptors
+import xyz.quaver.pupil.sources.sources
+import xyz.quaver.pupil.util.cleanCache
+import java.io.IOException
+import java.util.concurrent.ConcurrentHashMap
+
+private typealias ProgressListener = (Downloader.Tag, Long, Long, Boolean) -> Unit
+class Downloader private constructor(private val context: Context) {
+
+    data class Tag(val source: String, val itemID: String, val index: Int)
+
+    companion object {
+        var instance: Downloader? = null
+
+        fun getInstance(context: Context): Downloader {
+            return instance ?: synchronized(this) {
+                instance ?: Downloader(context).also {
+                    interceptors[Tag::class] = it.interceptor
+                    instance = it
+                }
+            }
+        }
+    }
+
+    //region ProgressListener
+    @Suppress("UNCHECKED_CAST")
+    private val progressListener: ProgressListener = { (source, itemID, index), bytesRead, contentLength, done ->
+        if (!done && progress["$source-$itemID"]?.get(index)?.isFinite() == true)
+            progress["$source-$itemID"]?.set(index, bytesRead * 100F / contentLength)
+    }
+
+    private class ProgressResponseBody(
+        val tag: Any?,
+        val responseBody: ResponseBody,
+        val progressListener : ProgressListener
+    ) : ResponseBody() {
+        private var bufferedSource : BufferedSource? = null
+
+        override fun contentLength() = responseBody.contentLength()
+        override fun contentType() = responseBody.contentType()
+
+        override fun source(): BufferedSource {
+            if (bufferedSource == null)
+                bufferedSource = Okio.buffer(source(responseBody.source()))
+
+            return bufferedSource!!
+        }
+
+        private fun source(source: Source) = object: ForwardingSource(source) {
+            var totalBytesRead = 0L
+
+            override fun read(sink: Buffer, byteCount: Long): Long {
+                val bytesRead = super.read(sink, byteCount)
+
+                totalBytesRead += if (bytesRead == -1L) 0L else bytesRead
+                progressListener.invoke(tag as Tag, totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
+
+                return bytesRead
+            }
+        }
+    }
+
+    private val interceptor: PupilInterceptor = { chain ->
+        val request = chain.request()
+        var response = chain.proceed(request)
+
+        var retry = 5
+        while (!response.isSuccessful && retry > 0) {
+            response = chain.proceed(request)
+            retry--
+        }
+
+        response.newBuilder()
+            .body(response.body()?.let {
+                ProgressResponseBody(request.tag(), it, progressListener)
+            }).build()
+    }
+    //endregion
+
+    private val callback = object : Callback {
+        override fun onFailure(call: Call, e: IOException) {
+            val (source, itemID, index) = call.request().tag() as Tag
+
+            FirebaseCrashlytics.getInstance().recordException(e)
+
+            progress["$source-$itemID"]?.set(index, Float.NEGATIVE_INFINITY)
+        }
+
+        override fun onResponse(call: Call, response: Response) {
+            val (source, itemID, index) = call.request().tag() as Tag
+            val ext = call.request().url().encodedPath().takeLastWhile { it != '.' }
+
+            if (response.code() != 200)
+                throw IOException()
+
+            response.body()?.use {
+                Cache.getInstance(context, source, itemID).putImage(index, "$index.$ext", it.byteStream())
+            }
+            progress["$source-$itemID"]?.set(index, Float.POSITIVE_INFINITY)
+        }
+    }
+
+    private val progress = ConcurrentHashMap<String, MutableList<Float>>()
+    fun getProgress(source: String, itemID: String): List<Float>? {
+        return progress["$source-$itemID"]
+    }
+
+    fun cancel() {
+        client.dispatcher().queuedCalls().filter {
+            it.request().tag() is Tag
+        }.forEach {
+            it.cancel()
+        }
+        client.dispatcher().runningCalls().filter {
+            it.request().tag() is Tag
+        }.forEach {
+            it.cancel()
+        }
+
+        progress.clear()
+    }
+
+    fun cancel(source: String, itemID: String) {
+        client.dispatcher().queuedCalls().filter {
+            (it.request().tag() as? Tag)?.let { tag ->
+                tag.source == source && tag.itemID == itemID
+            } == true
+        }.forEach {
+            it.cancel()
+        }
+        client.dispatcher().runningCalls().filter {
+            (it.request().tag() as? Tag)?.let { tag ->
+                tag.source == source && tag.itemID == itemID
+            } == true
+        }.forEach {
+            it.cancel()
+        }
+
+        progress.remove("$source-$itemID")
+    }
+
+    fun retry(source: String, itemID: String) {
+        cancel(source, itemID)
+        download(source, itemID)
+    }
+
+    var onImageListLoadedCallback: ((List<String>) -> Unit)? = null
+    fun download(source: String, itemID: String) = CoroutineScope(Dispatchers.IO).launch {
+        if (isDownloading(source, itemID))
+            return@launch
+
+        cleanCache(context)
+
+        val source = sources[source] ?: return@launch
+        val cache = Cache.getInstance(context, source.name, itemID)
+
+        source.images(itemID).also {
+            progress["${source.name}-$itemID"] = MutableList(it.size) { i ->
+                if (cache.metadata.imageList?.get(i) == null) 0F else Float.POSITIVE_INFINITY
+            }
+
+            with (Cache.getInstance(context, source.name, itemID).metadata) {
+                if (imageList == null)
+                    imageList = MutableList(it.size) { null }
+
+                imageList!!.forEachIndexed { index, s ->
+                    if (s != null)
+                        progress["${source.name}-$itemID"]?.set(index, Float.POSITIVE_INFINITY)
+                }
+            }
+
+            onImageListLoadedCallback?.invoke(it)
+        }.forEachIndexed { index, url ->
+            client.newCall(
+                Request.Builder()
+                    .tag(Tag(source.name, itemID, index))
+                    .url(url)
+                    .headers(Headers.of(source.getHeadersForImage(itemID, url)))
+                    .build()
+            ).enqueue(callback)
+        }
+    }
+
+    fun isDownloading(source: String, itemID: String): Boolean {
+        return (client.dispatcher().queuedCalls() + client.dispatcher().runningCalls()).any {
+            (it.request().tag() as? Tag)?.let { tag ->
+                tag.source == source && tag.itemID == itemID
+            } == true
+        }
+    }
+
+}
--- a/app/src/main/java/xyz/quaver/pupil/util/file.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/file.kt
@@ -57,9 +57,9 @@ fun cleanCache(context: Context) = CoroutineScope(Dispatchers.IO).launch {

                synchronized(histories) {
                    (histories.firstOrNull {
-                        caches.contains(it.toString()) && !downloadManager.isDownloading(it)
+                        TODO()
                    } ?: return@withLock).let {
-                        Cache.delete(context, it)
+                        TODO()
                    }
                }
            }
--- a/app/src/main/java/xyz/quaver/pupil/util/misc.kt
+++ b/app/src/main/java/xyz/quaver/pupil/util/misc.kt
@@ -19,19 +19,14 @@
 package xyz.quaver.pupil.util

 import android.annotation.SuppressLint
-import android.content.Context
-import android.content.Intent
-import android.os.Build
 import android.view.MenuItem
-import androidx.core.content.ContextCompat
 import kotlinx.serialization.json.*
 import okhttp3.OkHttpClient
 import okhttp3.Request
-import xyz.quaver.hitomi.GalleryBlock
 import xyz.quaver.hitomi.GalleryInfo
 import xyz.quaver.hitomi.getReferer
 import xyz.quaver.hitomi.imageUrlFromImage
-import xyz.quaver.hiyobi.createImgList
+import xyz.quaver.pupil.sources.ItemInfo
 import java.util.*
 import kotlin.collections.ArrayList

@@ -80,23 +75,23 @@ fun OkHttpClient.Builder.proxyInfo(proxyInfo: ProxyInfo) = this.apply {
    }
 }

-val formatMap = mapOf<String, GalleryBlock.() -> (String)>(
-    "-id-" to { id.toString() },
+val formatMap = mapOf<String, ItemInfo.() -> (String)>(
+    "-id-" to { id },
    "-title-" to { title },
-    "-artist-" to { artists.joinToString() }
+    "-artist-" to { artists }
    // TODO
 )
 /**
 * Formats download folder name with given Metadata
 */
-fun GalleryBlock.formatDownloadFolder(): String =
+fun ItemInfo.formatDownloadFolder(): String =
    Preferences["download_folder_name", "[-id-] -title-"].let {
        formatMap.entries.fold(it) { str, (k, v) ->
            str.replace(k, v.invoke(this), true)
        }
    }.replace(Regex("""[*\\|"?><:/]"""), "").ellipsize(127)

-fun GalleryBlock.formatDownloadFolderTest(format: String): String =
+fun ItemInfo.formatDownloadFolderTest(format: String): String =
    format.let {
        formatMap.entries.fold(it) { str, (k, v) ->
            str.replace(k, v.invoke(this), true)