diff --git a/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt b/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt index 6d4686e3..e76e2967 100644 --- a/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt +++ b/app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt @@ -37,6 +37,7 @@ import xyz.quaver.hiyobi.createImgList import xyz.quaver.hiyobi.getReader import xyz.quaver.hiyobi.user_agent import xyz.quaver.pupil.ui.LockActivity +import xyz.quaver.pupil.util.download.DownloadWorker import xyz.quaver.pupil.util.getDownloadDirectory import xyz.quaver.pupil.util.updateOldReaderGalleries import java.io.File @@ -118,4 +119,24 @@ class ExampleInstrumentedTest { updateOldReaderGalleries(context) } + + @Test + fun test_downloadWorker() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + + val galleryID = 515515 + + val worker = DownloadWorker.getInstance(context) + + worker.queue.add(galleryID) + + while(worker.progress.indexOfKey(galleryID) < 0 || worker.progress[galleryID] != null) { + Log.i("PUPILD", worker.progress[galleryID]?.joinToString(" ") ?: "null") + + if (worker.progress[galleryID]?.all { !it.isFinite() } == true) + break + } + + Log.i("PUPILD", "DONE!!") + } } \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt b/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt index f0022d54..eccc0556 100644 --- a/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt +++ b/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt @@ -19,7 +19,6 @@ package xyz.quaver.pupil.adapters import android.graphics.drawable.Drawable -import android.util.Log import android.util.SparseBooleanArray import android.view.LayoutInflater import android.view.View @@ -130,8 +129,6 @@ class GalleryBlockAdapter(private val glide: RequestManager, private val galleri with(view.galleryblock_progressbar) { progress = imageCache.invoke().list()?.size ?: 0 - Log.i("PUPILD", progress.toString()) - if (!readerCache.invoke().exists()) { visibility = View.GONE max = 0 diff --git a/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt b/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt index d45b1e3c..a843b4d1 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt @@ -20,6 +20,7 @@ package xyz.quaver.pupil.util.download import android.content.Context import android.content.ContextWrapper +import android.util.Base64 import android.util.SparseArray import androidx.core.content.ContextCompat import androidx.preference.PreferenceManager @@ -30,7 +31,10 @@ import kotlinx.serialization.parse import kotlinx.serialization.stringify import xyz.quaver.hitomi.GalleryBlock import xyz.quaver.hitomi.Reader +import xyz.quaver.pupil.util.getDownloadDirectory +import xyz.quaver.pupil.util.isParentOf import java.io.File +import java.net.URL class Cache(context: Context) : ContextWrapper(context) { @@ -86,6 +90,29 @@ class Cache(context: Context) : ContextWrapper(context) { } } + suspend fun getThumbnail(galleryID: Int): String? { + val metadata = Cache(this).getCachedMetadata(galleryID) + + val thumbnail = if (metadata?.thumbnail == null) + withContext(Dispatchers.IO) { + val thumbnails = getGalleryBlock(galleryID)?.thumbnails + try { + Base64.encodeToString(URL(thumbnails?.firstOrNull()).readBytes(), Base64.DEFAULT) + } catch (e: Exception) { + null + } + } + else + metadata.thumbnail + + setCachedMetadata( + galleryID, + Metadata(Cache(this).getCachedMetadata(galleryID), thumbnail = thumbnail) + ) + + return thumbnail + } + suspend fun getGalleryBlock(galleryID: Int): GalleryBlock? { val metadata = Cache(this).getCachedMetadata(galleryID) @@ -102,7 +129,7 @@ class Cache(context: Context) : ContextWrapper(context) { setCachedMetadata( galleryID, - Metadata(metadata, galleryBlock = galleryBlock) + Metadata(Cache(this).getCachedMetadata(galleryID), galleryBlock = galleryBlock) ) return galleryBlock @@ -129,7 +156,7 @@ class Cache(context: Context) : ContextWrapper(context) { if (readers.isNotEmpty()) setCachedMetadata( galleryID, - Metadata(metadata, readers = readers) + Metadata(Cache(this).getCachedMetadata(galleryID), readers = readers) ) val mirrors = preference.getString("mirrors", "")!!.split('>') @@ -140,22 +167,44 @@ class Cache(context: Context) : ContextWrapper(context) { } fun getImages(galleryID: Int): SparseArray? { - val regex = Regex("[0-9]+") val gallery = getCachedGallery(galleryID) ?: return null return SparseArray().apply { gallery.listFiles { file -> - file.nameWithoutExtension.matches(regex) + file.nameWithoutExtension.toIntOrNull() != null }?.forEach { append(it.nameWithoutExtension.toInt(), it) } } } - fun putImage(galleryID: Int, index: Int, data: ByteArray) { + fun putImage(galleryID: Int, name: String, data: ByteArray) { val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID") - File(cache, index.toString()).writeBytes(data) + with(File(cache, name)) { + + if (!parentFile!!.exists()) + parentFile!!.mkdirs() + + if (!exists()) + createNewFile() + + if (nameWithoutExtension.toIntOrNull() != null) + writeBytes(data) + else + IllegalArgumentException("File name is not a number") + } + } + + fun moveToDownload(galleryID: Int) { + val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID") + + val download = getDownloadDirectory(this) + + if (!download.isParentOf(cache)) { + cache.copyRecursively(download) + cache.deleteRecursively() + } } } \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt b/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt index 81e046b3..b14abc33 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt @@ -21,28 +21,39 @@ package xyz.quaver.pupil.util.download import android.content.Context import android.content.ContextWrapper import android.content.SharedPreferences +import android.util.SparseArray import androidx.preference.PreferenceManager import com.crashlytics.android.Crashlytics import io.fabric.sdk.android.Fabric -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.launch +import kotlinx.coroutines.* import okhttp3.* import okio.* import xyz.quaver.hitomi.Reader +import xyz.quaver.hitomi.getReferer import xyz.quaver.hitomi.urlFromUrlFromHash +import xyz.quaver.hiyobi.cookie import xyz.quaver.hiyobi.createImgList -import java.io.FileInputStream +import xyz.quaver.hiyobi.user_agent import java.io.IOException +import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue @UseExperimental(ExperimentalCoroutinesApi::class) class DownloadWorker private constructor(context: Context) : ContextWrapper(context) { - val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) + private val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this) //region ProgressListener + @Suppress("UNCHECKED_CAST") + private val progressListener = object: ProgressListener { + override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) { + val (galleryID, index) = (tag as? Pair) ?: return + + if (!done && progress[galleryID]!![index] != Float.POSITIVE_INFINITY) + progress[galleryID]!![index] = bytesRead * 100F / contentLength + } + } + interface ProgressListener { fun update(tag: Any?, bytesRead : Long, contentLength: Long, done: Boolean) } @@ -52,7 +63,7 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont val responseBody: ResponseBody, val progressListener : ProgressListener ) : ResponseBody() { - var bufferedSource : BufferedSource? = null + private var bufferedSource : BufferedSource? = null override fun contentLength() = responseBody.contentLength() override fun contentType() = responseBody.contentType() @@ -93,23 +104,46 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont } //endregion - val queue = Channel() - /* VALUE + val queue = LinkedBlockingQueue() + + /* + * KEY + * primary galleryID + * secondary index + * PRIMARY VALUE + * MutableList -> Download in progress + * null -> Loading / Gallery doesn't exist + * SECONDARY VALUE * 0 <= value < 100 -> Download in progress * Float.POSITIVE_INFINITY -> Download completed * Float.NaN -> Exception */ - val progress = mutableMapOf() - val result = mutableMapOf() - val exception = mutableMapOf() + val progress = SparseArray?>() + /* + * KEY + * primary galleryID + * secondary index + * PRIMARY VALUE + * MutableList -> Download in progress / Loading + * null -> Gallery doesn't exist + * SECONDARY VALUE + * Throwable -> Exception + * null -> Download in progress / Loading + */ + val exception = SparseArray?>() - val client = OkHttpClient.Builder() - .addNetworkInterceptor { chain -> + private val loop = loop() + private val worker = SparseArray() + @Volatile var nRunners = 0 + + private val client = OkHttpClient.Builder() + .addInterceptor { chain -> val request = chain.request() var response = chain.proceed(request) var retry = preferences.getInt("retry", 3) while (!response.isSuccessful && retry > 0) { + response.close() response = chain.proceed(request) retry-- } @@ -117,72 +151,121 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont response.newBuilder() .body(ProgressResponseBody(request.tag(), response.body!!, progressListener)) .build() - }.build() - - - val progressListener = object: ProgressListener { - override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) { - if (tag !is String) - return - - if (progress[tag] != Float.POSITIVE_INFINITY) - progress[tag] = bytesRead / contentLength.toFloat() } + .dispatcher(Dispatcher(Executors.newSingleThreadExecutor())) + .build() + + fun stop() { + loop.cancel() + for (i in 0..worker.size()) + worker[worker.keyAt(i)]?.cancel() + + client.dispatcher.cancelAll() } - init { - CoroutineScope(Dispatchers.Unconfined).launch { - while (!(queue.isEmpty && queue.isClosedForReceive)) { - val lowQuality = preferences.getBoolean("low_quality", false) - val galleryID = queue.receive() - launch(Dispatchers.IO) io@{ - val reader = Cache(context).getReader(galleryID) ?: return@io - val cache = Cache(context).getImages(galleryID) + fun cancel(galleryID: Int) { + worker[galleryID]?.cancel() - reader.galleryInfo.forEachIndexed { index, galleryInfo -> - val tag = "$galleryID-$index" - val url = when(reader.code) { - Reader.Code.HITOMI -> - urlFromUrlFromHash(galleryID, galleryInfo, if (lowQuality) "webp" else null) - Reader.Code.HIYOBI -> - createImgList(galleryID, reader, lowQuality)[index].path - else -> "" //Shouldn't be called anyways - } + client.dispatcher.queuedCalls() + .filter { it.request().tag(Pair::class.java)?.first == galleryID } + .forEach { + it.cancel() + } + } - //Cache exists :P - cache?.get(index)?.let { - result[tag] = FileInputStream(it).readBytes() - progress[tag] = Float.POSITIVE_INFINITY + private fun queueDownload(galleryID: Int, reader: Reader, index: Int, callback: Callback) { + val cache = Cache(this@DownloadWorker).getImages(galleryID) + val lowQuality = preferences.getBoolean("low_quality", false) - return@io - } + //Cache exists :P + cache?.get(index)?.let { + progress[galleryID]!![index] = Float.POSITIVE_INFINITY - val request = Request.Builder() - .url(url) - .tag(tag) - .build() + return + } - client.newCall(request).enqueue(object: Callback { - override fun onFailure(call: Call, e: IOException) { - if (Fabric.isInitialized()) - Crashlytics.logException(e) - - progress[tag] = Float.NaN - exception[tag] = e - } - - override fun onResponse(call: Call, response: Response) { - response.use { - val res = it.body!!.bytes() - result[tag] = res - Cache(context).putImage(galleryID, index, res) - progress[tag] = Float.POSITIVE_INFINITY - } - } - }) - } + val request = Request.Builder().apply { + when (reader.code) { + Reader.Code.HITOMI -> { + url( + urlFromUrlFromHash( + galleryID, + reader.galleryInfo[index], + if (lowQuality) "webp" else null + ) + ) + addHeader("Referer", getReferer(galleryID)) + } + Reader.Code.HIYOBI -> { + url(createImgList(galleryID, reader, lowQuality)[index].path) + addHeader("User-Agent", user_agent) + addHeader("Cookie", cookie) + } + else -> { + //shouldn't be called anyway } } + tag(galleryID to index) + }.build() + + client.newCall(request).enqueue(callback) + } + + private fun download(galleryID: Int) = CoroutineScope(Dispatchers.IO).launch { + val reader = Cache(this@DownloadWorker).getReader(galleryID) + + //gallery doesn't exist + if (reader == null) { + progress.put(galleryID, null) + exception.put(galleryID, null) + nRunners-- + return@launch + } + + progress.put(galleryID, reader.galleryInfo.map { 0F }.toMutableList()) + exception.put(galleryID, reader.galleryInfo.map { null }.toMutableList()) + + for (i in reader.galleryInfo.indices) { + val callback = object : Callback { + override fun onFailure(call: Call, e: IOException) { + if (Fabric.isInitialized()) + Crashlytics.logException(e) + + progress[galleryID]!![i] = Float.NaN + exception[galleryID]!![i] = e + + if (progress[galleryID]!!.all { !it.isFinite() }) + nRunners-- + } + + override fun onResponse(call: Call, response: Response) { + response.use { + val res = it.body!!.bytes() + val ext = + call.request().url.encodedPath.split('.').last() + + Cache(this@DownloadWorker).putImage(galleryID, "$i.$ext", res) + progress[galleryID]!![i] = Float.POSITIVE_INFINITY + } + + if (progress[galleryID]!!.all { !it.isFinite() }) + nRunners-- + } + } + + queueDownload(galleryID, reader, i, callback) + } + } + + private fun loop() = CoroutineScope(Dispatchers.Default).launch { + while (true) { + if (queue.isEmpty() || nRunners > preferences.getInt("max_download", 4)) + continue + + val galleryID = queue.poll() ?: continue + + worker.put(galleryID, download(galleryID)) + nRunners++ } } diff --git a/app/src/main/java/xyz/quaver/pupil/util/file.kt b/app/src/main/java/xyz/quaver/pupil/util/file.kt index 29d094f7..6bf6a88e 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/file.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/file.kt @@ -62,4 +62,6 @@ fun URL.download(to: File, onDownloadProgress: ((Long, Long) -> Unit)? = null) { } } -} \ No newline at end of file +} + +fun File.isParentOf(file: File) = file.absolutePath.startsWith(this.absolutePath) \ No newline at end of file diff --git a/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt b/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt index bc2121b9..5c097bd2 100644 --- a/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt +++ b/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt @@ -26,20 +26,16 @@ package xyz.quaver.pupil * See [testing documentation](http://d.android.com/tools/testing). */ +import android.util.SparseArray import org.junit.Test -import xyz.quaver.pupil.util.download -import java.io.File -import java.net.URL class ExampleUnitTest { @Test fun test() { - URL("https://github.com/tom5079/Pupil/releases/download/4.2-beta2-hotfix2/Pupil-v4.2-beta2-hotfix2.apk").download( - File(System.getenv("USERPROFILE"), "Pupil.apk") - ) { downloaded, fileSize -> - println("%.1f%%".format(downloaded*100.0/fileSize)) - } + val arr = SparseArray() + + print(arr.indexOfKey(34)) } }