Rebuilding Downloader

app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt

@@ -37,6 +37,7 @@ import xyz.quaver.hiyobi.createImgList
 import xyz.quaver.hiyobi.getReader
 import xyz.quaver.hiyobi.user_agent
 import xyz.quaver.pupil.ui.LockActivity
+import xyz.quaver.pupil.util.download.DownloadWorker
 import xyz.quaver.pupil.util.getDownloadDirectory
 import xyz.quaver.pupil.util.updateOldReaderGalleries
 import java.io.File
@@ -118,4 +119,24 @@ class ExampleInstrumentedTest {
 
         updateOldReaderGalleries(context)
     }
+
+    @Test
+    fun test_downloadWorker() {
+        val context = InstrumentationRegistry.getInstrumentation().targetContext
+
+        val galleryID = 515515
+
+        val worker = DownloadWorker.getInstance(context)
+
+        worker.queue.add(galleryID)
+
+        while(worker.progress.indexOfKey(galleryID) < 0 || worker.progress[galleryID] != null) {
+            Log.i("PUPILD", worker.progress[galleryID]?.joinToString(" ") ?: "null")
+
+            if (worker.progress[galleryID]?.all { !it.isFinite() } == true)
+                break
+        }
+
+        Log.i("PUPILD", "DONE!!")
+    }
 }

app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt

@@ -19,7 +19,6 @@
 package xyz.quaver.pupil.adapters
 
 import android.graphics.drawable.Drawable
-import android.util.Log
 import android.util.SparseBooleanArray
 import android.view.LayoutInflater
 import android.view.View
@@ -130,8 +129,6 @@ class GalleryBlockAdapter(private val glide: RequestManager, private val galleri
                             with(view.galleryblock_progressbar) {
                                 progress = imageCache.invoke().list()?.size ?: 0
 
-                                Log.i("PUPILD", progress.toString())
-
                                 if (!readerCache.invoke().exists()) {
                                     visibility = View.GONE
                                     max = 0

app/src/main/java/xyz/quaver/pupil/util/download/Cache.kt

@@ -20,6 +20,7 @@ package xyz.quaver.pupil.util.download
 
 import android.content.Context
 import android.content.ContextWrapper
+import android.util.Base64
 import android.util.SparseArray
 import androidx.core.content.ContextCompat
 import androidx.preference.PreferenceManager
@@ -30,7 +31,10 @@ import kotlinx.serialization.parse
 import kotlinx.serialization.stringify
 import xyz.quaver.hitomi.GalleryBlock
 import xyz.quaver.hitomi.Reader
+import xyz.quaver.pupil.util.getDownloadDirectory
+import xyz.quaver.pupil.util.isParentOf
 import java.io.File
+import java.net.URL
 
 class Cache(context: Context) : ContextWrapper(context) {
 
@@ -86,6 +90,29 @@ class Cache(context: Context) : ContextWrapper(context) {
         }
     }
 
+    suspend fun getThumbnail(galleryID: Int): String? {
+        val metadata = Cache(this).getCachedMetadata(galleryID)
+
+        val thumbnail = if (metadata?.thumbnail == null)
+            withContext(Dispatchers.IO) {
+                val thumbnails = getGalleryBlock(galleryID)?.thumbnails
+                try {
+                    Base64.encodeToString(URL(thumbnails?.firstOrNull()).readBytes(), Base64.DEFAULT)
+                } catch (e: Exception) {
+                    null
+                }
+            }
+        else
+            metadata.thumbnail
+
+        setCachedMetadata(
+            galleryID,
+            Metadata(Cache(this).getCachedMetadata(galleryID), thumbnail = thumbnail)
+        )
+
+        return thumbnail
+    }
+
     suspend fun getGalleryBlock(galleryID: Int): GalleryBlock? {
         val metadata = Cache(this).getCachedMetadata(galleryID)
 
@@ -102,7 +129,7 @@ class Cache(context: Context) : ContextWrapper(context) {
 
         setCachedMetadata(
             galleryID,
-            Metadata(metadata, galleryBlock = galleryBlock)
+            Metadata(Cache(this).getCachedMetadata(galleryID), galleryBlock = galleryBlock)
         )
 
         return galleryBlock
@@ -129,7 +156,7 @@ class Cache(context: Context) : ContextWrapper(context) {
         if (readers.isNotEmpty())
             setCachedMetadata(
                 galleryID,
-                Metadata(metadata, readers = readers)
+                Metadata(Cache(this).getCachedMetadata(galleryID), readers = readers)
             )
 
         val mirrors = preference.getString("mirrors", "")!!.split('>')
@@ -140,22 +167,44 @@ class Cache(context: Context) : ContextWrapper(context) {
     }
 
     fun getImages(galleryID: Int): SparseArray<File>? {
-        val regex = Regex("[0-9]+")
         val gallery = getCachedGallery(galleryID) ?: return null
 
         return SparseArray<File>().apply {
             gallery.listFiles { file ->
-                file.nameWithoutExtension.matches(regex)
+                file.nameWithoutExtension.toIntOrNull() != null
             }?.forEach {
                 append(it.nameWithoutExtension.toInt(), it)
             }
         }
     }
 
-    fun putImage(galleryID: Int, index: Int, data: ByteArray) {
+    fun putImage(galleryID: Int, name: String, data: ByteArray) {
         val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID")
 
-        File(cache, index.toString()).writeBytes(data)
+        with(File(cache, name)) {
+
+            if (!parentFile!!.exists())
+                parentFile!!.mkdirs()
+
+            if (!exists())
+                createNewFile()
+
+            if (nameWithoutExtension.toIntOrNull() != null)
+                writeBytes(data)
+            else
+                IllegalArgumentException("File name is not a number")
+        }
+    }
+
+    fun moveToDownload(galleryID: Int) {
+        val cache = getCachedGallery(galleryID) ?: File(cacheDir, "imageCache/$galleryID")
+
+        val download = getDownloadDirectory(this)
+
+        if (!download.isParentOf(cache)) {
+            cache.copyRecursively(download)
+            cache.deleteRecursively()
+        }
     }
 
 }

app/src/main/java/xyz/quaver/pupil/util/download/DownloadWorker.kt

@@ -21,28 +21,39 @@ package xyz.quaver.pupil.util.download
 import android.content.Context
 import android.content.ContextWrapper
 import android.content.SharedPreferences
+import android.util.SparseArray
 import androidx.preference.PreferenceManager
 import com.crashlytics.android.Crashlytics
 import io.fabric.sdk.android.Fabric
-import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.*
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.ExperimentalCoroutinesApi
-import kotlinx.coroutines.channels.Channel
-import kotlinx.coroutines.launch
 import okhttp3.*
 import okio.*
 import xyz.quaver.hitomi.Reader
+import xyz.quaver.hitomi.getReferer
 import xyz.quaver.hitomi.urlFromUrlFromHash
+import xyz.quaver.hiyobi.cookie
 import xyz.quaver.hiyobi.createImgList
-import java.io.FileInputStream
+import xyz.quaver.hiyobi.user_agent
 import java.io.IOException
+import java.util.concurrent.Executors
+import java.util.concurrent.LinkedBlockingQueue
 
 @UseExperimental(ExperimentalCoroutinesApi::class)
 class DownloadWorker private constructor(context: Context) : ContextWrapper(context) {
 
-    val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)
+    private val preferences : SharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)
 
     //region ProgressListener
+    @Suppress("UNCHECKED_CAST")
+    private val progressListener = object: ProgressListener {
+        override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) {
+            val (galleryID, index) = (tag as? Pair<Int, Int>) ?: return
+
+            if (!done && progress[galleryID]!![index] != Float.POSITIVE_INFINITY)
+                progress[galleryID]!![index] = bytesRead * 100F / contentLength
+        }
+    }
+
     interface ProgressListener {
         fun update(tag: Any?, bytesRead : Long, contentLength: Long, done: Boolean)
     }
@@ -52,7 +63,7 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
         val responseBody: ResponseBody,
         val progressListener : ProgressListener
     ) : ResponseBody() {
-        var bufferedSource : BufferedSource? = null
+        private var bufferedSource : BufferedSource? = null
 
         override fun contentLength() = responseBody.contentLength()
         override fun contentType() = responseBody.contentType()
@@ -93,23 +104,46 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
     }
     //endregion
 
-    val queue = Channel<Int>()
+    val queue = LinkedBlockingQueue<Int>()
-    /* VALUE
+
+    /*
+    * KEY
+    *  primary galleryID
+    *  secondary index
+    * PRIMARY VALUE
+    *  MutableList -> Download in progress
+    *  null -> Loading / Gallery doesn't exist
+    * SECONDARY VALUE
     *  0 <= value < 100 -> Download in progress
     *  Float.POSITIVE_INFINITY -> Download completed
     *  Float.NaN -> Exception
     */
-    val progress = mutableMapOf<String, Float>()
+    val progress = SparseArray<MutableList<Float>?>()
-    val result = mutableMapOf<String, ByteArray>()
+    /*
-    val exception = mutableMapOf<String, Throwable>()
+    * KEY
+    *  primary galleryID
+    *  secondary index
+    * PRIMARY VALUE
+    *  MutableList -> Download in progress / Loading
+    *  null -> Gallery doesn't exist
+    * SECONDARY VALUE
+    *  Throwable -> Exception
+    *  null -> Download in progress / Loading
+    */
+    val exception = SparseArray<MutableList<Throwable?>?>()
 
-    val client = OkHttpClient.Builder()
+    private val loop = loop()
-        .addNetworkInterceptor { chain ->
+    private val worker = SparseArray<Job?>()
+    @Volatile var nRunners = 0
+
+    private val client = OkHttpClient.Builder()
+        .addInterceptor { chain ->
             val request = chain.request()
             var response = chain.proceed(request)
 
             var retry = preferences.getInt("retry", 3)
             while (!response.isSuccessful && retry > 0) {
+                response.close()
                 response = chain.proceed(request)
                 retry--
             }
@@ -117,72 +151,121 @@ class DownloadWorker private constructor(context: Context) : ContextWrapper(cont
             response.newBuilder()
                 .body(ProgressResponseBody(request.tag(), response.body!!, progressListener))
                 .build()
-        }.build()
+        }
+        .dispatcher(Dispatcher(Executors.newSingleThreadExecutor()))
+        .build()
 
+    fun stop() {
+        loop.cancel()
+        for (i in 0..worker.size())
+            worker[worker.keyAt(i)]?.cancel()
 
-    val progressListener = object: ProgressListener {
+        client.dispatcher.cancelAll()
-        override fun update(tag: Any?, bytesRead: Long, contentLength: Long, done: Boolean) {
+    }
-            if (tag !is String)
-                return
 
-            if (progress[tag] != Float.POSITIVE_INFINITY)
+    fun cancel(galleryID: Int) {
-                progress[tag] = bytesRead / contentLength.toFloat()
+        worker[galleryID]?.cancel()
+
+        client.dispatcher.queuedCalls()
+            .filter { it.request().tag(Pair::class.java)?.first == galleryID }
+            .forEach {
+                it.cancel()
             }
     }
-    init {
+
-        CoroutineScope(Dispatchers.Unconfined).launch {
+    private fun queueDownload(galleryID: Int, reader: Reader, index: Int, callback: Callback) {
-            while (!(queue.isEmpty && queue.isClosedForReceive)) {
+        val cache = Cache(this@DownloadWorker).getImages(galleryID)
         val lowQuality = preferences.getBoolean("low_quality", false)
-                val galleryID = queue.receive()
-
-                launch(Dispatchers.IO) io@{
-                    val reader = Cache(context).getReader(galleryID) ?: return@io
-                    val cache = Cache(context).getImages(galleryID)
-
-                    reader.galleryInfo.forEachIndexed { index, galleryInfo ->
-                        val tag = "$galleryID-$index"
-                        val url = when(reader.code) {
-                            Reader.Code.HITOMI ->
-                                urlFromUrlFromHash(galleryID, galleryInfo, if (lowQuality) "webp" else null)
-                            Reader.Code.HIYOBI ->
-                                createImgList(galleryID, reader, lowQuality)[index].path
-                            else -> ""  //Shouldn't be called anyways
-                        }
 
         //Cache exists :P
         cache?.get(index)?.let {
-                            result[tag] = FileInputStream(it).readBytes()
+            progress[galleryID]!![index] = Float.POSITIVE_INFINITY
-                            progress[tag] = Float.POSITIVE_INFINITY
 
-                            return@io
+            return
         }
 
-                        val request = Request.Builder()
+        val request = Request.Builder().apply {
-                            .url(url)
+            when (reader.code) {
-                            .tag(tag)
+                Reader.Code.HITOMI -> {
-                            .build()
+                    url(
+                        urlFromUrlFromHash(
+                            galleryID,
+                            reader.galleryInfo[index],
+                            if (lowQuality) "webp" else null
+                        )
+                    )
+                    addHeader("Referer", getReferer(galleryID))
+                }
+                Reader.Code.HIYOBI -> {
+                    url(createImgList(galleryID, reader, lowQuality)[index].path)
+                    addHeader("User-Agent", user_agent)
+                    addHeader("Cookie", cookie)
+                }
+                else -> {
+                    //shouldn't be called anyway
+                }
+            }
+            tag(galleryID to index)
+        }.build()
 
-                        client.newCall(request).enqueue(object: Callback {
+        client.newCall(request).enqueue(callback)
+    }
+
+    private fun download(galleryID: Int) = CoroutineScope(Dispatchers.IO).launch {
+        val reader = Cache(this@DownloadWorker).getReader(galleryID)
+
+        //gallery doesn't exist
+        if (reader == null) {
+            progress.put(galleryID, null)
+            exception.put(galleryID, null)
+            nRunners--
+            return@launch
+        }
+
+        progress.put(galleryID, reader.galleryInfo.map { 0F }.toMutableList())
+        exception.put(galleryID, reader.galleryInfo.map { null }.toMutableList())
+
+        for (i in reader.galleryInfo.indices) {
+            val callback = object : Callback {
                 override fun onFailure(call: Call, e: IOException) {
                     if (Fabric.isInitialized())
                         Crashlytics.logException(e)
 
-                                progress[tag] = Float.NaN
+                    progress[galleryID]!![i] = Float.NaN
-                                exception[tag] = e
+                    exception[galleryID]!![i] = e
+
+                    if (progress[galleryID]!!.all { !it.isFinite() })
+                        nRunners--
                 }
 
                 override fun onResponse(call: Call, response: Response) {
                     response.use {
                         val res = it.body!!.bytes()
-                                    result[tag] = res
+                        val ext =
-                                    Cache(context).putImage(galleryID, index, res)
+                            call.request().url.encodedPath.split('.').last()
-                                    progress[tag] = Float.POSITIVE_INFINITY
+
+                        Cache(this@DownloadWorker).putImage(galleryID, "$i.$ext", res)
+                        progress[galleryID]!![i] = Float.POSITIVE_INFINITY
+                    }
+
+                    if (progress[galleryID]!!.all { !it.isFinite() })
+                        nRunners--
                 }
             }
-                        })
+
-                    }
+            queueDownload(galleryID, reader, i, callback)
         }
     }
+
+    private fun loop() = CoroutineScope(Dispatchers.Default).launch {
+        while (true) {
+            if (queue.isEmpty() || nRunners > preferences.getInt("max_download", 4))
+                continue
+
+            val galleryID = queue.poll() ?: continue
+
+            worker.put(galleryID, download(galleryID))
+            nRunners++
         }
     }
 

app/src/main/java/xyz/quaver/pupil/util/file.kt

@@ -63,3 +63,5 @@ fun URL.download(to: File, onDownloadProgress: ((Long, Long) -> Unit)? = null) {
 
     }
 }
+
+fun File.isParentOf(file: File) = file.absolutePath.startsWith(this.absolutePath)

app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt

@@ -26,20 +26,16 @@ package xyz.quaver.pupil
  * See [testing documentation](http://d.android.com/tools/testing).
  */
 
+import android.util.SparseArray
 import org.junit.Test
-import xyz.quaver.pupil.util.download
-import java.io.File
-import java.net.URL
 
 class ExampleUnitTest {
 
     @Test
     fun test() {
-        URL("https://github.com/tom5079/Pupil/releases/download/4.2-beta2-hotfix2/Pupil-v4.2-beta2-hotfix2.apk").download(
+        val arr = SparseArray<Float>()
-            File(System.getenv("USERPROFILE"), "Pupil.apk")
+
-        ) { downloaded, fileSize ->
+        print(arr.indexOfKey(34))
-            println("%.1f%%".format(downloaded*100.0/fileSize))
-        }
     }
 
 }