Merge remote-tracking branch 'origin/master'

Fixed images not loading

.idea/deploymentTargetDropDown.xml

@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="deploymentTargetDropDown">
-    <targetSelectedWithDropDown>
-      <Target>
-        <type value="QUICK_BOOT_TARGET" />
-        <deviceKey>
-          <Key>
-            <type value="VIRTUAL_DEVICE_PATH" />
-            <value value="$USER_HOME$/.android/avd/Pixel_2_API_30.avd" />
-          </Key>
-        </deviceKey>
-      </Target>
-    </targetSelectedWithDropDown>
-    <timeTargetWasSelectedWithDropDown value="2022-01-04T14:31:24.587053Z" />
-  </component>
-</project>

README.md

@@ -2,7 +2,7 @@
 *Pupil, Hitomi.la viewer for Android*  
 
 ![](https://img.shields.io/github/downloads/tom5079/Pupil/total)  
-[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.2.7/Pupil-v5.2.7.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.2.7/Pupil-v5.2.7.apk)
+[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.2.14/Pupil-v5.2.14.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.2.14/Pupil-v5.2.14.apk)  
 [![](https://discordapp.com/api/guilds/610452916612104194/embed.png?style=banner2)](https://discord.gg/Stj4b5v)
 
 # Features

app/build.gradle

@@ -38,7 +38,7 @@ android {
         minSdkVersion 16
         targetSdkVersion 31
         versionCode 69
-        versionName "5.2.8-BETA01"
+        versionName "5.2.15"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables.useSupportLibrary = true
     }
@@ -111,6 +111,7 @@ dependencies {
     implementation 'com.github.piasy:BigImageViewer:1.8.1'
     implementation 'com.github.piasy:FrescoImageLoader:1.8.1'
     implementation 'com.github.piasy:FrescoImageViewFactory:1.8.1'
+    implementation 'com.facebook.fresco:imagepipeline-okhttp3:2.6.0'
 
     //noinspection GradleDependency
     implementation "com.squareup.okhttp3:okhttp:$okhttp_version"

app/release/output-metadata.json

@@ -12,7 +12,7 @@
       "filters": [],
       "attributes": [],
       "versionCode": 69,
-      "versionName": "5.2.8-BETA01",
+      "versionName": "5.2.15",
       "outputFile": "app-release.apk"
     }
   ],

app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt

@@ -20,10 +20,13 @@
 
 package xyz.quaver.pupil
 
+import android.os.Build
 import android.util.Log
 import android.webkit.*
+import android.widget.Toast
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
+import com.google.firebase.crashlytics.FirebaseCrashlytics
 import kotlinx.coroutines.*
 import org.junit.Before
 import org.junit.Test
@@ -43,30 +46,7 @@ class ExampleInstrumentedTest {
 
         runBlocking {
             withContext(Dispatchers.Main) {
-                webView = WebView(appContext).apply {
-                    settings.javaScriptEnabled = true
-
-                    addJavascriptInterface(object {
-                        @JavascriptInterface
-                        fun onResult(uid: String, result: String) {
-                            _webViewFlow.tryEmit(uid to result)
-                        }
-                    }, "Callback")
-
-                    loadDataWithBaseURL(
-                        "https://hitomi.la/",
-                        """
-                            <script src="https://ltn.hitomi.la/jquery.min.js"></script>
-                            <script src="https://ltn.hitomi.la/common.js"></script>
-                            <script src="https://ltn.hitomi.la/search.js"></script>
-                            <script src="https://ltn.hitomi.la/searchlib.js"></script>
-                            <script src="https://ltn.hitomi.la/results.js></script>
-                        """.trimIndent(),
-                        "text/html",
-                        null,
-                        null
-                    )
-                }
+                initWebView(appContext)
             }
         }
     }
@@ -74,7 +54,7 @@ class ExampleInstrumentedTest {
     @Test
     fun test_getGalleryIDsFromNozomi() {
         runBlocking {
-            val result = getGalleryIDsFromNozomi(null, "index", "all")
+            val result = getGalleryIDsFromNozomi(null, "boten", "all")
 
             Log.d("PUPILD", "getGalleryIDsFromNozomi: ${result.size}")
         }
@@ -120,10 +100,19 @@ class ExampleInstrumentedTest {
         }
     }
 
+    @Test
+    fun test_getGallery() {
+        runBlocking {
+            val gallery = getGallery(2109479)
+
+            Log.d("PUPILD", gallery.toString())
+        }
+    }
+
     @Test
     fun test_getGalleryBlock() {
         runBlocking {
-            val block = getGalleryBlock(2102731)
+            val block = getGalleryBlock(2013877)
 
             Log.d("PUPILD", block.toString())
         }

app/src/main/AndroidManifest.xml

@@ -6,8 +6,8 @@
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
     <uses-permission android:name="android.permission.USE_BIOMETRIC" />
-    <uses-permission-sdk-23 android:name="android.permission.READ_EXTERNAL_STORAGE"/>
-    <uses-permission-sdk-23 android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="23"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="23"/>
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.VIBRATE"/>

app/src/main/java/xyz/quaver/pupil/Pupil.kt

@@ -28,32 +28,31 @@ import android.content.Intent
 import android.content.pm.ApplicationInfo
 import android.net.Uri
 import android.os.Build
-import android.util.Log
 import android.webkit.*
 import android.widget.Toast
 import androidx.appcompat.app.AppCompatDelegate
 import androidx.core.content.ContextCompat
 import androidx.preference.PreferenceManager
+import com.facebook.imagepipeline.backends.okhttp3.OkHttpImagePipelineConfigFactory
 import com.github.piasy.biv.BigImageViewer
 import com.github.piasy.biv.loader.fresco.FrescoImageLoader
 import com.google.android.gms.common.GooglePlayServicesNotAvailableException
 import com.google.android.gms.common.GooglePlayServicesRepairableException
 import com.google.android.gms.security.ProviderInstaller
-import com.google.android.material.snackbar.Snackbar
 import com.google.firebase.crashlytics.FirebaseCrashlytics
 import kotlinx.coroutines.*
-import kotlinx.coroutines.channels.BufferOverflow
 import kotlinx.coroutines.flow.MutableSharedFlow
 import kotlinx.coroutines.flow.asSharedFlow
 import okhttp3.*
 import xyz.quaver.io.FileX
-import xyz.quaver.pupil.hitomi.evaluations
+import xyz.quaver.pupil.hitomi.evaluationContext
+import xyz.quaver.pupil.types.JavascriptConsoleException
+import xyz.quaver.pupil.types.JavascriptOnErrorException
 import xyz.quaver.pupil.types.Tag
 import xyz.quaver.pupil.util.*
 import java.io.File
 import java.net.URL
 import java.util.*
-import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 import kotlin.reflect.KClass
@@ -81,30 +80,24 @@ val client: OkHttpClient
 
 @SuppressLint("StaticFieldLeak")
 lateinit var webView: WebView
-val _webViewFlow = MutableSharedFlow<Pair<String, String?>>(
-    extraBufferCapacity = 2,
-    onBufferOverflow = BufferOverflow.DROP_OLDEST
-)
+val _webViewFlow = MutableSharedFlow<Pair<String, String?>>()
 val webViewFlow = _webViewFlow.asSharedFlow()
 var webViewReady = false
-    private set
 var webViewFailed = false
-    private set
 private var reloadJob: Job? = null
 
 fun reloadWebView() {
     if (reloadJob?.isActive == true) return
 
     reloadJob = CoroutineScope(Dispatchers.IO).launch {
-        if (evaluations.isEmpty()) {
         webViewReady = false
         webViewFailed = false
 
-            while (evaluations.isNotEmpty()) yield()
+        evaluationContext.cancelChildren(CancellationException("reload"))
 
         runCatching {
             URL(
-                    if (isDebugBuild)
+                if (BuildConfig.DEBUG)
                     "https://tom5079.github.io/Pupil/hitomi-dev.html"
                 else
                     "https://tom5079.github.io/Pupil/hitomi.html"
@@ -123,7 +116,6 @@ fun reloadWebView() {
             }
         }
     }
-    }
 }
 
 private var htmlVersion: String = ""
@@ -133,7 +125,7 @@ fun reloadWhenFailedOrUpdate() = CoroutineScope(Dispatchers.Default).launch {
             webViewFailed ||
             runCatching {
                 URL(
-                    if (isDebugBuild)
+                    if (BuildConfig.DEBUG)
                         "https://tom5079.github.io/Pupil/hitomi-dev.html.ver"
                     else
                         "https://tom5079.github.io/Pupil/hitomi.html.ver"
@@ -151,24 +143,11 @@ fun reloadWhenFailedOrUpdate() = CoroutineScope(Dispatchers.Default).launch {
     }
 }
 
-var isDebugBuild: Boolean = false
-private lateinit var userAgent: String
+@SuppressLint("SetJavaScriptEnabled")
+fun initWebView(context: Context) {
+    if (BuildConfig.DEBUG) WebView.setWebContentsDebuggingEnabled(true)
 
-class Pupil : Application() {
-
-    companion object {
-        lateinit var instance: Pupil
-            private set
-    }
-
-    @SuppressLint("SetJavaScriptEnabled")
-    override fun onCreate() {
-        instance = this
-        isDebugBuild = applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE != 0
-
-        if (isDebugBuild) WebView.setWebContentsDebuggingEnabled(true)
-
-        webView = WebView(this).apply {
+    webView = WebView(context).apply {
         with (settings) {
             javaScriptEnabled = true
             domStorageEnabled = true
@@ -187,18 +166,17 @@ class Pupil : Application() {
                 error: WebResourceError?
             ) {
                 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
-                        FirebaseCrashlytics.getInstance().log(
-                            "onReceivedError: ${error?.description}"
+                    FirebaseCrashlytics.getInstance().recordException(
+                        JavascriptOnErrorException("onReceivedError: ${error?.description}")
                     )
                 }
-                    webViewFailed = true
             }
         }
 
         webChromeClient = object: WebChromeClient() {
             override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
-                    FirebaseCrashlytics.getInstance().log(
-                        "onConsoleMessage: ${consoleMessage?.message()} (${consoleMessage?.sourceId()}:${consoleMessage?.lineNumber()})"
+                FirebaseCrashlytics.getInstance().recordException(
+                    JavascriptConsoleException("onConsoleMessage: ${consoleMessage?.message()} (${consoleMessage?.sourceId()}:${consoleMessage?.lineNumber()})")
                 )
 
                 return super.onConsoleMessage(consoleMessage)
@@ -208,20 +186,40 @@ class Pupil : Application() {
         addJavascriptInterface(object {
             @JavascriptInterface
             fun onResult(uid: String, result: String) {
-                    _webViewFlow.tryEmit(uid to result)
+                CoroutineScope(Dispatchers.Unconfined).launch {
+                    _webViewFlow.emit(uid to result)
+                }
             }
             @JavascriptInterface
             fun onError(uid: String, message: String) {
-                    _webViewFlow.tryEmit(uid to null)
-                    Toast.makeText(this@Pupil, message, Toast.LENGTH_LONG).show()
+                CoroutineScope(Dispatchers.Unconfined).launch {
+                    _webViewFlow.emit(uid to null)
+                }
+                Toast.makeText(context, message, Toast.LENGTH_LONG).show()
                 FirebaseCrashlytics.getInstance().recordException(
-                        Exception(message)
+                    JavascriptOnErrorException(message)
                 )
             }
         }, "Callback")
     }
 
     reloadWhenFailedOrUpdate()
+}
+
+lateinit var userAgent: String
+
+class Pupil : Application() {
+
+    companion object {
+        lateinit var instance: Pupil
+            private set
+    }
+
+    @SuppressLint("SetJavaScriptEnabled")
+    override fun onCreate() {
+        instance = this
+
+        initWebView(this)
 
         AppCompatDelegate.setCompatVectorFromResourcesEnabled(true)
 
@@ -243,6 +241,7 @@ class Pupil : Application() {
             .addInterceptor { chain ->
                 val request = chain.request().newBuilder()
                     .header("User-Agent", userAgent)
+                    .header("Referer", "https://hitomi.la/")
                     .build()
 
                 val tag = request.tag() ?: return@addInterceptor chain.proceed(request)
@@ -257,6 +256,7 @@ class Pupil : Application() {
 
         try {
             Preferences.get<String>("download_folder").also {
+                if (it.startsWith("content://"))
                     contentResolver.takePersistableUriPermission(
                         Uri.parse(it),
                         Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION
@@ -295,7 +295,14 @@ class Pupil : Application() {
             e.printStackTrace()
         }
 
-        BigImageViewer.initialize(FrescoImageLoader.with(this))
+        BigImageViewer.initialize(
+            FrescoImageLoader.with(
+                this,
+                OkHttpImagePipelineConfigFactory
+                    .newBuilder(this, client)
+                    .build()
+            )
+        )
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
             val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager

app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt

@@ -104,10 +104,10 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
 
             val cache = Cache.getInstance(itemView.context, galleryID)
 
-            val galleryBlock = runBlocking {
-                cache.getGalleryBlock()
-            } ?: return
+            CoroutineScope(Dispatchers.IO).launch {
+                val galleryBlock = cache.getGalleryBlock() ?: return@launch
 
+                launch(Dispatchers.Main) {
                     val resources = itemView.context.resources
                     val languages = resources.getStringArray(R.array.languages).map {
                         it.split("|").let { split ->
@@ -128,10 +128,7 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
                         setFailureImage(ContextCompat.getDrawable(context, R.drawable.image_broken_variant))
                         setImageLoaderCallback(object: ImageLoader.Callback {
                             override fun onFail(error: Exception?) {
-                        Cache.getInstance(context, galleryID).let { cache ->
-                            cache.cacheFolder.getChild(".thumbnail").let { if (it.exists()) it.delete() }
-                            cache.downloadFolder?.getChild(".thumbnail")?.let { if (it.exists()) it.delete() }
-                        }
+                                Cache.delete(context, galleryID)
                             }
 
                             override fun onCacheHit(imageType: Int, image: File?) {}
@@ -266,6 +263,9 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
                     }
 
 
+                }
+            }
+
             // Make some views invisible to make it thinner
             if (thin) {
                 binding.galleryblockTagGroup.visibility = View.GONE

app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt

@@ -16,21 +16,11 @@
 
 package xyz.quaver
 
-import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.modules.SerializersModule
-import kotlinx.serialization.modules.plus
-import kotlinx.serialization.modules.polymorphic
-import kotlinx.serialization.modules.subclass
-import okhttp3.OkHttpClient
 import okhttp3.Request
 import xyz.quaver.pupil.client
 import java.io.IOException
 import java.net.URL
-import java.util.concurrent.TimeUnit
-import kotlin.time.Duration
 
 /**
  * kotlinx.serialization.json.Json object for global use  

app/src/main/java/xyz/quaver/pupil/hitomi/common.kt

@@ -16,6 +16,7 @@
 
 package xyz.quaver.pupil.hitomi
 
+import android.util.Log
 import android.webkit.WebView
 import android.widget.Toast
 import com.google.common.collect.ConcurrentHashMultiset
@@ -37,55 +38,71 @@ import kotlin.coroutines.suspendCoroutine
 
 const val protocol = "https:"
 
-val evaluations = Collections.newSetFromMap<String>(ConcurrentHashMap())
+val evaluationContext = Dispatchers.Main + Job()
 
-suspend fun WebView.evaluate(script: String): String = withContext(Dispatchers.Main) {
-    val result: String = withTimeout(10000) {
+suspend fun WebView.evaluate(script: String): String = coroutineScope {
+    var result: String? = null
+
+    while (result == null) {
+        try {
+            result = withContext(evaluationContext) {
                 while (webViewFailed || !webViewReady) yield()
 
-        val uid = UUID.randomUUID().toString()
-
-        evaluations.add(uid)
-
                 suspendCoroutine { continuation ->
                     evaluateJavascript(script) {
-                evaluations.remove(uid)
                         continuation.resume(it)
                     }
                 }
+
+            }
+        } catch (e: CancellationException) {
+            if (e.message != "reload") result = "null"
+        }
     }
 
     result
 }
 
 @OptIn(ExperimentalCoroutinesApi::class)
-suspend fun WebView.evaluatePromise(script: String, then: String = ".then(result => Callback.onResult(%uid, JSON.stringify(result))).catch(err => Callback.onError(%uid, JSON.stringify(error)))"): String? = withContext(Dispatchers.Main) {
-    val flow: Flow<Pair<String, String?>> = withTimeout(10000) {
+suspend fun WebView.evaluatePromise(
+    script: String,
+    then: String = ".then(result => Callback.onResult(%uid, JSON.stringify(result))).catch(err => { console.err(err); Callback.onError(%uid, JSON.stringify(err)); })"
+): String = coroutineScope {
+    var result: String? = null
+
+    while (result == null) {
+        try {
+            result = withContext(evaluationContext) {
                 while (webViewFailed || !webViewReady) yield()
 
                 val uid = UUID.randomUUID().toString()
 
-        evaluations.add(uid)
-
-        evaluateJavascript((script+then).replace("%uid", "'$uid'"), null)
-
-        webViewFlow.transformWhile { (currentUid, result) ->
+                val flow: Flow<Pair<String, String?>> = webViewFlow.transformWhile { (currentUid, result) ->
                     if (currentUid == uid) {
-                evaluations.remove(uid)
                         emit(currentUid to result)
                     }
                     currentUid != uid
                 }
+
+                launch {
+                    evaluateJavascript((script + then).replace("%uid", "'$uid'"), null)
                 }
 
                 flow.first().second
+            }
+        } catch (e: CancellationException) {
+            if (e.message != "reload") result = "null"
+        }
+    }
+
+    result
 }
 
 @Suppress("EXPERIMENTAL_API_USAGE")
 suspend fun getGalleryInfo(galleryID: Int): GalleryInfo {
     val result = webView.evaluatePromise("get_gallery_info($galleryID)")
 
-    return json.decodeFromString(result!!)
+    return json.decodeFromString(result)
 }
 
 //common.js
@@ -108,16 +125,6 @@ suspend fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String?
         """.trimIndent()
     )
 
-
-    FirebaseCrashlytics.getInstance().log(
-        """
-        url_from_url_from_hash(
-            ${galleryID.toString().js},
-            ${Json.encodeToString(image)},
-            ${dir.js}, ${ext.js}, ${base.js}
-        )
-        """.trimIndent()
-    )
     return Json.decodeFromString(result)
 }
 

app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt

@@ -17,7 +17,10 @@
 package xyz.quaver.pupil.hitomi
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import org.jsoup.Jsoup
+import xyz.quaver.pupil.webView
 import xyz.quaver.readText
 import java.net.URL
 import java.net.URLDecoder
@@ -25,7 +28,8 @@ import java.net.URLDecoder
 @Serializable
 data class Gallery(
     val related: List<Int>,
-    val langList: List<Pair<String, String>>,
+    val langList: Map<String, String>
+    ,
     val cover: String,
     val title: String,
     val artists: List<String>,
@@ -38,43 +42,6 @@ data class Gallery(
     val thumbnails: List<String>
 )
 suspend fun getGallery(galleryID: Int) : Gallery {
-    val url = Jsoup.parse(URL("https://hitomi.la/galleries/$galleryID.html").readText())
-        .select("link").attr("href")
-
-    val doc = Jsoup.parse(URL(url).readText())
-
-    val related = Regex("\\d+")
-        .findAll(doc.select("script").first()!!.html())
-        .map {
-            it.value.toInt()
-        }.toList()
-
-    val langList = doc.select("#lang-list a").map {
-            Pair(it.text(), "$protocol//hitomi.la${it.attr("href")}")
-        }
-
-    val cover = protocol + doc.selectFirst(".cover img")!!.attr("src")
-    val title = doc.selectFirst(".gallery h1 a")!!.text()
-    val artists = doc.select(".gallery h2 a").map { it.text() }
-    val groups = doc.select(".gallery-info a[href~=^/group/]").map { it.text() }
-    val type = doc.selectFirst(".gallery-info a[href~=^/type/]")!!.text()
-
-    val language = run {
-        val href = doc.select(".gallery-info a[href~=^/index.+\\.html\$]").attr("href")
-        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
-    }
-
-    val series = doc.select(".gallery-info a[href~=^/series/]").map { it.text() }
-    val characters = doc.select(".gallery-info a[href~=^/character/]").map { it.text() }
-
-    val tags = doc.select(".gallery-info a[href~=^/tag/]").map {
-        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
-        href.slice(5 until href.indexOf('-'))
-    }
-
-    val thumbnails = getGalleryInfo(galleryID).files.map { galleryInfo ->
-        urlFromUrlFromHash(galleryID, galleryInfo, "smalltn", "jpg", "tn")
-    }
-
-    return Gallery(related, langList, cover, title, artists, groups, type, language, series, characters, tags, thumbnails)
+    val result = webView.evaluatePromise("get_gallery($galleryID)")
+    return Json.decodeFromString(result)
 }

app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt

@@ -17,6 +17,8 @@
 package xyz.quaver.pupil.hitomi
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import org.jsoup.Jsoup
 import xyz.quaver.pupil.webView
 import xyz.quaver.readText
@@ -41,39 +43,6 @@ data class GalleryBlock(
 )
 
 suspend fun getGalleryBlock(galleryID: Int) : GalleryBlock {
-    val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension"
-
-    val html: String = webView.evaluatePromise(
-        """
-        $.get('$url').always(function(data, status) {
-            if (status === 'success') {
-                Callback.onResult(%uid, data);
-            }
-        }); 
-        """.trimIndent(),
-        then = ""
-    )!!
-
-    val doc = Jsoup.parse(html)
-
-    val galleryUrl = doc.selectFirst("h1 > a")!!.attr("href")
-
-    val thumbnails = doc.select(".dj-img-cont img").map { protocol + it.attr("src") }
-
-    val title = doc.selectFirst("h1 > a")!!.text()
-    val artists = doc.select(".artist-list a").map{ it.text() }
-    val series = doc.select(".dj-content a[href~=^/series/]").map { it.text() }
-    val type = doc.selectFirst("a[href~=^/type/]")!!.text()
-
-    val language = run {
-        val href = doc.select("a[href~=^/index.+\\.html\$]").attr("href")
-        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
-    }
-
-    val relatedTags = doc.select(".relatedtags a").map {
-        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
-        href.slice(5 until href.indexOf("-all"))
-    }
-
-    return GalleryBlock(galleryID, galleryUrl, thumbnails, title, artists, series, type, language, relatedTags)
+    val result = webView.evaluatePromise("get_gallery_block($galleryID)")
+    return Json.decodeFromString(result)
 }

app/src/main/java/xyz/quaver/pupil/hitomi/results.kt

@@ -16,7 +16,8 @@
 
 package xyz.quaver.pupil.hitomi
 
-import kotlinx.coroutines.*
+import kotlinx.coroutines.async
+import kotlinx.coroutines.coroutineScope
 import java.util.*
 
 suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> = coroutineScope {
@@ -47,7 +48,7 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
         }
     }
 
-    val negativeResults = negativeTerms.map {
+    val negativeResults = negativeTerms.mapIndexed { index, it ->
         async {
             runCatching {
                 getGalleryIDsForQuery(it)
@@ -55,21 +56,21 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
         }
     }
 
-    var results = when {
+    val results = when {
         sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
         positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
         else -> emptySet()
-    }
+    }.toMutableSet()
 
     fun filterPositive(newResults: Set<Int>) {
-        results = when {
-            results.isEmpty() -> newResults
-            else -> results intersect newResults
+        when {
+            results.isEmpty() -> results.addAll(newResults)
+            else -> results.retainAll(newResults)
         }
     }
 
     fun filterNegative(newResults: Set<Int>) {
-        results = results subtract newResults
+        results.removeAll(newResults)
     }
 
     //positive results
@@ -78,7 +79,7 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
     }
 
     //negative results
-    negativeResults.forEach {
+    negativeResults.forEachIndexed { index, it ->
         filterNegative(it.await())
     }
 

app/src/main/java/xyz/quaver/pupil/hitomi/search.kt

@@ -16,6 +16,7 @@
 
 package xyz.quaver.pupil.hitomi
 
+import android.util.Log
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.decodeFromString
@@ -27,7 +28,7 @@ const val extension = ".html"
 
 @OptIn(ExperimentalSerializationApi::class)
 suspend fun getGalleryIDsForQuery(query: String) : Set<Int> {
-    val result = webView.evaluatePromise("get_galleryids_for_query('$query')") ?: return emptySet()
+    val result = webView.evaluatePromise("get_galleryids_for_query('$query')")
 
     return Json.decodeFromString(result)
 }
@@ -37,14 +38,29 @@ data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
 
 @OptIn(ExperimentalSerializationApi::class)
 suspend fun getSuggestionsForQuery(query: String) : List<Suggestion> {
-    val result = webView.evaluatePromise("get_suggestions_for_query('$query')") ?: return emptyList()
+    val result = webView.evaluatePromise(
+        "get_suggestions_for_query('$query', ++search_serial)",
+        then = """
+            .then(r => {
+                let [results, results_serial] = r;
+                console.log(results_serial, r, search_serial);
+                if (search_serial !== results_serial) {
+                    Callback.onResult(%uid, '[]');
+                } else {
+                    Callback.onResult(%uid, JSON.stringify(results));
+                }
+            });
+        """.trimIndent()
+    )
 
-    return Json.decodeFromString<List<List<Suggestion>?>>(result)[0] ?: return emptyList()
+    return Json.decodeFromString(result) ?: return emptyList()
 }
 
 @OptIn(ExperimentalSerializationApi::class)
 suspend fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
     val jsArea = if (area == null) "null" else "'$area'"
 
-    return Json.decodeFromString(webView.evaluatePromise("""get_galleryids_from_nozomi($jsArea, '$tag', '$language')""") ?: return emptySet())
+    val json = webView.evaluatePromise("""get_galleryids_from_nozomi($jsArea, '$tag', '$language')""")
+
+    return Json.decodeFromString(json)
 }

app/src/main/java/xyz/quaver/pupil/services/DownloadService.kt

@@ -172,18 +172,16 @@ class DownloadService : Service() {
     private val interceptor: PupilInterceptor = { chain ->
         val request = chain.request()
 
-        if (rateLimitHost.matches(request.url().host()))
-            rateLimiter.acquire()
-
         var response = chain.proceed(request)
         var limit = 5
 
-        if (!response.isSuccessful && limit > 0) {
-            Thread.sleep(10000)
-            if (rateLimitHost.matches(request.url().host()))
-                rateLimiter.acquire()
+        while (!response.isSuccessful) {
+            if (response.code() == 503) {
+                Thread.sleep(200)
+            } else if (--limit > 0)
+                break
+
             response = chain.proceed(request)
-            limit -= 1
         }
 
         response.newBuilder()
@@ -224,31 +222,34 @@ class DownloadService : Service() {
             val (galleryID, index, startId) = call.request().tag() as Tag
             val ext = call.request().url().encodedPath().split('.').last()
 
-            kotlin.runCatching {
-                val image = response.also { if (it.code() != 200) throw IOException("$galleryID $index ${response.request().url()} CODE ${it.code()}") }.body()?.use { it.bytes() } ?: throw Exception("Response null")
+            CoroutineScope(Dispatchers.IO).launch {
+                runCatching {
+                    response.also {
+                        if (it.code() != 200) throw IOException(
+                            "$galleryID $index ${response.request().url()} CODE ${it.code()}"
+                        )
+                    }.body()?.use {
                         val padding = ceil(progress[galleryID]?.size?.let { log10(it.toFloat()) } ?: 0F).toInt()
 
-                CoroutineScope(Dispatchers.IO).launch {
-                    kotlin.runCatching {
-                        Cache.getInstance(this@DownloadService, galleryID).putImage(index, "${index.toString().padStart(padding, '0')}.$ext", image)
-                    }.onSuccess {
+                        Cache.getInstance(this@DownloadService, galleryID)
+                            .putImage(index, "${index.toString().padStart(padding, '0')}.$ext", it.byteStream())
+
                         progress[galleryID]?.set(index, Float.POSITIVE_INFINITY)
                         notify(galleryID)
 
                         if (isCompleted(galleryID)) {
                             if (DownloadManager.getInstance(this@DownloadService)
-                                    .getDownloadFolder(galleryID) != null)
+                                    .getDownloadFolder(galleryID) != null
+                            )
                                 Cache.getInstance(this@DownloadService, galleryID).moveToDownload()
 
                             startId?.let { stopSelf(it) }
                         }
+                    } ?: throw Exception("Response null")
                 }.onFailure {
                     FirebaseCrashlytics.getInstance().recordException(it)
                 }
             }
-            }.onFailure {
-                FirebaseCrashlytics.getInstance().recordException(it)
-            }
         }
     }
 

app/src/main/java/xyz/quaver/pupil/types/Exceptions.kt

@@ -0,0 +1,22 @@
+/*
+ *     Pupil, Hitomi.la viewer for Android
+ *     Copyright (C) 2022  tom5079
+ *
+ *     This program is free software: you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation, either version 3 of the License, or
+ *     (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package xyz.quaver.pupil.types
+
+class JavascriptConsoleException(message: String?): Exception(message)
+class JavascriptOnErrorException(message: String?): Exception(message)

app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt

@@ -801,7 +801,6 @@ class MainActivity :
                         throw Exception("No result")
                 }
             } catch (e: Exception) {
-
                 if (e !is CancellationException)
                     FirebaseCrashlytics.getInstance().recordException(e)
 

app/src/main/java/xyz/quaver/pupil/ui/dialog/DownloadFolderNameDialogFragment.kt

@@ -24,7 +24,9 @@ import android.view.ViewGroup
 import androidx.core.widget.addTextChangedListener
 import androidx.fragment.app.DialogFragment
 import com.google.android.material.snackbar.Snackbar
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import xyz.quaver.pupil.R
 import xyz.quaver.pupil.databinding.DownloadFolderNameDialogBinding
 import xyz.quaver.pupil.util.Preferences
@@ -56,15 +58,16 @@ class DownloadFolderNameDialogFragment : DialogFragment() {
 
     private fun initView() {
         val galleryID = Cache.instances.let { if (it.size == 0) 1199708 else it.keys.elementAt((0 until it.size).random()) }
-        val galleryBlock = runBlocking {
-            Cache.getInstance(requireContext(), galleryID).getGalleryBlock()
-        }
+        CoroutineScope(Dispatchers.IO).launch {
+            val galleryBlock = Cache.getInstance(requireContext(), galleryID).getGalleryBlock()
 
             binding.message.text = getString(R.string.settings_download_folder_name_message, formatMap.keys.toString(), galleryBlock?.formatDownloadFolder() ?: "")
-        binding.edittext.setText(Preferences["download_folder_name", "[-id-] -title-"])
             binding.edittext.addTextChangedListener {
                 binding.message.text = requireContext().getString(R.string.settings_download_folder_name_message, formatMap.keys.toString(), galleryBlock?.formatDownloadFolderTest(it.toString()) ?: "")
             }
+        }
+
+        binding.edittext.setText(Preferences["download_folder_name", "[-id-] -title-"])
         binding.okButton.setOnClickListener {
             val newValue = binding.edittext.text.toString()
 

app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt

@@ -40,8 +40,10 @@ import xyz.quaver.pupil.hitomi.GalleryBlock
 import xyz.quaver.pupil.hitomi.GalleryInfo
 import xyz.quaver.pupil.hitomi.getGalleryBlock
 import xyz.quaver.pupil.hitomi.getGalleryInfo
+import xyz.quaver.pupil.userAgent
 import java.io.File
 import java.io.IOException
+import java.io.InputStream
 import java.util.concurrent.ConcurrentHashMap
 
 @Serializable
@@ -172,6 +174,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
                 kotlin.runCatching {
                     val request = Request.Builder()
                         .url(it)
+                        .header("Referer", "https://hitomi.la/")
                         .build()
 
                     client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() }
@@ -208,12 +211,14 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
         metadata.imageList?.getOrNull(index)?.let { findFile(it) }
 
     @Suppress("BlockingMethodInNonBlockingContext")
-    fun putImage(index: Int, fileName: String, data: ByteArray) {
+    fun putImage(index: Int, fileName: String, data: InputStream) {
         val file = cacheFolder.getChild(fileName)
 
         if (!file.exists())
             file.createNewFile()
-        file.writeBytes(data)
+        file.outputStream()?.use {
+            data.copyTo(it)
+        }
         setMetadata { metadata -> metadata.imageList!![index] = fileName }
     }
 

app/src/main/java/xyz/quaver/pupil/util/downloader/DownloadManager.kt

@@ -20,8 +20,9 @@ package xyz.quaver.pupil.util.downloader
 
 import android.content.Context
 import android.content.ContextWrapper
-import android.util.Log
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
@@ -86,7 +87,7 @@ class DownloadManager private constructor(context: Context) : ContextWrapper(con
 
     @Synchronized
     fun isDownloading(galleryID: Int): Boolean {
-        val isThisGallery: (Call) -> Boolean = { (it.request().tag() as? DownloadService.Tag)?.galleryID == galleryID }
+        val isThisGallery: (Call) -> Boolean = { !it.isCanceled && (it.request().tag() as? DownloadService.Tag)?.galleryID == galleryID }
 
         return downloadFolderMap.containsKey(galleryID)
                 && client.dispatcher().let { it.queuedCalls().any(isThisGallery) || it.runningCalls().any(isThisGallery) }
@@ -96,16 +97,13 @@ class DownloadManager private constructor(context: Context) : ContextWrapper(con
     fun getDownloadFolder(galleryID: Int): FileX? =
         downloadFolderMap[galleryID]?.let { downloadFolder.getChild(it) }
 
-    @Synchronized
-    fun addDownloadFolder(galleryID: Int) {
-        val name = runBlocking {
-            Cache.getInstance(this@DownloadManager, galleryID).getGalleryBlock()
-        }?.formatDownloadFolder() ?: return
+    fun addDownloadFolder(galleryID: Int) = CoroutineScope(Dispatchers.IO).launch {
+        val name = Cache.getInstance(this@DownloadManager, galleryID).getGalleryBlock()
+            ?.formatDownloadFolder() ?: return@launch
 
         val folder = downloadFolder.getChild(name)
 
-        if (folder.exists())
-            return
+        if (folder.exists()) return@launch
 
         folder.mkdir()