Merge remote-tracking branch 'origin/master'

Fixed images not loading

.github/workflows/watchdiff.yml

@@ -0,0 +1,23 @@
+# This is a basic workflow that is manually triggered
+
+name: Watch hitomi.la file changes
+
+on:
+  schedule:
+    - cron: "*/10 * * * *"
+
+jobs:
+  watchdiff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: watchdiff
+      - name: Download files
+        run: ./fetch.sh
+      - name: Commit and Push
+        id: push
+        run: |
+            git config --global user.name 'Watchdiff bot'
+            git config --global user.email 'tom5079@naver.com'
+            { git add . && git commit -m "File update" && git push; } | tail -1 | grep -q "nothing to commit"

.idea/compiler.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="CompilerConfiguration">
-    <bytecodeTargetLevel target="1.8" />
+    <bytecodeTargetLevel target="11" />
   </component>
 </project>

.idea/gradle.xml

@@ -7,6 +7,8 @@
         <option name="testRunner" value="GRADLE" />
         <option name="distributionType" value="DEFAULT_WRAPPED" />
         <option name="externalProjectPath" value="$PROJECT_DIR$" />
+        <option name="gradleHome" value="/usr/share/java/gradle" />
+        <option name="gradleJvm" value="11" />
         <option name="modules">
           <set>
             <option value="$PROJECT_DIR$" />

.idea/misc.xml

@@ -1,6 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+  <component name="DesignSurface">
+    <option name="filePathToZoomLevelMap">
+      <map>
+        <entry key="app/src/main/res/layout/reader_activity.xml" value="0.14351851851851852" />
+      </map>
+    </option>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="1.8" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/build/classes" />
   </component>
   <component name="ProjectType">

README.md

@@ -2,7 +2,7 @@
 *Pupil, Hitomi.la viewer for Android*  
 
 ![](https://img.shields.io/github/downloads/tom5079/Pupil/total)  
-[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.1.36/Pupil-v5.1.36.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.1.36/Pupil-v5.1.36.apk)  
+[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.2.14/Pupil-v5.2.14.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.2.14/Pupil-v5.2.14.apk)  
 [![](https://discordapp.com/api/guilds/610452916612104194/embed.png?style=banner2)](https://discord.gg/Stj4b5v)
 
 # Features

app/build.gradle

@@ -32,13 +32,13 @@ configurations {
 }
 
 android {
-    compileSdkVersion 30
+    compileSdkVersion 31
     defaultConfig {
         applicationId "xyz.quaver.pupil"
         minSdkVersion 16
-        targetSdkVersion 30
+        targetSdkVersion 31
         versionCode 69
-        versionName "5.1.32"
+        versionName "5.2.15"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables.useSupportLibrary = true
     }
@@ -79,30 +79,30 @@ android {
 dependencies {
     implementation fileTree(dir: "libs", include: ["*.jar", "*.aar"])
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
-    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.4.3"
-    implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.2.0"
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.6.0"
+    implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.2"
     
-    implementation "androidx.appcompat:appcompat:1.3.0"
-    implementation "androidx.activity:activity-ktx:1.3.0-beta01"
-    implementation "androidx.fragment:fragment-ktx:1.3.4"
+    implementation "androidx.appcompat:appcompat:1.4.0"
+    implementation "androidx.activity:activity-ktx:1.4.0"
+    implementation "androidx.fragment:fragment-ktx:1.4.0"
     implementation "androidx.preference:preference-ktx:1.1.1"
     implementation "androidx.recyclerview:recyclerview:1.2.1"
-    implementation "androidx.constraintlayout:constraintlayout:2.0.4"
+    implementation "androidx.constraintlayout:constraintlayout:2.1.2"
     implementation "androidx.gridlayout:gridlayout:1.0.0"
     implementation "androidx.biometric:biometric:1.1.0"
-    implementation "androidx.work:work-runtime-ktx:2.6.0-beta01"
+    implementation "androidx.work:work-runtime-ktx:2.7.1"
 
     implementation "com.daimajia.swipelayout:library:1.2.0@aar"
 
-    implementation "com.google.android.material:material:1.3.0"
+    implementation "com.google.android.material:material:1.4.0"
 
-    implementation platform('com.google.firebase:firebase-bom:26.5.0')
+    implementation platform('com.google.firebase:firebase-bom:29.0.3')
     implementation "com.google.firebase:firebase-analytics-ktx"
     implementation "com.google.firebase:firebase-crashlytics-ktx"
     implementation "com.google.firebase:firebase-perf-ktx"
 
     implementation "com.google.android.gms:play-services-oss-licenses:17.0.0"
-    implementation "com.google.android.gms:play-services-mlkit-face-detection:16.1.7"
+    implementation "com.google.android.gms:play-services-mlkit-face-detection:16.2.1"
 
     implementation "com.github.clans:fab:1.6.4"
 
@@ -111,6 +111,7 @@ dependencies {
     implementation 'com.github.piasy:BigImageViewer:1.8.1'
     implementation 'com.github.piasy:FrescoImageLoader:1.8.1'
     implementation 'com.github.piasy:FrescoImageViewFactory:1.8.1'
+    implementation 'com.facebook.fresco:imagepipeline-okhttp3:2.6.0'
 
     //noinspection GradleDependency
     implementation "com.squareup.okhttp3:okhttp:$okhttp_version"
@@ -129,12 +130,13 @@ dependencies {
 
     implementation "com.google.guava:guava:31.0.1-android"
 
-    implementation "xyz.quaver:documentfilex:0.7.1"
+    implementation "xyz.quaver:documentfilex:0.7.2"
     implementation "xyz.quaver:floatingsearchview:1.1.7"
 
-    testImplementation "junit:junit:4.13.1"
-    androidTestImplementation "androidx.test.ext:junit:1.1.2"
-    androidTestImplementation "androidx.test:rules:1.3.0"
-    androidTestImplementation "androidx.test:runner:1.3.0"
-    androidTestImplementation "androidx.test.espresso:espresso-core:3.3.0"
+    testImplementation "junit:junit:4.13.2"
+    testImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-test:1.6.0"
+    androidTestImplementation "androidx.test.ext:junit:1.1.3"
+    androidTestImplementation "androidx.test:rules:1.4.0"
+    androidTestImplementation "androidx.test:runner:1.4.0"
+    androidTestImplementation "androidx.test.espresso:espresso-core:3.4.0"
 }

app/release/output-metadata.json

@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "artifactType": {
     "type": "APK",
     "kind": "Directory"
@@ -10,9 +10,11 @@
     {
       "type": "SINGLE",
       "filters": [],
+      "attributes": [],
       "versionCode": 69,
-      "versionName": "5.1.32",
+      "versionName": "5.2.15",
       "outputFile": "app-release.apk"
     }
-  ]
+  ],
+  "elementType": "File"
 }

app/src/androidTest/java/xyz/quaver/pupil/ExampleInstrumentedTest.kt

@@ -20,16 +20,18 @@
 
 package xyz.quaver.pupil
 
+import android.os.Build
 import android.util.Log
-import android.webkit.WebView
+import android.webkit.*
+import android.widget.Toast
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.platform.app.InstrumentationRegistry
-import kotlinx.coroutines.MainScope
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.withContext
+import com.google.firebase.crashlytics.FirebaseCrashlytics
+import kotlinx.coroutines.*
+import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
+import xyz.quaver.pupil.hitomi.*
 
 /**
  * Instrumented test, which will execute on an Android device.
@@ -38,22 +40,81 @@ import org.junit.runner.RunWith
  */
 @RunWith(AndroidJUnit4::class)
 class ExampleInstrumentedTest {
-
-    @Test
-    fun useAppContext() {
-        // Context of the app under test.
+    @Before
+    fun init() {
         val appContext = InstrumentationRegistry.getInstrumentation().targetContext
 
         runBlocking {
-            MainScope().launch {
-                val webView = WebView(appContext).apply {
-                    settings.javaScriptEnabled = true
+            withContext(Dispatchers.Main) {
+                initWebView(appContext)
             }
-                webView.evaluateJavascript("3") {
-                    Log.d("PUPILD", it)
         }
-                Log.d("PUPILD", "SYNC?")
-            }.join()
+    }
+
+    @Test
+    fun test_getGalleryIDsFromNozomi() {
+        runBlocking {
+            val result = getGalleryIDsFromNozomi(null, "boten", "all")
+
+            Log.d("PUPILD", "getGalleryIDsFromNozomi: ${result.size}")
+        }
+    }
+
+    @Test
+    fun test_getGalleryIDsForQuery() {
+        runBlocking {
+            val result = getGalleryIDsForQuery("female:crotch tattoo")
+
+            Log.d("PUPILD", "getGalleryIDsForQuery: ${result.size}")
+        }
+    }
+
+    @Test
+    fun test_getSuggestionsForQuery() {
+        runBlocking {
+            val result = getSuggestionsForQuery("fem")
+
+            Log.d("PUPILD", "getSuggestionsForQuery: ${result.size}")
+        }
+    }
+
+    @Test
+    fun test_urlFromUrlFromHash() {
+        runBlocking {
+            val galleryInfo = getGalleryInfo(2102416)
+
+            val result = galleryInfo.files.map {
+                imageUrlFromImage(2102416, it, false)
+            }
+
+            Log.d("PUPILD", result.toString())
+        }
+    }
+
+    @Test
+    fun test_getGalleryInfo() {
+        runBlocking {
+            val galleryInfo = getGalleryInfo(2102416)
+
+            Log.d("PUPILD", galleryInfo.toString())
+        }
+    }
+
+    @Test
+    fun test_getGallery() {
+        runBlocking {
+            val gallery = getGallery(2109479)
+
+            Log.d("PUPILD", gallery.toString())
+        }
+    }
+
+    @Test
+    fun test_getGalleryBlock() {
+        runBlocking {
+            val block = getGalleryBlock(2013877)
+
+            Log.d("PUPILD", block.toString())
         }
     }
 }

app/src/main/AndroidManifest.xml

@@ -6,8 +6,8 @@
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
     <uses-permission android:name="android.permission.USE_BIOMETRIC" />
-    <uses-permission-sdk-23 android:name="android.permission.READ_EXTERNAL_STORAGE"/>
-    <uses-permission-sdk-23 android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" android:maxSdkVersion="23"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" android:maxSdkVersion="23"/>
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.VIBRATE"/>
@@ -59,7 +59,8 @@
         <activity
             android:name=".ui.ReaderActivity"
             android:configChanges="keyboardHidden|orientation|screenSize"
-            android:parentActivityName=".ui.MainActivity">
+            android:parentActivityName=".ui.MainActivity"
+            android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.VIEW" />
 
@@ -223,7 +224,8 @@
         <activity
             android:name=".ui.MainActivity"
             android:configChanges="keyboardHidden|orientation|screenSize"
-            android:theme="@style/NoActionBarAppTheme">
+            android:theme="@style/NoActionBarAppTheme"
+            android:exported="true">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <action android:name="android.intent.action.VIEW" />

app/src/main/java/xyz/quaver/pupil/Pupil.kt

@@ -25,26 +25,33 @@ import android.app.NotificationChannel
 import android.app.NotificationManager
 import android.content.Context
 import android.content.Intent
+import android.content.pm.ApplicationInfo
 import android.net.Uri
 import android.os.Build
-import android.webkit.WebView
+import android.webkit.*
+import android.widget.Toast
 import androidx.appcompat.app.AppCompatDelegate
 import androidx.core.content.ContextCompat
 import androidx.preference.PreferenceManager
+import com.facebook.imagepipeline.backends.okhttp3.OkHttpImagePipelineConfigFactory
 import com.github.piasy.biv.BigImageViewer
 import com.github.piasy.biv.loader.fresco.FrescoImageLoader
 import com.google.android.gms.common.GooglePlayServicesNotAvailableException
 import com.google.android.gms.common.GooglePlayServicesRepairableException
 import com.google.android.gms.security.ProviderInstaller
 import com.google.firebase.crashlytics.FirebaseCrashlytics
-import okhttp3.Dispatcher
-import okhttp3.Interceptor
-import okhttp3.OkHttpClient
-import okhttp3.Response
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.asSharedFlow
+import okhttp3.*
 import xyz.quaver.io.FileX
+import xyz.quaver.pupil.hitomi.evaluationContext
+import xyz.quaver.pupil.types.JavascriptConsoleException
+import xyz.quaver.pupil.types.JavascriptOnErrorException
 import xyz.quaver.pupil.types.Tag
 import xyz.quaver.pupil.util.*
 import java.io.File
+import java.net.URL
 import java.util.*
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
@@ -73,6 +80,133 @@ val client: OkHttpClient
 
 @SuppressLint("StaticFieldLeak")
 lateinit var webView: WebView
+val _webViewFlow = MutableSharedFlow<Pair<String, String?>>()
+val webViewFlow = _webViewFlow.asSharedFlow()
+var webViewReady = false
+var webViewFailed = false
+private var reloadJob: Job? = null
+
+fun reloadWebView() {
+    if (reloadJob?.isActive == true) return
+
+    reloadJob = CoroutineScope(Dispatchers.IO).launch {
+        webViewReady = false
+        webViewFailed = false
+
+        evaluationContext.cancelChildren(CancellationException("reload"))
+
+        runCatching {
+            URL(
+                if (BuildConfig.DEBUG)
+                    "https://tom5079.github.io/Pupil/hitomi-dev.html"
+                else
+                    "https://tom5079.github.io/Pupil/hitomi.html"
+            ).readText()
+        }.onFailure {
+            webViewFailed = true
+        }.getOrNull()?.let { html ->
+            launch(Dispatchers.Main) {
+                webView.loadDataWithBaseURL(
+                    "https://hitomi.la/",
+                    html,
+                    "text/html",
+                    null,
+                    null
+                )
+            }
+        }
+    }
+}
+
+private var htmlVersion: String = ""
+fun reloadWhenFailedOrUpdate() = CoroutineScope(Dispatchers.Default).launch {
+    while (true) {
+        if (
+            webViewFailed ||
+            runCatching {
+                URL(
+                    if (BuildConfig.DEBUG)
+                        "https://tom5079.github.io/Pupil/hitomi-dev.html.ver"
+                    else
+                        "https://tom5079.github.io/Pupil/hitomi.html.ver"
+                ).readText()
+            }.getOrNull().let { version ->
+                (!version.isNullOrEmpty() && version != htmlVersion).also {
+                    if (it) htmlVersion = version!!
+                }
+            }
+        ) {
+            reloadWebView()
+        }
+
+        delay(if (webViewReady && !webViewFailed) 10000 else 1000)
+    }
+}
+
+@SuppressLint("SetJavaScriptEnabled")
+fun initWebView(context: Context) {
+    if (BuildConfig.DEBUG) WebView.setWebContentsDebuggingEnabled(true)
+
+    webView = WebView(context).apply {
+        with (settings) {
+            javaScriptEnabled = true
+            domStorageEnabled = true
+        }
+
+        userAgent = settings.userAgentString
+
+        webViewClient = object: WebViewClient() {
+            override fun onPageFinished(view: WebView?, url: String?) {
+                webViewReady = true
+            }
+
+            override fun onReceivedError(
+                view: WebView?,
+                request: WebResourceRequest?,
+                error: WebResourceError?
+            ) {
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+                    FirebaseCrashlytics.getInstance().recordException(
+                        JavascriptOnErrorException("onReceivedError: ${error?.description}")
+                    )
+                }
+            }
+        }
+
+        webChromeClient = object: WebChromeClient() {
+            override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
+                FirebaseCrashlytics.getInstance().recordException(
+                    JavascriptConsoleException("onConsoleMessage: ${consoleMessage?.message()} (${consoleMessage?.sourceId()}:${consoleMessage?.lineNumber()})")
+                )
+
+                return super.onConsoleMessage(consoleMessage)
+            }
+        }
+
+        addJavascriptInterface(object {
+            @JavascriptInterface
+            fun onResult(uid: String, result: String) {
+                CoroutineScope(Dispatchers.Unconfined).launch {
+                    _webViewFlow.emit(uid to result)
+                }
+            }
+            @JavascriptInterface
+            fun onError(uid: String, message: String) {
+                CoroutineScope(Dispatchers.Unconfined).launch {
+                    _webViewFlow.emit(uid to null)
+                }
+                Toast.makeText(context, message, Toast.LENGTH_LONG).show()
+                FirebaseCrashlytics.getInstance().recordException(
+                    JavascriptOnErrorException(message)
+                )
+            }
+        }, "Callback")
+    }
+
+    reloadWhenFailedOrUpdate()
+}
+
+lateinit var userAgent: String
 
 class Pupil : Application() {
 
@@ -85,11 +219,7 @@ class Pupil : Application() {
     override fun onCreate() {
         instance = this
 
-        webView = WebView(this).apply {
-            settings.javaScriptEnabled = true
-
-            loadData("""<script src="https://ltn.hitomi.la/gg.js"></script>""", "text/html", null)
-        }
+        initWebView(this)
 
         AppCompatDelegate.setCompatVectorFromResourcesEnabled(true)
 
@@ -109,7 +239,11 @@ class Pupil : Application() {
             .readTimeout(0, TimeUnit.SECONDS)
             .proxyInfo(proxyInfo)
             .addInterceptor { chain ->
-                val request = chain.request()
+                val request = chain.request().newBuilder()
+                    .header("User-Agent", userAgent)
+                    .header("Referer", "https://hitomi.la/")
+                    .build()
+
                 val tag = request.tag() ?: return@addInterceptor chain.proceed(request)
 
                 interceptors[tag::class]?.invoke(chain) ?: chain.proceed(request)
@@ -122,7 +256,7 @@ class Pupil : Application() {
 
         try {
             Preferences.get<String>("download_folder").also {
-                if (it.startsWith("content") && Build.VERSION.SDK_INT > 19)
+                if (it.startsWith("content://"))
                     contentResolver.takePersistableUriPermission(
                         Uri.parse(it),
                         Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_GRANT_WRITE_URI_PERMISSION
@@ -161,7 +295,14 @@ class Pupil : Application() {
             e.printStackTrace()
         }
 
-        BigImageViewer.initialize(FrescoImageLoader.with(this))
+        BigImageViewer.initialize(
+            FrescoImageLoader.with(
+                this,
+                OkHttpImagePipelineConfigFactory
+                    .newBuilder(this, client)
+                    .build()
+            )
+        )
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
             val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager

app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt

@@ -104,10 +104,10 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
 
             val cache = Cache.getInstance(itemView.context, galleryID)
 
-            val galleryBlock = runBlocking {
-                cache.getGalleryBlock()
-            } ?: return
+            CoroutineScope(Dispatchers.IO).launch {
+                val galleryBlock = cache.getGalleryBlock() ?: return@launch
 
+                launch(Dispatchers.Main) {
                     val resources = itemView.context.resources
                     val languages = resources.getStringArray(R.array.languages).map {
                         it.split("|").let { split ->
@@ -128,10 +128,7 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
                         setFailureImage(ContextCompat.getDrawable(context, R.drawable.image_broken_variant))
                         setImageLoaderCallback(object: ImageLoader.Callback {
                             override fun onFail(error: Exception?) {
-                        Cache.getInstance(context, galleryID).let { cache ->
-                            cache.cacheFolder.getChild(".thumbnail").let { if (it.exists()) it.delete() }
-                            cache.downloadFolder?.getChild(".thumbnail")?.let { if (it.exists()) it.delete() }
-                        }
+                                Cache.delete(context, galleryID)
                             }
 
                             override fun onCacheHit(imageType: Int, image: File?) {}
@@ -266,6 +263,9 @@ class GalleryBlockAdapter(private val galleries: List<Int>) : RecyclerSwipeAdapt
                     }
 
 
+                }
+            }
+
             // Make some views invisible to make it thinner
             if (thin) {
                 binding.galleryblockTagGroup.visibility = View.GONE

app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt

@@ -16,21 +16,11 @@
 
 package xyz.quaver
 
-import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.modules.SerializersModule
-import kotlinx.serialization.modules.plus
-import kotlinx.serialization.modules.polymorphic
-import kotlinx.serialization.modules.subclass
-import okhttp3.OkHttpClient
 import okhttp3.Request
 import xyz.quaver.pupil.client
 import java.io.IOException
 import java.net.URL
-import java.util.concurrent.TimeUnit
-import kotlin.time.Duration
 
 /**
  * kotlinx.serialization.json.Json object for global use  
@@ -54,12 +44,3 @@ fun URL.readText(settings: HeaderSetter? = null): String {
     
     return client.newCall(request).execute().also{ if (it.code() != 200) throw IOException() }.body()?.use { it.string() } ?: throw IOException()
 }
-
-fun URL.readBytes(settings: HeaderSetter? = null): ByteArray {
-    val request = Request.Builder()
-        .url(this).let {
-            settings?.invoke(it) ?: it
-        }.build()
-
-    return client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() } ?: throw IOException()
-}

app/src/main/java/xyz/quaver/pupil/hitomi/common.kt

@@ -16,145 +16,119 @@
 
 package xyz.quaver.pupil.hitomi
 
-import android.annotation.SuppressLint
 import android.util.Log
 import android.webkit.WebView
-import android.webkit.WebViewClient
+import android.widget.Toast
+import com.google.common.collect.ConcurrentHashMultiset
+import com.google.firebase.crashlytics.FirebaseCrashlytics
 import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.flow.transformWhile
+import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import xyz.quaver.json
-import xyz.quaver.pupil.Pupil
-import xyz.quaver.pupil.webView
-import xyz.quaver.readText
-import java.net.URL
-import java.nio.charset.Charset
+import xyz.quaver.pupil.*
+import java.util.*
+import java.util.concurrent.ConcurrentHashMap
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
 
 const val protocol = "https:"
 
+val evaluationContext = Dispatchers.Main + Job()
+
+suspend fun WebView.evaluate(script: String): String = coroutineScope {
+    var result: String? = null
+
+    while (result == null) {
+        try {
+            result = withContext(evaluationContext) {
+                while (webViewFailed || !webViewReady) yield()
+
+                suspendCoroutine { continuation ->
+                    evaluateJavascript(script) {
+                        continuation.resume(it)
+                    }
+                }
+
+            }
+        } catch (e: CancellationException) {
+            if (e.message != "reload") result = "null"
+        }
+    }
+
+    result
+}
+
+@OptIn(ExperimentalCoroutinesApi::class)
+suspend fun WebView.evaluatePromise(
+    script: String,
+    then: String = ".then(result => Callback.onResult(%uid, JSON.stringify(result))).catch(err => { console.err(err); Callback.onError(%uid, JSON.stringify(err)); })"
+): String = coroutineScope {
+    var result: String? = null
+
+    while (result == null) {
+        try {
+            result = withContext(evaluationContext) {
+                while (webViewFailed || !webViewReady) yield()
+
+                val uid = UUID.randomUUID().toString()
+
+                val flow: Flow<Pair<String, String?>> = webViewFlow.transformWhile { (currentUid, result) ->
+                    if (currentUid == uid) {
+                        emit(currentUid to result)
+                    }
+                    currentUid != uid
+                }
+
+                launch {
+                    evaluateJavascript((script + then).replace("%uid", "'$uid'"), null)
+                }
+
+                flow.first().second
+            }
+        } catch (e: CancellationException) {
+            if (e.message != "reload") result = "null"
+        }
+    }
+
+    result
+}
+
 @Suppress("EXPERIMENTAL_API_USAGE")
-fun getGalleryInfo(galleryID: Int) =
-    json.decodeFromString<GalleryInfo>(
-        URL("$protocol//$domain/galleries/$galleryID.js").readText()
-            .replace("var galleryinfo = ", "")
-    )
+suspend fun getGalleryInfo(galleryID: Int): GalleryInfo {
+    val result = webView.evaluatePromise("get_gallery_info($galleryID)")
+
+    return json.decodeFromString(result)
+}
 
 //common.js
 const val domain = "ltn.hitomi.la"
-const val galleryblockextension = ".html"
 const val galleryblockdir = "galleryblock"
 const val nozomiextension = ".nozomi"
 
-@SuppressLint("SetJavaScriptEnabled")
-interface gg {
-    fun m(g: Int): Int
-    val b: String
-    fun s(h: String): String
+val String?.js: String
+    get() = if (this == null) "null" else "'$this'"
 
-    companion object {
-        @Volatile private var instance: gg? = null
+@OptIn(ExperimentalSerializationApi::class)
+suspend fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null): String {
+    val result = webView.evaluate(
+        """
+        url_from_url_from_hash(
+            ${galleryID.toString().js},
+            ${Json.encodeToString(image)},
+            ${dir.js}, ${ext.js}, ${base.js}
+        )
+        """.trimIndent()
+    )
 
-        fun getInstance(): gg =
-            instance ?: synchronized(this) {
-                instance ?: object: gg {
-                    override fun m(g: Int): Int {
-                        var result: Int? = null
-
-                        MainScope().launch {
-                            while (webView.progress != 100) delay(100)
-                            webView.evaluateJavascript("gg.m($g)") {
-                                result = it.toInt()
-                            }
-                        }
-
-                        while (result == null) Thread.sleep(100)
-
-                        return result!!
-                    }
-
-                    override val b: String
-                        get() {
-                            var result: String? = null
-
-                            MainScope().launch {
-                                while (webView.progress != 100) delay(100)
-                                webView.evaluateJavascript("gg.b") {
-                                    result = it.replace("\"", "")
-                                }
-                            }
-
-                            while (result == null) Thread.sleep(100)
-
-                            return result!!
-                        }
-
-                    override fun s(h: String): String {
-                        var result: String? = null
-
-                        MainScope().launch {
-                            while (webView.progress != 100) delay(100)
-                            webView.evaluateJavascript("gg.s('$h')") {
-                                result = it.replace("\"", "")
-                            }
-                        }
-
-                        while (result == null) Thread.sleep(100)
-
-                        return result!!
-                    }
-                }.also { instance = it }
-            }
-    }
+    return Json.decodeFromString(result)
 }
 
-fun subdomainFromURL(url: String, base: String? = null) : String {
-    var retval = "b"
-
-    if (!base.isNullOrBlank())
-        retval = base
-
-    val b = 16
-
-    val r = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
-    val m = r.find(url) ?: return "a"
-
-    val g = m.groupValues.let { it[2]+it[1] }.toIntOrNull(b)
-
-    if (g != null) {
-        retval = (97+ gg.getInstance().m(g)).toChar().toString() + retval
-    }
-
-    return retval
-}
-
-fun urlFromUrl(url: String, base: String? = null) : String {
-    return url.replace(Regex("""//..?\.hitomi\.la/"""), "//${subdomainFromURL(url, base)}.hitomi.la/")
-}
-
-
-fun fullPathFromHash(hash: String) : String =
-    "${gg.getInstance().b}${gg.getInstance().s(hash)}/$hash"
-
-fun realFullPathFromHash(hash: String): String =
-    hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1/$hash")
-
-fun urlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null) : String {
-    val ext = ext ?: dir ?: image.name.takeLastWhile { it != '.' }
-    val dir = dir ?: "images"
-    return "https://a.hitomi.la/$dir/${fullPathFromHash(image.hash)}.$ext"
-}
-
-fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null) =
-    if (base == "tn")
-        urlFromUrl("https://a.hitomi.la/$dir/${realFullPathFromHash(image.hash)}.$ext", base)
-    else
-        urlFromUrl(urlFromHash(galleryID, image, dir, ext), base)
-
-fun rewriteTnPaths(html: String) =
-    html.replace(Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}""")) { url ->
-        urlFromUrl(url.value, "tn")
-    }
-
-fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String {
+suspend fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String {
     return when {
         noWebp ->
             urlFromUrlFromHash(galleryID, image)

app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt

@@ -17,7 +17,10 @@
 package xyz.quaver.pupil.hitomi
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import org.jsoup.Jsoup
+import xyz.quaver.pupil.webView
 import xyz.quaver.readText
 import java.net.URL
 import java.net.URLDecoder
@@ -25,7 +28,8 @@ import java.net.URLDecoder
 @Serializable
 data class Gallery(
     val related: List<Int>,
-    val langList: List<Pair<String, String>>,
+    val langList: Map<String, String>
+    ,
     val cover: String,
     val title: String,
     val artists: List<String>,
@@ -37,44 +41,7 @@ data class Gallery(
     val tags: List<String>,
     val thumbnails: List<String>
 )
-fun getGallery(galleryID: Int) : Gallery {
-    val url = Jsoup.parse(URL("https://hitomi.la/galleries/$galleryID.html").readText())
-        .select("link").attr("href")
-
-    val doc = Jsoup.parse(URL(url).readText())
-
-    val related = Regex("\\d+")
-        .findAll(doc.select("script").first()!!.html())
-        .map {
-            it.value.toInt()
-        }.toList()
-
-    val langList = doc.select("#lang-list a").map {
-            Pair(it.text(), "$protocol//hitomi.la${it.attr("href")}")
-        }
-
-    val cover = protocol + doc.selectFirst(".cover img")!!.attr("src")
-    val title = doc.selectFirst(".gallery h1 a")!!.text()
-    val artists = doc.select(".gallery h2 a").map { it.text() }
-    val groups = doc.select(".gallery-info a[href~=^/group/]").map { it.text() }
-    val type = doc.selectFirst(".gallery-info a[href~=^/type/]")!!.text()
-
-    val language = run {
-        val href = doc.select(".gallery-info a[href~=^/index.+\\.html\$]").attr("href")
-        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
-    }
-
-    val series = doc.select(".gallery-info a[href~=^/series/]").map { it.text() }
-    val characters = doc.select(".gallery-info a[href~=^/character/]").map { it.text() }
-
-    val tags = doc.select(".gallery-info a[href~=^/tag/]").map {
-        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
-        href.slice(5 until href.indexOf('-'))
-    }
-
-    val thumbnails = getGalleryInfo(galleryID).files.map { galleryInfo ->
-        urlFromUrlFromHash(galleryID, galleryInfo, "smalltn", "jpg", "tn")
-    }
-
-    return Gallery(related, langList, cover, title, artists, groups, type, language, series, characters, tags, thumbnails)
+suspend fun getGallery(galleryID: Int) : Gallery {
+    val result = webView.evaluatePromise("get_gallery($galleryID)")
+    return Json.decodeFromString(result)
 }

app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt

@@ -17,7 +17,10 @@
 package xyz.quaver.pupil.hitomi
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import org.jsoup.Jsoup
+import xyz.quaver.pupil.webView
 import xyz.quaver.readText
 import java.net.URL
 import java.net.URLDecoder
@@ -26,42 +29,6 @@ import java.nio.ByteOrder
 import java.util.*
 import javax.net.ssl.HttpsURLConnection
 
-//galleryblock.js
-fun fetchNozomi(area: String? = null, tag: String = "index", language: String = "all", start: Int = -1, count: Int = -1) : Pair<List<Int>, Int> {
-    val url =
-            when(area) {
-                null -> "$protocol//$domain/$tag-$language$nozomiextension"
-                else -> "$protocol//$domain/$area/$tag-$language$nozomiextension"
-            }
-
-    with(URL(url).openConnection() as HttpsURLConnection) {
-        requestMethod = "GET"
-
-        if (start != -1 && count != -1) {
-            val startByte = start*4
-            val endByte = (start+count)*4-1
-
-            setRequestProperty("Range", "bytes=$startByte-$endByte")
-        }
-
-        connect()
-
-        val totalItems = getHeaderField("Content-Range")
-            .replace(Regex("^[Bb]ytes \\d+-\\d+/"), "").toInt() / 4
-
-        val nozomi = ArrayList<Int>()
-
-        val arrayBuffer = ByteBuffer
-            .wrap(inputStream.readBytes())
-            .order(ByteOrder.BIG_ENDIAN)
-
-        while (arrayBuffer.hasRemaining())
-            nozomi.add(arrayBuffer.int)
-
-        return Pair(nozomi, totalItems)
-    }
-}
-
 @Serializable
 data class GalleryBlock(
     val id: Int,
@@ -75,31 +42,7 @@ data class GalleryBlock(
     val relatedTags: List<String>
 )
 
-fun getGalleryBlock(galleryID: Int) : GalleryBlock {
-    val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension"
-
-    val doc = Jsoup.parse(rewriteTnPaths(URL(url).readText()))
-
-    val galleryUrl = doc.selectFirst("h1 > a")!!.attr("href")
-
-    val thumbnails = doc.select(".dj-img-cont img").map { protocol + it.attr("src") }
-
-    val title = doc.selectFirst("h1 > a")!!.text()
-    val artists = doc.select(".artist-list a").map{ it.text() }
-    val series = doc.select(".dj-content a[href~=^/series/]").map { it.text() }
-    val type = doc.selectFirst("a[href~=^/type/]")!!.text()
-
-    val language = run {
-        val href = doc.select("a[href~=^/index.+\\.html\$]").attr("href")
-        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
-    }
-
-    val relatedTags = doc.select(".relatedtags a").map {
-        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
-        href.slice(5 until href.indexOf("-all"))
-    }
-
-    return GalleryBlock(galleryID, galleryUrl, thumbnails, title, artists, series, type, language, relatedTags)
+suspend fun getGalleryBlock(galleryID: Int) : GalleryBlock {
+    val result = webView.evaluatePromise("get_gallery_block($galleryID)")
+    return Json.decodeFromString(result)
 }
-
-fun getGalleryBlockOrNull(galleryID: Int) = runCatching { getGalleryBlock(galleryID) }.getOrNull()

app/src/main/java/xyz/quaver/pupil/hitomi/reader.kt

@@ -44,6 +44,6 @@ data class GalleryFiles(
 
 //Set header `Referer` to reader url to avoid 403 error
 @Deprecated("", replaceWith = ReplaceWith("getGalleryInfo"))
-fun getReader(galleryID: Int) : GalleryInfo {
+suspend fun getReader(galleryID: Int) : GalleryInfo {
    return getGalleryInfo(galleryID)
 }

app/src/main/java/xyz/quaver/pupil/hitomi/results.kt

@@ -16,13 +16,11 @@
 
 package xyz.quaver.pupil.hitomi
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.coroutineScope
 import java.util.*
 
-fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> {
+suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> = coroutineScope {
     val terms = query
         .trim()
         .replace(Regex("""^\?"""), "")
@@ -43,37 +41,36 @@ fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> {
     }
 
     val positiveResults = positiveTerms.map {
-        CoroutineScope(Dispatchers.IO).async {
-            kotlin.runCatching {
+        async {
+            runCatching {
                 getGalleryIDsForQuery(it)
             }.getOrElse { emptySet() }
         }
     }
 
-    val negativeResults = negativeTerms.map {
-        CoroutineScope(Dispatchers.IO).async {
-            kotlin.runCatching {
+    val negativeResults = negativeTerms.mapIndexed { index, it ->
+        async {
+            runCatching {
                 getGalleryIDsForQuery(it)
             }.getOrElse { emptySet() }
         }
     }
 
-    var results = when {
+    val results = when {
         sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
         positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
         else -> emptySet()
-    }
+    }.toMutableSet()
 
-    runBlocking {
-        @Synchronized fun filterPositive(newResults: Set<Int>) {
-            results = when {
-                results.isEmpty() -> newResults
-                else -> results intersect newResults
+    fun filterPositive(newResults: Set<Int>) {
+        when {
+            results.isEmpty() -> results.addAll(newResults)
+            else -> results.retainAll(newResults)
         }
     }
 
-        @Synchronized fun filterNegative(newResults: Set<Int>) {
-            results = results subtract newResults
+    fun filterNegative(newResults: Set<Int>) {
+        results.removeAll(newResults)
     }
 
     //positive results
@@ -82,10 +79,9 @@ fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> {
     }
 
     //negative results
-        negativeResults.forEach {
+    negativeResults.forEachIndexed { index, it ->
         filterNegative(it.await())
     }
-    }
 
-    return results
+    results
 }

app/src/main/java/xyz/quaver/pupil/hitomi/search.kt

@@ -16,315 +16,51 @@
 
 package xyz.quaver.pupil.hitomi
 
-import okhttp3.Request
-import xyz.quaver.pupil.client
-import xyz.quaver.readBytes
-import xyz.quaver.readText
-import java.net.URL
-import java.nio.ByteBuffer
-import java.nio.ByteOrder
-import java.security.MessageDigest
-import kotlin.math.min
+import android.util.Log
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import xyz.quaver.pupil.webView
 
 //searchlib.js
-const val separator = "-"
 const val extension = ".html"
-const val index_dir = "tagindex"
-const val galleries_index_dir = "galleriesindex"
-const val max_node_size = 464
-const val B = 16
-const val compressed_nozomi_prefix = "n"
 
-val tag_index_version: String by lazy { getIndexVersion("tagindex") }
-val galleries_index_version: String by lazy { getIndexVersion("galleriesindex") }
+@OptIn(ExperimentalSerializationApi::class)
+suspend fun getGalleryIDsForQuery(query: String) : Set<Int> {
+    val result = webView.evaluatePromise("get_galleryids_for_query('$query')")
 
-fun sha256(data: ByteArray) : ByteArray {
-    return MessageDigest.getInstance("SHA-256").digest(data)
-}
-
-@OptIn(ExperimentalUnsignedTypes::class)
-fun hashTerm(term: String) : UByteArray {
-    return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4)
-}
-
-fun sanitize(input: String) : String {
-    return input.replace(Regex("[/#]"), "")
-}
-
-fun getIndexVersion(name: String) =
-        URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText()
-
-//search.js
-fun getGalleryIDsForQuery(query: String) : Set<Int> {
-    query.replace("_", " ").let {
-        if (it.indexOf(':') > -1) {
-            val sides = it.split(":")
-            val ns = sides[0]
-            var tag = sides[1]
-
-            var area : String? = ns
-            var language = "all"
-            when (ns) {
-                "female", "male" -> {
-                    area = "tag"
-                    tag = it
-                }
-                "language" -> {
-                    area = null
-                    language = tag
-                    tag = "index"
-                }
-            }
-
-            return getGalleryIDsFromNozomi(area, tag, language)
-        }
-
-        val key = hashTerm(it)
-        val field = "galleries"
-
-        val node = getNodeAtAddress(field, 0) ?: return emptySet()
-
-        val data = bSearch(field, key, node)
-
-        if (data != null)
-            return getGalleryIDsFromData(data)
-
-        return emptySet()
-    }
-}
-
-fun getSuggestionsForQuery(query: String) : List<Suggestion> {
-    query.replace('_', ' ').let {
-        var field = "global"
-        var term = it
-
-        if (term.indexOf(':') > -1) {
-            val sides = it.split(':')
-            field = sides[0]
-            term = sides[1]
-        }
-
-        val key = hashTerm(term)
-        val node = getNodeAtAddress(field, 0) ?: return emptyList()
-        val data = bSearch(field, key, node)
-
-        if (data != null)
-            return getSuggestionsFromData(field, data)
-
-        return emptyList()
-    }
+    return Json.decodeFromString(result)
 }
 
+@Serializable
 data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
-fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggestion> {
-    val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data"
-    val (offset, length) = data
-    if (length > 10000 || length <= 0)
-        throw Exception("length $length is too long")
 
-    val inbuf = getURLAtRange(url, offset.until(offset+length))
-
-    val suggestions = ArrayList<Suggestion>()
-
-    val buffer = ByteBuffer
-        .wrap(inbuf)
-        .order(ByteOrder.BIG_ENDIAN)
-    val numberOfSuggestions = buffer.int
-
-    if (numberOfSuggestions > 100 || numberOfSuggestions <= 0)
-        throw Exception("number of suggestions $numberOfSuggestions is too long")
-
-    for (i in 0.until(numberOfSuggestions)) {
-        var top = buffer.int
-
-        val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
-        buffer.position(buffer.position()+top)
-
-        top = buffer.int
-
-        val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
-        buffer.position(buffer.position()+top)
-
-        val count = buffer.int
-
-        val tagname = sanitize(tag)
-        val u =
-                when(ns) {
-                    "female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
-                    "language" -> "/index-$tagname${separator}1$extension"
-                    else -> "/$ns/$tagname${separator}all${separator}1$extension"
+@OptIn(ExperimentalSerializationApi::class)
+suspend fun getSuggestionsForQuery(query: String) : List<Suggestion> {
+    val result = webView.evaluatePromise(
+        "get_suggestions_for_query('$query', ++search_serial)",
+        then = """
+            .then(r => {
+                let [results, results_serial] = r;
+                console.log(results_serial, r, search_serial);
+                if (search_serial !== results_serial) {
+                    Callback.onResult(%uid, '[]');
+                } else {
+                    Callback.onResult(%uid, JSON.stringify(results));
                 }
+            });
+        """.trimIndent()
+    )
 
-        suggestions.add(Suggestion(tag, count, u, ns))
-    }
-
-    return suggestions
+    return Json.decodeFromString(result) ?: return emptyList()
 }
 
-fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
-    val nozomiAddress =
-            when(area) {
-                null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension"
-                else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension"
-            }
+@OptIn(ExperimentalSerializationApi::class)
+suspend fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
+    val jsArea = if (area == null) "null" else "'$area'"
 
-    val bytes = try {
-        URL(nozomiAddress).readBytes()
-    } catch (e: Exception) {
-        return emptySet()
-    }
+    val json = webView.evaluatePromise("""get_galleryids_from_nozomi($jsArea, '$tag', '$language')""")
 
-    val nozomi = mutableSetOf<Int>()
-
-    val arrayBuffer = ByteBuffer
-        .wrap(bytes)
-        .order(ByteOrder.BIG_ENDIAN)
-
-    while (arrayBuffer.hasRemaining())
-        nozomi.add(arrayBuffer.int)
-
-    return nozomi
-}
-
-fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
-    val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data"
-    val (offset, length) = data
-    if (length > 100000000 || length <= 0)
-        throw Exception("length $length is too long")
-
-    val inbuf = getURLAtRange(url, offset.until(offset+length))
-
-    val galleryIDs = mutableSetOf<Int>()
-
-    val buffer = ByteBuffer
-        .wrap(inbuf)
-        .order(ByteOrder.BIG_ENDIAN)
-
-    val numberOfGalleryIDs = buffer.int
-
-    val expectedLength = numberOfGalleryIDs*4+4
-
-    if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0)
-        throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
-    else if (inbuf.size != expectedLength)
-        throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength")
-
-    for (i in 0.until(numberOfGalleryIDs))
-        galleryIDs.add(buffer.int)
-
-    return galleryIDs
-}
-
-fun getNodeAtAddress(field: String, address: Long) : Node? {
-    val url =
-            when(field) {
-                "galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"
-                "languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index"
-                "nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index"
-                else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index"
-            }
-
-    val nodedata = getURLAtRange(url, address.until(address+max_node_size))
-
-    return decodeNode(nodedata)
-}
-
-fun getURLAtRange(url: String, range: LongRange) : ByteArray {
-    val request = Request.Builder()
-        .url(url)
-        .header("Range", "bytes=${range.first}-${range.last}")
-        .build()
-    
-    return client.newCall(request).execute().body()?.use { it.bytes() } ?: byteArrayOf()
-}
-
-@OptIn(ExperimentalUnsignedTypes::class)
-data class Node(val keys: List<UByteArray>, val datas: List<Pair<Long, Int>>, val subNodeAddresses: List<Long>)
-@OptIn(ExperimentalUnsignedTypes::class)
-fun decodeNode(data: ByteArray) : Node {
-    val buffer = ByteBuffer
-        .wrap(data)
-        .order(ByteOrder.BIG_ENDIAN)
-
-    val uData = data.toUByteArray()
-
-    val numberOfKeys = buffer.int
-    val keys = ArrayList<UByteArray>()
-
-    for (i in 0.until(numberOfKeys)) {
-        val keySize = buffer.int
-
-        if (keySize == 0 || keySize > 32)
-            throw Exception("fatal: !keySize || keySize > 32")
-
-        keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize)))
-        buffer.position(buffer.position()+keySize)
-    }
-
-    val numberOfDatas = buffer.int
-    val datas = ArrayList<Pair<Long, Int>>()
-
-    for (i in 0.until(numberOfDatas)) {
-        val offset = buffer.long
-        val length = buffer.int
-
-        datas.add(Pair(offset, length))
-    }
-
-    val numberOfSubNodeAddresses = B+1
-    val subNodeAddresses = ArrayList<Long>()
-
-    for (i in 0.until(numberOfSubNodeAddresses)) {
-        val subNodeAddress = buffer.long
-        subNodeAddresses.add(subNodeAddress)
-    }
-
-    return Node(keys, datas, subNodeAddresses)
-}
-
-@OptIn(ExperimentalUnsignedTypes::class)
-fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
-    fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int {
-        val top = min(dv1.size, dv2.size)
-
-        for (i in 0.until(top)) {
-            if (dv1[i] < dv2[i])
-                return -1
-            else if (dv1[i] > dv2[i])
-                return 1
-        }
-
-        return 0
-    }
-
-    fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> {
-        for (i in node.keys.indices) {
-            val cmpResult = compareArrayBuffers(key, node.keys[i])
-
-            if (cmpResult <= 0)
-                return Pair(cmpResult==0, i)
-        }
-
-        return Pair(false, node.keys.size)
-    }
-
-    fun isLeaf(node: Node) : Boolean {
-        for (subnode in node.subNodeAddresses)
-            if (subnode != 0L)
-                return false
-
-        return true
-    }
-
-    if (node.keys.isEmpty())
-        return null
-
-    val (there, where) = locateKey(key, node)
-    if (there)
-        return node.datas[where]
-    else if (isLeaf(node))
-        return null
-
-    val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where]) ?: return null
-    return bSearch(field, key, nextNode)
+    return Json.decodeFromString(json)
 }

app/src/main/java/xyz/quaver/pupil/receiver/UpdateBroadcastReceiver.kt

@@ -24,6 +24,7 @@ import android.content.BroadcastReceiver
 import android.content.Context
 import android.content.Intent
 import android.net.Uri
+import android.os.Build
 import android.webkit.MimeTypeMap
 import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationManagerCompat
@@ -76,7 +77,7 @@ class UpdateBroadcastReceiver : BroadcastReceiver() {
                 val pendingIntent = PendingIntent.getActivity(context, 0, Intent(Intent.ACTION_VIEW).apply {
                     flags = Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_ACTIVITY_NEW_TASK
                     setDataAndType(uri, MimeTypeMap.getSingleton().getMimeTypeFromExtension("apk"))
-                }, 0)
+                }, if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) PendingIntent.FLAG_IMMUTABLE else 0)
 
                 val notification = NotificationCompat.Builder(context, "update")
                     .setSmallIcon(android.R.drawable.stat_sys_download_done)

app/src/main/java/xyz/quaver/pupil/services/DownloadService.kt

@@ -23,6 +23,7 @@ import android.app.PendingIntent
 import android.app.Service
 import android.content.Context
 import android.content.Intent
+import android.os.Build
 import android.util.Log
 import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationManagerCompat
@@ -41,12 +42,9 @@ import okhttp3.ResponseBody
 import okio.*
 import xyz.quaver.pupil.*
 import xyz.quaver.pupil.ui.ReaderActivity
-import xyz.quaver.pupil.util.cleanCache
+import xyz.quaver.pupil.util.*
 import xyz.quaver.pupil.util.downloader.Cache
 import xyz.quaver.pupil.util.downloader.DownloadManager
-import xyz.quaver.pupil.util.ellipsize
-import xyz.quaver.pupil.util.normalizeID
-import xyz.quaver.pupil.util.requestBuilders
 import java.io.IOException
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.math.ceil
@@ -77,7 +75,7 @@ class DownloadService : Service() {
 
         val pendingIntent = TaskStackBuilder.create(this).run {
             addNextIntentWithParentStack(intent)
-            getPendingIntent(galleryID, PendingIntent.FLAG_UPDATE_CURRENT)
+            getPendingIntent(galleryID, PendingIntent.FLAG_UPDATE_CURRENT or if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) PendingIntent.FLAG_MUTABLE else 0)
         }
         val action =
             NotificationCompat.Action.Builder(0, getText(android.R.string.cancel),
@@ -87,7 +85,7 @@ class DownloadService : Service() {
                     Intent(this, DownloadService::class.java)
                         .putExtra(KEY_COMMAND, COMMAND_CANCEL)
                         .putExtra(KEY_ID, galleryID),
-                    PendingIntent.FLAG_UPDATE_CURRENT),
+                    PendingIntent.FLAG_UPDATE_CURRENT or if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) PendingIntent.FLAG_MUTABLE else 0),
             ).build()
 
         notification[galleryID] = NotificationCompat.Builder(this, "download").apply {
@@ -174,18 +172,16 @@ class DownloadService : Service() {
     private val interceptor: PupilInterceptor = { chain ->
         val request = chain.request()
 
-        if (rateLimitHost.matches(request.url().host()))
-            rateLimiter.acquire()
-
         var response = chain.proceed(request)
         var limit = 5
 
-        if (!response.isSuccessful && limit > 0) {
-            Thread.sleep(10000)
-            if (rateLimitHost.matches(request.url().host()))
-                rateLimiter.acquire()
+        while (!response.isSuccessful) {
+            if (response.code() == 503) {
+                Thread.sleep(200)
+            } else if (--limit > 0)
+                break
+
             response = chain.proceed(request)
-            limit -= 1
         }
 
         response.newBuilder()
@@ -226,31 +222,34 @@ class DownloadService : Service() {
             val (galleryID, index, startId) = call.request().tag() as Tag
             val ext = call.request().url().encodedPath().split('.').last()
 
-            kotlin.runCatching {
-                val image = response.also { if (it.code() != 200) throw IOException("$galleryID $index ${response.request().url()} CODE ${it.code()}") }.body()?.use { it.bytes() } ?: throw Exception("Response null")
+            CoroutineScope(Dispatchers.IO).launch {
+                runCatching {
+                    response.also {
+                        if (it.code() != 200) throw IOException(
+                            "$galleryID $index ${response.request().url()} CODE ${it.code()}"
+                        )
+                    }.body()?.use {
                         val padding = ceil(progress[galleryID]?.size?.let { log10(it.toFloat()) } ?: 0F).toInt()
 
-                CoroutineScope(Dispatchers.IO).launch {
-                    kotlin.runCatching {
-                        Cache.getInstance(this@DownloadService, galleryID).putImage(index, "${index.toString().padStart(padding, '0')}.$ext", image)
-                    }.onSuccess {
+                        Cache.getInstance(this@DownloadService, galleryID)
+                            .putImage(index, "${index.toString().padStart(padding, '0')}.$ext", it.byteStream())
+
                         progress[galleryID]?.set(index, Float.POSITIVE_INFINITY)
                         notify(galleryID)
 
                         if (isCompleted(galleryID)) {
                             if (DownloadManager.getInstance(this@DownloadService)
-                                    .getDownloadFolder(galleryID) != null)
+                                    .getDownloadFolder(galleryID) != null
+                            )
                                 Cache.getInstance(this@DownloadService, galleryID).moveToDownload()
 
                             startId?.let { stopSelf(it) }
                         }
+                    } ?: throw Exception("Response null")
                 }.onFailure {
                     FirebaseCrashlytics.getInstance().recordException(it)
                 }
             }
-            }.onFailure {
-                FirebaseCrashlytics.getInstance().recordException(it)
-            }
         }
     }
 
@@ -357,7 +356,7 @@ class DownloadService : Service() {
             }
         }
 
-        galleryInfo.requestBuilders.forEachIndexed { index, it ->
+        galleryInfo.getRequestBuilders().forEachIndexed { index, it ->
             if (progress[galleryID]?.get(index)?.isInfinite() == false) {
                 val request = it.tag(Tag(galleryID, index, startId)).build()
                 client.newCall(request).enqueue(callback)

app/src/main/java/xyz/quaver/pupil/types/Exceptions.kt

@@ -0,0 +1,22 @@
+/*
+ *     Pupil, Hitomi.la viewer for Android
+ *     Copyright (C) 2022  tom5079
+ *
+ *     This program is free software: you can redistribute it and/or modify
+ *     it under the terms of the GNU General Public License as published by
+ *     the Free Software Foundation, either version 3 of the License, or
+ *     (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package xyz.quaver.pupil.types
+
+class JavascriptConsoleException(message: String?): Exception(message)
+class JavascriptOnErrorException(message: String?): Exception(message)

app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt

@@ -25,6 +25,7 @@ import android.os.Build
 import android.os.Bundle
 import android.text.InputType
 import android.text.util.Linkify
+import android.util.Log
 import android.view.KeyEvent
 import android.view.MenuItem
 import android.view.View
@@ -800,7 +801,6 @@ class MainActivity :
                         throw Exception("No result")
                 }
             } catch (e: Exception) {
-
                 if (e !is CancellationException)
                     FirebaseCrashlytics.getInstance().recordException(e)
 

app/src/main/java/xyz/quaver/pupil/ui/ReaderActivity.kt

@@ -161,12 +161,10 @@ class ReaderActivity : BaseActivity() {
         }
     }
 
-    override fun onCreateOptionsMenu(menu: Menu?): Boolean {
+    override fun onCreateOptionsMenu(menu: Menu): Boolean {
         menuInflater.inflate(R.menu.reader, menu)
 
-        with(menu?.findItem(R.id.reader_menu_favorite)) {
-            this ?: return@with
-
+        with(menu.findItem(R.id.reader_menu_favorite)) {
             if (favorites.contains(galleryID))
                 (icon as Animatable).start()
         }

app/src/main/java/xyz/quaver/pupil/ui/dialog/DownloadFolderNameDialogFragment.kt

@@ -24,7 +24,9 @@ import android.view.ViewGroup
 import androidx.core.widget.addTextChangedListener
 import androidx.fragment.app.DialogFragment
 import com.google.android.material.snackbar.Snackbar
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import xyz.quaver.pupil.R
 import xyz.quaver.pupil.databinding.DownloadFolderNameDialogBinding
 import xyz.quaver.pupil.util.Preferences
@@ -56,15 +58,16 @@ class DownloadFolderNameDialogFragment : DialogFragment() {
 
     private fun initView() {
         val galleryID = Cache.instances.let { if (it.size == 0) 1199708 else it.keys.elementAt((0 until it.size).random()) }
-        val galleryBlock = runBlocking {
-            Cache.getInstance(requireContext(), galleryID).getGalleryBlock()
-        }
+        CoroutineScope(Dispatchers.IO).launch {
+            val galleryBlock = Cache.getInstance(requireContext(), galleryID).getGalleryBlock()
 
             binding.message.text = getString(R.string.settings_download_folder_name_message, formatMap.keys.toString(), galleryBlock?.formatDownloadFolder() ?: "")
-        binding.edittext.setText(Preferences["download_folder_name", "[-id-] -title-"])
             binding.edittext.addTextChangedListener {
                 binding.message.text = requireContext().getString(R.string.settings_download_folder_name_message, formatMap.keys.toString(), galleryBlock?.formatDownloadFolderTest(it.toString()) ?: "")
             }
+        }
+
+        binding.edittext.setText(Preferences["download_folder_name", "[-id-] -title-"])
         binding.okButton.setOnClickListener {
             val newValue = binding.edittext.text.toString()
 

app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt

@@ -21,6 +21,7 @@ package xyz.quaver.pupil.util.downloader
 import android.content.Context
 import android.content.ContextWrapper
 import android.net.Uri
+import android.util.Log
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
@@ -39,8 +40,10 @@ import xyz.quaver.pupil.hitomi.GalleryBlock
 import xyz.quaver.pupil.hitomi.GalleryInfo
 import xyz.quaver.pupil.hitomi.getGalleryBlock
 import xyz.quaver.pupil.hitomi.getGalleryInfo
+import xyz.quaver.pupil.userAgent
 import java.io.File
 import java.io.IOException
+import java.io.InputStream
 import java.util.concurrent.ConcurrentHashMap
 
 @Serializable
@@ -171,6 +174,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
                 kotlin.runCatching {
                     val request = Request.Builder()
                         .url(it)
+                        .header("Referer", "https://hitomi.la/")
                         .build()
 
                     client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() }
@@ -207,12 +211,14 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
         metadata.imageList?.getOrNull(index)?.let { findFile(it) }
 
     @Suppress("BlockingMethodInNonBlockingContext")
-    fun putImage(index: Int, fileName: String, data: ByteArray) {
+    fun putImage(index: Int, fileName: String, data: InputStream) {
         val file = cacheFolder.getChild(fileName)
 
         if (!file.exists())
             file.createNewFile()
-        file.writeBytes(data)
+        file.outputStream()?.use {
+            data.copyTo(it)
+        }
         setMetadata { metadata -> metadata.imageList!![index] = fileName }
     }
 

app/src/main/java/xyz/quaver/pupil/util/downloader/DownloadManager.kt

@@ -20,8 +20,9 @@ package xyz.quaver.pupil.util.downloader
 
 import android.content.Context
 import android.content.ContextWrapper
-import android.util.Log
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
@@ -86,7 +87,7 @@ class DownloadManager private constructor(context: Context) : ContextWrapper(con
 
     @Synchronized
     fun isDownloading(galleryID: Int): Boolean {
-        val isThisGallery: (Call) -> Boolean = { (it.request().tag() as? DownloadService.Tag)?.galleryID == galleryID }
+        val isThisGallery: (Call) -> Boolean = { !it.isCanceled && (it.request().tag() as? DownloadService.Tag)?.galleryID == galleryID }
 
         return downloadFolderMap.containsKey(galleryID)
                 && client.dispatcher().let { it.queuedCalls().any(isThisGallery) || it.runningCalls().any(isThisGallery) }
@@ -96,16 +97,13 @@ class DownloadManager private constructor(context: Context) : ContextWrapper(con
     fun getDownloadFolder(galleryID: Int): FileX? =
         downloadFolderMap[galleryID]?.let { downloadFolder.getChild(it) }
 
-    @Synchronized
-    fun addDownloadFolder(galleryID: Int) {
-        val name = runBlocking {
-            Cache.getInstance(this@DownloadManager, galleryID).getGalleryBlock()
-        }?.formatDownloadFolder() ?: return
+    fun addDownloadFolder(galleryID: Int) = CoroutineScope(Dispatchers.IO).launch {
+        val name = Cache.getInstance(this@DownloadManager, galleryID).getGalleryBlock()
+            ?.formatDownloadFolder() ?: return@launch
 
         val folder = downloadFolder.getChild(name)
 
-        if (folder.exists())
-            return
+        if (folder.exists()) return@launch
 
         folder.mkdir()
 

app/src/main/java/xyz/quaver/pupil/util/misc.kt

@@ -19,6 +19,7 @@
 package xyz.quaver.pupil.util
 
 import android.annotation.SuppressLint
+import com.google.firebase.crashlytics.FirebaseCrashlytics
 import kotlinx.serialization.json.*
 import okhttp3.OkHttpClient
 import okhttp3.Request
@@ -97,18 +98,25 @@ fun GalleryBlock.formatDownloadFolderTest(format: String): String =
         }
     }.replace(Regex("""[*\\|"?><:/]"""), "").ellipsize(127)
 
-val GalleryInfo.requestBuilders: List<Request.Builder>
-    get() {
+suspend fun GalleryInfo.getRequestBuilders(): List<Request.Builder> {
     val galleryID = this.id ?: 0
     val lowQuality = Preferences["low_quality", true]
 
     return this.files.map {
         Request.Builder()
-                .url(imageUrlFromImage(galleryID, it, !lowQuality))
+            .url(
+                runCatching {
+                    imageUrlFromImage(galleryID, it, !lowQuality)
+                }
+                .onFailure {
+                    FirebaseCrashlytics.getInstance().recordException(it)
+                }
+                .getOrDefault("https://a/")
+            )
             .header("Referer", "https://hitomi.la/")
             .header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36")
     }
-    }
+}
 
 fun String.ellipsize(n: Int): String =
     if (this.length > n)

build.gradle

@@ -6,14 +6,14 @@ buildscript {
         mavenCentral()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:4.2.2'
+        classpath 'com.android.tools.build:gradle:7.0.4'
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath "org.jetbrains.kotlin:kotlin-android-extensions:$kotlin_version"
         classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlin_version"
-        classpath "com.google.gms:google-services:4.3.8"
+        classpath "com.google.gms:google-services:4.3.10"
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
-        classpath "com.google.firebase:firebase-crashlytics-gradle:2.7.0"
+        classpath "com.google.firebase:firebase-crashlytics-gradle:2.8.1"
         classpath "com.google.firebase:perf-plugin:1.4.0"
         classpath "com.google.android.gms:oss-licenses-plugin:0.10.4"
     }

gradle.properties

@@ -20,4 +20,4 @@ kotlin.code.style=official
 android.enableJetifier=true
 android.useAndroidX=true
 
-kotlin_version=1.5.10
+kotlin_version=1.6.10

gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-all.zip