Fixed images not loading

Fixes unable to download some images

.idea/deploymentTargetDropDown.xml

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="deploymentTargetDropDown">
-    <targetSelectedWithDropDown>
+    <runningDeviceTargetSelectedWithDropDown>
       <Target>
-        <type value="QUICK_BOOT_TARGET" />
+        <type value="RUNNING_DEVICE_TARGET" />
         <deviceKey>
           <Key>
             <type value="VIRTUAL_DEVICE_PATH" />
-            <value value="$USER_HOME$/.android/avd/Pixel_3a_API_30_x86.avd" />
+            <value value="$USER_HOME$/.android/avd/Pixel_2_API_30.avd" />
           </Key>
         </deviceKey>
       </Target>
-    </targetSelectedWithDropDown>
+    </runningDeviceTargetSelectedWithDropDown>
-    <timeTargetWasSelectedWithDropDown value="2021-10-25T00:27:52.904947Z" />
+    <timeTargetWasSelectedWithDropDown value="2022-01-01T07:38:11.679673Z" />
   </component>
 </project>

.idea/jarRepositories.xml

@@ -81,5 +81,10 @@
       <option name="name" value="MavenLocal" />
       <option name="url" value="file:$USER_HOME$/.m2/repository/" />
     </remote-repository>
+    <remote-repository>
+      <option name="id" value="maven2" />
+      <option name="name" value="maven2" />
+      <option name="url" value="https://oss.sonatype.org/content/repositories/snapshots" />
+    </remote-repository>
   </component>
 </project>

README.md

@@ -2,7 +2,7 @@
 *Pupil, Hitomi.la viewer for Android*  
 
 ![](https://img.shields.io/github/downloads/tom5079/Pupil/total)  
-[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.1.13/Pupil-v5.1.13.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.1.13/Pupil-v5.1.13.apk)  
+[![](https://img.shields.io/github/downloads/tom5079/Pupil/5.1.30/Pupil-v5.1.30.apk?color=%234fc3f7&label=DOWNLOAD%20APP&style=for-the-badge)](https://github.com/tom5079/Pupil/releases/download/5.1.30/Pupil-v5.1.30.apk)
 [![](https://discordapp.com/api/guilds/610452916612104194/embed.png?style=banner2)](https://discord.gg/Stj4b5v)
 
 # Features

app/build.gradle

@@ -37,8 +37,8 @@ android {
         applicationId "xyz.quaver.pupil"
         minSdkVersion 16
         targetSdkVersion 30
-        versionCode 67
+        versionCode 69
-        versionName "5.1.14"
+        versionName "5.1.30"
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables.useSupportLibrary = true
     }
@@ -125,8 +125,12 @@ dependencies {
 
     implementation "ru.noties.markwon:core:3.1.0"
 
-    implementation "xyz.quaver:libpupil:2.1.7-DEV"
+    implementation "org.jsoup:jsoup:1.14.3"
-    implementation "xyz.quaver:documentfilex:0.6.1"
+    implementation "app.cash.zipline:zipline:1.0.0-SNAPSHOT"
+
+    implementation "com.google.guava:guava:31.0.1-android"
+
+    implementation "xyz.quaver:documentfilex:0.7.1"
     implementation "xyz.quaver:floatingsearchview:1.1.7"
 
     testImplementation "junit:junit:4.13.1"

app/proguard-rules.pro

@@ -33,3 +33,4 @@
 }
 -keep class xyz.quaver.pupil.ui.fragment.ManageFavoritesFragment
 -keep class xyz.quaver.pupil.ui.fragment.ManageStorageFragment
+-keep class com.hippo.quickjs.** { *; }

app/release/output-metadata.json

@@ -10,8 +10,8 @@
     {
       "type": "SINGLE",
       "filters": [],
-      "versionCode": 67,
+      "versionCode": 69,
-      "versionName": "5.1.14",
+      "versionName": "5.1.30",
       "outputFile": "app-release.apk"
     }
   ]

app/src/main/java/xyz/quaver/pupil/Pupil.kt

@@ -26,7 +26,6 @@ import android.content.Context
 import android.content.Intent
 import android.net.Uri
 import android.os.Build
-import android.util.Log
 import androidx.appcompat.app.AppCompatDelegate
 import androidx.core.content.ContextCompat
 import androidx.preference.PreferenceManager
@@ -43,8 +42,6 @@ import okhttp3.Response
 import xyz.quaver.io.FileX
 import xyz.quaver.pupil.types.Tag
 import xyz.quaver.pupil.util.*
-import xyz.quaver.pupil.util.downloader.DownloadManager
-import xyz.quaver.setClient
 import java.io.File
 import java.util.*
 import java.util.concurrent.Executors
@@ -70,7 +67,6 @@ var clientHolder: OkHttpClient? = null
 val client: OkHttpClient
     get() = clientHolder ?: clientBuilder.build().also {
         clientHolder = it
-        setClient(it)
     }
 
 class Pupil : Application() {

app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt

@@ -35,14 +35,13 @@ import com.daimajia.swipe.adapters.RecyclerSwipeAdapter
 import com.daimajia.swipe.interfaces.SwipeAdapterInterface
 import com.github.piasy.biv.loader.ImageLoader
 import kotlinx.coroutines.*
-import xyz.quaver.hitomi.getGallery
-import xyz.quaver.hitomi.getGalleryInfo
-import xyz.quaver.hitomi.getReader
 import xyz.quaver.io.util.getChild
 import xyz.quaver.pupil.R
 import xyz.quaver.pupil.databinding.GalleryblockItemBinding
 import xyz.quaver.pupil.favoriteTags
 import xyz.quaver.pupil.favorites
+import xyz.quaver.pupil.hitomi.getGallery
+import xyz.quaver.pupil.hitomi.getGalleryInfo
 import xyz.quaver.pupil.types.Tag
 import xyz.quaver.pupil.ui.view.ProgressCard
 import xyz.quaver.pupil.util.Preferences

app/src/main/java/xyz/quaver/pupil/adapters/ReaderAdapter.kt

@@ -40,9 +40,9 @@ import com.github.piasy.biv.view.BigImageView
 import com.github.piasy.biv.view.ImageShownCallback
 import com.github.piasy.biv.view.ImageViewFactory
 import kotlinx.coroutines.*
-import xyz.quaver.hitomi.GalleryInfo
 import xyz.quaver.pupil.R
 import xyz.quaver.pupil.databinding.ReaderItemBinding
+import xyz.quaver.pupil.hitomi.GalleryInfo
 import xyz.quaver.pupil.ui.ReaderActivity
 import xyz.quaver.pupil.util.downloader.Cache
 import java.io.File

app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt

@@ -0,0 +1,65 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver
+
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.modules.plus
+import kotlinx.serialization.modules.polymorphic
+import kotlinx.serialization.modules.subclass
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import xyz.quaver.pupil.client
+import java.io.IOException
+import java.net.URL
+import java.util.concurrent.TimeUnit
+import kotlin.time.Duration
+
+/**
+ * kotlinx.serialization.json.Json object for global use  
+ * properties should not be changed
+ *
+ * @see [https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-core/kotlinx-serialization-core/kotlinx.serialization.json/-json/index.html]
+ */
+val json = Json {
+    isLenient = true
+    ignoreUnknownKeys = true
+    allowSpecialFloatingPointValues = true
+    useArrayPolymorphism = true
+}
+
+typealias HeaderSetter = (Request.Builder) -> Request.Builder
+fun URL.readText(settings: HeaderSetter? = null): String {
+    val request = Request.Builder()
+        .url(this).let { 
+            settings?.invoke(it) ?: it
+        }.build()
+    
+    return client.newCall(request).execute().also{ if (it.code() != 200) throw IOException() }.body()?.use { it.string() } ?: throw IOException()
+}
+
+fun URL.readBytes(settings: HeaderSetter? = null): ByteArray {
+    val request = Request.Builder()
+        .url(this).let {
+            settings?.invoke(it) ?: it
+        }.build()
+
+    return client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() } ?: throw IOException()
+}

app/src/main/java/xyz/quaver/pupil/hitomi/common.kt

@@ -0,0 +1,129 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import android.util.Log
+import app.cash.zipline.QuickJs
+import kotlinx.serialization.decodeFromString
+import xyz.quaver.json
+import xyz.quaver.readText
+import java.net.URL
+import java.nio.charset.Charset
+
+const val protocol = "https:"
+
+@Suppress("EXPERIMENTAL_API_USAGE")
+fun getGalleryInfo(galleryID: Int) =
+    json.decodeFromString<GalleryInfo>(
+        URL("$protocol//$domain/galleries/$galleryID.js").readText()
+            .replace("var galleryinfo = ", "")
+    )
+
+//common.js
+const val domain = "ltn.hitomi.la"
+const val galleryblockextension = ".html"
+const val galleryblockdir = "galleryblock"
+const val nozomiextension = ".nozomi"
+
+interface gg {
+    fun m(g: Int): Int
+    val b: String
+    fun s(h: String): String
+
+    companion object {
+        @Volatile private var instance: gg? = null
+
+        fun getInstance(): gg =
+            instance ?: synchronized(this) {
+                instance ?: object: gg {
+                    private val engine = QuickJs.create().also {
+                        it.evaluate(URL("https://ltn.hitomi.la/gg.js").readText(Charset.defaultCharset()).also {
+                            Log.d("PUPILD", it)
+                        })
+                    }
+
+                    override fun m(g: Int): Int = engine.evaluate("gg.m($g)") as Int
+
+                    override val b: String
+                        get() = engine.evaluate("gg.b") as String
+
+                    override fun s(h: String): String = engine.evaluate("gg.s('$h')") as String
+                }.also { instance = it }
+            }
+    }
+}
+
+fun subdomainFromURL(url: String, base: String? = null) : String {
+    var retval = "b"
+
+    if (!base.isNullOrBlank())
+        retval = base
+
+    val b = 16
+
+    val r = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
+    val m = r.find(url) ?: return "a"
+
+    val g = m.groupValues.let { it[2]+it[1] }.toIntOrNull(b)
+
+    if (g != null) {
+        retval = (97+ gg.getInstance().m(g)).toChar().toString() + retval
+    }
+
+    return retval
+}
+
+fun urlFromUrl(url: String, base: String? = null) : String {
+    return url.replace(Regex("""//..?\.hitomi\.la/"""), "//${subdomainFromURL(url, base)}.hitomi.la/")
+}
+
+
+fun fullPathFromHash(hash: String) : String =
+    "${gg.getInstance().b}${gg.getInstance().s(hash)}/$hash"
+
+fun realFullPathFromHash(hash: String): String =
+    hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1/$hash")
+
+fun urlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null) : String {
+    val ext = ext ?: dir ?: image.name.takeLastWhile { it != '.' }
+    val dir = dir ?: "images"
+    return "https://a.hitomi.la/$dir/${fullPathFromHash(image.hash)}.$ext"
+}
+
+fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null) =
+    if (base == "tn")
+        urlFromUrl("https://a.hitomi.la/$dir/${realFullPathFromHash(image.hash)}.$ext", base)
+    else
+        urlFromUrl(urlFromHash(galleryID, image, dir, ext), base)
+
+fun rewriteTnPaths(html: String) =
+    html.replace(Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}""")) { url ->
+        urlFromUrl(url.value, "tn")
+    }
+
+fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String {
+    return when {
+        noWebp ->
+            urlFromUrlFromHash(galleryID, image)
+//        image.hasavif != 0 ->
+//            urlFromUrlFromHash(galleryID, image, "avif", null, "a")
+        image.haswebp != 0 ->
+            urlFromUrlFromHash(galleryID, image, "webp", null, "a")
+        else ->
+            urlFromUrlFromHash(galleryID, image)
+    }
+}

app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt

@@ -0,0 +1,80 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import kotlinx.serialization.Serializable
+import org.jsoup.Jsoup
+import xyz.quaver.readText
+import java.net.URL
+import java.net.URLDecoder
+
+@Serializable
+data class Gallery(
+    val related: List<Int>,
+    val langList: List<Pair<String, String>>,
+    val cover: String,
+    val title: String,
+    val artists: List<String>,
+    val groups: List<String>,
+    val type: String,
+    val language: String,
+    val series: List<String>,
+    val characters: List<String>,
+    val tags: List<String>,
+    val thumbnails: List<String>
+)
+fun getGallery(galleryID: Int) : Gallery {
+    val url = Jsoup.parse(URL("https://hitomi.la/galleries/$galleryID.html").readText())
+        .select("link").attr("href")
+
+    val doc = Jsoup.parse(URL(url).readText())
+
+    val related = Regex("\\d+")
+        .findAll(doc.select("script").first()!!.html())
+        .map {
+            it.value.toInt()
+        }.toList()
+
+    val langList = doc.select("#lang-list a").map {
+            Pair(it.text(), "$protocol//hitomi.la${it.attr("href")}")
+        }
+
+    val cover = protocol + doc.selectFirst(".cover img")!!.attr("src")
+    val title = doc.selectFirst(".gallery h1 a")!!.text()
+    val artists = doc.select(".gallery h2 a").map { it.text() }
+    val groups = doc.select(".gallery-info a[href~=^/group/]").map { it.text() }
+    val type = doc.selectFirst(".gallery-info a[href~=^/type/]")!!.text()
+
+    val language = run {
+        val href = doc.select(".gallery-info a[href~=^/index.+\\.html\$]").attr("href")
+        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
+    }
+
+    val series = doc.select(".gallery-info a[href~=^/series/]").map { it.text() }
+    val characters = doc.select(".gallery-info a[href~=^/character/]").map { it.text() }
+
+    val tags = doc.select(".gallery-info a[href~=^/tag/]").map {
+        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
+        href.slice(5 until href.indexOf('-'))
+    }
+
+    val thumbnails = getGalleryInfo(galleryID).files.map { galleryInfo ->
+        urlFromUrlFromHash(galleryID, galleryInfo, "smalltn", "jpg", "tn")
+    }
+
+    return Gallery(related, langList, cover, title, artists, groups, type, language, series, characters, tags, thumbnails)
+}

app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt

@@ -0,0 +1,105 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import kotlinx.serialization.Serializable
+import org.jsoup.Jsoup
+import xyz.quaver.readText
+import java.net.URL
+import java.net.URLDecoder
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.*
+import javax.net.ssl.HttpsURLConnection
+
+//galleryblock.js
+fun fetchNozomi(area: String? = null, tag: String = "index", language: String = "all", start: Int = -1, count: Int = -1) : Pair<List<Int>, Int> {
+    val url =
+            when(area) {
+                null -> "$protocol//$domain/$tag-$language$nozomiextension"
+                else -> "$protocol//$domain/$area/$tag-$language$nozomiextension"
+            }
+
+    with(URL(url).openConnection() as HttpsURLConnection) {
+        requestMethod = "GET"
+
+        if (start != -1 && count != -1) {
+            val startByte = start*4
+            val endByte = (start+count)*4-1
+
+            setRequestProperty("Range", "bytes=$startByte-$endByte")
+        }
+
+        connect()
+
+        val totalItems = getHeaderField("Content-Range")
+            .replace(Regex("^[Bb]ytes \\d+-\\d+/"), "").toInt() / 4
+
+        val nozomi = ArrayList<Int>()
+
+        val arrayBuffer = ByteBuffer
+            .wrap(inputStream.readBytes())
+            .order(ByteOrder.BIG_ENDIAN)
+
+        while (arrayBuffer.hasRemaining())
+            nozomi.add(arrayBuffer.int)
+
+        return Pair(nozomi, totalItems)
+    }
+}
+
+@Serializable
+data class GalleryBlock(
+    val id: Int,
+    val galleryUrl: String,
+    val thumbnails: List<String>,
+    val title: String,
+    val artists: List<String>,
+    val series: List<String>,
+    val type: String,
+    val language: String,
+    val relatedTags: List<String>
+)
+
+fun getGalleryBlock(galleryID: Int) : GalleryBlock {
+    val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension"
+
+    val doc = Jsoup.parse(rewriteTnPaths(URL(url).readText()))
+
+    val galleryUrl = doc.selectFirst("h1 > a")!!.attr("href")
+
+    val thumbnails = doc.select(".dj-img-cont img").map { protocol + it.attr("src") }
+
+    val title = doc.selectFirst("h1 > a")!!.text()
+    val artists = doc.select(".artist-list a").map{ it.text() }
+    val series = doc.select(".dj-content a[href~=^/series/]").map { it.text() }
+    val type = doc.selectFirst("a[href~=^/type/]")!!.text()
+
+    val language = run {
+        val href = doc.select("a[href~=^/index.+\\.html\$]").attr("href")
+        Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: ""
+    }
+
+    val relatedTags = doc.select(".relatedtags a").map {
+        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
+        href.slice(5 until href.indexOf("-all"))
+    }
+
+    return GalleryBlock(galleryID, galleryUrl, thumbnails, title, artists, series, type, language, relatedTags)
+}
+
+fun getGalleryBlockOrNull(galleryID: Int) = runCatching { getGalleryBlock(galleryID) }.getOrNull()

app/src/main/java/xyz/quaver/pupil/hitomi/reader.kt

@@ -0,0 +1,49 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import kotlinx.serialization.Serializable
+
+fun getReferer(galleryID: Int) = "https://hitomi.la/reader/$galleryID.html"
+
+@Serializable
+data class GalleryInfo(
+    val language_localname: String? = null,
+    val language: String? = null,
+    val date: String? = null,
+    val files: List<GalleryFiles>,
+    val id: Int? = null,
+    val type: String? = null,
+    val title: String? = null
+)
+
+@Serializable
+data class GalleryFiles(
+    val width: Int,
+    val hash: String,
+    val haswebp: Int = 0,
+    val name: String,
+    val height: Int,
+    val hasavif: Int = 0,
+    val hasavifsmalltn: Int? = 0
+)
+
+//Set header `Referer` to reader url to avoid 403 error
+@Deprecated("", replaceWith = ReplaceWith("getGalleryInfo"))
+fun getReader(galleryID: Int) : GalleryInfo {
+   return getGalleryInfo(galleryID)
+}

app/src/main/java/xyz/quaver/pupil/hitomi/results.kt

@@ -0,0 +1,91 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
+import java.util.*
+
+fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> {
+    val terms = query
+        .trim()
+        .replace(Regex("""^\?"""), "")
+        .lowercase()
+        .split(Regex("\\s+"))
+        .map {
+            it.replace('_', ' ')
+        }
+
+    val positiveTerms = LinkedList<String>()
+    val negativeTerms = LinkedList<String>()
+
+    for (term in terms) {
+        if (term.matches(Regex("^-.+")))
+            negativeTerms.push(term.replace(Regex("^-"), ""))
+        else if (term.isNotBlank())
+            positiveTerms.push(term)
+    }
+
+    val positiveResults = positiveTerms.map {
+        CoroutineScope(Dispatchers.IO).async {
+            kotlin.runCatching {
+                getGalleryIDsForQuery(it)
+            }.getOrElse { emptySet() }
+        }
+    }
+
+    val negativeResults = negativeTerms.map {
+        CoroutineScope(Dispatchers.IO).async {
+            kotlin.runCatching {
+                getGalleryIDsForQuery(it)
+            }.getOrElse { emptySet() }
+        }
+    }
+
+    var results = when {
+        sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
+        positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
+        else -> emptySet()
+    }
+
+    runBlocking {
+        @Synchronized fun filterPositive(newResults: Set<Int>) {
+            results = when {
+                results.isEmpty() -> newResults
+                else -> results intersect newResults
+            }
+        }
+
+        @Synchronized fun filterNegative(newResults: Set<Int>) {
+            results = results subtract newResults
+        }
+
+        //positive results
+        positiveResults.forEach {
+            filterPositive(it.await())
+        }
+
+        //negative results
+        negativeResults.forEach {
+            filterNegative(it.await())
+        }
+    }
+
+    return results
+}

app/src/main/java/xyz/quaver/pupil/hitomi/search.kt

@@ -0,0 +1,330 @@
+/*
+ *    Copyright 2019 tom5079
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+package xyz.quaver.pupil.hitomi
+
+import okhttp3.Request
+import xyz.quaver.pupil.client
+import xyz.quaver.readBytes
+import xyz.quaver.readText
+import java.net.URL
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.security.MessageDigest
+import kotlin.math.min
+
+//searchlib.js
+const val separator = "-"
+const val extension = ".html"
+const val index_dir = "tagindex"
+const val galleries_index_dir = "galleriesindex"
+const val max_node_size = 464
+const val B = 16
+const val compressed_nozomi_prefix = "n"
+
+val tag_index_version: String by lazy { getIndexVersion("tagindex") }
+val galleries_index_version: String by lazy { getIndexVersion("galleriesindex") }
+
+fun sha256(data: ByteArray) : ByteArray {
+    return MessageDigest.getInstance("SHA-256").digest(data)
+}
+
+@OptIn(ExperimentalUnsignedTypes::class)
+fun hashTerm(term: String) : UByteArray {
+    return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4)
+}
+
+fun sanitize(input: String) : String {
+    return input.replace(Regex("[/#]"), "")
+}
+
+fun getIndexVersion(name: String) =
+        URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText()
+
+//search.js
+fun getGalleryIDsForQuery(query: String) : Set<Int> {
+    query.replace("_", " ").let {
+        if (it.indexOf(':') > -1) {
+            val sides = it.split(":")
+            val ns = sides[0]
+            var tag = sides[1]
+
+            var area : String? = ns
+            var language = "all"
+            when (ns) {
+                "female", "male" -> {
+                    area = "tag"
+                    tag = it
+                }
+                "language" -> {
+                    area = null
+                    language = tag
+                    tag = "index"
+                }
+            }
+
+            return getGalleryIDsFromNozomi(area, tag, language)
+        }
+
+        val key = hashTerm(it)
+        val field = "galleries"
+
+        val node = getNodeAtAddress(field, 0) ?: return emptySet()
+
+        val data = bSearch(field, key, node)
+
+        if (data != null)
+            return getGalleryIDsFromData(data)
+
+        return emptySet()
+    }
+}
+
+fun getSuggestionsForQuery(query: String) : List<Suggestion> {
+    query.replace('_', ' ').let {
+        var field = "global"
+        var term = it
+
+        if (term.indexOf(':') > -1) {
+            val sides = it.split(':')
+            field = sides[0]
+            term = sides[1]
+        }
+
+        val key = hashTerm(term)
+        val node = getNodeAtAddress(field, 0) ?: return emptyList()
+        val data = bSearch(field, key, node)
+
+        if (data != null)
+            return getSuggestionsFromData(field, data)
+
+        return emptyList()
+    }
+}
+
+data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
+fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggestion> {
+    val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data"
+    val (offset, length) = data
+    if (length > 10000 || length <= 0)
+        throw Exception("length $length is too long")
+
+    val inbuf = getURLAtRange(url, offset.until(offset+length))
+
+    val suggestions = ArrayList<Suggestion>()
+
+    val buffer = ByteBuffer
+        .wrap(inbuf)
+        .order(ByteOrder.BIG_ENDIAN)
+    val numberOfSuggestions = buffer.int
+
+    if (numberOfSuggestions > 100 || numberOfSuggestions <= 0)
+        throw Exception("number of suggestions $numberOfSuggestions is too long")
+
+    for (i in 0.until(numberOfSuggestions)) {
+        var top = buffer.int
+
+        val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
+        buffer.position(buffer.position()+top)
+
+        top = buffer.int
+
+        val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
+        buffer.position(buffer.position()+top)
+
+        val count = buffer.int
+
+        val tagname = sanitize(tag)
+        val u =
+                when(ns) {
+                    "female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
+                    "language" -> "/index-$tagname${separator}1$extension"
+                    else -> "/$ns/$tagname${separator}all${separator}1$extension"
+                }
+
+        suggestions.add(Suggestion(tag, count, u, ns))
+    }
+
+    return suggestions
+}
+
+fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
+    val nozomiAddress =
+            when(area) {
+                null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension"
+                else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension"
+            }
+
+    val bytes = try {
+        URL(nozomiAddress).readBytes()
+    } catch (e: Exception) {
+        return emptySet()
+    }
+
+    val nozomi = mutableSetOf<Int>()
+
+    val arrayBuffer = ByteBuffer
+        .wrap(bytes)
+        .order(ByteOrder.BIG_ENDIAN)
+
+    while (arrayBuffer.hasRemaining())
+        nozomi.add(arrayBuffer.int)
+
+    return nozomi
+}
+
+fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
+    val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data"
+    val (offset, length) = data
+    if (length > 100000000 || length <= 0)
+        throw Exception("length $length is too long")
+
+    val inbuf = getURLAtRange(url, offset.until(offset+length))
+
+    val galleryIDs = mutableSetOf<Int>()
+
+    val buffer = ByteBuffer
+        .wrap(inbuf)
+        .order(ByteOrder.BIG_ENDIAN)
+
+    val numberOfGalleryIDs = buffer.int
+
+    val expectedLength = numberOfGalleryIDs*4+4
+
+    if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0)
+        throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
+    else if (inbuf.size != expectedLength)
+        throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength")
+
+    for (i in 0.until(numberOfGalleryIDs))
+        galleryIDs.add(buffer.int)
+
+    return galleryIDs
+}
+
+fun getNodeAtAddress(field: String, address: Long) : Node? {
+    val url =
+            when(field) {
+                "galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"
+                "languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index"
+                "nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index"
+                else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index"
+            }
+
+    val nodedata = getURLAtRange(url, address.until(address+max_node_size))
+
+    return decodeNode(nodedata)
+}
+
+fun getURLAtRange(url: String, range: LongRange) : ByteArray {
+    val request = Request.Builder()
+        .url(url)
+        .header("Range", "bytes=${range.first}-${range.last}")
+        .build()
+    
+    return client.newCall(request).execute().body()?.use { it.bytes() } ?: byteArrayOf()
+}
+
+@OptIn(ExperimentalUnsignedTypes::class)
+data class Node(val keys: List<UByteArray>, val datas: List<Pair<Long, Int>>, val subNodeAddresses: List<Long>)
+@OptIn(ExperimentalUnsignedTypes::class)
+fun decodeNode(data: ByteArray) : Node {
+    val buffer = ByteBuffer
+        .wrap(data)
+        .order(ByteOrder.BIG_ENDIAN)
+
+    val uData = data.toUByteArray()
+
+    val numberOfKeys = buffer.int
+    val keys = ArrayList<UByteArray>()
+
+    for (i in 0.until(numberOfKeys)) {
+        val keySize = buffer.int
+
+        if (keySize == 0 || keySize > 32)
+            throw Exception("fatal: !keySize || keySize > 32")
+
+        keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize)))
+        buffer.position(buffer.position()+keySize)
+    }
+
+    val numberOfDatas = buffer.int
+    val datas = ArrayList<Pair<Long, Int>>()
+
+    for (i in 0.until(numberOfDatas)) {
+        val offset = buffer.long
+        val length = buffer.int
+
+        datas.add(Pair(offset, length))
+    }
+
+    val numberOfSubNodeAddresses = B+1
+    val subNodeAddresses = ArrayList<Long>()
+
+    for (i in 0.until(numberOfSubNodeAddresses)) {
+        val subNodeAddress = buffer.long
+        subNodeAddresses.add(subNodeAddress)
+    }
+
+    return Node(keys, datas, subNodeAddresses)
+}
+
+@OptIn(ExperimentalUnsignedTypes::class)
+fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
+    fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int {
+        val top = min(dv1.size, dv2.size)
+
+        for (i in 0.until(top)) {
+            if (dv1[i] < dv2[i])
+                return -1
+            else if (dv1[i] > dv2[i])
+                return 1
+        }
+
+        return 0
+    }
+
+    fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> {
+        for (i in node.keys.indices) {
+            val cmpResult = compareArrayBuffers(key, node.keys[i])
+
+            if (cmpResult <= 0)
+                return Pair(cmpResult==0, i)
+        }
+
+        return Pair(false, node.keys.size)
+    }
+
+    fun isLeaf(node: Node) : Boolean {
+        for (subnode in node.subNodeAddresses)
+            if (subnode != 0L)
+                return false
+
+        return true
+    }
+
+    if (node.keys.isEmpty())
+        return null
+
+    val (there, where) = locateKey(key, node)
+    if (there)
+        return node.datas[where]
+    else if (isLeaf(node))
+        return null
+
+    val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where]) ?: return null
+    return bSearch(field, key, nextNode)
+}

app/src/main/java/xyz/quaver/pupil/services/DownloadService.kt

@@ -23,10 +23,13 @@ import android.app.PendingIntent
 import android.app.Service
 import android.content.Context
 import android.content.Intent
+import android.util.Log
 import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationManagerCompat
 import androidx.core.app.TaskStackBuilder
 import androidx.core.content.ContextCompat
+import com.google.common.util.concurrent.RateLimiter
+import com.google.firebase.crashlytics.FirebaseCrashlytics
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
@@ -146,7 +149,7 @@ class DownloadService : Service() {
 
         override fun source(): BufferedSource {
             if (bufferedSource == null)
-                bufferedSource = Okio.buffer(source(responseBody.source()))
+                bufferedSource = source(responseBody.source()).buffer()
 
             return bufferedSource!!
         }
@@ -165,14 +168,28 @@ class DownloadService : Service() {
         }
     }
 
+    private val rateLimiter = RateLimiter.create(2.0)
+    private val rateLimitHost = Regex("..?\\.hitomi.la")
+
     private val interceptor: PupilInterceptor = { chain ->
         val request = chain.request()
-        var response = chain.proceed(request)
 
-        var retry = 5
+        Log.d("PUPILD", "REQ")
-        while (!response.isSuccessful && retry > 0) {
+
+        if (rateLimitHost.matches(request.url().host()))
+            rateLimiter.acquire()
+
+        Log.d("PUPILD", "ACQ ${request.url()}")
+
+        var response = chain.proceed(request)
+        var limit = 5
+
+        if (!response.isSuccessful && limit > 0) {
+            Thread.sleep(10000)
+            if (rateLimitHost.matches(request.url().host()))
+                rateLimiter.acquire()
             response = chain.proceed(request)
-            retry--
+            limit -= 1
         }
 
         response.newBuilder()
@@ -202,14 +219,10 @@ class DownloadService : Service() {
     private val callback = object: Callback {
 
         override fun onFailure(call: Call, e: IOException) {
-            e.printStackTrace()
+            FirebaseCrashlytics.getInstance().recordException(e)
 
             if (e.message?.contains("cancel", true) == false) {
                 val galleryID = (call.request().tag() as Tag).galleryID
-
-                // Retry
-                cancel(galleryID)
-                download(galleryID)
             }
         }
 
@@ -218,7 +231,7 @@ class DownloadService : Service() {
             val ext = call.request().url().encodedPath().split('.').last()
 
             kotlin.runCatching {
-                val image = response.also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() } ?: throw Exception()
+                val image = response.also { if (it.code() != 200) throw IOException("$galleryID $index ${response.request().url()} CODE ${it.code()}") }.body()?.use { it.bytes() } ?: throw Exception("Response null")
                 val padding = ceil(progress[galleryID]?.size?.let { log10(it.toFloat()) } ?: 0F).toInt()
 
                 CoroutineScope(Dispatchers.IO).launch {
@@ -236,11 +249,11 @@ class DownloadService : Service() {
                             startId?.let { stopSelf(it) }
                         }
                     }.onFailure {
-                        it.printStackTrace()
+                        FirebaseCrashlytics.getInstance().recordException(it)
-                        cancel(galleryID)
-                        download(galleryID)
                     }
                 }
+            }.onFailure {
+                FirebaseCrashlytics.getInstance().recordException(it)
             }
         }
     }

app/src/main/java/xyz/quaver/pupil/types/Suggestions.kt

@@ -21,7 +21,7 @@ package xyz.quaver.pupil.types
 import kotlinx.parcelize.IgnoredOnParcel
 import kotlinx.parcelize.Parcelize
 import xyz.quaver.floatingsearchview.suggestions.model.SearchSuggestion
-import xyz.quaver.hitomi.Suggestion
+import xyz.quaver.pupil.hitomi.Suggestion
 import xyz.quaver.pupil.util.translations
 
 @Parcelize

app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt

@@ -44,9 +44,9 @@ import xyz.quaver.floatingsearchview.FloatingSearchView
 import xyz.quaver.floatingsearchview.suggestions.model.SearchSuggestion
 import xyz.quaver.floatingsearchview.util.view.MenuView
 import xyz.quaver.floatingsearchview.util.view.SearchInputView
-import xyz.quaver.hitomi.doSearch
+import xyz.quaver.pupil.hitomi.doSearch
-import xyz.quaver.hitomi.getGalleryIDsFromNozomi
+import xyz.quaver.pupil.hitomi.getGalleryIDsFromNozomi
-import xyz.quaver.hitomi.getSuggestionsForQuery
+import xyz.quaver.pupil.hitomi.getSuggestionsForQuery
 import xyz.quaver.pupil.*
 import xyz.quaver.pupil.adapters.GalleryBlockAdapter
 import xyz.quaver.pupil.databinding.MainActivityBinding

app/src/main/java/xyz/quaver/pupil/ui/dialog/GalleryDialog.kt

@@ -35,8 +35,8 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
-import xyz.quaver.hitomi.Gallery
+import xyz.quaver.pupil.hitomi.Gallery
-import xyz.quaver.hitomi.getGallery
+import xyz.quaver.pupil.hitomi.getGallery
 import xyz.quaver.pupil.R
 import xyz.quaver.pupil.adapters.GalleryBlockAdapter
 import xyz.quaver.pupil.adapters.ThumbnailPageAdapter

app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt

@@ -32,11 +32,13 @@ import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import okhttp3.Request
-import xyz.quaver.hitomi.GalleryBlock
-import xyz.quaver.hitomi.GalleryInfo
 import xyz.quaver.io.FileX
 import xyz.quaver.io.util.*
 import xyz.quaver.pupil.client
+import xyz.quaver.pupil.hitomi.GalleryBlock
+import xyz.quaver.pupil.hitomi.GalleryInfo
+import xyz.quaver.pupil.hitomi.getGalleryBlock
+import xyz.quaver.pupil.hitomi.getGalleryInfo
 import java.io.File
 import java.io.IOException
 import java.util.concurrent.ConcurrentHashMap
@@ -155,7 +157,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
         return metadata.galleryBlock
             ?: withContext(Dispatchers.IO) {
                 try {
-                    xyz.quaver.hitomi.getGalleryBlock(galleryID).also {
+                    getGalleryBlock(galleryID).also {
                         setMetadata { metadata -> metadata.galleryBlock = it }
                     }
                 } catch (e: Exception) { return@withContext null }
@@ -187,7 +189,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW
         return metadata.galleryInfo
             ?: withContext(Dispatchers.IO) {
                 try {
-                    xyz.quaver.hitomi.getGalleryInfo(galleryID).also {
+                    getGalleryInfo(galleryID).also {
                         setMetadata { metadata ->
                             metadata.galleryInfo = it
 

app/src/main/java/xyz/quaver/pupil/util/misc.kt

@@ -22,10 +22,10 @@ import android.annotation.SuppressLint
 import kotlinx.serialization.json.*
 import okhttp3.OkHttpClient
 import okhttp3.Request
-import xyz.quaver.hitomi.GalleryBlock
+import xyz.quaver.pupil.hitomi.GalleryBlock
-import xyz.quaver.hitomi.GalleryInfo
+import xyz.quaver.pupil.hitomi.GalleryInfo
-import xyz.quaver.hitomi.getReferer
+import xyz.quaver.pupil.hitomi.getReferer
-import xyz.quaver.hitomi.imageUrlFromImage
+import xyz.quaver.pupil.hitomi.imageUrlFromImage
 import java.util.*
 import kotlin.collections.ArrayList
 
@@ -105,7 +105,8 @@ val GalleryInfo.requestBuilders: List<Request.Builder>
         return this.files.map {
             Request.Builder()
                 .url(imageUrlFromImage(galleryID, it, !lowQuality))
-                .header("Referer", getReferer(galleryID))
+                .header("Referer", "https://hitomi.la/")
+                .header("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36")
         }
     }
 

app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt

@@ -26,21 +26,21 @@ package xyz.quaver.pupil
  * See [testing documentation](http://d.android.com/tools/testing).
  */
 
-import kotlinx.serialization.*
+import okhttp3.OkHttpClient
-import kotlinx.serialization.json.Json
+import okhttp3.Request
+import org.junit.Assert.assertEquals
+import org.junit.Before
 import org.junit.Test
+import xyz.quaver.pupil.hitomi.getGalleryInfo
+import xyz.quaver.pupil.hitomi.imageUrlFromImage
 import java.lang.reflect.ParameterizedType
-import kotlin.reflect.KClass
+import java.util.concurrent.TimeUnit
-import kotlin.reflect.KType
-import kotlin.reflect.typeOf
 
 class ExampleUnitTest {
-
     @Test
     fun test() {
         val a = mutableSetOf<Int>()
 
         print(a::class.java.methods.firstOrNull { it.name == "add" }?.genericParameterTypes?.firstOrNull() as? ParameterizedType)
     }
-
 }

build.gradle

@@ -24,8 +24,8 @@ allprojects {
         google()
         mavenCentral()
         jcenter()
-        mavenLocal()
         maven { url "https://jitpack.io" }
+        maven { url "https://oss.sonatype.org/content/repositories/snapshots" }
         maven { url "https://guardian.github.io/maven/repo-releases/" }
     }
 }