Added Offline mode

app/src/main/java/xyz/quaver/pupil/MainActivity.kt

@@ -6,7 +6,6 @@ import android.os.Bundle
 import android.preference.PreferenceManager
 import android.text.*
 import android.text.style.AlignmentSpan
-import android.util.Log
 import android.view.View
 import android.view.WindowManager
 import androidx.appcompat.app.AlertDialog
@@ -22,6 +21,8 @@ import com.google.android.material.appbar.AppBarLayout
 import kotlinx.android.synthetic.main.activity_main.*
 import kotlinx.android.synthetic.main.activity_main_content.*
 import kotlinx.coroutines.*
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonConfiguration
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.content
 import ru.noties.markwon.Markwon
@@ -34,6 +35,7 @@ import xyz.quaver.pupil.util.SetLineOverlap
 import xyz.quaver.pupil.util.checkUpdate
 import java.io.File
 import java.io.FileOutputStream
+import java.net.URL
 import java.util.*
 import javax.net.ssl.HttpsURLConnection
 import kotlin.collections.ArrayList
@@ -436,25 +438,48 @@ class MainActivity : AppCompatActivity() {
                     galleryIDs
                 else ->
                     galleryIDs.slice(galleries.size until Math.min(galleries.size+perPage, galleryIDs.size))
-            }.chunked(4).let { chunks ->
+            }.chunked(5).let { chunks ->
                 for (chunk in chunks)
                     chunk.map {
                         async {
                             try {
-                                val galleryBlock = getGalleryBlock(it)
+                                val json = Json(JsonConfiguration.Stable)
+                                val serializer = GalleryBlock.serializer()
+
+                                val galleryBlock =
+                                    File(cacheDir, "imageCache/$it/galleryBlock.json").let { cache ->
+                                        when {
+                                            cache.exists() -> json.parse(serializer, cache.readText())
+                                            else -> {
+                                                getGalleryBlock(it).apply {
+                                                    this ?: return@apply
+
+                                                    if (!cache.parentFile.exists())
+                                                        cache.parentFile.mkdirs()
+
+                                                    cache.writeText(json.stringify(serializer, this))
+                                                }
+                                            }
+                                        }
+                                    } ?: return@async null
 
                                 val thumbnail = async {
-                                    val cache = File(cacheDir, "imageCache/$it/thumbnail.${galleryBlock.thumbnails[0].path.split('.').last()}")
+                                    val ext = galleryBlock.thumbnails[0].split('.').last()
+                                    File(cacheDir, "imageCache/$it/thumbnail.$ext").apply {
+                                        val cache = this
 
-                                    if (!cache.exists())
+                                        if (!cache.exists())
-                                        with(galleryBlock.thumbnails[0].openConnection() as HttpsURLConnection) {
+                                            try {
-                                            if (!cache.parentFile.exists())
+                                                with(URL(galleryBlock.thumbnails[0]).openConnection() as HttpsURLConnection) {
-                                                cache.parentFile.mkdirs()
+                                                    if (!cache.parentFile.exists())
+                                                        cache.parentFile.mkdirs()
 
-                                            inputStream.copyTo(FileOutputStream(cache))
+                                                    inputStream.copyTo(FileOutputStream(cache))
-                                        }
+                                                }
-
+                                            } catch (e: Exception) {
-                                    cache.absolutePath
+                                                cache.delete()
+                                            }
+                                    }.absolutePath
                                 }
 
                                 Pair(galleryBlock, thumbnail)
@@ -463,16 +488,19 @@ class MainActivity : AppCompatActivity() {
                             }
                         }
                     }.forEach {
-                        val galleryBlock = it.await() ?: return@forEach
+                        val galleryBlock = it.await()
 
                         withContext(Dispatchers.Main) {
                             main_progressbar.hide()
 
-                            galleries.add(galleryBlock)
+                            if (galleryBlock != null) {
-                            main_recyclerview.adapter?.notifyItemInserted(galleries.size - 1)
+                                galleries.add(galleryBlock)
+
+                                main_recyclerview.adapter?.notifyItemInserted(galleries.size - 1)
+                            }
                         }
                     }
-                }
+            }
         }
     }
 }

app/src/main/java/xyz/quaver/pupil/ReaderActivity.kt

@@ -66,26 +66,36 @@ class ReaderActivity : AppCompatActivity() {
                 else -> File(cacheDir, "imageCache/$galleryID/reader.json")
             }
 
-            if (cache.exists())
+            if (cache.exists()) {
-                json.parse(serializer, cache.readText())
+                val cached = json.parse(serializer, cache.readText())
-            else {
-                val reader = when {
-                    isHiyobi -> {
-                        try {
-                            xyz.quaver.hiyobi.getReader(galleryID)
-                        } catch (e: Exception) {
-                            getReader(galleryID)
-                        }
-                    }
-                    else -> {
-                        getReader(galleryID)
-                    }
-                }
 
-                cache.writeText(json.stringify(serializer, reader))
+                if (cached.isNotEmpty())
-
+                    return@async cached
-                reader
             }
+
+            val reader = when {
+                isHiyobi -> {
+                        xyz.quaver.hiyobi.getReader(galleryID).let {
+                            when {
+                                it.isEmpty() -> getReader(galleryID)
+                                else -> it
+                            }
+                        }
+                }
+                else -> {
+                    getReader(galleryID)
+                }
+            }
+
+            if (reader.isEmpty())
+                finish()
+
+            if (!cache.parentFile.exists())
+                cache.parentFile.mkdirs()
+
+            cache.writeText(json.stringify(serializer, reader))
+
+            reader
         }
 
         initView()
@@ -257,13 +267,17 @@ class ReaderActivity : AppCompatActivity() {
                         val cache = File(cacheDir, "/imageCache/$galleryID/$fileName")
 
                         if (!cache.exists())
-                            with(URL(url).openConnection() as HttpsURLConnection) {
+                            try {
-                                setRequestProperty("Referer", getReferer(galleryID))
+                                with(URL(url).openConnection() as HttpsURLConnection) {
+                                    setRequestProperty("Referer", getReferer(galleryID))
 
-                                if (!cache.parentFile.exists())
+                                    if (!cache.parentFile.exists())
-                                    cache.parentFile.mkdirs()
+                                        cache.parentFile.mkdirs()
 
-                                inputStream.copyTo(FileOutputStream(cache))
+                                    inputStream.copyTo(FileOutputStream(cache))
+                                }
+                            } catch (e: Exception) {
+                                cache.delete()
                             }
 
                         cache.absolutePath

app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt

@@ -9,11 +9,13 @@ import androidx.cardview.widget.CardView
 import androidx.recyclerview.widget.RecyclerView
 import com.google.android.material.chip.Chip
 import kotlinx.android.synthetic.main.item_galleryblock.view.*
-import kotlinx.coroutines.*
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import xyz.quaver.hitomi.GalleryBlock
 import xyz.quaver.hitomi.toTag
 import xyz.quaver.pupil.R
-import java.io.File
 
 class GalleryBlockAdapter(private val galleries: List<Pair<GalleryBlock, Deferred<String>>>) : RecyclerView.Adapter<RecyclerView.ViewHolder>() {
 

app/src/main/java/xyz/quaver/pupil/util/update.kt

@@ -4,8 +4,12 @@ import kotlinx.serialization.json.*
 import java.net.URL
 
 fun getReleases(url: String) : JsonArray {
-    return URL(url).readText().let {
+    return try {
-        Json(JsonConfiguration.Stable).parse(JsonArray.serializer(), it)
+        URL(url).readText().let {
+            Json(JsonConfiguration.Stable).parse(JsonArray.serializer(), it)
+        }
+    } catch (e: Exception) {
+        JsonArray(emptyList())
     }
 }
 

app/src/main/res/drawable-anydpi/ic_placeholder.xml

@@ -0,0 +1,10 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="24dp"
+    android:height="24dp"
+    android:viewportWidth="24"
+    android:viewportHeight="24"
+    android:tint="#333333">
+    <path
+        android:fillColor="#FF000000"
+        android:pathData="M21,5v6.59l-3,-3.01 -4,4.01 -4,-4 -4,4 -3,-3.01L3,5c0,-1.1 0.9,-2 2,-2h14c1.1,0 2,0.9 2,2zM18,11.42l3,3.01L21,19c0,1.1 -0.9,2 -2,2L5,21c-1.1,0 -2,-0.9 -2,-2v-6.58l3,2.99 4,-4 4,4 4,-3.99z"/>
+</vector>

libpupil/src/main/java/xyz/quaver/hitomi/galleryblock.kt

@@ -1,5 +1,6 @@
 package xyz.quaver.hitomi
 
+import kotlinx.serialization.Serializable
 import org.jsoup.Jsoup
 import java.net.URL
 import java.net.URLDecoder
@@ -39,13 +40,14 @@ fun fetchNozomi(area: String? = null, tag: String = "index", language: String =
             return nozomi
         }
     } catch (e: Exception) {
-        return listOf()
+        return emptyList()
     }
 }
 
+@Serializable
 data class GalleryBlock(
     val id: Int,
-    val thumbnails: List<URL>,
+    val thumbnails: List<String>,
     val title: String,
     val artists: List<String>,
     val series: List<String>,
@@ -53,27 +55,31 @@ data class GalleryBlock(
     val language: String,
     val relatedTags: List<String>
 )
-fun getGalleryBlock(galleryID: Int) : GalleryBlock {
+fun getGalleryBlock(galleryID: Int) : GalleryBlock? {
     val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension"
 
-    val doc = Jsoup.connect(url).get()
+    try {
+        val doc = Jsoup.connect(url).get()
 
-    val thumbnails = doc.select("img").map { URL(protocol + it.attr("data-src")) }
+        val thumbnails = doc.select("img").map { protocol + it.attr("data-src") }
 
-    val title = doc.selectFirst("h1.lillie > a").text()
+        val title = doc.selectFirst("h1.lillie > a").text()
-    val artists = doc.select("div.artist-list a").map{ it.text() }
+        val artists = doc.select("div.artist-list a").map{ it.text() }
-    val series = doc.select("a[href~=^/series/]").map { it.text() }
+        val series = doc.select("a[href~=^/series/]").map { it.text() }
-    val type = doc.selectFirst("a[href~=^/type/]").text()
+        val type = doc.selectFirst("a[href~=^/type/]").text()
 
-    val language = {
+        val language = {
-        val href = doc.select("a[href~=^/index-.+-1.html]").attr("href")
+            val href = doc.select("a[href~=^/index-.+-1.html]").attr("href")
-        href.slice(7 until href.indexOf("-1"))
+            href.slice(7 until href.indexOf("-1"))
-    }.invoke()
+        }.invoke()
 
-    val relatedTags = doc.select(".relatedtags a").map {
+        val relatedTags = doc.select(".relatedtags a").map {
-        val href = URLDecoder.decode(it.attr("href"), "UTF-8")
+            val href = URLDecoder.decode(it.attr("href"), "UTF-8")
-        href.slice(5 until href.indexOf('-'))
+            href.slice(5 until href.indexOf('-'))
+        }
+
+        return GalleryBlock(galleryID, thumbnails, title, artists, series, type, language, relatedTags)
+    } catch (e: Exception) {
+        return null
     }
-
-    return GalleryBlock(galleryID, thumbnails, title, artists, series, type, language, relatedTags)
 }

libpupil/src/main/java/xyz/quaver/hitomi/readers.kt

@@ -27,27 +27,32 @@ fun getReader(galleryID: Int) : Reader {
     val readerUrl = "https://hitomi.la/reader/$galleryID.html"
     val galleryInfoUrl = "https://ltn.hitomi.la/galleries/$galleryID.js"
 
-    val doc = Jsoup.connect(readerUrl).get()
+    try {
 
-    val images = doc.select(".img-url").map {
+        val doc = Jsoup.connect(readerUrl).get()
-        protocol + urlFromURL(it.text())
-    }
 
-    val galleryInfo = ArrayList<GalleryInfo?>()
+        val images = doc.select(".img-url").map {
+            protocol + urlFromURL(it.text())
+        }
 
-    galleryInfo.addAll(
+        val galleryInfo = ArrayList<GalleryInfo?>()
-        Json(JsonConfiguration.Stable).parse(
+
-            GalleryInfo.serializer().list,
+        galleryInfo.addAll(
-            Regex("""\[.+]""").find(
+            Json(JsonConfiguration.Stable).parse(
-                URL(galleryInfoUrl).readText()
+                GalleryInfo.serializer().list,
-            )?.value ?: "[]"
+                Regex("""\[.+]""").find(
+                    URL(galleryInfoUrl).readText()
+                )?.value ?: "[]"
+            )
         )
-    )
 
-    if (images.size > galleryInfo.size)
+        if (images.size > galleryInfo.size)
-        galleryInfo.addAll(arrayOfNulls(images.size - galleryInfo.size))
+            galleryInfo.addAll(arrayOfNulls(images.size - galleryInfo.size))
 
-    return (images zip galleryInfo).map {
+        return (images zip galleryInfo).map {
-        ReaderItem(it.first, it.second)
+            ReaderItem(it.first, it.second)
+        }
+    } catch (e: Exception) {
+        return emptyList()
     }
 }

libpupil/src/main/java/xyz/quaver/hitomi/search.kt

@@ -15,8 +15,8 @@ const val max_node_size = 464
 const val B = 16
 const val compressed_nozomi_prefix = "n"
 
-val tag_index_version = getIndexVersion("tagindex")
+var tag_index_version = getIndexVersion("tagindex")
-val galleries_index_version = getIndexVersion("galleriesindex")
+var galleries_index_version = getIndexVersion("galleriesindex")
 
 fun sha256(data: ByteArray) : ByteArray {
     return MessageDigest.getInstance("SHA-256").digest(data)
@@ -32,8 +32,12 @@ fun sanitize(input: String) : String {
 }
 
 fun getIndexVersion(name: String) : String {
-    return URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}")
+    return try {
-        .readText()
+        URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}")
+            .readText()
+    } catch (e: Exception) {
+        ""
+    }
 }
 
 //search.js
@@ -64,14 +68,14 @@ fun getGalleryIDsForQuery(query: String) : List<Int> {
         val key = hashTerm(it)
         val field = "galleries"
 
-        val node = getNodeAtAddress(field, 0) ?: return listOf()
+        val node = getNodeAtAddress(field, 0) ?: return emptyList()
 
         val data = bSearch(field, key, node)
 
         if (data != null)
             return getGalleryIDsFromData(data)
 
-        return arrayListOf()
+        return emptyList()
     }
 }
 
@@ -87,24 +91,27 @@ fun getSuggestionsForQuery(query: String) : List<Suggestion> {
         }
 
         val key = hashTerm(term)
-        val node = getNodeAtAddress(field, 0) ?: return listOf()
+        val node = getNodeAtAddress(field, 0) ?: return emptyList()
         val data = bSearch(field, key, node)
 
         if (data != null)
             return getSuggestionsFromData(field, data)
 
-        return listOf()
+        return emptyList()
     }
 }
 
 data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
 fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggestion> {
+    if (tag_index_version.isEmpty())
+        tag_index_version = getIndexVersion("tagindex")
+
     val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data"
     val (offset, length) = data
     if (length > 10000 || length <= 0)
         throw Exception("length $length is too long")
 
-    val inbuf = getURLAtRange(url, offset.until(offset+length)) ?: return listOf()
+    val inbuf = getURLAtRange(url, offset.until(offset+length)) ?: return emptyList()
 
     val suggestions = ArrayList<Suggestion>()
 
@@ -166,17 +173,20 @@ fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : List
             return nozomi
         }
     } catch (e: Exception) {
-        return listOf()
+        return emptyList()
     }
 }
 
 fun getGalleryIDsFromData(data: Pair<Long, Int>) : List<Int> {
+    if (galleries_index_version.isEmpty())
+        galleries_index_version = getIndexVersion("galleriesindex")
+
     val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data"
     val (offset, length) = data
     if (length > 100000000 || length <= 0)
         throw Exception("length $length is too long")
 
-    val inbuf = getURLAtRange(url, offset.until(offset+length)) ?: return listOf()
+    val inbuf = getURLAtRange(url, offset.until(offset+length)) ?: return emptyList()
 
     val galleryIDs = ArrayList<Int>()
 
@@ -200,6 +210,11 @@ fun getGalleryIDsFromData(data: Pair<Long, Int>) : List<Int> {
 }
 
 fun getNodeAtAddress(field: String, address: Long) : Node? {
+    if (tag_index_version.isEmpty())
+        tag_index_version = getIndexVersion("tagindex")
+    if (galleries_index_version.isEmpty())
+        galleries_index_version = getIndexVersion("galleriesindex")
+
     val url =
             when(field) {
                 "galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"

libpupil/src/main/java/xyz/quaver/hiyobi/reader.kt

@@ -16,11 +16,15 @@ var cookie: String = ""
 fun renewCookie() : String {
     val url = "https://$hiyobi/"
 
-    with(URL(url).openConnection() as HttpsURLConnection) {
+    try {
-        setRequestProperty("User-Agent", user_agent)
+        with(URL(url).openConnection() as HttpsURLConnection) {
-        connectTimeout = 2000
+            setRequestProperty("User-Agent", user_agent)
-        connect()
+            connectTimeout = 2000
-        return headerFields["Set-Cookie"]!![0]
+            connect()
+            return headerFields["Set-Cookie"]!![0]
+        }
+    } catch (e: Exception) {
+        return ""
     }
 }
 
@@ -30,19 +34,23 @@ fun getReader(galleryId: Int) : Reader {
     if (cookie.isEmpty())
         cookie = renewCookie()
 
-    val json = Json(JsonConfiguration.Stable).parseJson(
+    try {
-        with(URL(url).openConnection() as HttpsURLConnection) {
+        val json = Json(JsonConfiguration.Stable).parseJson(
-            setRequestProperty("User-Agent", user_agent)
+            with(URL(url).openConnection() as HttpsURLConnection) {
-            setRequestProperty("Cookie", cookie)
+                setRequestProperty("User-Agent", user_agent)
-            connectTimeout = 2000
+                setRequestProperty("Cookie", cookie)
-            connect()
+                connectTimeout = 2000
+                connect()
 
-            inputStream.bufferedReader().use { it.readText() }
+                inputStream.bufferedReader().use { it.readText() }
+            }
+        )
+
+        return json.jsonArray.map {
+            val name = it.jsonObject["name"]!!.content
+            ReaderItem("https://$hiyobi/data/$galleryId/$name", null)
         }
-    )
+    } catch (e: Exception) {
-
+        return emptyList()
-    return json.jsonArray.map {
-        val name = it.jsonObject["name"]!!.content
-        ReaderItem("https://$hiyobi/data/$galleryId/$name", null)
     }
 }

libpupil/src/test/java/xyz/quaver/hitomi/UnitTest.kt

@@ -1,14 +1,15 @@
 package xyz.quaver.hitomi
 
 import org.junit.Test
+import java.io.File
 import java.net.URL
 
 class UnitTest {
     @Test
     fun test() {
-        val url = URL("https://ltn.hitomi.la/galleries/1411672.js")
+        val f = File("C:/Users/tom50/Workspace/Pupil/nodir/nodir/asdf.txt")
 
-        print(url.path.substring(url.path.lastIndexOf('/')+1))
+        f.delete()
     }
 
     @Test
@@ -63,6 +64,6 @@ class UnitTest {
 
     @Test
     fun test_hiyobi() {
-        xyz.quaver.hiyobi.getReader(1414061)
+        print(xyz.quaver.hiyobi.getReader(1415416).size)
     }
 }