diff --git a/.idea/deploymentTargetDropDown.xml b/.idea/deploymentTargetDropDown.xml deleted file mode 100644 index 4046d06a..00000000 --- a/.idea/deploymentTargetDropDown.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index 33797fc3..c104ba28 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -38,7 +38,7 @@ android { minSdkVersion 16 targetSdkVersion 30 versionCode 69 - versionName "5.1.24" + versionName "5.1.28" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" vectorDrawables.useSupportLibrary = true } @@ -125,7 +125,9 @@ dependencies { implementation "ru.noties.markwon:core:3.1.0" - implementation "xyz.quaver:libpupil:2.1.15" + implementation "org.jsoup:jsoup:1.14.3" + implementation "com.github.seven332:quickjs-android:0.1.0" + implementation "xyz.quaver:documentfilex:0.7.1" implementation "xyz.quaver:floatingsearchview:1.1.7" diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index 7b1b98e0..130b0cd1 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -32,4 +32,5 @@ kotlinx.serialization.KSerializer serializer(...); } -keep class xyz.quaver.pupil.ui.fragment.ManageFavoritesFragment --keep class xyz.quaver.pupil.ui.fragment.ManageStorageFragment \ No newline at end of file +-keep class xyz.quaver.pupil.ui.fragment.ManageStorageFragment +-keep class com.hippo.quickjs.** { *; } \ No newline at end of file diff --git a/app/release/output-metadata.json b/app/release/output-metadata.json index 25b94404..60463a89 100644 --- a/app/release/output-metadata.json +++ b/app/release/output-metadata.json @@ -11,7 +11,7 @@ "type": "SINGLE", "filters": [], "versionCode": 69, - "versionName": "5.1.24", + "versionName": "5.1.28", "outputFile": "app-release.apk" } ] diff --git a/app/src/main/java/xyz/quaver/pupil/Pupil.kt b/app/src/main/java/xyz/quaver/pupil/Pupil.kt index 5c1d169d..bf773328 100644 --- a/app/src/main/java/xyz/quaver/pupil/Pupil.kt +++ b/app/src/main/java/xyz/quaver/pupil/Pupil.kt @@ -36,6 +36,11 @@ import com.google.android.gms.common.GooglePlayServicesNotAvailableException import com.google.android.gms.common.GooglePlayServicesRepairableException import com.google.android.gms.security.ProviderInstaller import com.google.firebase.crashlytics.FirebaseCrashlytics +import com.hippo.quickjs.android.QuickJS +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.launch import okhttp3.Dispatcher import okhttp3.Interceptor import okhttp3.OkHttpClient @@ -43,9 +48,9 @@ import okhttp3.Response import xyz.quaver.io.FileX import xyz.quaver.pupil.types.Tag import xyz.quaver.pupil.util.* -import xyz.quaver.pupil.util.downloader.DownloadManager -import xyz.quaver.setClient +import xyz.quaver.readText import java.io.File +import java.net.URL import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -70,7 +75,6 @@ var clientHolder: OkHttpClient? = null val client: OkHttpClient get() = clientHolder ?: clientBuilder.build().also { clientHolder = it - setClient(it) } class Pupil : Application() { diff --git a/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt b/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt index eb5b2274..f540b9d9 100644 --- a/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt +++ b/app/src/main/java/xyz/quaver/pupil/adapters/GalleryBlockAdapter.kt @@ -35,14 +35,13 @@ import com.daimajia.swipe.adapters.RecyclerSwipeAdapter import com.daimajia.swipe.interfaces.SwipeAdapterInterface import com.github.piasy.biv.loader.ImageLoader import kotlinx.coroutines.* -import xyz.quaver.hitomi.getGallery -import xyz.quaver.hitomi.getGalleryInfo -import xyz.quaver.hitomi.getReader import xyz.quaver.io.util.getChild import xyz.quaver.pupil.R import xyz.quaver.pupil.databinding.GalleryblockItemBinding import xyz.quaver.pupil.favoriteTags import xyz.quaver.pupil.favorites +import xyz.quaver.pupil.hitomi.getGallery +import xyz.quaver.pupil.hitomi.getGalleryInfo import xyz.quaver.pupil.types.Tag import xyz.quaver.pupil.ui.view.ProgressCard import xyz.quaver.pupil.util.Preferences diff --git a/app/src/main/java/xyz/quaver/pupil/adapters/ReaderAdapter.kt b/app/src/main/java/xyz/quaver/pupil/adapters/ReaderAdapter.kt index b1b3ebf3..93892729 100644 --- a/app/src/main/java/xyz/quaver/pupil/adapters/ReaderAdapter.kt +++ b/app/src/main/java/xyz/quaver/pupil/adapters/ReaderAdapter.kt @@ -40,9 +40,9 @@ import com.github.piasy.biv.view.BigImageView import com.github.piasy.biv.view.ImageShownCallback import com.github.piasy.biv.view.ImageViewFactory import kotlinx.coroutines.* -import xyz.quaver.hitomi.GalleryInfo import xyz.quaver.pupil.R import xyz.quaver.pupil.databinding.ReaderItemBinding +import xyz.quaver.pupil.hitomi.GalleryInfo import xyz.quaver.pupil.ui.ReaderActivity import xyz.quaver.pupil.util.downloader.Cache import java.io.File diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt new file mode 100644 index 00000000..0ebcc8f5 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/Utils.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver + +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.json.Json +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.modules.plus +import kotlinx.serialization.modules.polymorphic +import kotlinx.serialization.modules.subclass +import okhttp3.OkHttpClient +import okhttp3.Request +import xyz.quaver.pupil.client +import java.io.IOException +import java.net.URL +import java.util.concurrent.TimeUnit +import kotlin.time.Duration + +/** + * kotlinx.serialization.json.Json object for global use + * properties should not be changed + * + * @see [https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-core/kotlinx-serialization-core/kotlinx.serialization.json/-json/index.html] + */ +val json = Json { + isLenient = true + ignoreUnknownKeys = true + allowSpecialFloatingPointValues = true + useArrayPolymorphism = true +} + +typealias HeaderSetter = (Request.Builder) -> Request.Builder +fun URL.readText(settings: HeaderSetter? = null): String { + val request = Request.Builder() + .url(this).let { + settings?.invoke(it) ?: it + }.build() + + return client.newCall(request).execute().also{ if (it.code() != 200) throw IOException() }.body()?.use { it.string() } ?: throw IOException() +} + +fun URL.readBytes(settings: HeaderSetter? = null): ByteArray { + val request = Request.Builder() + .url(this).let { + settings?.invoke(it) ?: it + }.build() + + return client.newCall(request).execute().also { if (it.code() != 200) throw IOException() }.body()?.use { it.bytes() } ?: throw IOException() +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/common.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/common.kt new file mode 100644 index 00000000..1bed8be5 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/common.kt @@ -0,0 +1,148 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import android.util.Log +import com.hippo.quickjs.android.QuickJS +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.decodeFromString +import xyz.quaver.json +import xyz.quaver.readText +import java.net.URL + +const val protocol = "https:" + +@Suppress("EXPERIMENTAL_API_USAGE") +fun getGalleryInfo(galleryID: Int) = + json.decodeFromString( + URL("$protocol//$domain/galleries/$galleryID.js").readText() + .replace("var galleryinfo = ", "") + ) + +//common.js +const val domain = "ltn.hitomi.la" +const val galleryblockextension = ".html" +const val galleryblockdir = "galleryblock" +const val nozomiextension = ".nozomi" + +interface gg { + fun m(g: Int): Int + val b: String + fun s(h: String): String + + companion object { + @Volatile private var instance: gg? = null + + fun getInstance(): gg = + instance ?: synchronized(this) { + instance ?: object: gg { + private val ggjs by lazy { URL("https://ltn.hitomi.la/gg.js").readText() } + private val quickJS = QuickJS.Builder().build() + + override fun m(g: Int): Int = + quickJS.createJSRuntime().use { runtime -> + runtime.createJSContext().use { context -> + context.evaluate(ggjs, "gg.js") + context.evaluate("gg.m($g)", "gg.js", Int::class.java) + } + } + + override val b: String + get() = + quickJS.createJSRuntime().use { runtime -> + runtime.createJSContext().use { context -> + context.evaluate(ggjs, "gg.js") + context.evaluate("gg.b", "gg.js", String::class.java) + } + } + + override fun s(h: String): String = + quickJS.createJSRuntime().use { runtime -> + runtime.createJSContext().use { context -> + context.evaluate(ggjs, "gg.js") + context.evaluate("gg.s('$h')", "gg.js", String::class.java) + } + } + }.also { instance = it } + } + } +} + +fun subdomainFromURL(url: String, base: String? = null) : String { + var retval = "b" + + if (!base.isNullOrBlank()) + retval = base + + val b = 16 + + val r = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""") + val m = r.find(url) ?: return "a" + + val g = m.groupValues.let { it[2]+it[1] }.toIntOrNull(b) + + if (g != null) { + retval = (97+ gg.getInstance().m(g)).toChar().toString() + retval + } + + return retval +} + +fun urlFromUrl(url: String, base: String? = null) : String { + return url.replace(Regex("""//..?\.hitomi\.la/"""), "//${subdomainFromURL(url, base)}.hitomi.la/") +} + + +fun fullPathFromHash(hash: String) : String = + "${gg.getInstance().b}${gg.getInstance().s(hash)}/$hash" + +fun realFullPathFromHash(hash: String): String = + hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1/$hash") + +fun urlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null) : String { + val ext = ext ?: dir ?: image.name.takeLastWhile { it != '.' } + val dir = dir ?: "images" + return "https://a.hitomi.la/$dir/${fullPathFromHash(image.hash)}.$ext" +} + +fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null) = + if (base == "tn") + urlFromUrl("https://a.hitomi.la/$dir/${realFullPathFromHash(image.hash)}.$ext", base) + else + urlFromUrl(urlFromHash(galleryID, image, dir, ext), base) + +fun rewriteTnPaths(html: String) = + html.replace(Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}""")) { url -> + urlFromUrl(url.value, "tn") + } + +fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String { + return when { + noWebp -> + urlFromUrlFromHash(galleryID, image) +// image.hasavif != 0 -> +// urlFromUrlFromHash(galleryID, image, "avif", null, "a") + image.haswebp != 0 -> + urlFromUrlFromHash(galleryID, image, "webp", null, "a") + else -> + urlFromUrlFromHash(galleryID, image) + } +} diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt new file mode 100644 index 00000000..2fad3212 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/galleries.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import kotlinx.serialization.Serializable +import org.jsoup.Jsoup +import xyz.quaver.readText +import java.net.URL +import java.net.URLDecoder + +@Serializable +data class Gallery( + val related: List, + val langList: List>, + val cover: String, + val title: String, + val artists: List, + val groups: List, + val type: String, + val language: String, + val series: List, + val characters: List, + val tags: List, + val thumbnails: List +) +fun getGallery(galleryID: Int) : Gallery { + val url = Jsoup.parse(URL("https://hitomi.la/galleries/$galleryID.html").readText()) + .select("link").attr("href") + + val doc = Jsoup.parse(URL(url).readText()) + + val related = Regex("\\d+") + .findAll(doc.select("script").first()!!.html()) + .map { + it.value.toInt() + }.toList() + + val langList = doc.select("#lang-list a").map { + Pair(it.text(), "$protocol//hitomi.la${it.attr("href")}") + } + + val cover = protocol + doc.selectFirst(".cover img")!!.attr("src") + val title = doc.selectFirst(".gallery h1 a")!!.text() + val artists = doc.select(".gallery h2 a").map { it.text() } + val groups = doc.select(".gallery-info a[href~=^/group/]").map { it.text() } + val type = doc.selectFirst(".gallery-info a[href~=^/type/]")!!.text() + + val language = run { + val href = doc.select(".gallery-info a[href~=^/index.+\\.html\$]").attr("href") + Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: "" + } + + val series = doc.select(".gallery-info a[href~=^/series/]").map { it.text() } + val characters = doc.select(".gallery-info a[href~=^/character/]").map { it.text() } + + val tags = doc.select(".gallery-info a[href~=^/tag/]").map { + val href = URLDecoder.decode(it.attr("href"), "UTF-8") + href.slice(5 until href.indexOf('-')) + } + + val thumbnails = getGalleryInfo(galleryID).files.map { galleryInfo -> + urlFromUrlFromHash(galleryID, galleryInfo, "smalltn", "jpg", "tn") + } + + return Gallery(related, langList, cover, title, artists, groups, type, language, series, characters, tags, thumbnails) +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt new file mode 100644 index 00000000..4d7f87f5 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/galleryblock.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import kotlinx.serialization.Serializable +import org.jsoup.Jsoup +import xyz.quaver.readText +import java.net.URL +import java.net.URLDecoder +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.* +import javax.net.ssl.HttpsURLConnection + +//galleryblock.js +fun fetchNozomi(area: String? = null, tag: String = "index", language: String = "all", start: Int = -1, count: Int = -1) : Pair, Int> { + val url = + when(area) { + null -> "$protocol//$domain/$tag-$language$nozomiextension" + else -> "$protocol//$domain/$area/$tag-$language$nozomiextension" + } + + with(URL(url).openConnection() as HttpsURLConnection) { + requestMethod = "GET" + + if (start != -1 && count != -1) { + val startByte = start*4 + val endByte = (start+count)*4-1 + + setRequestProperty("Range", "bytes=$startByte-$endByte") + } + + connect() + + val totalItems = getHeaderField("Content-Range") + .replace(Regex("^[Bb]ytes \\d+-\\d+/"), "").toInt() / 4 + + val nozomi = ArrayList() + + val arrayBuffer = ByteBuffer + .wrap(inputStream.readBytes()) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + return Pair(nozomi, totalItems) + } +} + +@Serializable +data class GalleryBlock( + val id: Int, + val galleryUrl: String, + val thumbnails: List, + val title: String, + val artists: List, + val series: List, + val type: String, + val language: String, + val relatedTags: List +) + +fun getGalleryBlock(galleryID: Int) : GalleryBlock { + val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension" + + val doc = Jsoup.parse(rewriteTnPaths(URL(url).readText())) + + val galleryUrl = doc.selectFirst("h1 > a")!!.attr("href") + + val thumbnails = doc.select(".dj-img-cont img").map { protocol + it.attr("src") } + + val title = doc.selectFirst("h1 > a")!!.text() + val artists = doc.select(".artist-list a").map{ it.text() } + val series = doc.select(".dj-content a[href~=^/series/]").map { it.text() } + val type = doc.selectFirst("a[href~=^/type/]")!!.text() + + val language = run { + val href = doc.select("a[href~=^/index.+\\.html\$]").attr("href") + Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: "" + } + + val relatedTags = doc.select(".relatedtags a").map { + val href = URLDecoder.decode(it.attr("href"), "UTF-8") + href.slice(5 until href.indexOf("-all")) + } + + return GalleryBlock(galleryID, galleryUrl, thumbnails, title, artists, series, type, language, relatedTags) +} + +fun getGalleryBlockOrNull(galleryID: Int) = runCatching { getGalleryBlock(galleryID) }.getOrNull() \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/reader.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/reader.kt new file mode 100644 index 00000000..7d33642d --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/reader.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import kotlinx.serialization.Serializable + +fun getReferer(galleryID: Int) = "https://hitomi.la/reader/$galleryID.html" + +@Serializable +data class GalleryInfo( + val language_localname: String? = null, + val language: String? = null, + val date: String? = null, + val files: List, + val id: Int? = null, + val type: String? = null, + val title: String? = null +) + +@Serializable +data class GalleryFiles( + val width: Int, + val hash: String, + val haswebp: Int = 0, + val name: String, + val height: Int, + val hasavif: Int = 0, + val hasavifsmalltn: Int? = 0 +) + +//Set header `Referer` to reader url to avoid 403 error +@Deprecated("", replaceWith = ReplaceWith("getGalleryInfo")) +fun getReader(galleryID: Int) : GalleryInfo { + return getGalleryInfo(galleryID) +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/results.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/results.kt new file mode 100644 index 00000000..ad7cb1c7 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/results.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.runBlocking +import java.util.* + +fun doSearch(query: String, sortByPopularity: Boolean = false) : Set { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .split(Regex("\\s+")) + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList() + val negativeTerms = LinkedList() + + for (term in terms) { + if (term.matches(Regex("^-.+"))) + negativeTerms.push(term.replace(Regex("^-"), "")) + else if (term.isNotBlank()) + positiveTerms.push(term) + } + + val positiveResults = positiveTerms.map { + CoroutineScope(Dispatchers.IO).async { + kotlin.runCatching { + getGalleryIDsForQuery(it) + }.getOrElse { emptySet() } + } + } + + val negativeResults = negativeTerms.map { + CoroutineScope(Dispatchers.IO).async { + kotlin.runCatching { + getGalleryIDsForQuery(it) + }.getOrElse { emptySet() } + } + } + + var results = when { + sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all") + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all") + else -> emptySet() + } + + runBlocking { + @Synchronized fun filterPositive(newResults: Set) { + results = when { + results.isEmpty() -> newResults + else -> results intersect newResults + } + } + + @Synchronized fun filterNegative(newResults: Set) { + results = results subtract newResults + } + + //positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + //negative results + negativeResults.forEach { + filterNegative(it.await()) + } + } + + return results +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/hitomi/search.kt b/app/src/main/java/xyz/quaver/pupil/hitomi/search.kt new file mode 100644 index 00000000..74d2fece --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/hitomi/search.kt @@ -0,0 +1,330 @@ +/* + * Copyright 2019 tom5079 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xyz.quaver.pupil.hitomi + +import okhttp3.Request +import xyz.quaver.pupil.client +import xyz.quaver.readBytes +import xyz.quaver.readText +import java.net.URL +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import kotlin.math.min + +//searchlib.js +const val separator = "-" +const val extension = ".html" +const val index_dir = "tagindex" +const val galleries_index_dir = "galleriesindex" +const val max_node_size = 464 +const val B = 16 +const val compressed_nozomi_prefix = "n" + +val tag_index_version: String by lazy { getIndexVersion("tagindex") } +val galleries_index_version: String by lazy { getIndexVersion("galleriesindex") } + +fun sha256(data: ByteArray) : ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) +} + +@OptIn(ExperimentalUnsignedTypes::class) +fun hashTerm(term: String) : UByteArray { + return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4) +} + +fun sanitize(input: String) : String { + return input.replace(Regex("[/#]"), "") +} + +fun getIndexVersion(name: String) = + URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText() + +//search.js +fun getGalleryIDsForQuery(query: String) : Set { + query.replace("_", " ").let { + if (it.indexOf(':') > -1) { + val sides = it.split(":") + val ns = sides[0] + var tag = sides[1] + + var area : String? = ns + var language = "all" + when (ns) { + "female", "male" -> { + area = "tag" + tag = it + } + "language" -> { + area = null + language = tag + tag = "index" + } + } + + return getGalleryIDsFromNozomi(area, tag, language) + } + + val key = hashTerm(it) + val field = "galleries" + + val node = getNodeAtAddress(field, 0) ?: return emptySet() + + val data = bSearch(field, key, node) + + if (data != null) + return getGalleryIDsFromData(data) + + return emptySet() + } +} + +fun getSuggestionsForQuery(query: String) : List { + query.replace('_', ' ').let { + var field = "global" + var term = it + + if (term.indexOf(':') > -1) { + val sides = it.split(':') + field = sides[0] + term = sides[1] + } + + val key = hashTerm(term) + val node = getNodeAtAddress(field, 0) ?: return emptyList() + val data = bSearch(field, key, node) + + if (data != null) + return getSuggestionsFromData(field, data) + + return emptyList() + } +} + +data class Suggestion(val s: String, val t: Int, val u: String, val n: String) +fun getSuggestionsFromData(field: String, data: Pair) : List { + val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data" + val (offset, length) = data + if (length > 10000 || length <= 0) + throw Exception("length $length is too long") + + val inbuf = getURLAtRange(url, offset.until(offset+length)) + + val suggestions = ArrayList() + + val buffer = ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + val numberOfSuggestions = buffer.int + + if (numberOfSuggestions > 100 || numberOfSuggestions <= 0) + throw Exception("number of suggestions $numberOfSuggestions is too long") + + for (i in 0.until(numberOfSuggestions)) { + var top = buffer.int + + val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8")) + buffer.position(buffer.position()+top) + + top = buffer.int + + val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8")) + buffer.position(buffer.position()+top) + + val count = buffer.int + + val tagname = sanitize(tag) + val u = + when(ns) { + "female", "male" -> "/tag/$ns:$tagname${separator}1$extension" + "language" -> "/index-$tagname${separator}1$extension" + else -> "/$ns/$tagname${separator}all${separator}1$extension" + } + + suggestions.add(Suggestion(tag, count, u, ns)) + } + + return suggestions +} + +fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set { + val nozomiAddress = + when(area) { + null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension" + else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension" + } + + val bytes = try { + URL(nozomiAddress).readBytes() + } catch (e: Exception) { + return emptySet() + } + + val nozomi = mutableSetOf() + + val arrayBuffer = ByteBuffer + .wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + return nozomi +} + +fun getGalleryIDsFromData(data: Pair) : Set { + val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data" + val (offset, length) = data + if (length > 100000000 || length <= 0) + throw Exception("length $length is too long") + + val inbuf = getURLAtRange(url, offset.until(offset+length)) + + val galleryIDs = mutableSetOf() + + val buffer = ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + + val numberOfGalleryIDs = buffer.int + + val expectedLength = numberOfGalleryIDs*4+4 + + if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) + throw Exception("number_of_galleryids $numberOfGalleryIDs is too long") + else if (inbuf.size != expectedLength) + throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength") + + for (i in 0.until(numberOfGalleryIDs)) + galleryIDs.add(buffer.int) + + return galleryIDs +} + +fun getNodeAtAddress(field: String, address: Long) : Node? { + val url = + when(field) { + "galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index" + "languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index" + "nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index" + else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index" + } + + val nodedata = getURLAtRange(url, address.until(address+max_node_size)) + + return decodeNode(nodedata) +} + +fun getURLAtRange(url: String, range: LongRange) : ByteArray { + val request = Request.Builder() + .url(url) + .header("Range", "bytes=${range.first}-${range.last}") + .build() + + return client.newCall(request).execute().body()?.use { it.bytes() } ?: byteArrayOf() +} + +@OptIn(ExperimentalUnsignedTypes::class) +data class Node(val keys: List, val datas: List>, val subNodeAddresses: List) +@OptIn(ExperimentalUnsignedTypes::class) +fun decodeNode(data: ByteArray) : Node { + val buffer = ByteBuffer + .wrap(data) + .order(ByteOrder.BIG_ENDIAN) + + val uData = data.toUByteArray() + + val numberOfKeys = buffer.int + val keys = ArrayList() + + for (i in 0.until(numberOfKeys)) { + val keySize = buffer.int + + if (keySize == 0 || keySize > 32) + throw Exception("fatal: !keySize || keySize > 32") + + keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize))) + buffer.position(buffer.position()+keySize) + } + + val numberOfDatas = buffer.int + val datas = ArrayList>() + + for (i in 0.until(numberOfDatas)) { + val offset = buffer.long + val length = buffer.int + + datas.add(Pair(offset, length)) + } + + val numberOfSubNodeAddresses = B+1 + val subNodeAddresses = ArrayList() + + for (i in 0.until(numberOfSubNodeAddresses)) { + val subNodeAddress = buffer.long + subNodeAddresses.add(subNodeAddress) + } + + return Node(keys, datas, subNodeAddresses) +} + +@OptIn(ExperimentalUnsignedTypes::class) +fun bSearch(field: String, key: UByteArray, node: Node) : Pair? { + fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int { + val top = min(dv1.size, dv2.size) + + for (i in 0.until(top)) { + if (dv1[i] < dv2[i]) + return -1 + else if (dv1[i] > dv2[i]) + return 1 + } + + return 0 + } + + fun locateKey(key: UByteArray, node: Node) : Pair { + for (i in node.keys.indices) { + val cmpResult = compareArrayBuffers(key, node.keys[i]) + + if (cmpResult <= 0) + return Pair(cmpResult==0, i) + } + + return Pair(false, node.keys.size) + } + + fun isLeaf(node: Node) : Boolean { + for (subnode in node.subNodeAddresses) + if (subnode != 0L) + return false + + return true + } + + if (node.keys.isEmpty()) + return null + + val (there, where) = locateKey(key, node) + if (there) + return node.datas[where] + else if (isLeaf(node)) + return null + + val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where]) ?: return null + return bSearch(field, key, nextNode) +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/types/Suggestions.kt b/app/src/main/java/xyz/quaver/pupil/types/Suggestions.kt index d3c02196..86cef085 100644 --- a/app/src/main/java/xyz/quaver/pupil/types/Suggestions.kt +++ b/app/src/main/java/xyz/quaver/pupil/types/Suggestions.kt @@ -21,7 +21,7 @@ package xyz.quaver.pupil.types import kotlinx.parcelize.IgnoredOnParcel import kotlinx.parcelize.Parcelize import xyz.quaver.floatingsearchview.suggestions.model.SearchSuggestion -import xyz.quaver.hitomi.Suggestion +import xyz.quaver.pupil.hitomi.Suggestion import xyz.quaver.pupil.util.translations @Parcelize diff --git a/app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt b/app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt index 2962518c..56148b87 100644 --- a/app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt +++ b/app/src/main/java/xyz/quaver/pupil/ui/MainActivity.kt @@ -44,9 +44,9 @@ import xyz.quaver.floatingsearchview.FloatingSearchView import xyz.quaver.floatingsearchview.suggestions.model.SearchSuggestion import xyz.quaver.floatingsearchview.util.view.MenuView import xyz.quaver.floatingsearchview.util.view.SearchInputView -import xyz.quaver.hitomi.doSearch -import xyz.quaver.hitomi.getGalleryIDsFromNozomi -import xyz.quaver.hitomi.getSuggestionsForQuery +import xyz.quaver.pupil.hitomi.doSearch +import xyz.quaver.pupil.hitomi.getGalleryIDsFromNozomi +import xyz.quaver.pupil.hitomi.getSuggestionsForQuery import xyz.quaver.pupil.* import xyz.quaver.pupil.adapters.GalleryBlockAdapter import xyz.quaver.pupil.databinding.MainActivityBinding diff --git a/app/src/main/java/xyz/quaver/pupil/ui/dialog/GalleryDialog.kt b/app/src/main/java/xyz/quaver/pupil/ui/dialog/GalleryDialog.kt index c12b7677..7509ea70 100644 --- a/app/src/main/java/xyz/quaver/pupil/ui/dialog/GalleryDialog.kt +++ b/app/src/main/java/xyz/quaver/pupil/ui/dialog/GalleryDialog.kt @@ -35,8 +35,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -import xyz.quaver.hitomi.Gallery -import xyz.quaver.hitomi.getGallery +import xyz.quaver.pupil.hitomi.Gallery +import xyz.quaver.pupil.hitomi.getGallery import xyz.quaver.pupil.R import xyz.quaver.pupil.adapters.GalleryBlockAdapter import xyz.quaver.pupil.adapters.ThumbnailPageAdapter diff --git a/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt b/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt index 4aa9b0df..f9a005df 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/downloader/Cache.kt @@ -32,11 +32,13 @@ import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import okhttp3.Request -import xyz.quaver.hitomi.GalleryBlock -import xyz.quaver.hitomi.GalleryInfo import xyz.quaver.io.FileX import xyz.quaver.io.util.* import xyz.quaver.pupil.client +import xyz.quaver.pupil.hitomi.GalleryBlock +import xyz.quaver.pupil.hitomi.GalleryInfo +import xyz.quaver.pupil.hitomi.getGalleryBlock +import xyz.quaver.pupil.hitomi.getGalleryInfo import java.io.File import java.io.IOException import java.util.concurrent.ConcurrentHashMap @@ -155,7 +157,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW return metadata.galleryBlock ?: withContext(Dispatchers.IO) { try { - xyz.quaver.hitomi.getGalleryBlock(galleryID).also { + getGalleryBlock(galleryID).also { setMetadata { metadata -> metadata.galleryBlock = it } } } catch (e: Exception) { return@withContext null } @@ -187,7 +189,7 @@ class Cache private constructor(context: Context, val galleryID: Int) : ContextW return metadata.galleryInfo ?: withContext(Dispatchers.IO) { try { - xyz.quaver.hitomi.getGalleryInfo(galleryID).also { + getGalleryInfo(galleryID).also { setMetadata { metadata -> metadata.galleryInfo = it diff --git a/app/src/main/java/xyz/quaver/pupil/util/misc.kt b/app/src/main/java/xyz/quaver/pupil/util/misc.kt index 687a2adf..ce9fc58a 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/misc.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/misc.kt @@ -22,10 +22,10 @@ import android.annotation.SuppressLint import kotlinx.serialization.json.* import okhttp3.OkHttpClient import okhttp3.Request -import xyz.quaver.hitomi.GalleryBlock -import xyz.quaver.hitomi.GalleryInfo -import xyz.quaver.hitomi.getReferer -import xyz.quaver.hitomi.imageUrlFromImage +import xyz.quaver.pupil.hitomi.GalleryBlock +import xyz.quaver.pupil.hitomi.GalleryInfo +import xyz.quaver.pupil.hitomi.getReferer +import xyz.quaver.pupil.hitomi.imageUrlFromImage import java.util.* import kotlin.collections.ArrayList diff --git a/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt b/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt index 16349253..54369f9a 100644 --- a/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt +++ b/app/src/test/java/xyz/quaver/pupil/ExampleUnitTest.kt @@ -26,21 +26,21 @@ package xyz.quaver.pupil * See [testing documentation](http://d.android.com/tools/testing). */ -import kotlinx.serialization.* -import kotlinx.serialization.json.Json +import okhttp3.OkHttpClient +import okhttp3.Request +import org.junit.Assert.assertEquals +import org.junit.Before import org.junit.Test +import xyz.quaver.pupil.hitomi.getGalleryInfo +import xyz.quaver.pupil.hitomi.imageUrlFromImage import java.lang.reflect.ParameterizedType -import kotlin.reflect.KClass -import kotlin.reflect.KType -import kotlin.reflect.typeOf +import java.util.concurrent.TimeUnit class ExampleUnitTest { - @Test fun test() { val a = mutableSetOf() print(a::class.java.methods.firstOrNull { it.name == "add" }?.genericParameterTypes?.firstOrNull() as? ParameterizedType) } - }