Fuck hitomi
This commit is contained in:
tom5079
@@ -16,31 +16,81 @@
|
||||
|
||||
package xyz.quaver.pupil.hitomi
|
||||
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.first
|
||||
import kotlinx.coroutines.flow.transformWhile
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.asCoroutineDispatcher
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.decodeFromString
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import xyz.quaver.pupil.*
|
||||
import java.util.*
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.suspendCoroutine
|
||||
import okhttp3.Request
|
||||
import xyz.quaver.pupil.client
|
||||
import xyz.quaver.pupil.runtime
|
||||
import java.io.IOException
|
||||
import java.net.URL
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
const val protocol = "https:"
|
||||
|
||||
val evaluationContext = Dispatchers.Main + Job()
|
||||
@Serializable
|
||||
data class Artist(
|
||||
val artist: String,
|
||||
val url: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class Group(
|
||||
val group: String,
|
||||
val url: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class Parody(
|
||||
val parody: String,
|
||||
val url: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class Character(
|
||||
val character: String,
|
||||
val url: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class Tag(
|
||||
val tag: String,
|
||||
val url: String,
|
||||
val female: String? = null,
|
||||
val male: String? = null
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class Language(
|
||||
val galleryid: String,
|
||||
val url: String,
|
||||
val language_localname: String,
|
||||
val name: String
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class GalleryInfo(
|
||||
val id: String,
|
||||
val title: String,
|
||||
val japanese_title: String? = null,
|
||||
val language: String? = null,
|
||||
val type: String,
|
||||
val date: String,
|
||||
val artists: List<Artist>? = null,
|
||||
val groups: List<Group>? = null,
|
||||
val parodys: List<Parody>? = null,
|
||||
val tags: List<Tag>? = null,
|
||||
val related: List<Int>,
|
||||
val languages: List<Language>,
|
||||
val characters: List<Character>? = null,
|
||||
val scene_indexes: List<Int>,
|
||||
val files: List<GalleryFiles>
|
||||
)
|
||||
|
||||
/**
|
||||
* kotlinx.serialization.json.Json object for global use
|
||||
* properties should not be changed
|
||||
*
|
||||
* @see [https://kotlin.github.io/kotlinx.serialization/kotlinx-serialization-core/kotlinx-serialization-core/kotlinx.serialization.json/-json/index.html]
|
||||
*/
|
||||
val json = Json {
|
||||
isLenient = true
|
||||
ignoreUnknownKeys = true
|
||||
@@ -48,89 +98,103 @@ val json = Json {
|
||||
useArrayPolymorphism = true
|
||||
}
|
||||
|
||||
suspend inline fun <reified T> WebView.evaluate(script: String): T = coroutineScope { withTimeout(60000) {
|
||||
var result: String? = null
|
||||
typealias HeaderSetter = (Request.Builder) -> Request.Builder
|
||||
fun URL.readText(settings: HeaderSetter? = null): String {
|
||||
val request = Request.Builder()
|
||||
.url(this).let {
|
||||
settings?.invoke(it) ?: it
|
||||
}.build()
|
||||
|
||||
while (result == null) {
|
||||
try {
|
||||
while (!oldWebView && !webViewReady) delay(1000)
|
||||
return client.newCall(request).execute().also{ if (it.code() != 200) throw IOException("CODE ${it.code()}") }.body()?.use { it.string() } ?: throw IOException()
|
||||
}
|
||||
|
||||
result = if (oldWebView)
|
||||
"null"
|
||||
else withContext(evaluationContext) {
|
||||
suspendCoroutine { continuation ->
|
||||
evaluateJavascript(script) {
|
||||
continuation.resume(it)
|
||||
}
|
||||
}
|
||||
fun URL.readBytes(settings: HeaderSetter? = null): ByteArray {
|
||||
val request = Request.Builder()
|
||||
.url(this).let {
|
||||
settings?.invoke(it) ?: it
|
||||
}.build()
|
||||
|
||||
}
|
||||
} catch (e: CancellationException) {
|
||||
if (e.message != "reload") result = "null"
|
||||
}
|
||||
}
|
||||
|
||||
json.decodeFromString(result)
|
||||
} }
|
||||
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
suspend inline fun <reified T> WebView.evaluatePromise(
|
||||
script: String,
|
||||
then: String = ".then(result => Callback.onResult(%uid, JSON.stringify(result))).catch(err => Callback.onError(%uid, String.raw`$script`, err.message, err.stack))"
|
||||
): T = coroutineScope { withTimeout(60000) {
|
||||
var result: String? = null
|
||||
|
||||
while (result == null) {
|
||||
try {
|
||||
while (!oldWebView && !webViewReady) delay(1000)
|
||||
|
||||
result = if (oldWebView)
|
||||
"null"
|
||||
else withContext(evaluationContext) {
|
||||
val uid = UUID.randomUUID().toString()
|
||||
|
||||
val flow: Flow<Pair<String, String?>> = webViewFlow.transformWhile { (currentUid, result) ->
|
||||
if (currentUid == uid) {
|
||||
emit(currentUid to result)
|
||||
}
|
||||
currentUid != uid
|
||||
}
|
||||
|
||||
launch {
|
||||
evaluateJavascript((script + then).replace("%uid", "'$uid'"), null)
|
||||
}
|
||||
|
||||
flow.first().second
|
||||
}
|
||||
} catch (e: CancellationException) {
|
||||
if (e.message != "reload") result = "null"
|
||||
}
|
||||
}
|
||||
|
||||
json.decodeFromString(result)
|
||||
} }
|
||||
return client.newCall(request).execute().also { if (it.code() != 200) throw IOException("CODE ${it.code()}") }.body()?.use { it.bytes() } ?: throw IOException()
|
||||
}
|
||||
|
||||
@Suppress("EXPERIMENTAL_API_USAGE")
|
||||
suspend fun getGalleryInfo(galleryID: Int): GalleryInfo =
|
||||
webView.evaluatePromise("get_gallery_info($galleryID)")
|
||||
fun getGalleryInfo(galleryID: Int) =
|
||||
json.decodeFromString<GalleryInfo>(
|
||||
URL("$protocol//$domain/galleries/$galleryID.js").readText()
|
||||
.replace("var galleryinfo = ", "")
|
||||
)
|
||||
|
||||
//common.js
|
||||
const val domain = "ltn.hitomi.la"
|
||||
const val galleryblockextension = ".html"
|
||||
const val galleryblockdir = "galleryblock"
|
||||
const val nozomiextension = ".nozomi"
|
||||
|
||||
val String?.js: String
|
||||
get() = if (this == null) "null" else "'$this'"
|
||||
val evaluationContext = Executors.newSingleThreadExecutor().asCoroutineDispatcher() + Job()
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
suspend fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null): String =
|
||||
webView.evaluate(
|
||||
"""
|
||||
url_from_url_from_hash(
|
||||
${galleryID.toString().js},
|
||||
${Json.encodeToString(image)},
|
||||
${dir.js}, ${ext.js}, ${base.js}
|
||||
)
|
||||
""".trimIndent()
|
||||
)
|
||||
object gg {
|
||||
|
||||
suspend fun m(g: Int): Int = withContext(evaluationContext) {
|
||||
runtime.evaluate("gg.m($g)").toString().toInt()
|
||||
}
|
||||
suspend fun b(): String = withContext(evaluationContext) {
|
||||
runtime.evaluate("gg.b").toString()
|
||||
}
|
||||
|
||||
suspend fun s(h: String): String = withContext(evaluationContext) {
|
||||
runtime.evaluate("gg.s('$h')").toString()
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun subdomainFromURL(url: String, base: String? = null) : String {
|
||||
var retval = "b"
|
||||
|
||||
if (!base.isNullOrBlank())
|
||||
retval = base
|
||||
|
||||
val b = 16
|
||||
|
||||
val r = Regex("""/[0-9a-f]{61}([0-9a-f]{2})([0-9a-f])""")
|
||||
val m = r.find(url) ?: return "a"
|
||||
|
||||
val g = m.groupValues.let { it[2]+it[1] }.toIntOrNull(b)
|
||||
|
||||
if (g != null) {
|
||||
retval = (97+ gg.m(g)).toChar().toString() + retval
|
||||
}
|
||||
|
||||
return retval
|
||||
}
|
||||
|
||||
suspend fun urlFromUrl(url: String, base: String? = null) : String {
|
||||
return url.replace(Regex("""//..?\.hitomi\.la/"""), "//${subdomainFromURL(url, base)}.hitomi.la/")
|
||||
}
|
||||
|
||||
suspend fun fullPathFromHash(hash: String) : String =
|
||||
"${gg.b()}${gg.s(hash)}/$hash"
|
||||
|
||||
fun realFullPathFromHash(hash: String): String =
|
||||
hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1/$hash")
|
||||
|
||||
suspend fun urlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null) : String {
|
||||
val ext = ext ?: dir ?: image.name.takeLastWhile { it != '.' }
|
||||
val dir = dir ?: "images"
|
||||
return "https://a.hitomi.la/$dir/${fullPathFromHash(image.hash)}.$ext"
|
||||
}
|
||||
|
||||
suspend fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null) =
|
||||
if (base == "tn")
|
||||
urlFromUrl("https://a.hitomi.la/$dir/${realFullPathFromHash(image.hash)}.$ext", base)
|
||||
else
|
||||
urlFromUrl(urlFromHash(galleryID, image, dir, ext), base)
|
||||
|
||||
suspend fun rewriteTnPaths(html: String) {
|
||||
html.replace(Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}""")) { url ->
|
||||
runBlocking {
|
||||
urlFromUrl(url.value, "tn")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String {
|
||||
return when {
|
||||
|
||||
@@ -17,14 +17,11 @@
|
||||
package xyz.quaver.pupil.hitomi
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.decodeFromString
|
||||
import kotlinx.serialization.json.Json
|
||||
import xyz.quaver.pupil.webView
|
||||
|
||||
@Serializable
|
||||
data class Gallery(
|
||||
val related: List<Int>,
|
||||
val langList: Map<String, String>,
|
||||
val langList: List<Pair<String, String>>,
|
||||
val cover: String,
|
||||
val title: String,
|
||||
val artists: List<String>,
|
||||
@@ -36,5 +33,22 @@ data class Gallery(
|
||||
val tags: List<String>,
|
||||
val thumbnails: List<String>
|
||||
)
|
||||
suspend fun getGallery(galleryID: Int) : Gallery =
|
||||
webView.evaluatePromise("get_gallery($galleryID)")
|
||||
|
||||
suspend fun getGallery(galleryID: Int) : Gallery {
|
||||
val info = getGalleryInfo(galleryID)
|
||||
|
||||
return Gallery(
|
||||
info.related,
|
||||
info.languages.map { it.name to it.galleryid },
|
||||
urlFromUrlFromHash(galleryID, info.files.first(), "webpbigtn", "webp", "tn"),
|
||||
info.title,
|
||||
info.artists?.map { it.artist }.orEmpty(),
|
||||
info.groups?.map { it.group }.orEmpty(),
|
||||
info.type,
|
||||
info.language.orEmpty(),
|
||||
info.parodys?.map { it.parody }.orEmpty(),
|
||||
info.characters?.map { it.character }.orEmpty(),
|
||||
info.tags?.map { "${if (it.female.isNullOrEmpty()) "" else "female:"}${if (it.male.isNullOrEmpty()) "" else "male:"}${it.tag}" }.orEmpty(),
|
||||
info.files.map { urlFromUrlFromHash(galleryID, it, "webpsmalltn", "webp", "tn") }
|
||||
)
|
||||
}
|
||||
@@ -17,7 +17,48 @@
|
||||
package xyz.quaver.pupil.hitomi
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
import xyz.quaver.pupil.webView
|
||||
import org.jsoup.Jsoup
|
||||
import java.net.URL
|
||||
import java.net.URLDecoder
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import javax.net.ssl.HttpsURLConnection
|
||||
import kotlin.io.readText
|
||||
|
||||
//galleryblock.js
|
||||
fun fetchNozomi(area: String? = null, tag: String = "index", language: String = "all", start: Int = -1, count: Int = -1) : Pair<List<Int>, Int> {
|
||||
val url = when(area) {
|
||||
null -> "$protocol//$domain/$tag-$language$nozomiextension"
|
||||
else -> "$protocol//$domain/$area/$tag-$language$nozomiextension"
|
||||
}
|
||||
|
||||
with(URL(url).openConnection() as HttpsURLConnection) {
|
||||
requestMethod = "GET"
|
||||
|
||||
if (start != -1 && count != -1) {
|
||||
val startByte = start*4
|
||||
val endByte = (start+count)*4-1
|
||||
|
||||
setRequestProperty("Range", "bytes=$startByte-$endByte")
|
||||
}
|
||||
|
||||
connect()
|
||||
|
||||
val totalItems = getHeaderField("Content-Range")
|
||||
.replace(Regex("^[Bb]ytes \\d+-\\d+/"), "").toInt() / 4
|
||||
|
||||
val nozomi = ArrayList<Int>()
|
||||
|
||||
val arrayBuffer = ByteBuffer
|
||||
.wrap(inputStream.readBytes())
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
|
||||
while (arrayBuffer.hasRemaining())
|
||||
nozomi.add(arrayBuffer.int)
|
||||
|
||||
return Pair(nozomi, totalItems)
|
||||
}
|
||||
}
|
||||
|
||||
@Serializable
|
||||
data class GalleryBlock(
|
||||
@@ -28,9 +69,24 @@ data class GalleryBlock(
|
||||
val artists: List<String>,
|
||||
val series: List<String>,
|
||||
val type: String,
|
||||
val language: String?,
|
||||
val relatedTags: List<String>
|
||||
val language: String,
|
||||
val relatedTags: List<String>,
|
||||
val groups: List<String>
|
||||
)
|
||||
|
||||
suspend fun getGalleryBlock(galleryID: Int) : GalleryBlock =
|
||||
webView.evaluatePromise("get_gallery_block($galleryID)")
|
||||
suspend fun getGalleryBlock(galleryID: Int) : GalleryBlock {
|
||||
val info = getGalleryInfo(galleryID)
|
||||
|
||||
return GalleryBlock(
|
||||
galleryID,
|
||||
"",
|
||||
listOf(urlFromUrlFromHash(galleryID, info.files.first(), "webpbigtn", "webp", "tn")),
|
||||
info.title,
|
||||
info.artists?.map { it.artist }.orEmpty(),
|
||||
info.parodys?.map { it.parody }.orEmpty(),
|
||||
info.type,
|
||||
info.language.orEmpty(),
|
||||
info.tags?.map { "${if (it.female.isNullOrEmpty()) "" else "female:"}${if (it.male.isNullOrEmpty()) "" else "male:"}${it.tag}" }.orEmpty(),
|
||||
info.groups?.map { it.group }.orEmpty()
|
||||
)
|
||||
}
|
||||
@@ -17,19 +17,8 @@
|
||||
package xyz.quaver.pupil.hitomi
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
fun getReferer(galleryID: Int) = "https://hitomi.la/reader/$galleryID.html"
|
||||
|
||||
@Serializable
|
||||
data class GalleryInfo(
|
||||
val language_localname: String? = null,
|
||||
val language: String? = null,
|
||||
val date: String? = null,
|
||||
val files: List<GalleryFiles>,
|
||||
val id: Int? = null,
|
||||
val type: String? = null,
|
||||
val title: String? = null
|
||||
)
|
||||
import xyz.quaver.pupil.hitomi.GalleryInfo
|
||||
import xyz.quaver.pupil.hitomi.getGalleryInfo
|
||||
|
||||
@Serializable
|
||||
data class GalleryFiles(
|
||||
@@ -44,6 +33,6 @@ data class GalleryFiles(
|
||||
|
||||
//Set header `Referer` to reader url to avoid 403 error
|
||||
@Deprecated("", replaceWith = ReplaceWith("getGalleryInfo"))
|
||||
suspend fun getReader(galleryID: Int) : GalleryInfo {
|
||||
fun getReader(galleryID: Int) : GalleryInfo {
|
||||
return getGalleryInfo(galleryID)
|
||||
}
|
||||
@@ -16,42 +16,313 @@
|
||||
|
||||
package xyz.quaver.pupil.hitomi
|
||||
|
||||
import android.util.Log
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.decodeFromString
|
||||
import kotlinx.serialization.json.Json
|
||||
import xyz.quaver.pupil.webView
|
||||
import okhttp3.Request
|
||||
import xyz.quaver.pupil.client
|
||||
import java.net.URL
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.security.MessageDigest
|
||||
import kotlin.math.min
|
||||
|
||||
//searchlib.js
|
||||
const val separator = "-"
|
||||
const val extension = ".html"
|
||||
const val index_dir = "tagindex"
|
||||
const val galleries_index_dir = "galleriesindex"
|
||||
const val max_node_size = 464
|
||||
const val B = 16
|
||||
const val compressed_nozomi_prefix = "n"
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
suspend fun getGalleryIDsForQuery(query: String) : Set<Int> =
|
||||
webView.evaluatePromise("get_galleryids_for_query('$query')")
|
||||
val tag_index_version: String by lazy { getIndexVersion("tagindex") }
|
||||
val galleries_index_version: String by lazy { getIndexVersion("galleriesindex") }
|
||||
|
||||
@Serializable
|
||||
data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
|
||||
fun sha256(data: ByteArray) : ByteArray {
|
||||
return MessageDigest.getInstance("SHA-256").digest(data)
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
suspend fun getSuggestionsForQuery(query: String) : List<Suggestion> =
|
||||
webView.evaluatePromise(
|
||||
"get_suggestions_for_query('$query', ++search_serial)",
|
||||
then = """
|
||||
.then(r => {
|
||||
let [results, results_serial] = r;
|
||||
if (search_serial !== results_serial) {
|
||||
Callback.onResult(%uid, '[]');
|
||||
} else {
|
||||
Callback.onResult(%uid, JSON.stringify(results));
|
||||
@OptIn(ExperimentalUnsignedTypes::class)
|
||||
fun hashTerm(term: String) : UByteArray {
|
||||
return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4)
|
||||
}
|
||||
|
||||
fun sanitize(input: String) : String {
|
||||
return input.replace(Regex("[/#]"), "")
|
||||
}
|
||||
|
||||
fun getIndexVersion(name: String) =
|
||||
URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText()
|
||||
|
||||
//search.js
|
||||
fun getGalleryIDsForQuery(query: String) : Set<Int> {
|
||||
query.replace("_", " ").let {
|
||||
if (it.indexOf(':') > -1) {
|
||||
val sides = it.split(":")
|
||||
val ns = sides[0]
|
||||
var tag = sides[1]
|
||||
|
||||
var area : String? = ns
|
||||
var language = "all"
|
||||
when (ns) {
|
||||
"female", "male" -> {
|
||||
area = "tag"
|
||||
tag = it
|
||||
}
|
||||
});
|
||||
""".trimIndent()
|
||||
)
|
||||
"language" -> {
|
||||
area = null
|
||||
language = tag
|
||||
tag = "index"
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
suspend fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
|
||||
val jsArea = if (area == null) "null" else "'$area'"
|
||||
return getGalleryIDsFromNozomi(area, tag, language)
|
||||
}
|
||||
|
||||
return webView.evaluatePromise("""get_galleryids_from_nozomi($jsArea, '$tag', '$language')""")
|
||||
val key = hashTerm(it)
|
||||
val field = "galleries"
|
||||
|
||||
val node = getNodeAtAddress(field, 0) ?: return emptySet()
|
||||
|
||||
val data = bSearch(field, key, node)
|
||||
|
||||
if (data != null)
|
||||
return getGalleryIDsFromData(data)
|
||||
|
||||
return emptySet()
|
||||
}
|
||||
}
|
||||
|
||||
fun getSuggestionsForQuery(query: String) : List<Suggestion> {
|
||||
query.replace('_', ' ').let {
|
||||
var field = "global"
|
||||
var term = it
|
||||
|
||||
if (term.indexOf(':') > -1) {
|
||||
val sides = it.split(':')
|
||||
field = sides[0]
|
||||
term = sides[1]
|
||||
}
|
||||
|
||||
val key = hashTerm(term)
|
||||
val node = getNodeAtAddress(field, 0) ?: return emptyList()
|
||||
val data = bSearch(field, key, node)
|
||||
|
||||
if (data != null)
|
||||
return getSuggestionsFromData(field, data)
|
||||
|
||||
return emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
|
||||
fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggestion> {
|
||||
val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data"
|
||||
val (offset, length) = data
|
||||
if (length > 10000 || length <= 0)
|
||||
throw Exception("length $length is too long")
|
||||
|
||||
val inbuf = getURLAtRange(url, offset.until(offset+length))
|
||||
|
||||
val suggestions = ArrayList<Suggestion>()
|
||||
|
||||
val buffer = ByteBuffer
|
||||
.wrap(inbuf)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
val numberOfSuggestions = buffer.int
|
||||
|
||||
if (numberOfSuggestions > 100 || numberOfSuggestions <= 0)
|
||||
throw Exception("number of suggestions $numberOfSuggestions is too long")
|
||||
|
||||
for (i in 0.until(numberOfSuggestions)) {
|
||||
var top = buffer.int
|
||||
|
||||
val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
|
||||
buffer.position(buffer.position()+top)
|
||||
|
||||
top = buffer.int
|
||||
|
||||
val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
|
||||
buffer.position(buffer.position()+top)
|
||||
|
||||
val count = buffer.int
|
||||
|
||||
val tagname = sanitize(tag)
|
||||
val u =
|
||||
when(ns) {
|
||||
"female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
|
||||
"language" -> "/index-$tagname${separator}1$extension"
|
||||
else -> "/$ns/$tagname${separator}all${separator}1$extension"
|
||||
}
|
||||
|
||||
suggestions.add(Suggestion(tag, count, u, ns))
|
||||
}
|
||||
|
||||
return suggestions
|
||||
}
|
||||
|
||||
fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
|
||||
val nozomiAddress =
|
||||
when(area) {
|
||||
null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension"
|
||||
else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension"
|
||||
}
|
||||
|
||||
val bytes = try {
|
||||
URL(nozomiAddress).readBytes()
|
||||
} catch (e: Exception) {
|
||||
return emptySet()
|
||||
}
|
||||
|
||||
val nozomi = mutableSetOf<Int>()
|
||||
|
||||
val arrayBuffer = ByteBuffer
|
||||
.wrap(bytes)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
|
||||
while (arrayBuffer.hasRemaining())
|
||||
nozomi.add(arrayBuffer.int)
|
||||
|
||||
return nozomi
|
||||
}
|
||||
|
||||
fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
|
||||
val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data"
|
||||
val (offset, length) = data
|
||||
if (length > 100000000 || length <= 0)
|
||||
throw Exception("length $length is too long")
|
||||
|
||||
val inbuf = getURLAtRange(url, offset.until(offset+length))
|
||||
|
||||
val galleryIDs = mutableSetOf<Int>()
|
||||
|
||||
val buffer = ByteBuffer
|
||||
.wrap(inbuf)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
|
||||
val numberOfGalleryIDs = buffer.int
|
||||
|
||||
val expectedLength = numberOfGalleryIDs*4+4
|
||||
|
||||
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0)
|
||||
throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
|
||||
else if (inbuf.size != expectedLength)
|
||||
throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength")
|
||||
|
||||
for (i in 0.until(numberOfGalleryIDs))
|
||||
galleryIDs.add(buffer.int)
|
||||
|
||||
return galleryIDs
|
||||
}
|
||||
|
||||
fun getNodeAtAddress(field: String, address: Long) : Node? {
|
||||
val url =
|
||||
when(field) {
|
||||
"galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"
|
||||
"languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index"
|
||||
"nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index"
|
||||
else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index"
|
||||
}
|
||||
|
||||
val nodedata = getURLAtRange(url, address.until(address+ max_node_size))
|
||||
|
||||
return decodeNode(nodedata)
|
||||
}
|
||||
|
||||
fun getURLAtRange(url: String, range: LongRange) : ByteArray {
|
||||
val request = Request.Builder()
|
||||
.url(url)
|
||||
.header("Range", "bytes=${range.first}-${range.last}")
|
||||
.build()
|
||||
|
||||
return client.newCall(request).execute().body()?.use { it.bytes() } ?: byteArrayOf()
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalUnsignedTypes::class)
|
||||
data class Node(val keys: List<UByteArray>, val datas: List<Pair<Long, Int>>, val subNodeAddresses: List<Long>)
|
||||
@OptIn(ExperimentalUnsignedTypes::class)
|
||||
fun decodeNode(data: ByteArray) : Node {
|
||||
val buffer = ByteBuffer
|
||||
.wrap(data)
|
||||
.order(ByteOrder.BIG_ENDIAN)
|
||||
|
||||
val uData = data.toUByteArray()
|
||||
|
||||
val numberOfKeys = buffer.int
|
||||
val keys = ArrayList<UByteArray>()
|
||||
|
||||
for (i in 0.until(numberOfKeys)) {
|
||||
val keySize = buffer.int
|
||||
|
||||
if (keySize == 0 || keySize > 32)
|
||||
throw Exception("fatal: !keySize || keySize > 32")
|
||||
|
||||
keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize)))
|
||||
buffer.position(buffer.position()+keySize)
|
||||
}
|
||||
|
||||
val numberOfDatas = buffer.int
|
||||
val datas = ArrayList<Pair<Long, Int>>()
|
||||
|
||||
for (i in 0.until(numberOfDatas)) {
|
||||
val offset = buffer.long
|
||||
val length = buffer.int
|
||||
|
||||
datas.add(Pair(offset, length))
|
||||
}
|
||||
|
||||
val numberOfSubNodeAddresses = B +1
|
||||
val subNodeAddresses = ArrayList<Long>()
|
||||
|
||||
for (i in 0.until(numberOfSubNodeAddresses)) {
|
||||
val subNodeAddress = buffer.long
|
||||
subNodeAddresses.add(subNodeAddress)
|
||||
}
|
||||
|
||||
return Node(keys, datas, subNodeAddresses)
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalUnsignedTypes::class)
|
||||
fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
|
||||
fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int {
|
||||
val top = min(dv1.size, dv2.size)
|
||||
|
||||
for (i in 0.until(top)) {
|
||||
if (dv1[i] < dv2[i])
|
||||
return -1
|
||||
else if (dv1[i] > dv2[i])
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> {
|
||||
for (i in node.keys.indices) {
|
||||
val cmpResult = compareArrayBuffers(key, node.keys[i])
|
||||
|
||||
if (cmpResult <= 0)
|
||||
return Pair(cmpResult==0, i)
|
||||
}
|
||||
|
||||
return Pair(false, node.keys.size)
|
||||
}
|
||||
|
||||
fun isLeaf(node: Node) : Boolean {
|
||||
for (subnode in node.subNodeAddresses)
|
||||
if (subnode != 0L)
|
||||
return false
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
if (node.keys.isEmpty())
|
||||
return null
|
||||
|
||||
val (there, where) = locateKey(key, node)
|
||||
if (there)
|
||||
return node.datas[where]
|
||||
else if (isLeaf(node))
|
||||
return null
|
||||
|
||||
val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where]) ?: return null
|
||||
return bSearch(field, key, nextNode)
|
||||
}
|
||||
Reference in New Issue
Block a user