user01/Pupil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sort mode

Browse Source
This commit is contained in:
tom5079
2025-02-24 23:00:26 -08:00
parent f888535389
commit 83d6058f2b
9 changed files with 524 additions and 379 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
app/src/main/java/xyz/quaver/pupil/hitomi/results.kt
View File

@@ -18,9 +18,9 @@ package xyz.quaver.pupil.hitomi
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import java.util.*
import java.util.LinkedList
suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int> = coroutineScope {
suspend fun doSearch(query: String, sortMode: SortMode): List<Int> = coroutineScope {
val terms = query
.trim()
.replace(Regex("""^\?"""), "")
@@ -34,8 +34,8 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
val negativeTerms = LinkedList<String>()
for (term in terms) {
if (term.matches(Regex("^-.+")))
negativeTerms.push(term.replace(Regex("^-"), ""))
if (term.startsWith("-"))
negativeTerms.push(term.substring(1))
else if (term.isNotBlank())
positiveTerms.push(term)
}
@@ -43,22 +43,25 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
val positiveResults = positiveTerms.map {
async {
runCatching {
getGalleryIDsForQuery(it)
getGalleryIDsForQuery(it, sortMode)
}.getOrElse { emptySet() }
}
}
val negativeResults = negativeTerms.mapIndexed { index, it ->
val negativeResults = negativeTerms.map {
async {
runCatching {
getGalleryIDsForQuery(it)
getGalleryIDsForQuery(it, sortMode)
}.getOrElse { emptySet() }
}
}
val results = when {
sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", "all")
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", "all")
positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(
SearchArgs("all", "index", "all"),
sortMode
)
else -> emptySet()
}.toMutableSet()
@@ -79,9 +82,13 @@ suspend fun doSearch(query: String, sortByPopularity: Boolean = false) : Set<Int
}
//negative results
negativeResults.forEachIndexed { index, it ->
negativeResults.forEach {
filterNegative(it.await())
}
results
return@coroutineScope if (sortMode != SortMode.RANDOM) {
results.toList()
} else {
results.shuffled()
}
}

205
app/src/main/java/xyz/quaver/pupil/hitomi/search.kt
View File

@@ -26,6 +26,54 @@ import java.nio.ByteOrder
import java.security.MessageDigest
import kotlin.math.min
data class SearchArgs(
val area: String?,
val tag: String,
val language: String,
) {
companion object {
fun fromQuery(query: String): SearchArgs? {
if (!query.contains(':')) {
return null
}
val (left, right) = query.split(':')
return when (left) {
"male", "female" -> SearchArgs("tag", query, "all")
"language" -> SearchArgs(null, "index", right)
else -> SearchArgs(left, right, "all")
}
}
}
}
enum class SortMode {
DATE_ADDED,
DATE_PUBLISHED,
POPULAR_TODAY,
POPULAR_WEEK,
POPULAR_MONTH,
POPULAR_YEAR,
RANDOM;
val orderBy: String
get() = when (this) {
DATE_ADDED, DATE_PUBLISHED, RANDOM -> "date"
POPULAR_TODAY, POPULAR_WEEK, POPULAR_MONTH, POPULAR_YEAR -> "popular"
}
val orderByKey: String
get() = when (this) {
DATE_ADDED, RANDOM -> "added"
DATE_PUBLISHED -> "published"
POPULAR_TODAY -> "today"
POPULAR_WEEK -> "week"
POPULAR_MONTH -> "month"
POPULAR_YEAR -> "year"
}
}
//searchlib.js
const val separator = "-"
const val extension = ".html"
@@ -39,51 +87,35 @@ val tag_index_version: String by lazy { getIndexVersion("tagindex") }
val galleries_index_version: String by lazy { getIndexVersion("galleriesindex") }
val tagIndexDomain = "tagindex.hitomi.la"
fun sha256(data: ByteArray) : ByteArray {
fun sha256(data: ByteArray): ByteArray {
return MessageDigest.getInstance("SHA-256").digest(data)
}
@OptIn(ExperimentalUnsignedTypes::class)
fun hashTerm(term: String) : UByteArray {
fun hashTerm(term: String): UByteArray {
return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4)
}
fun sanitize(input: String) : String {
fun sanitize(input: String): String {
return input.replace(Regex("[/#]"), "")
}
fun getIndexVersion(name: String) =
URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText()
URL("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}").readText()
//search.js
fun getGalleryIDsForQuery(query: String) : Set<Int> {
query.replace("_", " ").let {
if (it.indexOf(':') > -1) {
val sides = it.split(":")
val ns = sides[0]
var tag = sides[1]
fun getGalleryIDsForQuery(query: String, sortMode: SortMode): Set<Int> {
val sanitizedQuery = query.replace("_", " ")
var area : String? = ns
var language = "all"
when (ns) {
"female", "male" -> {
area = "tag"
tag = it
}
"language" -> {
area = null
language = tag
tag = "index"
}
}
val args = SearchArgs.fromQuery(sanitizedQuery)
return getGalleryIDsFromNozomi(area, tag, language)
}
val key = hashTerm(it)
return if (args != null) {
getGalleryIDsFromNozomi(args, sortMode)
} else {
val key = hashTerm(sanitizedQuery)
val field = "galleries"
val node = getNodeAtAddress(field, 0) ?: return emptySet()
val node = getNodeAtAddress(field, 0)
val data = bSearch(field, key, node)
@@ -95,14 +127,14 @@ fun getGalleryIDsForQuery(query: String) : Set<Int> {
}
fun encodeSearchQueryForUrl(s: Char) =
when(s) {
when (s) {
' ' -> "_"
'/' -> "slash"
'.' -> "dot"
else -> s.toString()
}
fun getSuggestionsForQuery(query: String) : List<Suggestion> {
fun getSuggestionsForQuery(query: String): List<Suggestion> {
query.replace('_', ' ').let {
var field = "global"
var term = it
@@ -114,13 +146,16 @@ fun getSuggestionsForQuery(query: String) : List<Suggestion> {
}
val chars = term.map(::encodeSearchQueryForUrl)
val url = "https://$tagIndexDomain/$field${if (chars.isNotEmpty()) "/${chars.joinToString("/")}" else ""}.json"
val url =
"https://$tagIndexDomain/$field${if (chars.isNotEmpty()) "/${chars.joinToString("/")}" else ""}.json"
val request = Request.Builder()
.url(url)
.build()
val suggestions = json.parseToJsonElement(client.newCall(request).execute().body()?.use { body -> body.string() } ?: return emptyList())
val suggestions = json.parseToJsonElement(
client.newCall(request).execute().body()?.use { body -> body.string() }
?: return emptyList())
return buildList {
suggestions.jsonArray.forEach { suggestionRaw ->
@@ -131,26 +166,34 @@ fun getSuggestionsForQuery(query: String) : List<Suggestion> {
val ns = suggestion[2].content ?: ""
val tagname = sanitize(suggestion[0].content ?: return@forEach)
val url = when(ns) {
val url = when (ns) {
"female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
"language" -> "/index-$tagname${separator}1$extension"
else -> "/$ns/$tagname${separator}all${separator}1$extension"
}
add(Suggestion(suggestion[0].content ?: "", suggestion[1].content?.toIntOrNull() ?: 0, url, ns))
add(
Suggestion(
suggestion[0].content ?: "",
suggestion[1].content?.toIntOrNull() ?: 0,
url,
ns
)
)
}
}
}
}
data class Suggestion(val s: String, val t: Int, val u: String, val n: String)
fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggestion> {
fun getSuggestionsFromData(field: String, data: Pair<Long, Int>): List<Suggestion> {
val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data"
val (offset, length) = data
if (length > 10000 || length <= 0)
throw Exception("length $length is too long")
val inbuf = getURLAtRange(url, offset.until(offset+length))
val inbuf = getURLAtRange(url, offset.until(offset + length))
val suggestions = ArrayList<Suggestion>()
@@ -165,23 +208,25 @@ fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggesti
for (i in 0.until(numberOfSuggestions)) {
var top = buffer.int
val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
buffer.position(buffer.position()+top)
val ns = inbuf.sliceArray(buffer.position().until(buffer.position() + top))
.toString(charset("UTF-8"))
buffer.position(buffer.position() + top)
top = buffer.int
val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8"))
buffer.position(buffer.position()+top)
val tag = inbuf.sliceArray(buffer.position().until(buffer.position() + top))
.toString(charset("UTF-8"))
buffer.position(buffer.position() + top)
val count = buffer.int
val tagname = sanitize(tag)
val u =
when(ns) {
"female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
"language" -> "/index-$tagname${separator}1$extension"
else -> "/$ns/$tagname${separator}all${separator}1$extension"
}
when (ns) {
"female", "male" -> "/tag/$ns:$tagname${separator}1$extension"
"language" -> "/index-$tagname${separator}1$extension"
else -> "/$ns/$tagname${separator}all${separator}1$extension"
}
suggestions.add(Suggestion(tag, count, u, ns))
}
@@ -189,12 +234,17 @@ fun getSuggestionsFromData(field: String, data: Pair<Long, Int>) : List<Suggesti
return suggestions
}
fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<Int> {
val nozomiAddress =
when(area) {
null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension"
else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension"
}
fun nozomiAddressFromArgs(args: SearchArgs, sortMode: SortMode) = when {
sortMode != SortMode.DATE_ADDED && sortMode != SortMode.RANDOM ->
if (args.area == "all") "$protocol//$domain/$compressed_nozomi_prefix/${sortMode.orderBy}/${sortMode.orderByKey}-${args.language}$nozomiextension"
else "$protocol//$domain/$compressed_nozomi_prefix/${args.area}/${sortMode.orderBy}/${sortMode.orderByKey}/${args.tag}-${args.language}$nozomiextension"
args.area == "all" -> "$protocol//$domain/$compressed_nozomi_prefix/${args.tag}-${args.language}$nozomiextension"
else -> "$protocol//$domain/$compressed_nozomi_prefix/${args.area}/${args.tag}-${args.language}$nozomiextension"
}
fun getGalleryIDsFromNozomi(args: SearchArgs, sortMode: SortMode): Set<Int> {
val nozomiAddress = nozomiAddressFromArgs(args, sortMode)
val bytes = URL(nozomiAddress).readBytes()
@@ -210,13 +260,13 @@ fun getGalleryIDsFromNozomi(area: String?, tag: String, language: String) : Set<
return nozomi
}
fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> {
val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data"
val (offset, length) = data
if (length > 100000000 || length <= 0)
throw Exception("length $length is too long")
val inbuf = getURLAtRange(url, offset.until(offset+length))
val inbuf = getURLAtRange(url, offset.until(offset + length))
val galleryIDs = mutableSetOf<Int>()
@@ -226,7 +276,7 @@ fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
val numberOfGalleryIDs = buffer.int
val expectedLength = numberOfGalleryIDs*4+4
val expectedLength = numberOfGalleryIDs * 4 + 4
if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0)
throw Exception("number_of_galleryids $numberOfGalleryIDs is too long")
@@ -239,33 +289,38 @@ fun getGalleryIDsFromData(data: Pair<Long, Int>) : Set<Int> {
return galleryIDs
}
fun getNodeAtAddress(field: String, address: Long) : Node? {
fun getNodeAtAddress(field: String, address: Long): Node {
val url =
when(field) {
"galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"
"languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index"
"nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index"
else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index"
}
when (field) {
"galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index"
"languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index"
"nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index"
else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index"
}
val nodedata = getURLAtRange(url, address.until(address+ max_node_size))
val nodedata = getURLAtRange(url, address.until(address + max_node_size))
return decodeNode(nodedata)
}
fun getURLAtRange(url: String, range: LongRange) : ByteArray {
fun getURLAtRange(url: String, range: LongRange): ByteArray {
val request = Request.Builder()
.url(url)
.header("Range", "bytes=${range.first}-${range.last}")
.build()
return client.newCall(request).execute().body()?.use { it.bytes() } ?: byteArrayOf()
}
@OptIn(ExperimentalUnsignedTypes::class)
data class Node(val keys: List<UByteArray>, val datas: List<Pair<Long, Int>>, val subNodeAddresses: List<Long>)
data class Node(
val keys: List<UByteArray>,
val datas: List<Pair<Long, Int>>,
val subNodeAddresses: List<Long>
)
@OptIn(ExperimentalUnsignedTypes::class)
fun decodeNode(data: ByteArray) : Node {
fun decodeNode(data: ByteArray): Node {
val buffer = ByteBuffer
.wrap(data)
.order(ByteOrder.BIG_ENDIAN)
@@ -281,8 +336,8 @@ fun decodeNode(data: ByteArray) : Node {
if (keySize == 0 || keySize > 32)
throw Exception("fatal: !keySize || keySize > 32")
keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize)))
buffer.position(buffer.position()+keySize)
keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize)))
buffer.position(buffer.position() + keySize)
}
val numberOfDatas = buffer.int
@@ -295,7 +350,7 @@ fun decodeNode(data: ByteArray) : Node {
datas.add(Pair(offset, length))
}
val numberOfSubNodeAddresses = B +1
val numberOfSubNodeAddresses = B + 1
val subNodeAddresses = ArrayList<Long>()
for (i in 0.until(numberOfSubNodeAddresses)) {
@@ -307,8 +362,8 @@ fun decodeNode(data: ByteArray) : Node {
}
@OptIn(ExperimentalUnsignedTypes::class)
fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int {
fun bSearch(field: String, key: UByteArray, node: Node): Pair<Long, Int>? {
fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray): Int {
val top = min(dv1.size, dv2.size)
for (i in 0.until(top)) {
@@ -321,18 +376,18 @@ fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
return 0
}
fun locateKey(key: UByteArray, node: Node) : Pair<Boolean, Int> {
fun locateKey(key: UByteArray, node: Node): Pair<Boolean, Int> {
for (i in node.keys.indices) {
val cmpResult = compareArrayBuffers(key, node.keys[i])
if (cmpResult <= 0)
return Pair(cmpResult==0, i)
return Pair(cmpResult == 0, i)
}
return Pair(false, node.keys.size)
}
fun isLeaf(node: Node) : Boolean {
fun isLeaf(node: Node): Boolean {
for (subnode in node.subNodeAddresses)
if (subnode != 0L)
return false
@@ -349,6 +404,6 @@ fun bSearch(field: String, key: UByteArray, node: Node) : Pair<Long, Int>? {
else if (isLeaf(node))
return null
val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where]) ?: return null
val nextNode = getNodeAtAddress(field, node.subNodeAddresses[where])
return bSearch(field, key, nextNode)
}