diff --git a/.idea/deploymentTargetDropDown.xml b/.idea/deploymentTargetDropDown.xml index a3d5c4b6..ec0d83c0 100644 --- a/.idea/deploymentTargetDropDown.xml +++ b/.idea/deploymentTargetDropDown.xml @@ -1,17 +1,17 @@ - + - + - - + + - - + + \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 3f11696b..69c047e2 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -135,7 +135,6 @@ dependencies { implementation("ru.noties.markwon:core:3.1.0") - implementation("xyz.quaver:libpupil:2.1.11") implementation("xyz.quaver:documentfilex:0.7.1") implementation("xyz.quaver:subsampledimage:0.0.1-alpha11-SNAPSHOT") diff --git a/app/src/main/java/xyz/quaver/pupil/sources/composable/SearchBase.kt b/app/src/main/java/xyz/quaver/pupil/sources/composable/SearchBase.kt index 850d2562..e2a53cde 100644 --- a/app/src/main/java/xyz/quaver/pupil/sources/composable/SearchBase.kt +++ b/app/src/main/java/xyz/quaver/pupil/sources/composable/SearchBase.kt @@ -21,20 +21,24 @@ package xyz.quaver.pupil.sources.composable import android.app.Application import androidx.appcompat.graphics.drawable.DrawerArrowDrawable import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.animateFloatAsState import androidx.compose.animation.core.updateTransition +import androidx.compose.foundation.Canvas import androidx.compose.foundation.clickable import androidx.compose.foundation.gestures.awaitFirstDown import androidx.compose.foundation.gestures.forEachGesture import androidx.compose.foundation.interaction.MutableInteractionSource import androidx.compose.foundation.layout.* -import androidx.compose.material.CircularProgressIndicator -import androidx.compose.material.Icon -import androidx.compose.material.Scaffold +import androidx.compose.material.* +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.NavigateBefore +import androidx.compose.material.icons.filled.NavigateNext import androidx.compose.material.ripple.rememberRipple import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.hapticfeedback.HapticFeedbackType import androidx.compose.ui.input.nestedscroll.NestedScrollConnection import androidx.compose.ui.input.nestedscroll.NestedScrollSource import androidx.compose.ui.input.nestedscroll.nestedScroll @@ -44,14 +48,16 @@ import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.platform.LocalHapticFeedback +import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp import androidx.compose.ui.util.fastFirstOrNull import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewmodel.compose.viewModel import com.google.accompanist.drawablepainter.rememberDrawablePainter -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToInt +import xyz.quaver.pupil.R +import xyz.quaver.pupil.ui.theme.LightBlue300 +import kotlin.math.* private enum class NavigationIconState { MENU, @@ -67,10 +73,8 @@ open class SearchBaseViewModel(app: Application) : AndroidViewModel(app) { var currentPage by mutableStateOf(1) var totalItems by mutableStateOf(0) - private set var maxPage by mutableStateOf(0) - private set val prevPageAvailable by derivedStateOf { currentPage > 1 } val nextPageAvailable by derivedStateOf { currentPage <= maxPage } @@ -78,7 +82,6 @@ open class SearchBaseViewModel(app: Application) : AndroidViewModel(app) { var query by mutableStateOf("") var loading by mutableStateOf(false) - private set //region UI var isFabVisible by mutableStateOf(true) @@ -96,6 +99,7 @@ fun SearchBase( ) { val context = LocalContext.current val focusManager = LocalFocusManager.current + val haptic = LocalHapticFeedback.current var isFabExpanded by remember { mutableStateOf(FloatingActionButtonState.COLLAPSED) } @@ -133,11 +137,65 @@ fun SearchBase( } ) { Box(Modifier.fillMaxSize()) { + val topCircleRadius by animateFloatAsState(if (overscroll?.let { it >= pageTurnIndicatorHeight } == true) 1000f else 0f) + val bottomCircleRadius by animateFloatAsState(if (overscroll?.let { it <= -pageTurnIndicatorHeight } == true) 1000f else 0f) + + if (topCircleRadius != 0f || bottomCircleRadius != 0f) + Canvas(modifier = Modifier.fillMaxSize()) { + drawCircle( + LightBlue300.copy(alpha = 0.6f), + center = Offset(this.center.x, searchBarHeight.toFloat()), + radius = topCircleRadius + ) + drawCircle( + LightBlue300.copy(alpha = 0.6f), + center = Offset(this.center.x, this.size.height-pageTurnIndicatorHeight), + radius = bottomCircleRadius + ) + } + + val isOverscrollOverHeight = overscroll?.let { abs(it) >= pageTurnIndicatorHeight } == true + LaunchedEffect(isOverscrollOverHeight) { + if (isOverscrollOverHeight) haptic.performHapticFeedback(HapticFeedbackType.LongPress) + } + + overscroll?.let { overscroll -> + if (overscroll > 0f) + Row( + modifier = Modifier + .align(Alignment.TopCenter) + .offset(0.dp, 64.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + Icons.Default.NavigateBefore, + contentDescription = null, + tint = MaterialTheme.colors.secondary, + modifier = Modifier.size(48.dp) + ) + Text(stringResource(R.string.main_move_to_page, model.currentPage-1)) + } + + if (overscroll < 0f) + Row( + modifier = Modifier.align(Alignment.BottomCenter), + verticalAlignment = Alignment.CenterVertically + ) { + Text(stringResource(R.string.main_move_to_page, model.currentPage+1)) + Icon( + Icons.Default.NavigateNext, + contentDescription = null, + tint = MaterialTheme.colors.secondary, + modifier = Modifier.size(48.dp) + ) + } + } + Box( modifier = Modifier .offset( 0.dp, - overscroll?.let { overscroll -> LocalDensity.current.run { overscroll.toDp() } } + overscroll?.coerceIn(-pageTurnIndicatorHeight, pageTurnIndicatorHeight)?.let { overscroll -> LocalDensity.current.run { overscroll.toDp() } } ?: 0.dp) .nestedScroll(object : NestedScrollConnection { override fun onPreScroll( @@ -147,7 +205,11 @@ fun SearchBase( val overscrollSnapshot = overscroll if (overscrollSnapshot == null || overscrollSnapshot == 0f) { - model.searchBarOffset = (model.searchBarOffset + available.y.roundToInt()).coerceIn(-searchBarHeight, 0) + model.searchBarOffset = + (model.searchBarOffset + available.y.roundToInt()).coerceIn( + -searchBarHeight, + 0 + ) model.isFabVisible = available.y > 0f @@ -172,20 +234,20 @@ fun SearchBase( available: Offset, source: NestedScrollSource ): Offset { - if (available.y == 0f || source == NestedScrollSource.Fling) return Offset.Zero + if ( + available.y == 0f || + source == NestedScrollSource.Fling || + !model.prevPageAvailable && available.y > 0f || + !model.nextPageAvailable && available.y < 0f + ) return Offset.Zero return overscroll?.let { - val newOverscroll = (it + available.y).coerceIn( - -pageTurnIndicatorHeight, - pageTurnIndicatorHeight - ) - - Offset(0f, newOverscroll - it).also { - overscroll = newOverscroll - } + overscroll = it + available.y + Offset(0f, available.y) } ?: Offset.Zero } - }).pointerInput(Unit) { + }) + .pointerInput(Unit) { forEachGesture { awaitPointerEventScope { val down = awaitFirstDown(requireUnconsumed = false) @@ -194,12 +256,16 @@ fun SearchBase( while (true) { val event = awaitPointerEvent() - val dragEvent = event.changes.fastFirstOrNull { it.id == pointer }!! + val dragEvent = + event.changes.fastFirstOrNull { it.id == pointer }!! if (dragEvent.changedToUpIgnoreConsumed()) { val otherDown = event.changes.fastFirstOrNull { it.pressed } if (otherDown == null) { dragEvent.consumePositionChange() + overscroll?.let { + model.currentPage -= it.sign.toInt() + } overscroll = null break } else diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/Hitomi.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/Hitomi.kt index 432b344f..dd09de31 100644 --- a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/Hitomi.kt +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/Hitomi.kt @@ -27,6 +27,7 @@ import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.lifecycle.viewmodel.compose.viewModel import androidx.navigation.NavController +import io.ktor.client.* import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import org.kodein.di.DIAware @@ -35,18 +36,20 @@ import org.kodein.di.compose.rememberInstance import org.kodein.di.instance import org.kodein.log.LoggerFactory import org.kodein.log.newLogger -import xyz.quaver.hitomi.getGalleryInfo -import xyz.quaver.hitomi.getReferer -import xyz.quaver.hitomi.imageUrlFromImage import xyz.quaver.pupil.R import xyz.quaver.pupil.db.AppDatabase import xyz.quaver.pupil.sources.Source import xyz.quaver.pupil.sources.composable.* import xyz.quaver.pupil.sources.hitomi.composable.DetailedSearchResult +import xyz.quaver.pupil.sources.hitomi.lib.getGalleryInfo +import xyz.quaver.pupil.sources.hitomi.lib.getReferer +import xyz.quaver.pupil.sources.hitomi.lib.imageUrlFromImage class Hitomi(app: Application) : Source(), DIAware { override val di by closestDI(app) + private val client: HttpClient by instance() + private val logger = newLogger(LoggerFactory.default) private val database: AppDatabase by instance() @@ -75,6 +78,10 @@ class Hitomi(app: Application) : Source(), DIAware { bookmarks?.toSet() ?: emptySet() } + LaunchedEffect(model.currentPage) { + model.search() + } + SearchBase( model, fabSubMenu = listOf( @@ -133,7 +140,7 @@ class Hitomi(app: Application) : Source(), DIAware { kotlin.runCatching { val galleryID = itemID.toInt() - val galleryInfo = getGalleryInfo(galleryID) + val galleryInfo = getGalleryInfo(client, galleryID) model.title = galleryInfo.title @@ -141,6 +148,7 @@ class Hitomi(app: Application) : Source(), DIAware { append("Referer", getReferer(galleryID)) } }.onFailure { + logger.warning(it) model.error = true } } diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/HitomiSearchResultViewModel.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/HitomiSearchResultViewModel.kt index 0e72eaa0..15df10f5 100644 --- a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/HitomiSearchResultViewModel.kt +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/HitomiSearchResultViewModel.kt @@ -19,38 +19,77 @@ package xyz.quaver.pupil.sources.hitomi import android.app.Application -import kotlinx.coroutines.* +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.lifecycle.viewModelScope +import io.ktor.client.* +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield import org.kodein.di.DIAware import org.kodein.di.android.closestDI import org.kodein.di.instance -import xyz.quaver.hitomi.GalleryBlock -import xyz.quaver.hitomi.doSearch -import xyz.quaver.hitomi.getGalleryBlock import xyz.quaver.pupil.db.AppDatabase import xyz.quaver.pupil.sources.composable.SearchBaseViewModel +import xyz.quaver.pupil.sources.hitomi.lib.GalleryBlock +import xyz.quaver.pupil.sources.hitomi.lib.doSearch +import xyz.quaver.pupil.sources.hitomi.lib.getGalleryBlock +import kotlin.math.ceil class HitomiSearchResultViewModel(app: Application) : SearchBaseViewModel(app), DIAware { override val di by closestDI(app) + private val client: HttpClient by instance() + private val database: AppDatabase by instance() private val bookmarkDao = database.bookmarkDao() - init { - search() - } + private var cachedQuery: String? = null + private var cachedSortByPopularity: Boolean? = null + private val cache = mutableListOf() + + var sortByPopularity by mutableStateOf(false) private var searchJob: Job? = null fun search() { - searchJob?.cancel() - searchResults.clear() - searchJob = CoroutineScope(Dispatchers.IO).launch { - val result = doSearch("female:loli") + val resultsPerPage = 25 - yield() + viewModelScope.launch { + searchJob?.cancelAndJoin() + + searchResults.clear() + searchBarOffset = 0 + loading = true + + searchJob = launch { + if (cachedQuery != query || cachedSortByPopularity != sortByPopularity || cache.isEmpty()) { + cachedQuery = null + cache.clear() + + yield() + + val result = doSearch(client, query, sortByPopularity) + + yield() + + cache.addAll(result) + cachedQuery = query + totalItems = result.size + maxPage = ceil(result.size / resultsPerPage.toDouble()).toInt() + } - result.take(25).forEach { yield() - searchResults.add(transform(getGalleryBlock(it))) + + cache.slice((currentPage-1)*resultsPerPage until currentPage*resultsPerPage).forEach { galleryID -> + searchResults.add(transform(getGalleryBlock(client, galleryID))) + } + } + + viewModelScope.launch { + searchJob?.join() + loading = false } } } diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/common.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/common.kt new file mode 100644 index 00000000..19708ae3 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/common.kt @@ -0,0 +1,121 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2021 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package xyz.quaver.pupil.sources.hitomi.lib + +import io.ktor.client.* +import io.ktor.client.request.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json + +const val protocol = "https:" + +private val json = Json { + isLenient = true + ignoreUnknownKeys = true + allowSpecialFloatingPointValues = true + useArrayPolymorphism = true +} + +suspend fun getGalleryInfo(client: HttpClient, galleryID: Int): GalleryInfo = withContext(Dispatchers.IO) { + json.decodeFromString( + client.get("$protocol//$domain/galleries/$galleryID.js") + .replace("var galleryinfo = ", "") + ) +} + +//common.js +const val domain = "ltn.hitomi.la" +const val galleryblockextension = ".html" +const val galleryblockdir = "galleryblock" +const val nozomiextension = ".nozomi" + +fun subdomainFromGalleryID(g: Int, numberOfFrontends: Int) : String { + val o = g % numberOfFrontends + + return (97+o).toChar().toString() +} + +fun subdomainFromURL(url: String, base: String? = null) : String { + var retval = "b" + + if (!base.isNullOrBlank()) + retval = base + + var numberOfFrontends = 2 + val b = 16 + + val r = Regex("""/[0-9a-f]/([0-9a-f]{2})/""") + val m = r.find(url) ?: return "a" + + val g = m.groupValues[1].toIntOrNull(b) + + if (g != null) { + val o = when { + g < 0x7c -> 1 + else -> 0 + } + + // retval = subdomainFromGalleryID(g, numberOfFrontends) + retval + retval = (97+o).toChar().toString() + retval + } + + return retval +} + +fun urlFromURL(url: String, base: String? = null) : String { + return url.replace(Regex("""//..?\.hitomi\.la/"""), "//${subdomainFromURL(url, base)}.hitomi.la/") +} + + +fun fullPathFromHash(hash: String?) : String? { + return when { + (hash?.length ?: 0) < 3 -> hash + else -> hash!!.replace(Regex("^.*(..)(.)$"), "$2/$1/$hash") + } +} + +@Suppress("NAME_SHADOWING", "UNUSED_PARAMETER") +fun urlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null) : String { + val ext = ext ?: dir ?: image.name.split('.').last() + val dir = dir ?: "images" + return "$protocol//a.hitomi.la/$dir/${fullPathFromHash(image.hash)}.$ext" +} + +fun urlFromUrlFromHash(galleryID: Int, image: GalleryFiles, dir: String? = null, ext: String? = null, base: String? = null) = + urlFromURL(urlFromHash(galleryID, image, dir, ext), base) + +fun rewriteTnPaths(html: String) = + html.replace(Regex("""//tn\.hitomi\.la/[^/]+/[0-9a-f]/[0-9a-f]{2}/""")) { url -> + urlFromURL(url.value, "tn") + } + +fun imageUrlFromImage(galleryID: Int, image: GalleryFiles, noWebp: Boolean) : String { + return when { + noWebp -> + urlFromUrlFromHash(galleryID, image) + //image.hasavif != 0 -> + // urlFromUrlFromHash(galleryID, image, "avif", null, "a") + image.haswebp != 0 -> + urlFromUrlFromHash(galleryID, image, "webp", null, "a") + else -> + urlFromUrlFromHash(galleryID, image) + } +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleries.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleries.kt new file mode 100644 index 00000000..7ffe0d58 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleries.kt @@ -0,0 +1,84 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2021 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package xyz.quaver.pupil.sources.hitomi.lib + +import io.ktor.client.* +import io.ktor.client.request.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable +import org.jsoup.Jsoup +import java.net.URLDecoder + +@Serializable +data class Gallery( + val related: List, + val langList: List>, + val cover: String, + val title: String, + val artists: List, + val groups: List, + val type: String, + val language: String, + val series: List, + val characters: List, + val tags: List, + val thumbnails: List +) +suspend fun getGallery(client: HttpClient, galleryID: Int) : Gallery = withContext(Dispatchers.IO) { + val url = Jsoup.parse(client.get("https://hitomi.la/galleries/$galleryID.html")) + .select("link").attr("href") + + val doc = Jsoup.parse(client.get(url)) + + val related = Regex("\\d+") + .findAll(doc.select("script").first()!!.html()) + .map { + it.value.toInt() + }.toList() + + val langList = doc.select("#lang-list a").map { + Pair(it.text(), "$protocol//hitomi.la${it.attr("href")}") + } + + val cover = protocol + doc.selectFirst(".cover img")!!.attr("src") + val title = doc.selectFirst(".gallery h1 a")!!.text() + val artists = doc.select(".gallery h2 a").map { it.text() } + val groups = doc.select(".gallery-info a[href~=^/group/]").map { it.text() } + val type = doc.selectFirst(".gallery-info a[href~=^/type/]")!!.text() + + val language = run { + val href = doc.select(".gallery-info a[href~=^/index.+\\.html\$]").attr("href") + Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: "" + } + + val series = doc.select(".gallery-info a[href~=^/series/]").map { it.text() } + val characters = doc.select(".gallery-info a[href~=^/character/]").map { it.text() } + + val tags = doc.select(".gallery-info a[href~=^/tag/]").map { + val href = URLDecoder.decode(it.attr("href"), "UTF-8") + href.slice(5 until href.indexOf('-')) + } + + val thumbnails = getGalleryInfo(client, galleryID).files.map { galleryInfo -> + urlFromUrlFromHash(galleryID, galleryInfo, "smalltn", "jpg", "tn") + } + + Gallery(related, langList, cover, title, artists, groups, type, language, series, characters, tags, thumbnails) +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleryblock.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleryblock.kt new file mode 100644 index 00000000..e7ccb3cb --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/galleryblock.kt @@ -0,0 +1,105 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2021 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package xyz.quaver.pupil.sources.hitomi.lib + +import io.ktor.client.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable +import org.jsoup.Jsoup +import java.net.URLDecoder + +//galleryblock.js +suspend fun fetchNozomi( + client: HttpClient, + area: String? = null, + tag: String = "index", + language: String = "all", + start: Int = -1, + count: Int = -1 +) : Pair, Int> = withContext(Dispatchers.IO) { + val url = + when(area) { + null -> "$protocol//$domain/$tag-$language$nozomiextension" + else -> "$protocol//$domain/$area/$tag-$language$nozomiextension" + } + + val response: HttpResponse = client.get(url) { + headers { + if (start != -1 && count != -1) { + val startByte = start*4 + val endByte = (start+count)*4-1 + + append("Range", "bytes=$startByte-$endByte") + } + } + } + + val totalItems = response.headers["Content-Range"]!! + .replace(Regex("^[Bb]ytes \\d+-\\d+/"), "").toInt() / 4 + + response.readBytes().asIterable().chunked(4) { + (it[0].toInt() and 0xFF) or + ((it[1].toInt() and 0xFF) shl 8) or + ((it[2].toInt() and 0xFF) shl 16) or + ((it[3].toInt() and 0xFF) shl 24) + } to totalItems +} + +@Serializable +data class GalleryBlock( + val id: Int, + val galleryUrl: String, + val thumbnails: List, + val title: String, + val artists: List, + val series: List, + val type: String, + val language: String, + val relatedTags: List +) + +suspend fun getGalleryBlock(client: HttpClient, galleryID: Int) : GalleryBlock = withContext(Dispatchers.IO) { + val url = "$protocol//$domain/$galleryblockdir/$galleryID$extension" + + val doc = Jsoup.parse(rewriteTnPaths(client.get(url))) + + val galleryUrl = doc.selectFirst("h1 > a")!!.attr("href") + + val thumbnails = doc.select(".dj-img-cont img").map { protocol + it.attr("src") } + + val title = doc.selectFirst("h1 > a")!!.text() + val artists = doc.select(".artist-list a").map{ it.text() } + val series = doc.select(".dj-content a[href~=^/series/]").map { it.text() } + val type = doc.selectFirst("a[href~=^/type/]")!!.text() + + val language = run { + val href = doc.select("a[href~=^/index.+\\.html\$]").attr("href") + Regex("""index-([^-]+)(-.+)?\.html""").find(href)?.groupValues?.getOrNull(1) ?: "" + } + + val relatedTags = doc.select(".relatedtags a").map { + val href = URLDecoder.decode(it.attr("href"), "UTF-8") + href.slice(5 until href.indexOf("-all")) + } + + GalleryBlock(galleryID, galleryUrl, thumbnails, title, artists, series, type, language, relatedTags) +} diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/reader.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/reader.kt new file mode 100644 index 00000000..70e4d833 --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/reader.kt @@ -0,0 +1,55 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2021 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package xyz.quaver.pupil.sources.hitomi.lib + +import kotlinx.serialization.Serializable + +fun getReferer(galleryID: Int) = "https://hitomi.la/reader/$galleryID.html" + +@Serializable +data class Tag( + val male: String? = null, + val female: String? = null, + val url: String, + val tag: String +) + +@Serializable +data class GalleryInfo( + val id: Int? = null, + val language_localname: String? = null, + val tags: List = emptyList(), + val title: String? = null, + val files: List, + val date: String? = null, + val type: String? = null, + val language: String? = null, + val japanese_title: String? = null +) + +@Serializable +data class GalleryFiles( + val width: Int, + val hash: String? = null, + val haswebp: Int = 0, + val name: String, + val height: Int, + val hasavif: Int = 0, + val hasavifsmalltn: Int? = 0 +) \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/results.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/results.kt new file mode 100644 index 00000000..ea8bb28a --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/results.kt @@ -0,0 +1,76 @@ +package xyz.quaver.pupil.sources.hitomi.lib + +import io.ktor.client.* +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import java.util.* + +suspend fun doSearch( + client: HttpClient, + query: String, + sortByPopularity: Boolean = false +) : Set = coroutineScope { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .split(Regex("\\s+")) + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList() + val negativeTerms = LinkedList() + + for (term in terms) { + if (term.matches(Regex("^-.+"))) + negativeTerms.push(term.replace(Regex("^-"), "")) + else if (term.isNotBlank()) + positiveTerms.push(term) + } + + val positiveResults = positiveTerms.map { + async { + runCatching { + getGalleryIDsForQuery(client, it) + }.getOrElse { emptySet() } + } + } + + val negativeResults = negativeTerms.map { + async { + runCatching { + getGalleryIDsForQuery(client, it) + }.getOrElse { emptySet() } + } + } + + var results = when { + sortByPopularity -> getGalleryIDsFromNozomi(client, null, "popular", "all") + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(client, null, "index", "all") + else -> emptySet() + } + + fun filterPositive(newResults: Set) { + results = when { + results.isEmpty() -> newResults + else -> results intersect newResults + } + } + + fun filterNegative(newResults: Set) { + results = results subtract newResults + } + + //positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + //negative results + negativeResults.forEach { + filterNegative(it.await()) + } + + results +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/search.kt b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/search.kt new file mode 100644 index 00000000..fc5d9ccc --- /dev/null +++ b/app/src/main/java/xyz/quaver/pupil/sources/hitomi/lib/search.kt @@ -0,0 +1,330 @@ +/* + * Pupil, Hitomi.la viewer for Android + * Copyright (C) 2021 tom5079 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package xyz.quaver.pupil.sources.hitomi.lib + +import io.ktor.client.* +import io.ktor.client.request.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import kotlin.math.min + +//searchlib.js +const val separator = "-" +const val extension = ".html" +const val index_dir = "tagindex" +const val galleries_index_dir = "galleriesindex" +const val max_node_size = 464 +const val B = 16 +const val compressed_nozomi_prefix = "n" + +var tag_index_version: String? = null +var galleries_index_version: String? = null + +fun sha256(data: ByteArray) : ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) +} + +@OptIn(ExperimentalUnsignedTypes::class) +fun hashTerm(term: String) : UByteArray { + return sha256(term.toByteArray()).toUByteArray().sliceArray(0 until 4) +} + +fun sanitize(input: String) : String { + return input.replace(Regex("[/#]"), "") +} + +suspend fun getIndexVersion(client: HttpClient, name: String): String = + client.get("$protocol//$domain/$name/version?_=${System.currentTimeMillis()}") + +//search.js +suspend fun getGalleryIDsForQuery(client: HttpClient, query: String) : Set { + query.replace("_", " ").let { + if (it.indexOf(':') > -1) { + val sides = it.split(":") + val ns = sides[0] + var tag = sides[1] + + var area : String? = ns + var language = "all" + when (ns) { + "female", "male" -> { + area = "tag" + tag = it + } + "language" -> { + area = null + language = tag + tag = "index" + } + } + + return getGalleryIDsFromNozomi(client, area, tag, language) + } + + val key = hashTerm(it) + val field = "galleries" + + val node = getNodeAtAddress(client, field, 0) ?: return emptySet() + + val data = bSearch(client, field, key, node) + + if (data != null) + return getGalleryIDsFromData(client, data) + + return emptySet() + } +} + +suspend fun getSuggestionsForQuery(client: HttpClient, query: String) : List { + query.replace('_', ' ').let { + var field = "global" + var term = it + + if (term.indexOf(':') > -1) { + val sides = it.split(':') + field = sides[0] + term = sides[1] + } + + val key = hashTerm(term) + val node = getNodeAtAddress(client, field, 0) ?: return emptyList() + val data = bSearch(client, field, key, node) + + if (data != null) + return getSuggestionsFromData(client, field, data) + + return emptyList() + } +} + +data class Suggestion(val s: String, val t: Int, val u: String, val n: String) +suspend fun getSuggestionsFromData(client: HttpClient, field: String, data: Pair) : List { + val url = "$protocol//$domain/$index_dir/$field.$tag_index_version.data" + val (offset, length) = data + if (length > 10000 || length <= 0) + throw Exception("length $length is too long") + + val inbuf = getURLAtRange(client, url, offset.until(offset+length)) + + val suggestions = ArrayList() + + val buffer = ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + val numberOfSuggestions = buffer.int + + if (numberOfSuggestions > 100 || numberOfSuggestions <= 0) + throw Exception("number of suggestions $numberOfSuggestions is too long") + + for (i in 0.until(numberOfSuggestions)) { + var top = buffer.int + + val ns = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8")) + buffer.position(buffer.position()+top) + + top = buffer.int + + val tag = inbuf.sliceArray(buffer.position().until(buffer.position()+top)).toString(charset("UTF-8")) + buffer.position(buffer.position()+top) + + val count = buffer.int + + val tagname = sanitize(tag) + val u = + when(ns) { + "female", "male" -> "/tag/$ns:$tagname${separator}1$extension" + "language" -> "/index-$tagname${separator}1$extension" + else -> "/$ns/$tagname${separator}all${separator}1$extension" + } + + suggestions.add(Suggestion(tag, count, u, ns)) + } + + return suggestions +} + +suspend fun getGalleryIDsFromNozomi(client: HttpClient, area: String?, tag: String, language: String) : Set = withContext(Dispatchers.IO) { + val nozomiAddress = + when(area) { + null -> "$protocol//$domain/$compressed_nozomi_prefix/$tag-$language$nozomiextension" + else -> "$protocol//$domain/$compressed_nozomi_prefix/$area/$tag-$language$nozomiextension" + } + + val bytes: ByteArray = try { + client.get(nozomiAddress) + } catch (e: Exception) { + return@withContext emptySet() + } + + val nozomi = mutableSetOf() + + val arrayBuffer = ByteBuffer + .wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + nozomi +} + +suspend fun getGalleryIDsFromData(client: HttpClient, data: Pair) : Set { + val url = "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.data" + val (offset, length) = data + if (length > 100000000 || length <= 0) + throw Exception("length $length is too long") + + val inbuf = getURLAtRange(client, url, offset.until(offset+length)) + + val galleryIDs = mutableSetOf() + + val buffer = ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + + val numberOfGalleryIDs = buffer.int + + val expectedLength = numberOfGalleryIDs*4+4 + + if (numberOfGalleryIDs > 10000000 || numberOfGalleryIDs <= 0) + throw Exception("number_of_galleryids $numberOfGalleryIDs is too long") + else if (inbuf.size != expectedLength) + throw Exception("inbuf.byteLength ${inbuf.size} != expected_length $expectedLength") + + for (i in 0.until(numberOfGalleryIDs)) + galleryIDs.add(buffer.int) + + return galleryIDs +} + +suspend fun getNodeAtAddress(client: HttpClient, field: String, address: Long) : Node? { + val url = + when(field) { + "galleries" -> "$protocol//$domain/$galleries_index_dir/galleries.$galleries_index_version.index" + "languages" -> "$protocol//$domain/$galleries_index_dir/languages.$galleries_index_version.index" + "nozomiurl" -> "$protocol//$domain/$galleries_index_dir/nozomiurl.$galleries_index_version.index" + else -> "$protocol//$domain/$index_dir/$field.$tag_index_version.index" + } + + val nodedata = getURLAtRange(client, url, address.until(address+max_node_size)) + + return decodeNode(nodedata) +} + +suspend fun getURLAtRange(client: HttpClient, url: String, range: LongRange) : ByteArray = withContext(Dispatchers.IO) { + client.get(url) { + headers { + append("Range", "bytes=${range.first}-${range.last}") + } + } +} + +@OptIn(ExperimentalUnsignedTypes::class) +data class Node(val keys: List, val datas: List>, val subNodeAddresses: List) +@OptIn(ExperimentalUnsignedTypes::class) +fun decodeNode(data: ByteArray) : Node { + val buffer = ByteBuffer + .wrap(data) + .order(ByteOrder.BIG_ENDIAN) + + val uData = data.toUByteArray() + + val numberOfKeys = buffer.int + val keys = ArrayList() + + for (i in 0.until(numberOfKeys)) { + val keySize = buffer.int + + if (keySize == 0 || keySize > 32) + throw Exception("fatal: !keySize || keySize > 32") + + keys.add(uData.sliceArray(buffer.position().until(buffer.position()+keySize))) + buffer.position(buffer.position()+keySize) + } + + val numberOfDatas = buffer.int + val datas = ArrayList>() + + for (i in 0.until(numberOfDatas)) { + val offset = buffer.long + val length = buffer.int + + datas.add(Pair(offset, length)) + } + + val numberOfSubNodeAddresses = B+1 + val subNodeAddresses = ArrayList() + + for (i in 0.until(numberOfSubNodeAddresses)) { + val subNodeAddress = buffer.long + subNodeAddresses.add(subNodeAddress) + } + + return Node(keys, datas, subNodeAddresses) +} + +@OptIn(ExperimentalUnsignedTypes::class) +suspend fun bSearch(client: HttpClient, field: String, key: UByteArray, node: Node) : Pair? { + fun compareArrayBuffers(dv1: UByteArray, dv2: UByteArray) : Int { + val top = min(dv1.size, dv2.size) + + for (i in 0.until(top)) { + if (dv1[i] < dv2[i]) + return -1 + else if (dv1[i] > dv2[i]) + return 1 + } + + return 0 + } + + fun locateKey(key: UByteArray, node: Node) : Pair { + for (i in node.keys.indices) { + val cmpResult = compareArrayBuffers(key, node.keys[i]) + + if (cmpResult <= 0) + return Pair(cmpResult==0, i) + } + + return Pair(false, node.keys.size) + } + + fun isLeaf(node: Node) : Boolean { + for (subnode in node.subNodeAddresses) + if (subnode != 0L) + return false + + return true + } + + if (node.keys.isEmpty()) + return null + + val (there, where) = locateKey(key, node) + if (there) + return node.datas[where] + else if (isLeaf(node)) + return null + + val nextNode = getNodeAtAddress(client, field, node.subNodeAddresses[where]) ?: return null + return bSearch(client, field, key, nextNode) +} \ No newline at end of file diff --git a/app/src/main/java/xyz/quaver/pupil/util/NetworkCache.kt b/app/src/main/java/xyz/quaver/pupil/util/NetworkCache.kt index 89e0ba7e..805b55c5 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/NetworkCache.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/NetworkCache.kt @@ -37,7 +37,6 @@ import org.kodein.di.android.closestDI import org.kodein.di.instance import org.kodein.log.LoggerFactory import org.kodein.log.newLogger -import xyz.quaver.hitomi.sha256 import java.io.File import java.util.* import java.util.concurrent.ConcurrentHashMap diff --git a/app/src/main/java/xyz/quaver/pupil/util/misc.kt b/app/src/main/java/xyz/quaver/pupil/util/misc.kt index 639ea60d..721c8940 100644 --- a/app/src/main/java/xyz/quaver/pupil/util/misc.kt +++ b/app/src/main/java/xyz/quaver/pupil/util/misc.kt @@ -38,6 +38,7 @@ import xyz.quaver.io.FileX import xyz.quaver.io.util.inputStream import xyz.quaver.pupil.db.AppDatabase import xyz.quaver.pupil.sources.SourceEntries +import java.security.MessageDigest operator fun JsonElement.get(index: Int) = this.jsonArray[index] @@ -76,4 +77,8 @@ class FileXImageSource(val file: FileX): ImageSource { @Composable fun rememberFileXImageSource(file: FileX) = remember { FileXImageSource(file) -} \ No newline at end of file +} + +fun sha256(data: ByteArray) : ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) +}