diff --git a/cps/db.py b/cps/db.py index 0cd891a2b..ccf28a7a0 100644 --- a/cps/db.py +++ b/cps/db.py @@ -30,7 +30,7 @@ from sqlite3 import OperationalError as sqliteOperationalError from sqlalchemy import create_engine from sqlalchemy import Table, Column, ForeignKey, CheckConstraint from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float -from sqlalchemy.orm import relationship, sessionmaker, scoped_session +from sqlalchemy.orm import relationship, sessionmaker, scoped_session, selectinload from sqlalchemy.orm.collections import InstrumentedList from sqlalchemy.ext.declarative import DeclarativeMeta from sqlalchemy.exc import OperationalError @@ -901,29 +901,36 @@ class CalibreDB: for entry in entries: if combined: sort_authors = entry.Books.author_sort.split('&') - ids = [a.id for a in entry.Books.authors] - + authors_list = entry.Books.authors else: sort_authors = entry.author_sort.split('&') - ids = [a.id for a in entry.authors] - authors_ordered = list() - # error = False + authors_list = entry.authors + + # Create dictionary for O(1) lookup instead of nested loops + authors_by_sort = {} + authors_by_id = {} + for author in authors_list: + authors_by_sort[author.sort] = author + authors_by_id[author.id] = author + + authors_ordered = [] + ids_remaining = set(authors_by_id.keys()) + + # Order authors based on sort field using dictionary lookup for auth in sort_authors: auth = strip_whitespaces(auth) - results = self.session.query(Authors).filter(Authors.sort == auth).all() - # ToDo: How to handle not found author name - if not len(results): - book_id = entry.id if isinstance(entry, Books) else entry[0].id - log.error("Author '{}' of book {} not found to display name in right order".format(auth, book_id)) - # error = True - break - for r in results: - if r.id in ids: - authors_ordered.append(r) - ids.remove(r.id) - for author_id in ids: - result = self.session.query(Authors).filter(Authors.id == author_id).first() - authors_ordered.append(result) + if auth in authors_by_sort: + author = authors_by_sort[auth] + authors_ordered.append(author) + ids_remaining.discard(author.id) + else: + # This can happen if author_sort has stale data or formatting issues + book_id = entry.id if isinstance(entry, Books) else (entry.Books.id if combined else entry.id) + log.warning("Author '{}' of book {} not found in author list, skipping in sort order".format(auth, book_id)) + + # Add any remaining authors not in sort order + for author_id in ids_remaining: + authors_ordered.append(authors_by_id[author_id]) if list_return: if combined: @@ -956,36 +963,93 @@ class CalibreDB: .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() def search_query(self, term, config, *join): - strip_whitespaces(term).lower() + term = strip_whitespaces(term).lower() self.create_functions() - # self.session.connection().connection.connection.create_function("lower", 1, lcase) - q = list() - author_terms = re.split("[, ]+", term) - for author_term in author_terms: - q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) - query = self.generate_linked_query(config.config_read_column, Books) - if len(join) == 6: - query = query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) - if len(join) == 3: - query = query.outerjoin(join[0], join[1]).outerjoin(join[2]) - elif len(join) == 2: - query = query.outerjoin(join[0], join[1]) - elif len(join) == 1: - query = query.outerjoin(join[0]) + # Try FTS5 search first for better performance + fts_ids = None + # Check if FTS5 table exists before attempting search + if not hasattr(self, '_fts_available'): + try: + result = self.session.execute( + text("SELECT name FROM sqlite_master WHERE type='table' AND name='books_fts'") + ).fetchone() + self._fts_available = result is not None + except Exception: + self._fts_available = False + + if self._fts_available: + try: + # Escape FTS5 special characters to prevent query errors + term_fts = term.replace('"', '""') + # Wrap in quotes for phrase matching and better accuracy + fts_results = self.session.execute( + text("SELECT DISTINCT rowid FROM books_fts WHERE books_fts MATCH :term"), + {"term": f'"{term_fts}"'} + ).fetchall() + if fts_results: + fts_ids = [r[0] for r in fts_results] + except Exception as ex: + # FTS5 query failed, fall back to traditional search + log.debug("FTS5 search failed for term '{}', using fallback: {}".format(term, ex)) + + # Build base query with optimized joins + base_query = self.generate_linked_query(config.config_read_column, Books) + base_query = base_query.filter(self.common_filters(True)) + + # Apply eager loading for authors to avoid N+1 queries + base_query = base_query.options(selectinload(Books.authors)) + + if len(join) == 6: + base_query = base_query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) + if len(join) == 3: + base_query = base_query.outerjoin(join[0], join[1]).outerjoin(join[2]) + elif len(join) == 2: + base_query = base_query.outerjoin(join[0], join[1]) + elif len(join) == 1: + base_query = base_query.outerjoin(join[0]) + + # If FTS5 found results, use those IDs + if fts_ids: + return base_query.filter(Books.id.in_(fts_ids)) + + # Fallback to traditional search with optimized subqueries + author_terms = re.split("[, ]+", term) + + # Use subquery for authors to avoid expensive .any() with OR + author_subquery = self.session.query(books_authors_link.c.book).join( + Authors, books_authors_link.c.author == Authors.id + ) + author_filters = [] + for author_term in author_terms: + author_filters.append(func.lower(Authors.name).ilike("%" + author_term + "%")) + if author_filters: + author_subquery = author_subquery.filter(and_(*author_filters)) + + # Build optimized filter expressions cc = self.get_cc_columns(config, filter_config_custom_read=True) - filter_expression = [Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), - Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), - Books.authors.any(and_(*q)), - Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), - func.lower(Books.title).ilike("%" + term + "%")] + filter_expression = [ + Books.id.in_(self.session.query(books_tags_link.c.book).join( + Tags, books_tags_link.c.tag == Tags.id + ).filter(func.lower(Tags.name).ilike("%" + term + "%"))), + Books.id.in_(self.session.query(books_series_link.c.book).join( + Series, books_series_link.c.series == Series.id + ).filter(func.lower(Series.name).ilike("%" + term + "%"))), + Books.id.in_(author_subquery), + Books.id.in_(self.session.query(books_publishers_link.c.book).join( + Publishers, books_publishers_link.c.publisher == Publishers.id + ).filter(func.lower(Publishers.name).ilike("%" + term + "%"))), + func.lower(Books.title).ilike("%" + term + "%") + ] + for c in cc: if c.datatype not in ["datetime", "rating", "bool", "int", "float"]: filter_expression.append( getattr(Books, 'custom_column_' + str(c.id)).any( func.lower(cc_classes[c.id].value).ilike("%" + term + "%"))) - return query.filter(self.common_filters(True)).filter(or_(*filter_expression)) + + return base_query.filter(or_(*filter_expression)) def get_cc_columns(self, config, filter_config_custom_read=False): tmp_cc = self.session.query(CustomColumns).filter(CustomColumns.datatype.notin_(cc_exceptions)).all() @@ -1007,18 +1071,32 @@ class CalibreDB: def get_search_results(self, term, config, offset=None, order=None, limit=None, *join): order = order[0] if order else [Books.sort] pagination = None - result = self.search_query(term, config, *join).order_by(*order).all() - result_count = len(result) + if offset is not None and limit is not None: offset = int(offset) - limit_all = offset + int(limit) - pagination = Pagination((offset / (int(limit)) + 1), limit, result_count) + limit_int = int(limit) + + # Use LIMIT+1 pattern to estimate total count without expensive count() + query = self.search_query(term, config, *join).order_by(*order) + result = query.limit(offset + limit_int + 1).all() + + # Check if there are more results + has_more = len(result) > (offset + limit_int) + if has_more: + result_count = offset + limit_int + 1 # Estimate: at least this many + else: + result_count = len(result) + + # Extract the page of results + result = result[offset:offset + limit_int] + pagination = Pagination((offset / limit_int + 1), limit_int, result_count) else: - offset = 0 - limit_all = result_count + # No pagination, fetch all results + result = self.search_query(term, config, *join).order_by(*order).all() + result_count = len(result) ub.store_combo_ids(result) - entries = self.order_authors(result[offset:limit_all], list_return=True, combined=True) + entries = self.order_authors(result, list_return=True, combined=True) return entries, result_count, pagination diff --git a/pyproject.toml b/pyproject.toml index f5ea31aaa..875c6a230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "Flask>=1.0.2,<3.2.0", "iso-639>=0.4.5,<0.5.0;python_version<'3.12'", "pycountry>=20.0.0,<25.0.0;python_version>='3.12'", - "PyPDF>=3.15.6,<5.5.0", + "PyPDF>=6.1.3,<6.5.0", "pytz>=2016.10", "requests>=2.32.0,<2.33.0", "SQLAlchemy>=1.3.0,<2.1.0", diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 0ac0e55fd..d6831bdc3 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
Start Time: 2025-09-21 09:23:41
+Start Time: 2025-12-06 19:29:38
Stop Time: 2025-09-21 16:46:10
+Stop Time: 2025-12-07 02:46:00
Duration: 6h 10 min
+Duration: 6h 5 min
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_kobo_sync.py", line 373, in test_sync_shelf - self.assertEqual(1, len(data), data) -AssertionError: 1 != 0 : []-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_mass_edit_books_list.py", line 221, in test_invalid_author_title - self.assertEqual(bl['table'][4]['Authors']['text'], "Kurt Jilo") -AssertionError: 'Sigurd Lindgren' != 'Kurt Jilo' -- Sigurd Lindgren -+ Kurt Jilo-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_mass_edit_books_list.py", line 462, in test_mass_edit_series - self.assertEqual(bl['table'][5]['Series']['text'], "Tuto") -AssertionError: '+' != 'Tuto' -- + -+ Tuto-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_mass_edit_books_list.py", line 322, in test_protected_author_title - self.assertEqual(bl['table'][3]['Authors']['text'], "Kurt Jilo") -AssertionError: 'Peter Parker' != 'Kurt Jilo' -- Peter Parker -+ Kurt Jilo-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_merge_books_list.py", line 124, in test_book_merge - self.assertTrue(bl['table'][0]['selector']['element'].is_selected()) -AssertionError: False is not true-
Traceback (most recent call last):
+ File "/home/ozzie/Development/calibre-web-test/test/test_pdf_metadata.py", line 133, in test_upload_invalid_pdf
+ self.check_uploaded_pdf({'author': "Mani Mücks",
+ File "/home/ozzie/Development/calibre-web-test/test/test_pdf_metadata.py", line 76, in check_uploaded_pdf
+ self.assertTrue('languages' not in details)
+AssertionError: False is not true
+ Traceback (most recent call last):
+ File "/home/ozzie/Development/calibre-web-test/test/test_shelf.py", line 569, in test_adv_search_shelf
+ self.assertEqual(len(self.adv_search({u'include_shelf': u'Search', 'title': 'book'})), 2)
+AssertionError: 7 != 2
+ Traceback (most recent call last):
File "/home/ozzie/Development/calibre-web-test/test/test_thumbnails.py", line 317, in test_sideloaded_book
self.assertAlmostEqual(diff(BytesIO(list_cover), BytesIO(old_list_cover), delete_diff_file=True), 0.0,
-AssertionError: 0.004562894976745623 != 0.0 within 0.0001 delta (0.004562894976745623 difference)
+AssertionError: 0.049274509803921576 != 0.0 within 0.0001 delta (0.049274509803921576 difference)
@@ -5010,11 +4946,11 @@ AssertionError: 0.004562894976745623 != 0.0 within 0.0001 delta (0.0045628949767
- Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 437, in test_upload_aac + self.assertAlmostEqual(diff(BytesIO(self.jpg_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.0026100147661403746 != 0.0 within 0.001 delta (0.0026100147661403746 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 251, in test_upload_aiff + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 483, in test_upload_asf + self.assertAlmostEqual(diff(BytesIO(self.jpg_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.0026100147661403746 != 0.0 within 0.001 delta (0.0026100147661403746 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 379, in test_upload_flac + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 565, in test_upload_m4a + self.assertAlmostEqual(diff(BytesIO(self.jpg_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.0026100147661403746 != 0.0 within 0.001 delta (0.0026100147661403746 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 605, in test_upload_m4b + self.assertAlmostEqual(diff(BytesIO(self.jpg_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.0026100147661403746 != 0.0 within 0.001 delta (0.0026100147661403746 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 120, in test_upload_mp3 + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.006) +AssertionError: 0.0075997182498730486 != 0.0 within 0.006 delta (0.0075997182498730486 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 525, in test_upload_mp4 + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 316, in test_upload_oggvorbis + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 710, in test_upload_ogv + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_audio.py", line 655, in test_upload_opus + self.assertAlmostEqual(diff(BytesIO(self.png_original), BytesIO(cover_image), delete_diff_file=True), 0.0, delta=0.001) +AssertionError: 0.004005578839723024 != 0.0 within 0.001 delta (0.004005578839723024 difference)+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_upload_epubs.py", line 214, in test_upload_epub_cover_formats + self.assertAlmostEqual(diff(BytesIO(original), BytesIO(cover_image), delete_diff_file=True), 0.0, +AssertionError: 0.004005578839723024 != 0.0 within 0.0001 delta (0.004005578839723024 difference)+
Traceback (most recent call last):
+ File "/home/ozzie/Development/calibre-web-test/test/test_upload_epubs.py", line 104, in test_upload_epub_lang
+ self.assertEqual(len(self.adv_search({u'include_language': u'Tedesco'})), 1)
+AssertionError: 12 != 1
+