user01/calibre-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed thumbnail generate tasks, added thumbnail cleanup task, added reconnect db scheduled job

Browse Source
This commit is contained in:
mmonkey
2020-12-22 17:49:21 -06:00
parent e48bdf9d5a
commit 541fc7e14e
7 changed files with 245 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cps/tasks/thumbnail.py
+136 -41
View File
@@ -19,13 +19,15 @@
from __future__ import division, print_function, unicode_literals
import os
from cps import config, db, gdriveutils, logger, ub
from cps.constants import CACHE_DIR as _CACHE_DIR
from cps import config, db, fs, gdriveutils, logger, ub
from cps.services.worker import CalibreTask
from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X
from datetime import datetime, timedelta
from sqlalchemy import func
from urllib.request import urlopen
try:
from urllib.request import urlopen
except ImportError as e:
from urllib2 import urlopen
try:
from wand.image import Image
@@ -33,73 +35,92 @@ try:
except (ImportError, RuntimeError) as e:
use_IM = False
THUMBNAIL_RESOLUTION_1X = 1
THUMBNAIL_RESOLUTION_2X = 2
class TaskThumbnail(CalibreTask):
class TaskGenerateCoverThumbnails(CalibreTask):
def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
super(TaskThumbnail, self).__init__(task_message)
super(TaskGenerateCoverThumbnails, self).__init__(task_message)
self.self_cleanup = True
self.limit = limit
self.log = logger.create()
self.app_db_session = ub.get_new_session_instance()
self.worker_db = db.CalibreDB(expire_on_commit=False)
self.calibre_db = db.CalibreDB(expire_on_commit=False)
self.cache = fs.FileSystem()
self.resolutions = [
THUMBNAIL_RESOLUTION_1X,
THUMBNAIL_RESOLUTION_2X
]
def run(self, worker_thread):
if self.worker_db.session and use_IM:
thumbnails = self.get_thumbnail_book_ids()
thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails))
if self.calibre_db.session and use_IM:
expired_thumbnails = self.get_expired_thumbnails()
thumbnail_book_ids = self.get_thumbnail_book_ids()
books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
count = len(books_without_thumbnails)
for i, book in enumerate(books_without_thumbnails):
thumbnails = self.get_thumbnails_for_book(thumbnails, book)
if thumbnails:
for thumbnail in thumbnails:
self.update_book_thumbnail(book, thumbnail)
else:
self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X)
self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X)
for resolution in self.resolutions:
expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution(
book,
resolution,
expired_thumbnails
)
if expired_thumbnail:
self.update_book_thumbnail(book, expired_thumbnail)
else:
self.create_book_thumbnail(book, resolution)
self.progress = (1.0 / count) * i
self._handleSuccess()
self.app_db_session.close()
self.app_db_session.remove()
def get_expired_thumbnails(self):
return self.app_db_session\
.query(ub.Thumbnail)\
.filter(ub.Thumbnail.expiration < datetime.utcnow())\
.all()
def get_thumbnail_book_ids(self):
return self.app_db_session\
.query(ub.Thumbnail)\
.query(ub.Thumbnail.book_id)\
.group_by(ub.Thumbnail.book_id)\
.having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
.all()
.distinct()
def get_books_without_thumbnails(self, thumbnail_book_ids):
return self.worker_db.session\
return self.calibre_db.session\
.query(db.Books)\
.filter(db.Books.has_cover == 1)\
.filter(db.Books.id.notin_(thumbnail_book_ids))\
.limit(self.limit)\
.all()
def get_thumbnails_for_book(self, thumbnails, book):
results = list()
for thumbnail in thumbnails:
if thumbnail.book_id == book.id:
results.append(thumbnail)
def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails):
for thumbnail in expired_thumbnails:
if thumbnail.book_id == book.id and thumbnail.resolution == resolution:
return thumbnail
return results
return None
def update_book_thumbnail(self, book, thumbnail):
thumbnail.generated_at = datetime.utcnow()
thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
try:
self.app_db_session.commit()
self.generate_book_thumbnail(book, thumbnail)
except Exception as ex:
self.log.info(u'Error updating book thumbnail: ' + str(ex))
self._handleError(u'Error updating book thumbnail: ' + str(ex))
self.app_db_session.rollback()
def create_book_thumbnail(self, book, resolution):
thumbnail = ub.Thumbnail()
thumbnail.book_id = book.id
thumbnail.format = 'jpeg'
thumbnail.resolution = resolution
self.app_db_session.add(thumbnail)
@@ -107,6 +128,7 @@ class TaskThumbnail(CalibreTask):
self.app_db_session.commit()
self.generate_book_thumbnail(book, thumbnail)
except Exception as ex:
self.log.info(u'Error creating book thumbnail: ' + str(ex))
self._handleError(u'Error creating book thumbnail: ' + str(ex))
self.app_db_session.rollback()
@@ -128,9 +150,12 @@ class TaskThumbnail(CalibreTask):
if img.height > height:
width = self.get_thumbnail_width(height, img)
img.resize(width=width, height=height, filter='lanczos')
img.save(filename=self.get_thumbnail_cache_path(thumbnail))
img.format = thumbnail.format
filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
img.save(filename=filename)
except Exception as ex:
# Bubble exception to calling function
self.log.info(u'Error generating thumbnail file: ' + str(ex))
raise ex
finally:
stream.close()
@@ -144,7 +169,9 @@ class TaskThumbnail(CalibreTask):
if img.height > height:
width = self.get_thumbnail_width(height, img)
img.resize(width=width, height=height, filter='lanczos')
img.save(filename=self.get_thumbnail_cache_path(thumbnail))
img.format = thumbnail.format
filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
img.save(filename=filename)
def get_thumbnail_height(self, thumbnail):
return int(225 * thumbnail.resolution)
@@ -153,20 +180,88 @@ class TaskThumbnail(CalibreTask):
percent = (height / float(img.height))
return int((float(img.width) * float(percent)))
def get_thumbnail_cache_dir(self):
if not os.path.isdir(_CACHE_DIR):
os.makedirs(_CACHE_DIR)
@property
def name(self):
return "GenerateCoverThumbnails"
if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')):
os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails'))
return os.path.join(_CACHE_DIR, 'thumbnails')
class TaskCleanupCoverThumbnailCache(CalibreTask):
def __init__(self, task_message=u'Validating cover thumbnail cache'):
super(TaskCleanupCoverThumbnailCache, self).__init__(task_message)
self.log = logger.create()
self.app_db_session = ub.get_new_session_instance()
self.calibre_db = db.CalibreDB(expire_on_commit=False)
self.cache = fs.FileSystem()
def get_thumbnail_cache_path(self, thumbnail):
if thumbnail:
return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename)
return None
def run(self, worker_thread):
cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS)
# Expire thumbnails in the database if the cached file is missing
# This case will happen if a user deletes the cache dir or cached files
if self.app_db_session:
self.expire_missing_thumbnails(cached_thumbnail_files)
self.progress = 0.33
# Delete thumbnails in the database if the book has been removed
# This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem
if self.app_db_session and self.calibre_db:
book_ids = self.get_book_ids()
self.delete_thumbnails_for_missing_books(book_ids)
self.progress = 0.66
# Delete extraneous cached thumbnail files
# This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally
if self.app_db_session:
db_thumbnail_files = self.get_thumbnail_filenames()
self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files)
self._handleSuccess()
self.app_db_session.remove()
def expire_missing_thumbnails(self, filenames):
try:
self.app_db_session\
.query(ub.Thumbnail)\
.filter(ub.Thumbnail.filename.notin_(filenames))\
.update({"expiration": datetime.utcnow()}, synchronize_session=False)
self.app_db_session.commit()
except Exception as ex:
self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex))
self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex))
self.app_db_session.rollback()
def get_book_ids(self):
results = self.calibre_db.session\
.query(db.Books.id)\
.filter(db.Books.has_cover == 1)\
.distinct()
return [value for value, in results]
def delete_thumbnails_for_missing_books(self, book_ids):
try:
self.app_db_session\
.query(ub.Thumbnail)\
.filter(ub.Thumbnail.book_id.notin_(book_ids))\
.delete(synchronize_session=False)
self.app_db_session.commit()
except Exception as ex:
self.log.info(str(ex))
self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex))
self.app_db_session.rollback()
def get_thumbnail_filenames(self):
results = self.app_db_session\
.query(ub.Thumbnail.filename)\
.all()
return [thumbnail for thumbnail, in results]
def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files):
extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files))
for file in extraneous_files:
self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS)
@property
def name(self):
return "Thumbnail"
return "CleanupCoverThumbnailCache"