From 5e9ec706c5554e0128b958c196c35d4da6708a12 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Sun, 19 Jun 2022 22:59:54 +0000 Subject: [PATCH 01/93] first draft of embedding metadata on conversion --- cps/tasks/convert.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 3062850dd..9d2ee6361 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,12 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +#donrar +from tempfile import gettempdir +from time import time +#enddonrar from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -39,6 +43,7 @@ from cps import gdriveutils log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, kindle_mail, user=None): @@ -223,6 +228,7 @@ class TaskConvert(CalibreTask): return check, None def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters @@ -230,8 +236,18 @@ class TaskConvert(CalibreTask): # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux quotes = [1, 2] + + # TODO: Clean up and make cli work with windows. Also, implement changing covers. + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") + opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command) + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] + (file_path + format_new_ext), '--from-opf', path_tmp_opf] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 8c781ad4a438450ca1a1d79ac1d84f31ac3e7596 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Sun, 19 Jun 2022 23:20:59 +0000 Subject: [PATCH 02/93] code cleanup and implements cover change --- cps/tasks/convert.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 9d2ee6361..6d723ac8f 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -21,10 +21,8 @@ import re from glob import glob from shutil import copyfile, copyfileobj from markupsafe import escape -#donrar from tempfile import gettempdir from time import time -#enddonrar from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -237,7 +235,7 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [1, 2] - # TODO: Clean up and make cli work with windows. Also, implement changing covers. + # TODO: Clean up and make cli work with windows. tmp_dir = os.path.join(gettempdir(), 'calibre_web') path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] @@ -247,7 +245,8 @@ class TaskConvert(CalibreTask): copyfileobj(p.stdout, fd) command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf] + (file_path + format_new_ext), '--from-opf', path_tmp_opf, + '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 3c4330ba510aa31bb60e58c00a24226225576a73 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Tue, 21 Jun 2022 17:04:44 +0000 Subject: [PATCH 03/93] refactoring of calibre binary detection --- cps/admin.py | 1 + cps/config_sql.py | 55 +++++++++++++++++++++++++++------- cps/constants.py | 3 ++ cps/tasks/convert.py | 7 +++-- cps/templates/config_edit.html | 7 +++++ 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 76275922f..43103eedb 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1721,6 +1721,7 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") diff --git a/cps/config_sql.py b/cps/config_sql.py index 743b2ce7c..9e3472bdf 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -131,6 +131,7 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) @@ -166,12 +167,16 @@ class _ConfigSQL(object): self.cli = cli change = False + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition + change = True + self.config_binariesdir = autodetect_calibre_binaries() + if self.config_converterpath == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_converterpath = autodetect_converter_binary() if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition - change = True self.config_kepubifypath = autodetect_kepubify_binary() @@ -266,6 +271,21 @@ class _ConfigSQL(object): def get_scheduled_task_settings(self): return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} + def get_calibre_binarypath(self, binary): + binariesdir = self.config_binariesdir + if binariesdir: + # TODO: Need to make sure that all supported calibre binaries are actually in the specified directory when set via UI + if sys.platform == "win32": + extension = ".exe" + else: + extension = "" + if binary in constants.SUPPORTED_CALIBRE_BINARIES: + return os.path.join(binariesdir, binary + extension) + else: + # TODO: Error handling + pass + return "" + def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. @@ -407,17 +427,31 @@ def _migrate_table(session, orm_class): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + extension = ".exe" + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + extension = "" + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary + extension) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + return element + return "" + + +def autodetect_converter_binary(): + calibre_path = autodetect_calibre_binaries() + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -462,6 +496,7 @@ def load_configuration(conf, session, cli): conf.init_config(session, cli) # return conf + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 0f3b25308..c684453ce 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -152,6 +152,9 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'opus', 'wav', 'flac', 'm4a', 'm4b'} +SUPPORTED_CALIBRE_BINARIES = ["ebook-convert", "calibredb"] + + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 6d723ac8f..744558378 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -235,10 +235,11 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [1, 2] - # TODO: Clean up and make cli work with windows. + # TODO: Clean up. + # TODO: Maybe delete/clean-up tmp files directly. tmp_dir = os.path.join(gettempdir(), 'calibre_web') - path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") - opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + calibredb_binarypath = config.get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index f61ca9a5a..725fb88d2 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -323,6 +323,13 @@
+ +
+ + + + +
From 03359599edcc91dd9dffa410265640253a6141b6 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 23 Jun 2022 20:02:54 +0000 Subject: [PATCH 04/93] input validation for calibre binary directory --- cps/admin.py | 4 ++++ cps/config_sql.py | 18 ++++++++---------- cps/constants.py | 6 ++++-- cps/helper.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 43103eedb..3b24e5185 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1724,6 +1724,10 @@ def _configuration_update_helper(): _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 9e3472bdf..a37987656 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -30,6 +30,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -274,13 +275,8 @@ class _ConfigSQL(object): def get_calibre_binarypath(self, binary): binariesdir = self.config_binariesdir if binariesdir: - # TODO: Need to make sure that all supported calibre binaries are actually in the specified directory when set via UI - if sys.platform == "win32": - extension = ".exe" - else: - extension = "" if binary in constants.SUPPORTED_CALIBRE_BINARIES: - return os.path.join(binariesdir, binary + extension) + return os.path.join(binariesdir, binary) else: # TODO: Error handling pass @@ -429,18 +425,20 @@ def _migrate_table(session, orm_class): def autodetect_calibre_binaries(): if sys.platform == "win32": - extension = ".exe" calibre_path = ["C:\\program files\\calibre\\", "C:\\program files(x86)\\calibre\\", "C:\\program files(x86)\\calibre2\\", "C:\\program files\\calibre2\\"] else: - extension = "" calibre_path = ["/opt/calibre/"] for element in calibre_path: - supported_binary_paths = [os.path.join(element, binary + extension) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES] if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): - return element + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element return "" diff --git a/cps/constants.py b/cps/constants.py index c684453ce..849f95623 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -151,8 +151,10 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} - -SUPPORTED_CALIBRE_BINARIES = ["ebook-convert", "calibredb"] +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = [binary + _extension for binary in ["ebook-convert", "calibredb"]] def has_flag(value, bit_flag): diff --git a/cps/helper.py b/cps/helper.py index ed11e1c0a..aba18dcc7 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -53,7 +53,7 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES from .subproc_wrapper import process_wait from .services.worker import WorkerThread from .tasks.mail import TaskEmail @@ -940,6 +940,33 @@ def check_unrar(unrar_location): return _('Error excecuting UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + return _('Missing calibre binaries in the specified directory') + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" From e39c6130c34a1bc4bb93cffa77a55865a9c21e36 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Wed, 29 Jun 2022 19:54:53 +0000 Subject: [PATCH 05/93] cleanup and better error handling --- cps/config_sql.py | 3 +-- cps/helper.py | 6 ++++-- cps/tasks/convert.py | 11 ++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index a37987656..9dc0abddf 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -278,8 +278,7 @@ class _ConfigSQL(object): if binary in constants.SUPPORTED_CALIBRE_BINARIES: return os.path.join(binariesdir, binary) else: - # TODO: Error handling - pass + raise ValueError("'{}' is not a supported Calibre binary".format(binary)) return "" def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): diff --git a/cps/helper.py b/cps/helper.py index aba18dcc7..31b15eb96 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -952,7 +952,8 @@ def check_calibre(calibre_location): try: supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] - if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): version = values[0].group(1) @@ -960,7 +961,8 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: - return _('Missing calibre binaries in the specified directory') + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES, binaries_available) if not available] + return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: log.error_or_exception(err) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 744558378..1c9975e10 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -233,18 +233,19 @@ class TaskConvert(CalibreTask): # windows py2.7 encode as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux - quotes = [1, 2] - # TODO: Clean up. - # TODO: Maybe delete/clean-up tmp files directly. + quotes = [3, 5] tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) calibredb_binarypath = config.get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] - p = process_open(opf_command) + p = process_open(opf_command, quotes) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext), '--from-opf', path_tmp_opf, '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] @@ -257,7 +258,7 @@ class TaskConvert(CalibreTask): quotes_index += 1 p = process_open(command, quotes, newlines=False) - except OSError as e: + except (ValueError, OSError) as e: return 1, N_(u"Ebook-converter failed: %(error)s", error=e) while p.poll() is None: From a77aef83c67f292c6ddac7e6c57d77cb372a996c Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 30 Jun 2022 13:05:36 +0000 Subject: [PATCH 06/93] automatically set `config_converterpath` --- cps/admin.py | 4 +++- cps/templates/config_edit.html | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 3b24e5185..149d4aa48 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1722,12 +1722,14 @@ def _configuration_update_helper(): _config_string(to_save, "config_calibre") _config_string(to_save, "config_binariesdir") - _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") if "config_binariesdir" in to_save: calibre_status = helper.check_calibre(config.config_binariesdir) if calibre_status: return _configuration_result(calibre_status) + # ToDo: Remove this and 'self.config_converterpath' and replace with 'config.get_calibre_binarypath("ebook-convert")' everywhere + to_save["config_converterpath"] = config.get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 725fb88d2..c9cf9a29b 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -330,13 +330,6 @@
- -
- - - - -
From 0d34f41a48e3056c0cd3c8bb0cc5882878481642 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Fri, 1 Jul 2022 12:06:33 +0000 Subject: [PATCH 07/93] cleanup of `autodetect_converter_binary` --- cps/config_sql.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index 9dc0abddf..7867b6f30 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -172,10 +172,7 @@ class _ConfigSQL(object): if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True self.config_binariesdir = autodetect_calibre_binaries() - - if self.config_converterpath == None: # pylint: disable=access-member-before-definition - change = True - self.config_converterpath = autodetect_converter_binary() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -441,13 +438,12 @@ def autodetect_calibre_binaries(): return "" -def autodetect_converter_binary(): - calibre_path = autodetect_calibre_binaries() +def autodetect_converter_binary(calibre_path): if sys.platform == "win32": converter_path = os.path.join(calibre_path, "ebook-convert.exe") else: converter_path = os.path.join(calibre_path, "ebook-convert") - if os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): return converter_path return "" From c5c3874243c45436975dd780c96198c3ee5199b6 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Fri, 1 Jul 2022 16:04:25 +0000 Subject: [PATCH 08/93] first implementation --- cps/helper.py | 25 +++++++++++++++++++++++-- cps/tasks/convert.py | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 31b15eb96..885ffba5c 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,7 +54,7 @@ from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES -from .subproc_wrapper import process_wait +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -911,7 +911,28 @@ def do_download_file(book, book_format, client, data, headers): if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) + if config.config_binariesdir: + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = config.get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + except (ValueError, OSError) as e: + # ToDo real error handling + log.error_or_exception(e) + + response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 1c9975e10..594f0c38f 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -241,6 +241,7 @@ class TaskConvert(CalibreTask): calibredb_binarypath = config.get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) + p.wait() path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) From fc004f4f0c9d4bbb0e9817f6983f82bce76dd2ac Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 7 Jul 2022 11:41:51 +0000 Subject: [PATCH 09/93] moved `get_calibre_binarypath()` to `helper.py` --- cps/admin.py | 5 ++--- cps/config_sql.py | 11 +---------- cps/constants.py | 2 +- cps/helper.py | 15 +++++++++++++-- cps/tasks/convert.py | 6 +++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 149d4aa48..e487b5ca1 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -43,7 +43,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -1727,8 +1727,7 @@ def _configuration_update_helper(): calibre_status = helper.check_calibre(config.config_binariesdir) if calibre_status: return _configuration_result(calibre_status) - # ToDo: Remove this and 'self.config_converterpath' and replace with 'config.get_calibre_binarypath("ebook-convert")' everywhere - to_save["config_converterpath"] = config.get_calibre_binarypath("ebook-convert") + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 7867b6f30..9b8c29fa6 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -269,15 +269,6 @@ class _ConfigSQL(object): def get_scheduled_task_settings(self): return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} - def get_calibre_binarypath(self, binary): - binariesdir = self.config_binariesdir - if binariesdir: - if binary in constants.SUPPORTED_CALIBRE_BINARIES: - return os.path.join(binariesdir, binary) - else: - raise ValueError("'{}' is not a supported Calibre binary".format(binary)) - return "" - def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. @@ -428,7 +419,7 @@ def autodetect_calibre_binaries(): else: calibre_path = ["/opt/calibre/"] for element in calibre_path: - supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): diff --git a/cps/constants.py b/cps/constants.py index 849f95623..b613d0aab 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -154,7 +154,7 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' _extension = "" if sys.platform == "win32": _extension = ".exe" -SUPPORTED_CALIBRE_BINARIES = [binary + _extension for binary in ["ebook-convert", "calibredb"]] +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} def has_flag(value, bit_flag): diff --git a/cps/helper.py b/cps/helper.py index 31b15eb96..a16245fa8 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -951,7 +951,7 @@ def check_calibre(calibre_location): return _('Please specify a directory, not a file') try: - supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] if all(binaries_available): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] @@ -961,7 +961,7 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: - missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES, binaries_available) if not available] + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: @@ -1032,6 +1032,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 1c9975e10..25cf562d2 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -37,7 +37,7 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils - +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() @@ -238,7 +238,7 @@ class TaskConvert(CalibreTask): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) - calibredb_binarypath = config.get_calibre_binarypath("calibredb") + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") @@ -258,7 +258,7 @@ class TaskConvert(CalibreTask): quotes_index += 1 p = process_open(command, quotes, newlines=False) - except (ValueError, OSError) as e: + except OSError as e: return 1, N_(u"Ebook-converter failed: %(error)s", error=e) while p.poll() is None: From 4913673e8fe4f394b48926252402b66c95a7a1fd Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 7 Jul 2022 11:47:10 +0000 Subject: [PATCH 10/93] added `subprocess.wait()` when getting metadata --- cps/tasks/convert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 25cf562d2..79d7ddbd9 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -241,6 +241,7 @@ class TaskConvert(CalibreTask): calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) + p.wait() path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) From fc7ce8da2d2dee4f2c235ebce8c3ab59be832778 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Wed, 13 Jul 2022 15:39:01 +0000 Subject: [PATCH 11/93] cleanup --- cps/helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 77323d0bf..52174b020 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -917,7 +917,7 @@ def do_download_file(book, book_format, client, data, headers): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) - calibredb_binarypath = config.get_calibre_binarypath("calibredb") + calibredb_binarypath = get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] @@ -928,9 +928,9 @@ def do_download_file(book, book_format, client, data, headers): _, err = p.communicate() if err: log.error('Metadata embedder encountered an error: %s', err) - except (ValueError, OSError) as e: + except OSError as ex: # ToDo real error handling - log.error_or_exception(e) + log.error_or_exception(ex) response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) # ToDo Check headers parameter From 0b4731913eadcd511246aa286867e7b9850d52f9 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 14 Jul 2022 09:25:37 +0000 Subject: [PATCH 12/93] created `do_calibre_export` function --- cps/helper.py | 55 +++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 52174b020..282a22d37 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -894,9 +894,10 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: return gd.do_gdrive_download(df, headers) @@ -904,41 +905,47 @@ def do_download_file(book, book_format, client, data, headers): abort(404) else: filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") if config.config_binariesdir: - try: - quotes = [3, 5, 7, 9] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - calibredb_binarypath = get_calibre_binarypath("calibredb") - opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), - '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, - '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] - file_name = book.title - if len(book.authors) > 0: - file_name = file_name + ' - ' + book.authors[0].name - p = process_open(opf_command, quotes) - _, err = p.communicate() - if err: - log.error('Metadata embedder encountered an error: %s', err) - except OSError as ex: - # ToDo real error handling - log.error_or_exception(ex) + filename, book_name = do_calibre_export(book, book_format) - response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) + response = make_response(send_from_directory(filename, book_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) return response + +def do_calibre_export(book, book_format): + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + + ################################## From 80b0e8865079a076939f3547405b556b485abb59 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 14 Jul 2022 13:34:42 +0000 Subject: [PATCH 13/93] fix for GDrive integration --- cps/tasks/convert.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 79d7ddbd9..ac0d3a2a4 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -64,15 +64,20 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if df: + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df and df_cover: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + u"." + self.settings['old_book_format'].lower()) + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _(u"%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) @@ -82,6 +87,7 @@ class TaskConvert(CalibreTask): filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower()) + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: From 9bcbe523d7660c4d14bf2742ee998212cbe230d6 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Fri, 22 Jul 2022 08:58:28 +0000 Subject: [PATCH 14/93] (draft) metadata embedding when sending to device --- cps/constants.py | 2 +- cps/helper.py | 13 ++++++++++++- cps/tasks/convert.py | 10 ++++++++++ cps/tasks/mail.py | 27 ++++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/cps/constants.py b/cps/constants.py index b613d0aab..d8b05cb3f 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -154,7 +154,7 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' _extension = "" if sys.platform == "win32": _extension = ".exe" -SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb", "ebook-meta"]} def has_flag(value, bit_flag): diff --git a/cps/helper.py b/cps/helper.py index a16245fa8..800562441 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,7 +54,7 @@ from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES -from .subproc_wrapper import process_wait +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -213,6 +213,17 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): # returns None if success, otherwise errormessage return convert_book_format(book_id, calibrepath, u'azw3', book_format.lower(), user_id, kindle_mail) + # ToDo: Delete when OPF creation has been implemented + if config.config_binariesdir: + quotes = [3, 5] + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_opf = os.path.join(config.config_calibre_dir, book.path, "metadata.opf") + with open(path_opf, 'w') as fd: + shutil.copyfileobj(p.stdout, fd) + for entry in iter(book.data): if entry.format.upper() == book_format.upper(): converted_file_name = entry.name + '.' + book_format.lower() diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 79d7ddbd9..23af302a7 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -93,6 +93,16 @@ class TaskConvert(CalibreTask): # todo: figure out how to incorporate this into the progress try: EmailText = N_(u"%(book)s send to Kindle", book=escape(self.title)) + # ToDo: Delete when OPF creation has been implemented + if config.config_binariesdir: + quotes = [3, 5] + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_opf = os.path.join(config.config_calibre_dir, cur_book.path, "metadata.opf") + with open(path_opf, 'w') as fd: + copyfileobj(p.stdout, fd) worker_thread.add(self.user, TaskEmail(self.settings['subject'], self.results["path"], filename, diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index be240c795..6c94faeef 100755 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -20,6 +20,7 @@ import os import smtplib import threading import socket +from shutil import copy import mimetypes from io import StringIO @@ -32,6 +33,8 @@ from email.utils import formatdate from cps.services.worker import CalibreTask from cps.services import gmail from cps import logger, config +from cps.subproc_wrapper import process_open +from cps.constants import SUPPORTED_CALIBRE_BINARIES from cps import gdriveutils import uuid @@ -245,15 +248,23 @@ class TaskEmail(CalibreTask): df.GetContentFile(datafile) else: return None + if config.config_binariesdir: + datafile = cls._embed_metadata(calibre_path, book_path, filename, datafile) + os.remove(os.path.join(calibre_path, book_path, filename)) file_ = open(datafile, 'rb') data = file_.read() file_.close() os.remove(datafile) else: + datafile = os.path.join(calibre_path, book_path, filename) try: - file_ = open(os.path.join(calibre_path, book_path, filename), 'rb') + if config.config_binariesdir: + datafile = cls._embed_metadata(calibre_path, book_path, filename, datafile) + file_ = open(datafile, 'rb') data = file_.read() file_.close() + if config.config_binariesdir: + os.remove(datafile) except IOError as e: log.error_or_exception(e, stacklevel=3) log.error(u'The requested file could not be read. Maybe wrong permissions?') @@ -270,3 +281,17 @@ class TaskEmail(CalibreTask): def __str__(self): return "E-mail {}, {}".format(self.name, self.subject) + + def _embed_metadata(self, calibre_path, book_path, filename, datafile): + datafile_tmp = os.path.join(calibre_path, book_path, "tmp_" + filename) + path_opf = os.path.join(calibre_path, book_path, "metadata.opf") + copy(datafile, datafile_tmp) + + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["ebook-meta"]) + opf_command = [calibredb_binarypath, datafile_tmp, "--from-opf", path_opf] + p = process_open(opf_command) + _, err = p.communicate() + if err: + # ToDo: Improve error handling + log.error('Metadata embedder encountered an error: %s', err) + return datafile_tmp From b7aaa0f24dd5142b05fff19b58913930f9910b28 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 2 Nov 2023 17:05:02 +0100 Subject: [PATCH 15/93] Add metadata change code --- cps/admin.py | 10 ++++- cps/config_sql.py | 40 ++++++++++++----- cps/constants.py | 5 +++ cps/helper.py | 82 +++++++++++++++++++++++++++++++--- cps/tasks/convert.py | 34 +++++++++++--- cps/templates/config_edit.html | 10 ++--- 6 files changed, 152 insertions(+), 29 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 045a95236..51fddbeec 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -1761,8 +1761,14 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") - _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 21644ccd6..485c3fc27 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -138,6 +139,7 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) @@ -184,9 +186,11 @@ class ConfigSQL(object): self.load() change = False - if self.config_converterpath == None: # pylint: disable=access-member-before-definition + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -469,17 +473,32 @@ def _migrate_table(session, orm_class, secret_key=None): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -521,6 +540,7 @@ def load_configuration(session, secret_key): session.commit() + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 18c4f1b1c..d8842e78b 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) diff --git a/cps/helper.py b/cps/helper.py index 0c526d01a..cabc03632 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,8 +54,8 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES -from .subproc_wrapper import process_wait +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -938,9 +938,10 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: return gd.do_gdrive_download(df, headers) @@ -948,20 +949,47 @@ def do_download_file(book, book_format, client, data, headers): abort(404) else: filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) + if config.config_binariesdir: + filename, book_name = do_calibre_export(book, book_format) + + response = make_response(send_from_directory(filename, book_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) return response + +def do_calibre_export(book, book_format): + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + + ################################## @@ -984,6 +1012,35 @@ def check_unrar(unrar_location): return _('Error executing UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" @@ -1047,6 +1104,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index df6ae1043..2bef9a209 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,10 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +from tempfile import gettempdir +from time import time from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -35,10 +37,11 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils - +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): @@ -61,15 +64,20 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if df: + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df and df_cover: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) @@ -79,6 +87,7 @@ class TaskConvert(CalibreTask): filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -225,15 +234,30 @@ class TaskConvert(CalibreTask): return check, None def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters # windows py2.7 encode as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux - quotes = [1, 2] + + quotes = [3, 5] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] + (file_path + format_new_ext), '--from-opf', path_tmp_opf, + '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index d101f9608..2ec0575cf 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -323,12 +323,12 @@
- +
- - - - + + + +
From cf6810db87ffd7d8f4ce48c707561e72789e3dd6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 11 Nov 2023 14:48:59 +0100 Subject: [PATCH 16/93] Refactored get_temp_dir bugfixes export_metadata --- cps/file_helper.py | 26 +++++++++++++++++++++++ cps/gdrive.py | 6 ++---- cps/gdriveutils.py | 1 - cps/helper.py | 49 +++++++++++++++++++++++--------------------- cps/tasks/convert.py | 9 ++++---- cps/updater.py | 6 +++--- cps/uploader.py | 7 ++----- 7 files changed, 64 insertions(+), 40 deletions(-) create mode 100644 cps/file_helper.py diff --git a/cps/file_helper.py b/cps/file_helper.py new file mode 100644 index 000000000..717cbb7d6 --- /dev/null +++ b/cps/file_helper.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tempfile import gettempdir +import os + +def get_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + return tmp_dir diff --git a/cps/gdrive.py b/cps/gdrive.py index 832350e13..4d110f834 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -23,7 +23,6 @@ import os import hashlib import json -import tempfile from uuid import uuid4 from time import time from shutil import move, copyfile @@ -34,6 +33,7 @@ from flask_login import login_required from . import logger, gdriveutils, config, ub, calibre_db, csrf from .admin import admin_required +from .file_helper import get_temp_dir gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive') log = logger.create() @@ -139,9 +139,7 @@ try: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() if not response['deleted'] and response['file']['title'] == 'metadata.db' \ and response['file']['md5Checksum'] != hashlib.md5(dbpath): # nosec - tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time()))) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 08ead47dc..b1d30596f 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -34,7 +34,6 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError from sqlalchemy.orm.exc import StaleDataError -from sqlalchemy.sql.expression import text try: from httplib2 import __version__ as httplib2_version diff --git a/cps/helper.py b/cps/helper.py index cabc03632..013a6fc2f 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -25,9 +25,9 @@ import re import shutil import socket from datetime import datetime, timedelta -from tempfile import gettempdir import requests import unidecode +from uuid import uuid4 from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ @@ -60,6 +60,7 @@ from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails from .tasks.metadata_backup import TaskBackupMetadata +from .file_helper import get_temp_dir log = logger.create() @@ -921,10 +922,7 @@ def save_cover(img, book_path): return False, _("Only jpg/jpeg files are supported as coverfile") if config.config_use_google_drive: - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img) if ret is True: gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"), @@ -944,7 +942,13 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - return gd.do_gdrive_download(df, headers) + # ToDo check:!!!!!!!! + if config.config_binariesdir: + output = os.path.join(config.config_calibre_dir, book.path, data.name) + gd.ownloadFile(book.path, book_name + "." + book_format, output) + filename, download_name = do_calibre_export(book, book_format) + else: + return gd.do_gdrive_download(df, headers) else: abort(404) else: @@ -957,34 +961,33 @@ def do_download_file(book, book_format, client, data, headers): headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") if config.config_binariesdir: - filename, book_name = do_calibre_export(book, book_format) + filename, download_name = do_calibre_export(book, book_format) + else: + download_name = book_name + + response = make_response(send_from_directory(filename, download_name + "." + book_format)) + # ToDo Check headers parameter + for element in headers: + response.headers[element[0]] = element[1] + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) + return response - response = make_response(send_from_directory(filename, book_name + "." + book_format)) - # ToDo Check headers parameter - for element in headers: - response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) - return response def do_calibre_export(book, book_format): try: quotes = [3, 5, 7, 9] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() calibredb_binarypath = get_calibre_binarypath("calibredb") - opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), - '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, - '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] - file_name = book.title - if len(book.authors) > 0: - file_name = file_name + ' - ' + book.authors[0].name + temp_file_name = str(uuid4()) + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir, + '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name), + str(book.id)] p = process_open(opf_command, quotes) _, err = p.communicate() if err: log.error('Metadata embedder encountered an error: %s', err) - return tmp_dir, file_name + return tmp_dir, temp_file_name except OSError as ex: # ToDo real error handling log.error_or_exception(ex) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 2bef9a209..61a2bda07 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -21,7 +21,6 @@ import re from glob import glob from shutil import copyfile, copyfileobj from markupsafe import escape -from tempfile import gettempdir from time import time from sqlalchemy.exc import SQLAlchemyError @@ -34,6 +33,7 @@ from cps.subproc_wrapper import process_open from flask_babel import gettext as _ from cps.kobo_sync_status import remove_synced_book from cps.ub import init_db_thread +from cps.file_helper import get_temp_dir from cps.tasks.mail import TaskEmail from cps import gdriveutils @@ -243,9 +243,10 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [3, 5] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() + #tmp_dir = os.path.join(gettempdir(), 'calibre_web') + #if not os.path.isdir(tmp_dir): + # os.mkdir(tmp_dir) calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) diff --git a/cps/updater.py b/cps/updater.py index 6d6e408f0..4369e18f0 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -25,13 +25,13 @@ import threading import time import zipfile from io import BytesIO -from tempfile import gettempdir - import requests + from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server +from .file_helper import gettempdir log = logger.create() @@ -85,7 +85,7 @@ class Updater(threading.Thread): z = zipfile.ZipFile(BytesIO(r.content)) self.status = 3 log.debug('Extracting zipfile') - tmp_dir = gettempdir() + tmp_dir = get_temp_dir() z.extractall(tmp_dir) folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1] if not os.path.isdir(folder_name): diff --git a/cps/uploader.py b/cps/uploader.py index 23dfc4a6b..8f20762f2 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -18,12 +18,12 @@ import os import hashlib -from tempfile import gettempdir from flask_babel import gettext as _ from . import logger, comic, isoLanguages from .constants import BookMeta from .helper import split_authors +from .file_helper import get_temp_dir log = logger.create() @@ -249,10 +249,7 @@ def get_magick_version(): def upload(uploadfile, rar_excecutable): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() filename = uploadfile.filename filename_root, file_extension = os.path.splitext(filename) From 7fd1d10fca3220329d6d762d3851486150e93405 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 11 Nov 2023 15:26:05 +0100 Subject: [PATCH 17/93] Implement gdrive metadata on download --- cps/helper.py | 9 +++++---- cps/updater.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 013a6fc2f..ab3123ec1 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -942,10 +942,12 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - # ToDo check:!!!!!!!! if config.config_binariesdir: - output = os.path.join(config.config_calibre_dir, book.path, data.name) - gd.ownloadFile(book.path, book_name + "." + book_format, output) + output_path = os.path.join(config.config_calibre_dir, book.path) + if not os.path.exists(output_path): + os.makedirs(output_path) + output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) + gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) else: return gd.do_gdrive_download(df, headers) @@ -973,7 +975,6 @@ def do_download_file(book, book_format, client, data, headers): return response - def do_calibre_export(book, book_format): try: quotes = [3, 5, 7, 9] diff --git a/cps/updater.py b/cps/updater.py index 4369e18f0..8150aa0df 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -31,7 +31,7 @@ from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server -from .file_helper import gettempdir +from .file_helper import get_temp_dir log = logger.create() From 794cd354ca8be3eb307c7c1dd7e913b81f0e8c81 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 12 Nov 2023 11:02:30 +0100 Subject: [PATCH 18/93] Added ToDo's --- cps/helper.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index ab3123ec1..c06ab4021 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -949,6 +949,7 @@ def do_download_file(book, book_format, client, data, headers): output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) + # ToDo: delete path in calibre-folder structure else: return gd.do_gdrive_download(df, headers) else: @@ -1027,17 +1028,21 @@ def check_calibre(calibre_location): return _('Please specify a directory, not a file') try: - supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] - binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + supported_binary_paths = [os.path.join(calibre_location, binary) + for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) + for binary_path in supported_binary_paths] if all(binaries_available): - values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') + for binary_path in supported_binary_paths] if all(values): version = values[0].group(1) log.debug("calibre version %s", version) else: return _('Calibre binaries not viable') else: - missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + missing_binaries=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: From 7fbbb85f475e00af351bff8f27172747c321f28e Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 4 Dec 2023 19:26:43 +0100 Subject: [PATCH 19/93] Temp folder is deleted on regular base --- cps/file_helper.py | 6 ++++++ cps/schedule.py | 6 ++++++ cps/tasks/tempFolder.py | 47 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 cps/tasks/tempFolder.py diff --git a/cps/file_helper.py b/cps/file_helper.py index 717cbb7d6..7c3e52917 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -18,9 +18,15 @@ from tempfile import gettempdir import os +import shutil def get_temp_dir(): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) return tmp_dir + + +def del_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + shutil.rmtree(tmp_dir) diff --git a/cps/schedule.py b/cps/schedule.py index 05367e995..c1c4aafcb 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,6 +21,7 @@ import datetime from . import config, constants from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler from .tasks.database import TaskReconnectDatabase +from .tasks.tempFolder import TaskDeleteTempFolder from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache from .services.worker import WorkerThread from .tasks.metadata_backup import TaskBackupMetadata @@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True): if reconnect: tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) + # Delete temp folder + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', False]) + # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False]) @@ -86,6 +90,8 @@ def register_startup_tasks(): # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + else: + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', False]]) def should_task_be_running(start, duration): diff --git a/cps/tasks/tempFolder.py b/cps/tasks/tempFolder.py new file mode 100644 index 000000000..e740cd1e8 --- /dev/null +++ b/cps/tasks/tempFolder.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from urllib.request import urlopen + +from flask_babel import lazy_gettext as N_ + +from cps import logger, file_helper +from cps.services.worker import CalibreTask + + +class TaskDeleteTempFolder(CalibreTask): + def __init__(self, task_message=N_('Delete temp folder contents')): + super(TaskDeleteTempFolder, self).__init__(task_message) + self.log = logger.create() + + def run(self, worker_thread): + try: + file_helper.del_temp_dir() + except FileNotFoundError: + pass + except (PermissionError, OSError) as e: + self.log.error("Error deleting temp folder: {}".format(e)) + self._handleSuccess() + + @property + def name(self): + return "Delete Temp Folder" + + @property + def is_cancellable(self): + return False From fd90d6e3754ff3004678e44a6c406f727eb36942 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 6 Dec 2023 19:20:42 +0100 Subject: [PATCH 20/93] Delete temp dir not shown in tasks overview Handle case no cover during convert ebook --- cps/schedule.py | 4 ++-- cps/tasks/convert.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cps/schedule.py b/cps/schedule.py index c1c4aafcb..bf622b36a 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -33,7 +33,7 @@ def get_scheduled_tasks(reconnect=True): tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) # Delete temp folder - tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', False]) + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True]) # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: @@ -91,7 +91,7 @@ def register_startup_tasks(): if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) else: - scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', False]]) + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]]) def should_task_be_running(start, duration): diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 61a2bda07..5bee07736 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -49,6 +49,7 @@ class TaskConvert(CalibreTask): self.file_path = file_path self.book_id = book_id self.title = "" + self.has_cover = None self.settings = settings self.ereader_mail = ereader_mail self.user = user @@ -161,7 +162,8 @@ class TaskConvert(CalibreTask): if not os.path.exists(config.config_converterpath): self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) return - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext) + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) if check == 0: cur_book = local_db.get_book(book_id) @@ -233,7 +235,7 @@ class TaskConvert(CalibreTask): folder=os.path.dirname(file_path)) return check, None - def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters @@ -257,8 +259,9 @@ class TaskConvert(CalibreTask): quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf, - '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] + (file_path + format_new_ext), '--from-opf', path_tmp_opf] + if has_cover: + command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 90ad570578ba8d2aad45c61e79ff26519e8ceba0 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 7 Dec 2023 16:22:34 +0100 Subject: [PATCH 21/93] Show only folders for selecting converter binaries --- cps/templates/config_edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 2ec0575cf..736062f44 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -327,7 +327,7 @@
- +
From 2334e8f9c958f8f933f92dbbece197dda6dd3f27 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 7 Dec 2023 16:47:10 +0100 Subject: [PATCH 22/93] Refactored calibre executable detection for better error messages --- cps/helper.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index c06ab4021..a8acc7502 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -1030,9 +1030,9 @@ def check_calibre(calibre_location): try: supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] - binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) - for binary_path in supported_binary_paths] - if all(binaries_available): + binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths] + binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available) and all(binaries_executable): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): @@ -1041,9 +1041,17 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: + ret_val = [] missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] - return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) + + missing_perms=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available] + if missing_binaries: + ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))) + if missing_perms: + ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms))) + return ", ".join(ret_val) except (OSError, UnicodeDecodeError) as err: log.error_or_exception(err) From d341faf204a73ecfe17c988cc43cc50a9b4cbbb0 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 9 Dec 2023 09:36:28 +0100 Subject: [PATCH 23/93] Make Version setuptools compatible and still have the "Beta" in the User interface --- cps/about.py | 4 ++-- cps/admin.py | 2 +- cps/cli.py | 4 ++-- cps/constants.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cps/about.py b/cps/about.py index 7b6cc71af..1d081fe27 100644 --- a/cps/about.py +++ b/cps/about.py @@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol def collect_stats(): if constants.NIGHTLY_VERSION[0] == "$Format:%H$": - calibre_web_version = constants.STABLE_VERSION['version'] + calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta") else: - calibre_web_version = (constants.STABLE_VERSION['version'] + ' - ' + calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - ' + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - ' + constants.NIGHTLY_VERSION[1].replace('%', '%%')) diff --git a/cps/admin.py b/cps/admin.py index 51fddbeec..a933f8bc4 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -214,7 +214,7 @@ def admin(): form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:])) commit = format_datetime(form_date - tz, format='short') else: - commit = version['version'] + commit = version['version'].replace("b", " Beta") all_user = ub.session.query(ub.User).all() # email_settings = mail_config.get_mail_settings() diff --git a/cps/cli.py b/cps/cli.py index e9b97b9de..855ad8990 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE def version_info(): if _NIGHTLY_VERSION[1].startswith('$Format'): - return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'] - return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) + return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta") + return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1]) class CliParameter(object): diff --git a/cps/constants.py b/cps/constants.py index d8842e78b..09f5cf535 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -173,7 +173,7 @@ def selected_roles(dictionary): BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, ' 'series_id, languages, publisher, pubdate, identifiers') -STABLE_VERSION = {'version': '0.6.22 Beta'} +STABLE_VERSION = {'version': '0.6.22b'} NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' From 1086296d1dd44854493fd8c9d673ac7fd698f22b Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 9 Dec 2023 11:23:26 +0100 Subject: [PATCH 24/93] Make embed metadata configurable --- cps/admin.py | 1 + cps/config_sql.py | 1 + cps/helper.py | 5 ++--- cps/tasks/convert.py | 33 ++++++++++++++------------------- cps/templates/config_edit.html | 4 ++++ 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index a933f8bc4..3af3e4227 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1745,6 +1745,7 @@ def _configuration_update_helper(): _config_checkbox_int(to_save, "config_uploading") _config_checkbox_int(to_save, "config_unicode_filename") + _config_checkbox_int(to_save, "config_embed_metadata") # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse") and config.config_login_type == constants.LOGIN_LDAP) diff --git a/cps/config_sql.py b/cps/config_sql.py index 485c3fc27..ac8f7b5d0 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -144,6 +144,7 @@ class _Settings(_Base): config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_unicode_filename = Column(Boolean, default=False) + config_embed_metadata = Column(Boolean, default=True) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) diff --git a/cps/helper.py b/cps/helper.py index a8acc7502..326e2355d 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -942,14 +942,13 @@ def do_download_file(book, book_format, client, data, headers): df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - if config.config_binariesdir: + if config.config_binariesdir and config.config_embed_metadata: output_path = os.path.join(config.config_calibre_dir, book.path) if not os.path.exists(output_path): os.makedirs(output_path) output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) gd.downloadFile(book.path, book_name + "." + book_format, output) filename, download_name = do_calibre_export(book, book_format) - # ToDo: delete path in calibre-folder structure else: return gd.do_gdrive_download(df, headers) else: @@ -963,7 +962,7 @@ def do_download_file(book, book_format, client, data, headers): if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - if config.config_binariesdir: + if config.config_binariesdir and config.config_embed_metadata: filename, download_name = do_calibre_export(book, book_format) else: download_name = book_name diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 5bee07736..a39b3f612 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -238,28 +238,23 @@ class TaskConvert(CalibreTask): def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): book_id = self.book_id try: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - - quotes = [3, 5] - tmp_dir = get_temp_dir() - #tmp_dir = os.path.join(gettempdir(), 'calibre_web') - #if not os.path.isdir(tmp_dir): - # os.mkdir(tmp_dir) - calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) - opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] - p = process_open(opf_command, quotes) - p.wait() - path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") - with open(path_tmp_opf, 'w') as fd: - copyfileobj(p.stdout, fd) + if config.config_embed_metadata: + quotes = [3, 5] + tmp_dir = get_temp_dir() + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), + '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf] + (file_path + format_new_ext)] + if config.config_embed_metadata: + command.extend('--from-opf', path_tmp_opf) if has_cover: command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 736062f44..8e37b2e58 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -103,6 +103,10 @@
+
+ + +
From 9ef89dbcc383099bbcc5573e97eef73efb181b37 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 10 Dec 2023 07:26:26 +0100 Subject: [PATCH 25/93] Bugfix convert file from gdrive without cover Update Teststatus --- cps/tasks/convert.py | 19 +- test/Calibre-Web TestSummary_Linux.html | 6263 +++++++++++++++++++++-- 2 files changed, 5783 insertions(+), 499 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index a39b3f612..913b92dd9 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -66,29 +66,32 @@ class TaskConvert(CalibreTask): df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") - if df and df_cover: + if df: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - datafile_cover = os.path.join(config.config_calibre_dir, - cur_book.path, "cover.jpg") + if df_cover: + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) - df_cover.GetContentFile(datafile_cover) + if df_cover: + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: - # ToDo Include cover in error handling + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) worker_db.session.close() - return error_message + return self._handleError(self, error_message) filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) - os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -254,7 +257,7 @@ class TaskConvert(CalibreTask): command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext)] if config.config_embed_metadata: - command.extend('--from-opf', path_tmp_opf) + command.extend(['--from-opf', path_tmp_opf]) if has_cover: command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 7ca3dad54..03883e3bb 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2023-10-16 19:38:22

+

Start Time: 2023-12-10 09:21:07

-

Stop Time: 2023-10-17 02:18:49

+

Stop Time: 2023-12-10 14:53:07

-

Duration: 5h 37 min

+

Duration: 4h 31 min

@@ -236,13 +236,13 @@ TestBackupMetadata - 22 - 22 + 21 + 21 0 0 0 - Detail + Detail @@ -429,15 +429,6 @@ - -
TestBackupMetadata - test_gdrive
- - PASS - - - - -
TestBackupMetadata - test_upload_book
@@ -861,12 +852,12 @@ - + TestEbookConvertCalibreGDrive 6 - 6 - 0 - 0 + 4 + 1 + 1 0 Detail @@ -875,20 +866,62 @@ - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- PASS + +
+ FAIL +
+ + + + @@ -1932,13 +1965,13 @@ - + TestLoadMetadata 1 - 0 - 0 1 0 + 0 + 0 Detail @@ -1946,42 +1979,22 @@ - +
TestLoadMetadata - test_load_metadata
- -
- ERROR -
- - - - + PASS - + TestEditBooksOnGdrive 18 - 17 + 18 + 0 0 - 1 0 Detail @@ -2125,31 +2138,11 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_edit_title
- -
- ERROR -
- - - - + PASS @@ -2691,12 +2684,12 @@ KeyError: 'title' - + TestLdapLogin 13 - 13 - 0 + 11 0 + 2 0 Detail @@ -2804,20 +2797,91 @@ KeyError: 'title' - +
TestLdapLogin - test_ldap_opds_anonymous
- PASS + +
+ ERROR +
+ + + + - +
TestLdapLogin - test_ldap_opds_download_book
- PASS + +
+ ERROR +
+ + + + @@ -3092,15 +3156,15 @@ KeyError: 'title' - + TestLogin + 18 17 - 17 - 0 + 1 0 0 - Detail + Detail @@ -3241,7 +3305,36 @@ KeyError: 'title' - + + +
TestLogin - test_proxy_login_multi_user
+ + +
+ FAIL +
+ + + + + + + + +
TestLogin - test_proxy_login_opds
@@ -3250,7 +3343,7 @@ KeyError: 'title' - +
TestLogin - test_robots
@@ -3623,82 +3716,65 @@ KeyError: 'title' - + TestReader - 6 + 8 5 1 - 0 - 0 + 1 + 1 - Detail + Detail - + + +
TestReader - test_cb7_reader
+ + SKIP + + + + +
TestReader - test_comic_MACOS_files
- PASS - - - - - - -
TestReader - test_comic_reader
- - PASS - - - - - - -
TestReader - test_epub_reader
- - PASS - - - - - - -
TestReader - test_pdf_reader
- - PASS - - - - - - -
TestReader - test_sound_listener
-
- FAIL + ERROR
-