Merge branch 'Develop' into embed_metadata_on_convert

cps/MyLoginManager.py

@@ -21,15 +21,32 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 
-from flask_login import LoginManager
-from flask import session
-
+from flask_login import LoginManager, confirm_login
+from flask import session, current_app
+from flask_login.utils import decode_cookie
+from flask_login.signals import user_loaded_from_cookie
 
 class MyLoginManager(LoginManager):
     def _session_protection_failed(self):
-        _session = session._get_current_object()
+        sess = session._get_current_object()
         ident = self._session_identifier_generator()
-        if(_session and not (len(_session) == 1
-                             and _session.get('csrf_token', None))) and ident != _session.get('_id', None):
+        if(sess and not (len(sess) == 1
+                             and sess.get('csrf_token', None))) and ident != sess.get('_id', None):
             return super(). _session_protection_failed()
         return False
+
+    def _load_user_from_remember_cookie(self, cookie):
+        user_id = decode_cookie(cookie)
+        if user_id is not None:
+            session["_user_id"] = user_id
+            session["_fresh"] = False
+            user = None
+            if self._user_callback:
+                user = self._user_callback(user_id)
+            if user is not None:
+                app = current_app._get_current_object()
+                user_loaded_from_cookie.send(app, user=user)
+                # if session was restored from remember me cookie make login valid
+                confirm_login()
+                return user
+        return None

cps/__init__.py

@@ -36,11 +36,16 @@ from .reverseproxy import ReverseProxied
 from .server import WebServer
 from .dep_check import dependency_check
 from .updater import Updater
-from .babel import babel
+from .babel import babel, get_locale
 from . import config_sql
 from . import cache_buster
 from . import ub, db
 
+try:
+    from flask_limiter import Limiter
+    limiter_present = True
+except ImportError:
+    limiter_present = False
 try:
     from flask_wtf.csrf import CSRFProtect
     wtf_present = True
@@ -59,7 +64,8 @@ mimetypes.add_type('application/x-mobi8-ebook', '.azw3')
 mimetypes.add_type('application/x-cbr', '.cbr')
 mimetypes.add_type('application/x-cbz', '.cbz')
 mimetypes.add_type('application/x-cbt', '.cbt')
-mimetypes.add_type('image/vnd.djvu', '.djvu')
+mimetypes.add_type('application/x-cb7', '.cb7')
+mimetypes.add_type('image/vnd.djv', '.djv')
 mimetypes.add_type('application/mpeg', '.mpeg')
 mimetypes.add_type('application/mpeg', '.mp3')
 mimetypes.add_type('application/mp4', '.m4a')
@@ -81,10 +87,10 @@ app.config.update(
 
 lm = MyLoginManager()
 
-config = config_sql._ConfigSQL()
-
 cli_param = CliParameter()
 
+config = config_sql.ConfigSQL()
+
 if wtf_present:
     csrf = CSRFProtect()
 else:
@@ -96,33 +102,36 @@ web_server = WebServer()
 
 updater_thread = Updater()
 
+if limiter_present:
+    limiter = Limiter(key_func=True, headers_enabled=True, auto_check=False, swallow_errors=True)
+else:
+    limiter = None
 
 def create_app():
-    lm.login_view = 'web.login'
-    lm.anonymous_user = ub.Anonymous
-    lm.session_protection = 'strong'
-
     if csrf:
         csrf.init_app(app)
 
     cli_param.init()
 
-    ub.init_db(cli_param.settings_path, cli_param.user_credentials)
-
+    ub.init_db(cli_param.settings_path)
     # pylint: disable=no-member
-    config_sql.load_configuration(config, ub.session, cli_param)
+    encrypt_key, error = config_sql.get_encryption_key(os.path.dirname(cli_param.settings_path))
 
-    db.CalibreDB.update_config(config)
-    db.CalibreDB.setup_db(config.config_calibre_dir, cli_param.settings_path)
-    calibre_db.init_db()
+    config_sql.load_configuration(ub.session, encrypt_key)
+    config.init_config(ub.session, encrypt_key, cli_param)
 
-    updater_thread.init_updater(config, web_server)
-    # Perform dry run of updater and exit afterwards
-    if cli_param.dry_run:
-        updater_thread.dry_run()
-        sys.exit(0)
-    updater_thread.start()
+    if error:
+        log.error(error)
 
+    ub.password_change(cli_param.user_credentials)
+
+    if not limiter:
+        log.info('*** "flask-limiter" is needed for calibre-web to run. '
+                 'Please install it using pip: "pip install flask-limiter" ***')
+        print('*** "flask-limiter" is needed for calibre-web to run. '
+              'Please install it using pip: "pip install flask-limiter" ***')
+        web_server.stop(True)
+        sys.exit(8)
     if sys.version_info < (3, 0):
         log.info(
             '*** Python2 is EOL since end of 2019, this version of Calibre-Web is no longer supporting Python2, '
@@ -139,8 +148,24 @@ def create_app():
               'Please install it using pip: "pip install flask-WTF" ***')
         web_server.stop(True)
         sys.exit(7)
+
+    lm.login_view = 'web.login'
+    lm.anonymous_user = ub.Anonymous
+    lm.session_protection = 'strong' if config.config_session == 1 else "basic"
+
+    db.CalibreDB.update_config(config)
+    db.CalibreDB.setup_db(config.config_calibre_dir, cli_param.settings_path)
+    calibre_db.init_db()
+
+    updater_thread.init_updater(config, web_server)
+    # Perform dry run of updater and exit afterwards
+    if cli_param.dry_run:
+        updater_thread.dry_run()
+        sys.exit(0)
+    updater_thread.start()
+
     for res in dependency_check() + dependency_check(True):
-        log.info('*** "{}" version does not fit the requirements. '
+        log.info('*** "{}" version does not meet the requirements. '
                  'Should: {}, Found: {}, please consider installing required version ***'
                  .format(res['name'],
                          res['target'],
@@ -150,14 +175,16 @@ def create_app():
     if os.environ.get('FLASK_DEBUG'):
         cache_buster.init_cache_busting(app)
     log.info('Starting Calibre Web...')
-
     Principal(app)
     lm.init_app(app)
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-
-    babel.init_app(app)
+    if hasattr(babel, "localeselector"):
+        babel.init_app(app)
+        babel.localeselector(get_locale)
+    else:
+        babel.init_app(app, locale_selector=get_locale)
 
     from . import services
 
@@ -165,9 +192,13 @@ def create_app():
         services.ldap.init_app(app, config)
     if services.goodreads_support:
         services.goodreads_support.connect(config.config_goodreads_api_key,
-                                           config.config_goodreads_api_secret,
+                                           config.config_goodreads_api_secret_e,
                                            config.config_use_goodreads)
     config.store_calibre_uuid(calibre_db, db.Library_Id)
+    # Configure rate limiter
+    app.config.update(RATELIMIT_ENABLED=config.config_ratelimiter)
+    limiter.init_app(app)
+
     # Register scheduled tasks
     from .schedule import register_scheduled_tasks, register_startup_tasks
     register_scheduled_tasks(config.schedule_reconnect)

cps/about.py

@@ -25,7 +25,6 @@ import platform
 import sqlite3
 from collections import OrderedDict
 
-import werkzeug
 import flask
 import flask_login
 import jinja2
@@ -37,41 +36,40 @@ from .render_template import render_title_template
 
 about = flask.Blueprint('about', __name__)
 
-ret = dict()
-req = dep_check.load_dependencys(False)
-opt = dep_check.load_dependencys(True)
+modules = dict()
+req = dep_check.load_dependencies(False)
+opt = dep_check.load_dependencies(True)
 for i in (req + opt):
-    ret[i[1]] = i[0]
-
-if constants.NIGHTLY_VERSION[0] == "$Format:%H$":
-    calibre_web_version = constants.STABLE_VERSION['version']
-else:
-    calibre_web_version = (constants.STABLE_VERSION['version'] + ' - '
-                           + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - '
-                           + constants.NIGHTLY_VERSION[1].replace('%', '%%'))
-
-if getattr(sys, 'frozen', False):
-    calibre_web_version += " - Exe-Version"
-elif constants.HOME_CONFIG:
-    calibre_web_version += " - pyPi"
-
-_VERSIONS = OrderedDict(
-    Platform='{0[0]} {0[2]} {0[3]} {0[4]} {0[5]}'.format(platform.uname()),
-    Python=sys.version,
-    Calibre_Web=calibre_web_version,
-    Werkzeug=werkzeug.__version__,
-    Jinja2=jinja2.__version__,
-    pySqlite=sqlite3.version,
-    SQLite=sqlite3.sqlite_version,
-)
-_VERSIONS.update(ret)
-_VERSIONS.update(uploader.get_versions())
+    modules[i[1]] = i[0]
+modules['Jinja2'] = jinja2.__version__
+modules['pySqlite'] = sqlite3.version
+modules['SQLite'] = sqlite3.sqlite_version
+sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefold())))
 
 
 def collect_stats():
-    _VERSIONS['ebook converter'] = converter.get_calibre_version()
-    _VERSIONS['unrar'] = converter.get_unrar_version()
-    _VERSIONS['kepubify'] = converter.get_kepubify_version()
+    if constants.NIGHTLY_VERSION[0] == "$Format:%H$":
+        calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta")
+    else:
+        calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - '
+                               + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - '
+                               + constants.NIGHTLY_VERSION[1].replace('%', '%%'))
+
+    if getattr(sys, 'frozen', False):
+        calibre_web_version += " - Exe-Version"
+    elif constants.HOME_CONFIG:
+        calibre_web_version += " - pyPi"
+
+    _VERSIONS = {'Calibre Web': calibre_web_version}
+    _VERSIONS.update(OrderedDict(
+        Python=sys.version,
+        Platform='{0[0]} {0[2]} {0[3]} {0[4]} {0[5]}'.format(platform.uname()),
+    ))
+    _VERSIONS.update(uploader.get_magick_version())
+    _VERSIONS['Unrar'] = converter.get_unrar_version()
+    _VERSIONS['Ebook converter'] = converter.get_calibre_version()
+    _VERSIONS['Kepubify'] = converter.get_kepubify_version()
+    _VERSIONS.update(sorted_modules)
     return _VERSIONS
 
 
@@ -80,7 +78,7 @@ def collect_stats():
 def stats():
     counter = calibre_db.session.query(db.Books).count()
     authors = calibre_db.session.query(db.Authors).count()
-    categorys = calibre_db.session.query(db.Tags).count()
+    categories = calibre_db.session.query(db.Tags).count()
     series = calibre_db.session.query(db.Series).count()
     return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=collect_stats(),
-                                 categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")
+                                 categorycounter=categories, seriecounter=series, title=_("Statistics"), page="stat")

cps/admin.py

@@ -22,17 +22,21 @@
 
 import os
 import re
-import base64
 import json
 import operator
-from datetime import datetime, timedelta, time
+import time
+import sys
+import string
+from datetime import datetime, timedelta
+from datetime import time as datetime_time
 from functools import wraps
-
+from urllib.parse import urlparse
 
 from flask import Blueprint, flash, redirect, url_for, abort, request, make_response, send_from_directory, g, Response
-from flask_login import login_required, current_user, logout_user, confirm_login
+from markupsafe import Markup
+from flask_login import login_required, current_user, logout_user
 from flask_babel import gettext as _
-from flask_babel import get_locale,  format_time, format_datetime, format_timedelta
+from flask_babel import get_locale, format_time, format_datetime, format_timedelta
 from flask import session as flask_session
 from sqlalchemy import and_
 from sqlalchemy.orm.attributes import flag_modified
@@ -50,27 +54,28 @@ from .services.worker import WorkerThread
 from .babel import get_available_translations, get_available_locale, get_user_locale_language
 from . import debug_info
 
-
 log = logger.create()
 
 feature_support = {
-        'ldap': bool(services.ldap),
-        'goodreads': bool(services.goodreads_support),
-        'kobo':  bool(services.kobo),
-        'updater': constants.UPDATER_AVAILABLE,
-        'gmail': bool(services.gmail),
-        'scheduler': schedule.use_APScheduler,
-        'gdrive': gdrive_support
-    }
+    'ldap': bool(services.ldap),
+    'goodreads': bool(services.goodreads_support),
+    'kobo': bool(services.kobo),
+    'updater': constants.UPDATER_AVAILABLE,
+    'gmail': bool(services.gmail),
+    'scheduler': schedule.use_APScheduler,
+    'gdrive': gdrive_support
+}
 
 try:
     import rarfile  # pylint: disable=unused-import
+
     feature_support['rar'] = True
 except (ImportError, SyntaxError):
     feature_support['rar'] = False
 
 try:
     from .oauth_bb import oauth_check, oauthblueprints
+
     feature_support['oauth'] = True
 except ImportError as err:
     log.debug('Cannot import Flask-Dance, login with Oauth will not work: %s', err)
@@ -78,7 +83,6 @@ except ImportError as err:
     oauthblueprints = []
     oauth_check = {}
 
-
 admi = Blueprint('admin', __name__)
 
 
@@ -98,25 +102,26 @@ def admin_required(f):
 
 @admi.before_app_request
 def before_request():
-    # make remember me function work
-    if current_user.is_authenticated:
-        confirm_login()
-    if not ub.check_user_session(current_user.id, flask_session.get('_id')) and 'opds' not in request.path:
-        logout_user()
+    try:
+        if not ub.check_user_session(current_user.id,
+                                     flask_session.get('_id')) and 'opds' not in request.path \
+          and config.config_session == 1:
+            logout_user()
+    except AttributeError:
+        pass    # ? fails on requesting /ajax/emailstat during restart ?
     g.constants = constants
-    g.user = current_user
+    g.google_site_verification = os.getenv('GOOGLE_SITE_VERIFICATION', '')
     g.allow_registration = config.config_public_reg
     g.allow_anonymous = config.config_anonbrowse
     g.allow_upload = config.config_uploading
     g.current_theme = config.config_theme
     g.config_authors_max = config.config_authors_max
-    g.shelves_access = ub.session.query(ub.Shelf).filter(
-        or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == current_user.id)).order_by(ub.Shelf.name).all()
     if '/static/' not in request.path and not config.db_configured and \
         request.endpoint not in ('admin.ajax_db_config',
                                  'admin.simulatedbchange',
                                  'admin.db_configuration',
                                  'web.login',
+                                 'web.login_post',
                                  'web.logout',
                                  'admin.load_dialogtexts',
                                  'admin.ajax_pathchooser'):
@@ -134,31 +139,42 @@ def admin_forbidden():
 @admin_required
 def shutdown():
     task = request.get_json().get('parameter', -1)
-    showtext = {}
+    show_text = {}
     if task in (0, 1):  # valid commandos received
         # close all database connections
         calibre_db.dispose()
         ub.dispose()
 
         if task == 0:
-            showtext['text'] = _(u'Server restarted, please reload page')
+            show_text['text'] = _('Server restarted, please reload page.')
         else:
-            showtext['text'] = _(u'Performing shutdown of server, please close window')
+            show_text['text'] = _('Performing Server shutdown, please close window.')
         # stop gevent/tornado server
         web_server.stop(task == 0)
-        return json.dumps(showtext)
+        return json.dumps(show_text)
 
     if task == 2:
         log.warning("reconnecting to calibre database")
         calibre_db.reconnect_db(config, ub.app_DB_path)
-        showtext['text'] = _(u'Reconnect successful')
-        return json.dumps(showtext)
+        show_text['text'] = _('Success! Database Reconnected')
+        return json.dumps(show_text)
 
-    showtext['text'] = _(u'Unknown command')
-    return json.dumps(showtext), 400
+    show_text['text'] = _('Unknown command')
+    return json.dumps(show_text), 400
 
 
-# method is available without login and not protected by CSRF to make it easy reachable, is per default switched of
+@admi.route("/metadata_backup", methods=["POST"])
+@login_required
+@admin_required
+def queue_metadata_backup():
+    show_text = {}
+    log.warning("Queuing all books for metadata backup")
+    helper.set_all_metadata_dirty()
+    show_text['text'] = _('Success! Books queued for Metadata Backup, please check Tasks for result')
+    return json.dumps(show_text)
+
+
+# method is available without login and not protected by CSRF to make it easy reachable, is per default switched off
 # needed for docker applications, as changes on metadata.db from host are not visible to application
 @admi.route("/reconnect", methods=['GET'])
 def reconnect():
@@ -187,32 +203,32 @@ def update_thumbnails():
 def admin():
     version = updater_thread.get_current_version_info()
     if version is False:
-        commit = _(u'Unknown')
+        commit = _('Unknown')
     else:
         if 'datetime' in version:
             commit = version['datetime']
 
             tz = timedelta(seconds=time.timezone if (time.localtime().tm_isdst == 0) else time.altzone)
             form_date = datetime.strptime(commit[:19], "%Y-%m-%dT%H:%M:%S")
-            if len(commit) > 19:    # check if string has timezone
+            if len(commit) > 19:  # check if string has timezone
                 if commit[19] == '+':
                     form_date -= timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
                 elif commit[19] == '-':
                     form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
             commit = format_datetime(form_date - tz, format='short')
         else:
-            commit = version['version']
+            commit = version['version'].replace("b", " Beta")
 
     all_user = ub.session.query(ub.User).all()
-    email_settings = config.get_mail_settings()
-    schedule_time = format_time(time(hour=config.schedule_start_time), format="short")
+    # email_settings = mail_config.get_mail_settings()
+    schedule_time = format_time(datetime_time(hour=config.schedule_start_time), format="short")
     t = timedelta(hours=config.schedule_duration // 60, minutes=config.schedule_duration % 60)
     schedule_duration = format_timedelta(t, threshold=.99)
 
-    return render_title_template("admin.html", allUser=all_user, email=email_settings, config=config, commit=commit,
+    return render_title_template("admin.html", allUser=all_user, config=config, commit=commit,
                                  feature_support=feature_support, schedule_time=schedule_time,
                                  schedule_duration=schedule_duration,
-                                 title=_(u"Admin page"), page="admin")
+                                 title=_("Admin page"), page="admin")
 
 
 @admi.route("/admin/dbconfig", methods=["GET", "POST"])
@@ -232,7 +248,7 @@ def configuration():
                                  config=config,
                                  provider=oauthblueprints,
                                  feature_support=feature_support,
-                                 title=_(u"Basic Configuration"), page="config")
+                                 title=_("Basic Configuration"), page="config")
 
 
 @admi.route("/admin/ajaxconfig", methods=["POST"])
@@ -260,9 +276,9 @@ def calibreweb_alive():
 @login_required
 @admin_required
 def view_configuration():
-    read_column = calibre_db.session.query(db.CustomColumns)\
+    read_column = calibre_db.session.query(db.CustomColumns) \
         .filter(and_(db.CustomColumns.datatype == 'bool', db.CustomColumns.mark_for_delete == 0)).all()
-    restrict_columns = calibre_db.session.query(db.CustomColumns)\
+    restrict_columns = calibre_db.session.query(db.CustomColumns) \
         .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all()
     languages = calibre_db.speaking_language()
     translations = get_available_locale()
@@ -270,7 +286,7 @@ def view_configuration():
                                  restrictColumns=restrict_columns,
                                  languages=languages,
                                  translations=translations,
-                                 title=_(u"UI Configuration"), page="uiconfig")
+                                 title=_("UI Configuration"), page="uiconfig")
 
 
 @admi.route("/admin/usertable")
@@ -281,11 +297,11 @@ def edit_user_table():
     languages = calibre_db.speaking_language()
     translations = get_available_locale()
     all_user = ub.session.query(ub.User)
-    tags = calibre_db.session.query(db.Tags)\
-        .join(db.books_tags_link)\
-        .join(db.Books)\
+    tags = calibre_db.session.query(db.Tags) \
+        .join(db.books_tags_link) \
+        .join(db.Books) \
         .filter(calibre_db.common_filters()) \
-        .group_by(text('books_tags_link.tag'))\
+        .group_by(text('books_tags_link.tag')) \
         .order_by(db.Tags.name).all()
     if config.config_restricted_column:
         custom_values = calibre_db.session.query(db.cc_classes[config.config_restricted_column]).all()
@@ -304,7 +320,7 @@ def edit_user_table():
                                  all_roles=constants.ALL_ROLES,
                                  kobo_support=kobo_support,
                                  sidebar_settings=constants.sidebar_settings,
-                                 title=_(u"Edit Users"),
+                                 title=_("Edit Users"),
                                  page="usertable")
 
 
@@ -462,20 +478,20 @@ def edit_list_user(param):
                 elif param.endswith('role'):
                     value = int(vals['field_index'])
                     if user.name == "Guest" and value in \
-                            [constants.ROLE_ADMIN, constants.ROLE_PASSWD, constants.ROLE_EDIT_SHELFS]:
+                        [constants.ROLE_ADMIN, constants.ROLE_PASSWD, constants.ROLE_EDIT_SHELFS]:
                         raise Exception(_("Guest can't have this role"))
                     # check for valid value, last on checks for power of 2 value
-                    if value > 0 and value <= constants.ROLE_VIEWER and (value & value-1 == 0 or value == 1):
+                    if value > 0 and value <= constants.ROLE_VIEWER and (value & value - 1 == 0 or value == 1):
                         if vals['value'] == 'true':
                             user.role |= value
                         elif vals['value'] == 'false':
                             if value == constants.ROLE_ADMIN:
-                                if not ub.session.query(ub.User).\
-                                       filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
-                                              ub.User.id != user.id).count():
+                                if not ub.session.query(ub.User). \
+                                    filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
+                                           ub.User.id != user.id).count():
                                     return Response(
                                         json.dumps([{'type': "danger",
-                                                     'message': _(u"No admin user remaining, can't remove admin role",
+                                                     'message': _("No admin user remaining, can't remove admin role",
                                                                   nick=user.name)}]), mimetype='application/json')
                             user.role &= ~value
                         else:
@@ -487,7 +503,7 @@ def edit_list_user(param):
                     if user.name == "Guest" and value == constants.SIDEBAR_READ_AND_UNREAD:
                         raise Exception(_("Guest can't have this view"))
                     # check for valid value, last on checks for power of 2 value
-                    if value > 0 and value <= constants.SIDEBAR_LIST and (value & value-1 == 0 or value == 1):
+                    if value > 0 and value <= constants.SIDEBAR_LIST and (value & value - 1 == 0 or value == 1):
                         if vals['value'] == 'true':
                             user.sidebar_view |= value
                         elif vals['value'] == 'false':
@@ -552,13 +568,13 @@ def update_view_configuration():
         calibre_db.update_title_sort(config)
 
     if not check_valid_read_column(to_save.get("config_read_column", "0")):
-        flash(_(u"Invalid Read Column"), category="error")
+        flash(_("Invalid Read Column"), category="error")
         log.debug("Invalid Read column")
         return view_configuration()
     _config_int(to_save, "config_read_column")
 
     if not check_valid_restricted_column(to_save.get("config_restricted_column", "0")):
-        flash(_(u"Invalid Restricted Column"), category="error")
+        flash(_("Invalid Restricted Column"), category="error")
         log.debug("Invalid Restricted Column")
         return view_configuration()
     _config_int(to_save, "config_restricted_column")
@@ -578,7 +594,7 @@ def update_view_configuration():
         config.config_default_show |= constants.DETAIL_RANDOM
 
     config.save()
-    flash(_(u"Calibre-Web configuration updated"), category="success")
+    flash(_("Calibre-Web configuration updated"), category="success")
     log.debug("Calibre-Web configuration updated")
     before_request()
 
@@ -613,7 +629,8 @@ def load_dialogtexts(element_id):
     elif element_id == "db_submit":
         texts["main"] = _('Are you sure you want to change Calibre library location?')
     elif element_id == "admin_refresh_cover_cache":
-        texts["main"] = _('Calibre-Web will search for updated Covers and update Cover Thumbnails, this may take a while?')
+        texts["main"] = _('Calibre-Web will search for updated Covers '
+                          'and update Cover Thumbnails, this may take a while?')
     elif element_id == "btnfullsync":
         texts["main"] = _("Are you sure you want delete Calibre-Web's sync database "
                           "to force a full sync with your Kobo Reader?")
@@ -639,7 +656,7 @@ def edit_domain(allow):
 @admin_required
 def add_domain(allow):
     domain_name = request.form.to_dict()['domainname'].replace('*', '%').replace('?', '_').lower()
-    check = ub.session.query(ub.Registration).filter(ub.Registration.domain == domain_name)\
+    check = ub.session.query(ub.Registration).filter(ub.Registration.domain == domain_name) \
         .filter(ub.Registration.allow == allow).first()
     if not check:
         new_domain = ub.Registration(domain=domain_name, allow=allow)
@@ -744,6 +761,7 @@ def edit_restriction(res_type, user_id):
             ub.session_commit("Changed denied columns of user {} to {}".format(usr.name, usr.denied_column_value))
     return ""
 
+
 @admi.route("/ajax/addrestriction/<int:res_type>", methods=['POST'])
 @login_required
 @admin_required
@@ -856,16 +874,16 @@ def delete_restriction(res_type, user_id):
 @login_required
 @admin_required
 def list_restriction(res_type, user_id):
-    if res_type == 0:   # Tags as template
-        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd'+str(i)}
+    if res_type == 0:  # Tags as template
+        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd' + str(i)}
                     for i, x in enumerate(config.list_denied_tags()) if x != '']
-        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a'+str(i)}
+        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a' + str(i)}
                  for i, x in enumerate(config.list_allowed_tags()) if x != '']
         json_dumps = restrict + allow
     elif res_type == 1:  # CustomC as template
-        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd'+str(i)}
+        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd' + str(i)}
                     for i, x in enumerate(config.list_denied_column_values()) if x != '']
-        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a'+str(i)}
+        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a' + str(i)}
                  for i, x in enumerate(config.list_allowed_column_values()) if x != '']
         json_dumps = restrict + allow
     elif res_type == 2:  # Tags per user
@@ -873,9 +891,9 @@ def list_restriction(res_type, user_id):
             usr = ub.session.query(ub.User).filter(ub.User.id == user_id).first()
         else:
             usr = current_user
-        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd'+str(i)}
+        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd' + str(i)}
                     for i, x in enumerate(usr.list_denied_tags()) if x != '']
-        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a'+str(i)}
+        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a' + str(i)}
                  for i, x in enumerate(usr.list_allowed_tags()) if x != '']
         json_dumps = restrict + allow
     elif res_type == 3:  # CustomC per user
@@ -883,9 +901,9 @@ def list_restriction(res_type, user_id):
             usr = ub.session.query(ub.User).filter(ub.User.id == user_id).first()
         else:
             usr = current_user
-        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd'+str(i)}
+        restrict = [{'Element': x, 'type': _('Deny'), 'id': 'd' + str(i)}
                     for i, x in enumerate(usr.list_denied_column_values()) if x != '']
-        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a'+str(i)}
+        allow = [{'Element': x, 'type': _('Allow'), 'id': 'a' + str(i)}
                  for i, x in enumerate(usr.list_allowed_column_values()) if x != '']
         json_dumps = restrict + allow
     else:
@@ -915,7 +933,7 @@ def ajax_pathchooser():
 def check_valid_read_column(column):
     if column != "0":
         if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \
-              .filter(and_(db.CustomColumns.datatype == 'bool', db.CustomColumns.mark_for_delete == 0)).all():
+            .filter(and_(db.CustomColumns.datatype == 'bool', db.CustomColumns.mark_for_delete == 0)).all():
             return False
     return True
 
@@ -923,7 +941,7 @@ def check_valid_read_column(column):
 def check_valid_restricted_column(column):
     if column != "0":
         if not calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.id == column) \
-              .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all():
+            .filter(and_(db.CustomColumns.datatype == 'text', db.CustomColumns.mark_for_delete == 0)).all():
             return False
     return True
 
@@ -951,7 +969,7 @@ def prepare_tags(user, action, tags_name, id_list):
             raise Exception(_("Tag not found"))
         new_tags_list = [x.name for x in tags]
     else:
-        tags = calibre_db.session.query(db.cc_classes[config.config_restricted_column])\
+        tags = calibre_db.session.query(db.cc_classes[config.config_restricted_column]) \
             .filter(db.cc_classes[config.config_restricted_column].id.in_(id_list)).all()
         new_tags_list = [x.value for x in tags]
     saved_tags_list = user.__dict__[tags_name].split(",") if len(user.__dict__[tags_name]) else []
@@ -964,6 +982,19 @@ def prepare_tags(user, action, tags_name, id_list):
     return ",".join(saved_tags_list)
 
 
+def get_drives(current):
+    drive_letters = []
+    for d in string.ascii_uppercase:
+        if os.path.exists('{}:'.format(d)) and current[0].lower() != d.lower():
+            drive = "{}:\\".format(d)
+            data = {"name": drive, "fullpath": drive}
+            data["sort"] = "_" + data["fullpath"].lower()
+            data["type"] = "dir"
+            data["size"] = ""
+            drive_letters.append(data)
+    return drive_letters
+
+
 def pathchooser():
     browse_for = "folder"
     folder_only = request.args.get('folder', False) == "true"
@@ -971,43 +1002,45 @@ def pathchooser():
     path = os.path.normpath(request.args.get('path', ""))
 
     if os.path.isfile(path):
-        oldfile = path
+        old_file = path
         path = os.path.dirname(path)
     else:
-        oldfile = ""
+        old_file = ""
 
     absolute = False
 
     if os.path.isdir(path):
-        # if os.path.isabs(path):
         cwd = os.path.realpath(path)
         absolute = True
-        # else:
-        #    cwd = os.path.relpath(path)
     else:
         cwd = os.getcwd()
 
     cwd = os.path.normpath(os.path.realpath(cwd))
-    parentdir = os.path.dirname(cwd)
+    parent_dir = os.path.dirname(cwd)
     if not absolute:
         if os.path.realpath(cwd) == os.path.realpath("/"):
             cwd = os.path.relpath(cwd)
         else:
             cwd = os.path.relpath(cwd) + os.path.sep
-        parentdir = os.path.relpath(parentdir) + os.path.sep
+        parent_dir = os.path.relpath(parent_dir) + os.path.sep
 
-    if os.path.realpath(cwd) == os.path.realpath("/"):
-        parentdir = ""
+    files = []
+    if os.path.realpath(cwd) == os.path.realpath("/") \
+            or (sys.platform == "win32" and os.path.realpath(cwd)[1:] == os.path.realpath("/")[1:]):
+        # we are in root
+        parent_dir = ""
+        if sys.platform == "win32":
+            files = get_drives(cwd)
 
     try:
         folders = os.listdir(cwd)
     except Exception:
         folders = []
 
-    files = []
     for f in folders:
         try:
-            data = {"name": f, "fullpath": os.path.join(cwd, f)}
+            sanitized_f = str(Markup.escape(f))
+            data = {"name": sanitized_f, "fullpath": os.path.join(cwd, sanitized_f)}
             data["sort"] = data["fullpath"].lower()
         except Exception:
             continue
@@ -1037,9 +1070,9 @@ def pathchooser():
     context = {
         "cwd": cwd,
         "files": files,
-        "parentdir": parentdir,
+        "parentdir": parent_dir,
         "type": browse_for,
-        "oldfile": oldfile,
+        "oldfile": old_file,
         "absolute": absolute,
     }
     return json.dumps(context)
@@ -1058,7 +1091,7 @@ def _config_checkbox_int(to_save, x):
 
 
 def _config_string(to_save, x):
-    return config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
+    return config.set_from_dictionary(to_save, x, lambda y: y.strip().strip(u'\u200B\u200C\u200D\ufeff') if y else y)
 
 
 def _configuration_gdrive_helper(to_save):
@@ -1077,12 +1110,12 @@ def _configuration_gdrive_helper(to_save):
             if not gdrive_secrets:
                 return _configuration_result(_('client_secrets.json Is Not Configured For Web Application'))
             gdriveutils.update_settings(
-                                gdrive_secrets['client_id'],
-                                gdrive_secrets['client_secret'],
-                                gdrive_secrets['redirect_uris'][0]
-                            )
+                gdrive_secrets['client_id'],
+                gdrive_secrets['client_secret'],
+                gdrive_secrets['redirect_uris'][0]
+            )
 
-    # always show google drive settings, but in case of error deny support
+    # always show Google Drive settings, but in case of error deny support
     new_gdrive_value = (not gdrive_error) and ("config_use_google_drive" in to_save)
     if config.config_use_google_drive and not new_gdrive_value:
         config.config_google_drive_watch_changes_response = {}
@@ -1097,12 +1130,12 @@ def _configuration_oauth_helper(to_save):
     reboot_required = False
     for element in oauthblueprints:
         if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \
-                or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
+            or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
             reboot_required = True
             element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"]
             element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"]
         if to_save["config_" + str(element['id']) + "_oauth_client_id"] \
-                and to_save["config_" + str(element['id']) + "_oauth_client_secret"]:
+            and to_save["config_" + str(element['id']) + "_oauth_client_secret"]:
             active_oauths += 1
             element["active"] = 1
         else:
@@ -1132,7 +1165,6 @@ def _configuration_logfile_helper(to_save):
 
 def _configuration_ldap_helper(to_save):
     reboot_required = False
-    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
     reboot_required |= _config_int(to_save, "config_ldap_port")
     reboot_required |= _config_int(to_save, "config_ldap_authentication")
     reboot_required |= _config_string(to_save, "config_ldap_dn")
@@ -1147,21 +1179,26 @@ def _configuration_ldap_helper(to_save):
     reboot_required |= _config_string(to_save, "config_ldap_cert_path")
     reboot_required |= _config_string(to_save, "config_ldap_key_path")
     _config_string(to_save, "config_ldap_group_name")
-    if to_save.get("config_ldap_serv_password", "") != "":
+
+    address = urlparse(to_save.get("config_ldap_provider_url", ""))
+    to_save["config_ldap_provider_url"] = (address.hostname or address.path).strip("/")
+    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
+
+    if to_save.get("config_ldap_serv_password_e", "") != "":
         reboot_required |= 1
-        config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
+        config.set_from_dictionary(to_save, "config_ldap_serv_password_e")
     config.save()
 
     if not config.config_ldap_provider_url \
         or not config.config_ldap_port \
         or not config.config_ldap_dn \
-            or not config.config_ldap_user_object:
+        or not config.config_ldap_user_object:
         return reboot_required, _configuration_result(_('Please Enter a LDAP Provider, '
                                                         'Port, DN and User Object Identifier'))
 
     if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
         if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
-            if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password):
+            if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password_e):
                 return reboot_required, _configuration_result(_('Please Enter a LDAP Service Account and Password'))
         else:
             if not config.config_ldap_serv_username:
@@ -1225,16 +1262,16 @@ def new_user():
         content.default_language = config.config_default_language
     return render_title_template("user_edit.html", new_user=1, content=content,
                                  config=config, translations=translations,
-                                 languages=languages, title=_(u"Add new user"), page="newuser",
+                                 languages=languages, title=_("Add New User"), page="newuser",
                                  kobo_support=kobo_support, registered_oauth=oauth_check)
 
 
-@admi.route("/admin/mailsettings")
+@admi.route("/admin/mailsettings", methods=["GET"])
 @login_required
 @admin_required
 def edit_mailsettings():
     content = config.get_mail_settings()
-    return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"),
+    return render_title_template("email_edit.html", content=content, title=_("Edit Email Server Settings"),
                                  page="mailset", feature_support=feature_support)
 
 
@@ -1253,40 +1290,44 @@ def update_mailsettings():
     elif to_save.get("gmail"):
         try:
             config.mail_gmail_token = services.gmail.setup_gmail(config.mail_gmail_token)
-            flash(_(u"Gmail Account Verification Successful"), category="success")
+            flash(_("Success! Gmail Account Verified."), category="success")
         except Exception as ex:
             flash(str(ex), category="error")
             log.error(ex)
             return edit_mailsettings()
 
     else:
-        _config_string(to_save, "mail_server")
         _config_int(to_save, "mail_port")
         _config_int(to_save, "mail_use_ssl")
-        _config_string(to_save, "mail_login")
-        _config_string(to_save, "mail_password")
-        _config_string(to_save, "mail_from")
-        _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
+        if to_save.get("mail_password_e", ""):
+            _config_string(to_save, "mail_password_e")
+        _config_int(to_save, "mail_size", lambda y: int(y) * 1024 * 1024)
+        config.mail_server = to_save.get('mail_server', "").strip()
+        config.mail_from = to_save.get('mail_from', "").strip()
+        config.mail_login = to_save.get('mail_login', "").strip()
     try:
         config.save()
     except (OperationalError, InvalidRequestError) as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
+        return edit_mailsettings()
+    except Exception as e:
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
         return edit_mailsettings()
 
     if to_save.get("test"):
         if current_user.email:
             result = send_test_mail(current_user.email, current_user.name)
             if result is None:
-                flash(_(u"Test e-mail queued for sending to %(email)s, please check Tasks for result",
+                flash(_("Test e-mail queued for sending to %(email)s, please check Tasks for result",
                         email=current_user.email), category="info")
             else:
-                flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error")
+                flash(_("There was an error sending the Test e-mail: %(res)s", res=result), category="error")
         else:
-            flash(_(u"Please configure your e-mail address first..."), category="error")
+            flash(_("Please configure your e-mail address first..."), category="error")
     else:
-        flash(_(u"E-mail server settings updated"), category="success")
+        flash(_("Email Server Settings updated"), category="success")
 
     return edit_mailsettings()
 
@@ -1300,16 +1341,16 @@ def edit_scheduledtasks():
     duration_field = list()
 
     for n in range(24):
-        time_field.append((n , format_time(time(hour=n), format="short",)))
+        time_field.append((n, format_time(datetime_time(hour=n), format="short", )))
     for n in range(5, 65, 5):
         t = timedelta(hours=n // 60, minutes=n % 60)
-        duration_field.append((n, format_timedelta(t, threshold=.9)))
+        duration_field.append((n, format_timedelta(t, threshold=.97)))
 
     return render_title_template("schedule_edit.html",
                                  config=content,
                                  starttime=time_field,
                                  duration=duration_field,
-                                 title=_(u"Edit Scheduled Tasks Settings"))
+                                 title=_("Edit Scheduled Tasks Settings"))
 
 
 @admi.route("/admin/scheduledtasks", methods=["POST"])
@@ -1319,23 +1360,24 @@ def update_scheduledtasks():
     error = False
     to_save = request.form.to_dict()
     if 0 <= int(to_save.get("schedule_start_time")) <= 23:
-        _config_int(to_save, "schedule_start_time")
+        _config_int( to_save, "schedule_start_time")
     else:
-        flash(_(u"Invalid start time for task specified"), category="error")
+        flash(_("Invalid start time for task specified"), category="error")
         error = True
     if 0 < int(to_save.get("schedule_duration")) <= 60:
         _config_int(to_save, "schedule_duration")
     else:
-        flash(_(u"Invalid duration for task specified"), category="error")
+        flash(_("Invalid duration for task specified"), category="error")
         error = True
     _config_checkbox(to_save, "schedule_generate_book_covers")
     _config_checkbox(to_save, "schedule_generate_series_covers")
+    _config_checkbox(to_save, "schedule_metadata_backup")
     _config_checkbox(to_save, "schedule_reconnect")
 
     if not error:
         try:
             config.save()
-            flash(_(u"Scheduled tasks settings updated"), category="success")
+            flash(_("Scheduled tasks settings updated"), category="success")
 
             # Cancel any running tasks
             schedule.end_scheduled_tasks()
@@ -1345,7 +1387,7 @@ def update_scheduledtasks():
         except IntegrityError:
             ub.session.rollback()
             log.error("An unknown error occurred while saving scheduled tasks settings")
-            flash(_(u"An unknown error occurred. Please try again later."), category="error")
+            flash(_("Oops! An unknown error occurred. Please try again later."), category="error")
         except OperationalError:
             ub.session.rollback()
             log.error("Settings DB is not Writeable")
@@ -1360,7 +1402,7 @@ def update_scheduledtasks():
 def edit_user(user_id):
     content = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first()  # type: ub.User
     if not content or (not config.config_anonbrowse and content.name == "Guest"):
-        flash(_(u"User not found"), category="error")
+        flash(_("User not found"), category="error")
         return redirect(url_for('admin.admin'))
     languages = calibre_db.speaking_language(return_all_languages=True)
     translations = get_available_locale()
@@ -1379,7 +1421,7 @@ def edit_user(user_id):
                                  registered_oauth=oauth_check,
                                  mail_configured=config.get_mail_server_configured(),
                                  kobo_support=kobo_support,
-                                 title=_(u"Edit User %(nick)s", nick=content.name),
+                                 title=_("Edit User %(nick)s", nick=content.name),
                                  page="edituser")
 
 
@@ -1390,14 +1432,14 @@ def reset_user_password(user_id):
     if current_user is not None and current_user.is_authenticated:
         ret, message = reset_password(user_id)
         if ret == 1:
-            log.debug(u"Password for user %s reset", message)
-            flash(_(u"Password for user %(user)s reset", user=message), category="success")
+            log.debug("Password for user %s reset", message)
+            flash(_("Success! Password for user %(user)s reset", user=message), category="success")
         elif ret == 0:
-            log.error(u"An unknown error occurred. Please try again later.")
-            flash(_(u"An unknown error occurred. Please try again later."), category="error")
+            log.error("An unknown error occurred. Please try again later.")
+            flash(_("Oops! An unknown error occurred. Please try again later."), category="error")
         else:
-            log.error(u"Please configure the SMTP mail settings first...")
-            flash(_(u"Please configure the SMTP mail settings first..."), category="error")
+            log.error("Please configure the SMTP mail settings.")
+            flash(_("Oops! Please configure the SMTP mail settings."), category="error")
     return redirect(url_for('admin.admin'))
 
 
@@ -1408,7 +1450,7 @@ def view_logfile():
     logfiles = {0: logger.get_logfile(config.config_logfile),
                 1: logger.get_accesslogfile(config.config_access_logfile)}
     return render_title_template("logviewer.html",
-                                 title=_(u"Logfile viewer"),
+                                 title=_("Logfile viewer"),
                                  accesslog_enable=config.config_access_log,
                                  log_enable=bool(config.config_logfile != logger.LOG_TO_STDOUT),
                                  logfiles=logfiles,
@@ -1458,7 +1500,7 @@ def download_debug():
 @admin_required
 def get_update_status():
     if feature_support['updater']:
-        log.info(u"Update status requested")
+        log.info("Update status requested")
         return updater_thread.get_available_updates(request.method)
     else:
         return ''
@@ -1519,11 +1561,11 @@ def ldap_import_create_user(user, user_data):
         log.warning("LDAP User  %s Already in Database", user_data)
         return 0, None
 
-    kindlemail = ''
+    ereader_mail = ''
     if 'mail' in user_data:
         useremail = user_data['mail'][0].decode('utf-8')
         if len(user_data['mail']) > 1:
-            kindlemail = user_data['mail'][1].decode('utf-8')
+            ereader_mail = user_data['mail'][1].decode('utf-8')
 
     else:
         log.debug('No Mail Field Found in LDAP Response')
@@ -1539,7 +1581,7 @@ def ldap_import_create_user(user, user_data):
     content.name = username
     content.password = ''  # dummy password which will be replaced by ldap one
     content.email = useremail
-    content.kindle_mail = kindlemail
+    content.kindle_mail = ereader_mail
     content.default_language = config.config_default_language
     content.locale = config.config_default_locale
     content.role = config.config_default_role
@@ -1551,7 +1593,7 @@ def ldap_import_create_user(user, user_data):
     ub.session.add(content)
     try:
         ub.session.commit()
-        return 1, None    # increase no of users
+        return 1, None  # increase no of users
     except Exception as ex:
         log.warning("Failed to create LDAP user: %s - %s", user, ex)
         ub.session.rollback()
@@ -1653,7 +1695,7 @@ def _db_configuration_update_helper():
     except (OperationalError, InvalidRequestError) as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        _db_configuration_result(_(u"Database error: %(error)s.", error=e.orig), gdrive_error)
+        _db_configuration_result(_("Oops! Database Error: %(error)s.", error=e.orig), gdrive_error)
     try:
         metadata_db = os.path.join(to_save['config_calibre_dir'], "metadata.db")
         if config.config_use_google_drive and is_gdrive_ready() and not os.path.exists(metadata_db):
@@ -1663,10 +1705,11 @@ def _db_configuration_update_helper():
         return _db_configuration_result('{}'.format(ex), gdrive_error)
 
     if db_change or not db_valid or not config.db_configured \
-            or config.config_calibre_dir != to_save["config_calibre_dir"]:
-        if not calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path):
-            return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'),
-                                            gdrive_error)
+       or config.config_calibre_dir != to_save["config_calibre_dir"]:
+        if not os.path.exists(metadata_db) or not to_save['config_calibre_dir']:
+            return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdrive_error)
+        else:
+            calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path)
         config.store_calibre_uuid(calibre_db, db.Library_Id)
         # if db changed -> delete shelfs, delete download books, delete read books, kobo sync...
         if db_change:
@@ -1684,7 +1727,10 @@ def _db_configuration_update_helper():
         _config_string(to_save, "config_calibre_dir")
         calibre_db.update_config(config)
         if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK):
-            flash(_(u"DB is not Writeable"), category="warning")
+            flash(_("DB is not Writeable"), category="warning")
+    _config_string(to_save, "config_calibre_split_dir")
+    config.config_calibre_split = to_save.get('config_calibre_split', 0) == "on"
+    calibre_db.update_config(config)
     config.save()
     return _db_configuration_result(None, gdrive_error)
 
@@ -1705,6 +1751,7 @@ def _configuration_update_helper():
 
         _config_checkbox_int(to_save, "config_uploading")
         _config_checkbox_int(to_save, "config_unicode_filename")
+        _config_checkbox_int(to_save, "config_embed_metadata")
         # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case
         reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse")
                             and config.config_login_type == constants.LOGIN_LDAP)
@@ -1747,10 +1794,11 @@ def _configuration_update_helper():
         # Goodreads configuration
         _config_checkbox(to_save, "config_use_goodreads")
         _config_string(to_save, "config_goodreads_api_key")
-        _config_string(to_save, "config_goodreads_api_secret")
+        if to_save.get("config_goodreads_api_secret_e", ""):
+            _config_string(to_save, "config_goodreads_api_secret_e")
         if services.goodreads_support:
             services.goodreads_support.connect(config.config_goodreads_api_key,
-                                               config.config_goodreads_api_secret,
+                                               config.config_goodreads_api_secret_e,
                                                config.config_use_goodreads)
 
         _config_int(to_save, "config_updatechannel")
@@ -1763,10 +1811,25 @@ def _configuration_update_helper():
         if config.config_login_type == constants.LOGIN_OAUTH:
             reboot_required |= _configuration_oauth_helper(to_save)
 
+        # logfile configuration
         reboot, message = _configuration_logfile_helper(to_save)
         if message:
             return message
         reboot_required |= reboot
+
+        # security configuration
+        _config_checkbox(to_save, "config_password_policy")
+        _config_checkbox(to_save, "config_password_number")
+        _config_checkbox(to_save, "config_password_lower")
+        _config_checkbox(to_save, "config_password_upper")
+        _config_checkbox(to_save, "config_password_special")
+        if 0 < int(to_save.get("config_password_min_length", "0")) < 41:
+            _config_int(to_save, "config_password_min_length")
+        else:
+            return _configuration_result(_('Password length has to be between 1 and 40'))
+        reboot_required |= _config_int(to_save, "config_session")
+        reboot_required |= _config_checkbox(to_save, "config_ratelimiter")
+
         # Rarfile Content configuration
         _config_string(to_save, "config_rarfile_location")
         if "config_rarfile_location" in to_save:
@@ -1776,7 +1839,7 @@ def _configuration_update_helper():
     except (OperationalError, InvalidRequestError) as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        _configuration_result(_(u"Database error: %(error)s.", error=e.orig))
+        _configuration_result(_("Oops! Database Error: %(error)s.", error=e.orig))
 
     config.save()
     if reboot_required:
@@ -1792,7 +1855,7 @@ def _configuration_result(error_flash=None, reboot=False):
         config.load()
         resp['result'] = [{'type': "danger", 'message': error_flash}]
     else:
-        resp['result'] = [{'type': "success", 'message': _(u"Calibre-Web configuration updated")}]
+        resp['result'] = [{'type': "success", 'message': _("Calibre-Web configuration updated")}]
     resp['reboot'] = reboot
     resp['config_upload'] = config.config_upload_formats
     return Response(json.dumps(resp), mimetype='application/json')
@@ -1823,7 +1886,7 @@ def _db_configuration_result(error_flash=None, gdrive_error=None):
                                  gdriveError=gdrive_error,
                                  gdrivefolders=gdrivefolders,
                                  feature_support=feature_support,
-                                 title=_(u"Database Configuration"), page="dbconfig")
+                                 title=_("Database Configuration"), page="dbconfig")
 
 
 def _handle_new_user(to_save, content, languages, translations, kobo_support):
@@ -1835,25 +1898,25 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support):
         content.sidebar_view |= constants.DETAIL_RANDOM
 
     content.role = constants.selected_roles(to_save)
-    content.password = generate_password_hash(to_save["password"])
     try:
         if not to_save["name"] or not to_save["email"] or not to_save["password"]:
             log.info("Missing entries on new user")
-            raise Exception(_(u"Please fill out all fields!"))
+            raise Exception(_("Oops! Please complete all fields."))
+        content.password = generate_password_hash(helper.valid_password(to_save.get("password", "")))
         content.email = check_email(to_save["email"])
-        # Query User name, if not existing, change
+        # Query username, if not existing, change
         content.name = check_username(to_save["name"])
         if to_save.get("kindle_mail"):
             content.kindle_mail = valid_email(to_save["kindle_mail"])
         if config.config_public_reg and not check_valid_domain(content.email):
             log.info("E-mail: {} for new user is not from valid domain".format(content.email))
-            raise Exception(_(u"E-mail is not from valid domain"))
+            raise Exception(_("E-mail is not from valid domain"))
     except Exception as ex:
         flash(str(ex), category="error")
         return render_title_template("user_edit.html", new_user=1, content=content,
                                      config=config,
                                      translations=translations,
-                                     languages=languages, title=_(u"Add new user"), page="newuser",
+                                     languages=languages, title=_("Add new user"), page="newuser",
                                      kobo_support=kobo_support, registered_oauth=oauth_check)
     try:
         content.allowed_tags = config.config_allowed_tags
@@ -1864,17 +1927,17 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support):
         content.kobo_only_shelves_sync = to_save.get("kobo_only_shelves_sync", 0) == "on"
         ub.session.add(content)
         ub.session.commit()
-        flash(_(u"User '%(user)s' created", user=content.name), category="success")
+        flash(_("User '%(user)s' created", user=content.name), category="success")
         log.debug("User {} created".format(content.name))
         return redirect(url_for('admin.admin'))
     except IntegrityError:
         ub.session.rollback()
         log.error("Found an existing account for {} or {}".format(content.name, content.email))
-        flash(_("Found an existing account for this e-mail address or name."), category="error")
+        flash(_("Oops! An account already exists for this Email. or name."), category="error")
     except OperationalError as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
 
 
 def _delete_user(content):
@@ -1902,10 +1965,10 @@ def _delete_user(content):
             log.info("User {} deleted".format(content.name))
             return _("User '%(nick)s' deleted", nick=content.name)
         else:
-            log.warning(_("Can't delete Guest User"))
+            # log.warning(_("Can't delete Guest User"))
             raise Exception(_("Can't delete Guest User"))
     else:
-        log.warning("No admin user remaining, can't delete user")
+        # log.warning("No admin user remaining, can't delete user")
         raise Exception(_("No admin user remaining, can't delete user"))
 
 
@@ -1923,14 +1986,6 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
             log.warning("No admin user remaining, can't remove admin role from {}".format(content.name))
             flash(_("No admin user remaining, can't remove admin role"), category="error")
             return redirect(url_for('admin.admin'))
-        if to_save.get("password"):
-            content.password = generate_password_hash(to_save["password"])
-        anonymous = content.is_anonymous
-        content.role = constants.selected_roles(to_save)
-        if anonymous:
-            content.role |= constants.ROLE_ANONYMOUS
-        else:
-            content.role &= ~constants.ROLE_ANONYMOUS
 
         val = [int(k[5:]) for k in to_save if k.startswith('show_')]
         sidebar, __ = get_sidebar_config()
@@ -1958,9 +2013,21 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
         if to_save.get("locale"):
             content.locale = to_save["locale"]
         try:
-            if to_save.get("email", content.email) != content.email:
-                content.email = check_email(to_save["email"])
-            # Query User name, if not existing, change
+            anonymous = content.is_anonymous
+            content.role = constants.selected_roles(to_save)
+            if anonymous:
+                content.role |= constants.ROLE_ANONYMOUS
+            else:
+                content.role &= ~constants.ROLE_ANONYMOUS
+                if to_save.get("password", ""):
+                    content.password = generate_password_hash(helper.valid_password(to_save.get("password", "")))
+
+            new_email = valid_email(to_save.get("email", content.email))
+            if not new_email:
+                raise Exception(_("Email can't be empty and has to be a valid Email"))
+            if new_email != content.email:
+                content.email = check_email(new_email)
+            # Query username, if not existing, change
             if to_save.get("name", content.name) != content.name:
                 if to_save.get("name") == "Guest":
                     raise Exception(_("Guest Name can't be changed"))
@@ -1979,24 +2046,24 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support):
                                          content=content,
                                          config=config,
                                          registered_oauth=oauth_check,
-                                         title=_(u"Edit User %(nick)s", nick=content.name),
+                                         title=_("Edit User %(nick)s", nick=content.name),
                                          page="edituser")
     try:
         ub.session_commit()
-        flash(_(u"User '%(nick)s' updated", nick=content.name), category="success")
+        flash(_("User '%(nick)s' updated", nick=content.name), category="success")
     except IntegrityError as ex:
         ub.session.rollback()
         log.error("An unknown error occurred while changing user: {}".format(str(ex)))
-        flash(_(u"An unknown error occurred. Please try again later."), category="error")
+        flash(_("Oops! An unknown error occurred. Please try again later."), category="error")
     except OperationalError as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
     return ""
 
 
 def extract_user_data_from_field(user, field):
-    match = re.search(field + r"=([\.\d\s\w-]+)", user, re.IGNORECASE | re.UNICODE)
+    match = re.search(field + r"=([@\.\d\s\w-]+)", user, re.IGNORECASE | re.UNICODE)
     if match:
         return match.group(1)
     else:

cps/babel.py

@@ -1,7 +1,8 @@
 from babel import negotiate_locale
 from flask_babel import Babel, Locale
 from babel.core import UnknownLocaleError
-from flask import request, g
+from flask import request
+from flask_login import current_user
 
 from . import logger
 
@@ -9,14 +10,12 @@ log = logger.create()
 
 babel = Babel()
 
-
-@babel.localeselector
 def get_locale():
     # if a user is logged in, use the locale from the user settings
-    user = getattr(g, 'user', None)
-    if user is not None and hasattr(user, "locale"):
-        if user.name != 'Guest':   # if the account is the guest account bypass the config lang settings
-            return user.locale
+    if current_user is not None and hasattr(current_user, "locale"):
+        # if the account is the guest account bypass the config lang settings
+        if current_user.name != 'Guest':
+            return current_user.locale
 
     preferred = list()
     if request.accept_languages:
@@ -32,8 +31,10 @@ def get_locale():
 def get_user_locale_language(user_language):
     return Locale.parse(user_language).get_language_name(get_locale())
 
+
 def get_available_locale():
     return [Locale('en')] + babel.list_translations()
 
+
 def get_available_translations():
     return set(str(item) for item in get_available_locale())

cps/cache_buster.py

@@ -49,7 +49,7 @@ def init_cache_busting(app):
             rooted_filename = os.path.join(dirpath, filename)
             try:
                 with open(rooted_filename, 'rb') as f:
-                    file_hash = hashlib.md5(f.read()).hexdigest()[:7] # nosec
+                    file_hash = hashlib.md5(f.read()).hexdigest()[:7]  # nosec
                 # save version to tables
                 file_path = rooted_filename.replace(static_folder, "")
                 file_path = file_path.replace("\\", "/")  # Convert Windows path to web path
@@ -59,11 +59,11 @@ def init_cache_busting(app):
 
     log.debug('Finished computing cache-busting values')
 
-    def bust_filename(filename):
-        return hash_table.get(filename, "")
+    def bust_filename(file_name):
+        return hash_table.get(file_name, "")
 
-    def unbust_filename(filename):
-        return filename.split("?", 1)[0]
+    def unbust_filename(file_name):
+        return file_name.split("?", 1)[0]
 
     @app.url_defaults
     # pylint: disable=unused-variable

cps/cli.py

@@ -26,10 +26,12 @@ from .constants import STABLE_VERSION as _STABLE_VERSION
 from .constants import NIGHTLY_VERSION as _NIGHTLY_VERSION
 from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE
 
+
 def version_info():
     if _NIGHTLY_VERSION[1].startswith('$Format'):
-        return "Calibre-Web version: %s - unkown git-clone" % _STABLE_VERSION['version']
-    return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1])
+        return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta")
+    return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1])
+
 
 class CliParameter(object):
 
@@ -37,15 +39,16 @@ class CliParameter(object):
         self.arg_parser()
 
     def arg_parser(self):
-        parser = argparse.ArgumentParser(description='Calibre Web is a web app'
-                                                     ' providing a interface for browsing, reading and downloading eBooks\n',
+        parser = argparse.ArgumentParser(description='Calibre Web is a web app providing '
+                                                     'a interface for browsing, reading and downloading eBooks\n',
                                          prog='cps.py')
         parser.add_argument('-p', metavar='path', help='path and name to settings db, e.g. /opt/cw.db')
         parser.add_argument('-g', metavar='path', help='path and name to gdrive db, e.g. /opt/gd.db')
-        parser.add_argument('-c', metavar='path',
-                            help='path and name to SSL certfile, e.g. /opt/test.cert, works only in combination with keyfile')
-        parser.add_argument('-k', metavar='path',
-                            help='path and name to SSL keyfile, e.g. /opt/test.key, works only in combination with certfile')
+        parser.add_argument('-c', metavar='path', help='path and name to SSL certfile, e.g. /opt/test.cert, '
+                                                       'works only in combination with keyfile')
+        parser.add_argument('-k', metavar='path', help='path and name to SSL keyfile, e.g. /opt/test.key, '
+                                                       'works only in combination with certfile')
+        parser.add_argument('-o', metavar='path', help='path and name Calibre-Web logfile')
         parser.add_argument('-v', '--version', action='version', help='Shows version number and exits Calibre-Web',
                             version=version_info())
         parser.add_argument('-i', metavar='ip-address', help='Server IP-Address to listen')
@@ -58,6 +61,7 @@ class CliParameter(object):
         parser.add_argument('-r', action='store_true', help='Enable public database reconnect route under /reconnect')
         args = parser.parse_args()
 
+        self.logpath = args.o or ""
         self.settings_path = args.p or os.path.join(_CONFIG_DIR, DEFAULT_SETTINGS_FILE)
         self.gd_path = args.g or os.path.join(_CONFIG_DIR, DEFAULT_GDRIVE_FILE)
 
@@ -67,7 +71,6 @@ class CliParameter(object):
         if os.path.isdir(self.gd_path):
             self.gd_path = os.path.join(self.gd_path, DEFAULT_GDRIVE_FILE)
 
-
         # handle and check parameter for ssl encryption
         self.certfilepath = None
         self.keyfilepath = None
@@ -96,7 +99,7 @@ class CliParameter(object):
             self.keyfilepath = ""
 
         # dry run updater
-        self.dry_run =args.d or None
+        self.dry_run = args.d or None
         # enable reconnect endpoint for docker database reconnect
         self.reconnect_enable = args.r or os.environ.get("CALIBRE_RECONNECT", None)
         # load covers from localhost
@@ -112,7 +115,7 @@ class CliParameter(object):
                     else:
                         socket.inet_pton(socket.AF_INET, self.ip_address)
                 else:
-                    # on windows python < 3.4, inet_pton is not available
+                    # on Windows python < 3.4, inet_pton is not available
                     # inet_atom only handles IPv4 addresses
                     socket.inet_aton(self.ip_address)
             except socket.error as err:

cps/comic.py

@@ -36,6 +36,12 @@ try:
         from comicapi import __version__ as comic_version
     except ImportError:
         comic_version = ''
+    try:
+        from comicapi.comicarchive import load_archive_plugins
+        import comicapi.utils
+        comicapi.utils.add_rar_paths()
+    except ImportError:
+        load_archive_plugins = None
 except (ImportError, LookupError) as e:
     log.debug('Cannot import comicapi, extracting comic metadata will not work: %s', e)
     import zipfile
@@ -46,6 +52,12 @@ except (ImportError, LookupError) as e:
     except (ImportError, SyntaxError) as e:
         log.debug('Cannot import rarfile, extracting cover files from rar files will not work: %s', e)
         use_rarfile = False
+    try:
+        import py7zr
+        use_7zip = True
+    except (ImportError, SyntaxError) as e:
+        log.debug('Cannot import py7zr, extracting cover files from CB7 files will not work: %s', e)
+        use_7zip = False
     use_comic_meta = False
 
 
@@ -78,23 +90,40 @@ def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_exec
                 if len(ext) > 1:
                     extension = ext[1].lower()
                     if extension in cover.COVER_EXTENSIONS:
-                        cover_data = cf.read(name)
+                        cover_data = cf.read([name])
                         break
         except Exception as ex:
-            log.debug('Rarfile failed with error: {}'.format(ex))
+            log.error('Rarfile failed with error: {}'.format(ex))
+    elif original_file_extension.upper() == '.CB7' and use_7zip:
+        cf = py7zr.SevenZipFile(tmp_file_name)
+        for name in cf.getnames():
+            ext = os.path.splitext(name)
+            if len(ext) > 1:
+                extension = ext[1].lower()
+                if extension in cover.COVER_EXTENSIONS:
+                    try:
+                        cover_data = cf.read(name)[name].read()
+                    except (py7zr.Bad7zFile, OSError) as ex:
+                        log.error('7Zip file failed with error: {}'.format(ex))
+                    break
     return cover_data, extension
 
 
 def _extract_cover(tmp_file_name, original_file_extension, rar_executable):
     cover_data = extension = None
     if use_comic_meta:
-        archive = ComicArchive(tmp_file_name, rar_exe_path=rar_executable)
-        for index, name in enumerate(archive.getPageNameList()):
+        try:
+            archive = ComicArchive(tmp_file_name, rar_exe_path=rar_executable)
+        except TypeError:
+            archive = ComicArchive(tmp_file_name)
+        name_list = archive.getPageNameList if hasattr(archive, "getPageNameList") else archive.get_page_name_list
+        for index, name in enumerate(name_list()):
             ext = os.path.splitext(name)
             if len(ext) > 1:
                 extension = ext[1].lower()
                 if extension in cover.COVER_EXTENSIONS:
-                    cover_data = archive.getPage(index)
+                    get_page = archive.getPage if hasattr(archive, "getPageNameList") else archive.get_page
+                    cover_data = get_page(index)
                     break
     else:
         cover_data, extension = _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_executable)
@@ -103,17 +132,26 @@ def _extract_cover(tmp_file_name, original_file_extension, rar_executable):
 
 def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rar_executable):
     if use_comic_meta:
-        archive = ComicArchive(tmp_file_path, rar_exe_path=rar_executable)
-        if archive.seemsToBeAComicArchive():
-            if archive.hasMetadata(MetaDataStyle.CIX):
+        try:
+            archive = ComicArchive(tmp_file_path, rar_exe_path=rar_executable)
+        except TypeError:
+            load_archive_plugins(force=True, rar=rar_executable)
+            archive = ComicArchive(tmp_file_path)
+        if hasattr(archive, "seemsToBeAComicArchive"):
+            seems_archive = archive.seemsToBeAComicArchive
+        else:
+            seems_archive = archive.seems_to_be_a_comic_archive
+        if seems_archive():
+            has_metadata = archive.hasMetadata if hasattr(archive, "hasMetadata") else archive.has_metadata
+            if has_metadata(MetaDataStyle.CIX):
                 style = MetaDataStyle.CIX
-            elif archive.hasMetadata(MetaDataStyle.CBI):
+            elif has_metadata(MetaDataStyle.CBI):
                 style = MetaDataStyle.CBI
             else:
                 style = None
 
-            # if style is not None:
-            loaded_metadata = archive.readMetadata(style)
+            read_metadata = archive.readMetadata if hasattr(archive, "readMetadata") else archive.read_metadata
+            loaded_metadata = read_metadata(style)
 
             lang = loaded_metadata.language or ""
             loaded_metadata.language = isoLanguages.get_lang3(lang)
@@ -138,7 +176,7 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
         file_path=tmp_file_path,
         extension=original_file_extension,
         title=original_file_name,
-        author=u'Unknown',
+        author='Unknown',
         cover=_extract_cover(tmp_file_path, original_file_extension, rar_executable),
         description="",
         tags="",

cps/config_sql.py

@@ -23,6 +23,10 @@ import json
 from sqlalchemy import Column, String, Integer, SmallInteger, Boolean, BLOB, JSON
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.sql.expression import text
+from sqlalchemy import exists
+from cryptography.fernet import Fernet
+import cryptography.exceptions
+from base64 import urlsafe_b64decode
 try:
     # Compatibility with sqlalchemy 2.0
     from sqlalchemy.orm import declarative_base
@@ -36,6 +40,7 @@ from .subproc_wrapper import process_wait
 log = logger.create()
 _Base = declarative_base()
 
+
 class _Flask_Settings(_Base):
     __tablename__ = 'flask_settings'
 
@@ -56,7 +61,8 @@ class _Settings(_Base):
     mail_port = Column(Integer, default=25)
     mail_use_ssl = Column(SmallInteger, default=0)
     mail_login = Column(String, default='mail@example.com')
-    mail_password = Column(String, default='mypassword')
+    mail_password_e = Column(String)
+    mail_password = Column(String)
     mail_from = Column(String, default='automailer <mail@example.com>')
     mail_size = Column(Integer, default=25*1024*1024)
     mail_server_type = Column(SmallInteger, default=0)
@@ -64,24 +70,25 @@ class _Settings(_Base):
 
     config_calibre_dir = Column(String)
     config_calibre_uuid = Column(String)
+    config_calibre_split = Column(Boolean, default=False)
+    config_calibre_split_dir = Column(String)
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
     config_external_port = Column(Integer, default=constants.DEFAULT_PORT)
     config_certfile = Column(String)
     config_keyfile = Column(String)
-    config_trustedhosts = Column(String,default='')
-    config_calibre_web_title = Column(String, default=u'Calibre-Web')
+    config_trustedhosts = Column(String, default='')
+    config_calibre_web_title = Column(String, default='Calibre-Web')
     config_books_per_page = Column(Integer, default=60)
     config_random_books = Column(Integer, default=4)
     config_authors_max = Column(Integer, default=0)
     config_read_column = Column(Integer, default=0)
-    config_title_regex = Column(String, default=r'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+')
-    config_mature_content_tags = Column(String, default='')
+    config_title_regex = Column(String, default=r'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines|Le|La|Les|L\'|Un|Une)\s+')    
     config_theme = Column(Integer, default=0)
 
     config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL)
-    config_logfile = Column(String)
+    config_logfile = Column(String, default=logger.DEFAULT_LOG_FILE)
     config_access_log = Column(SmallInteger, default=0)
-    config_access_logfile = Column(String)
+    config_access_logfile = Column(String, default=logger.DEFAULT_ACCESS_LOG)
 
     config_uploading = Column(SmallInteger, default=0)
     config_anonbrowse = Column(SmallInteger, default=0)
@@ -107,6 +114,7 @@ class _Settings(_Base):
 
     config_use_goodreads = Column(Boolean, default=False)
     config_goodreads_api_key = Column(String)
+    config_goodreads_api_secret_e = Column(String)
     config_goodreads_api_secret = Column(String)
     config_register_email = Column(Boolean, default=False)
     config_login_type = Column(Integer, default=0)
@@ -117,14 +125,15 @@ class _Settings(_Base):
     config_ldap_port = Column(SmallInteger, default=389)
     config_ldap_authentication = Column(SmallInteger, default=constants.LDAP_AUTH_SIMPLE)
     config_ldap_serv_username = Column(String, default='cn=admin,dc=example,dc=org')
-    config_ldap_serv_password = Column(String, default="")
+    config_ldap_serv_password_e = Column(String)
+    config_ldap_serv_password = Column(String)
     config_ldap_encryption = Column(SmallInteger, default=0)
     config_ldap_cacert_path = Column(String, default="")
     config_ldap_cert_path = Column(String, default="")
     config_ldap_key_path = Column(String, default="")
     config_ldap_dn = Column(String, default='dc=example,dc=org')
     config_ldap_user_object = Column(String, default='uid=%s')
-    config_ldap_member_user_object = Column(String, default='') #
+    config_ldap_member_user_object = Column(String, default='')
     config_ldap_openldap = Column(Boolean, default=True)
     config_ldap_group_object_filter = Column(String, default='(&(objectclass=posixGroup)(cn=%s))')
     config_ldap_group_members_field = Column(String, default='memberUid')
@@ -137,6 +146,7 @@ class _Settings(_Base):
     config_rarfile_location = Column(String, default=None)
     config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
     config_unicode_filename = Column(Boolean, default=False)
+    config_embed_metadata = Column(Boolean, default=True)
 
     config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
 
@@ -148,24 +158,35 @@ class _Settings(_Base):
     schedule_generate_book_covers = Column(Boolean, default=False)
     schedule_generate_series_covers = Column(Boolean, default=False)
     schedule_reconnect = Column(Boolean, default=False)
+    schedule_metadata_backup = Column(Boolean, default=False)
+
+    config_password_policy = Column(Boolean, default=True)
+    config_password_min_length = Column(Integer, default=8)
+    config_password_number = Column(Boolean, default=True)
+    config_password_lower = Column(Boolean, default=True)
+    config_password_upper = Column(Boolean, default=True)
+    config_password_special = Column(Boolean, default=True)
+    config_session = Column(Integer, default=1)
+    config_ratelimiter = Column(Boolean, default=True)
 
     def __repr__(self):
         return self.__class__.__name__
 
 
 # Class holds all application specific settings in calibre-web
-class _ConfigSQL(object):
+class ConfigSQL(object):
     # pylint: disable=no-member
     def __init__(self):
-        pass
+        self.__dict__["dirty"] = list()
 
-    def init_config(self, session, cli):
+    def init_config(self, session, secret_key, cli):
         self._session = session
         self._settings = None
         self.db_configured = None
         self.config_calibre_dir = None
-        self.load()
+        self._fernet = Fernet(secret_key)
         self.cli = cli
+        self.load()
 
         change = False
 
@@ -260,14 +281,14 @@ class _ConfigSQL(object):
         return logger.get_level_name(self.config_log_level)
 
     def get_mail_settings(self):
-        return {k:v for k, v in self.__dict__.items() if k.startswith('mail_')}
+        return {k: v for k, v in self.__dict__.items() if k.startswith('mail_')}
 
     def get_mail_server_configured(self):
         return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0)
                     or (self.mail_gmail_token != {} and self.mail_server_type == 1))
 
     def get_scheduled_task_settings(self):
-        return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')}
+        return {k: v for k, v in self.__dict__.items() if k.startswith('schedule_')}
 
     def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None):
         """Possibly updates a field of this object.
@@ -296,15 +317,15 @@ class _ConfigSQL(object):
         setattr(self, field, new_value)
         return True
 
-    def toDict(self):
+    def to_dict(self):
         storage = {}
         for k, v in self.__dict__.items():
-            if k[0] != '_' and not k.endswith("password") and not k.endswith("secret") and not k == "cli":
+            if k[0] != '_' and not k.endswith("_e") and not k == "cli":
                 storage[k] = v
         return storage
 
     def load(self):
-        '''Load all configuration values from the underlying storage.'''
+        """Load all configuration values from the underlying storage."""
         s = self._read_from_storage()  # type: _Settings
         for k, v in s.__dict__.items():
             if k[0] != '_':
@@ -313,7 +334,13 @@ class _ConfigSQL(object):
                     column = s.__class__.__dict__.get(k)
                     if column.default is not None:
                         v = column.default.arg
-                setattr(self, k, v)
+                if k.endswith("_e") and v is not None:
+                    try:
+                        setattr(self, k, self._fernet.decrypt(v).decode())
+                    except cryptography.fernet.InvalidToken:
+                        setattr(self, k, "")
+                else:
+                    setattr(self, k, v)
 
         have_metadata_db = bool(self.config_calibre_dir)
         if have_metadata_db:
@@ -321,30 +348,37 @@ class _ConfigSQL(object):
             have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
         constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')]
+        from . import cli_param
         if os.environ.get('FLASK_DEBUG'):
             logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG)
         else:
             # pylint: disable=access-member-before-definition
-            logfile = logger.setup(self.config_logfile, self.config_log_level)
-        if logfile != self.config_logfile:
-            log.warning("Log path %s not valid, falling back to default", self.config_logfile)
+            logfile = logger.setup(cli_param.logpath or self.config_logfile, self.config_log_level)
+        if logfile != os.path.abspath(self.config_logfile):
+            if logfile != os.path.abspath(cli_param.logpath):
+                log.warning("Log path %s not valid, falling back to default", self.config_logfile)
             self.config_logfile = logfile
+            s.config_logfile = logfile
             self._session.merge(s)
             try:
                 self._session.commit()
             except OperationalError as e:
                 log.error('Database error: %s', e)
                 self._session.rollback()
+        self.__dict__["dirty"] = list()
 
     def save(self):
-        '''Apply all configuration values to the underlying storage.'''
+        """Apply all configuration values to the underlying storage."""
         s = self._read_from_storage()  # type: _Settings
 
-        for k, v in self.__dict__.items():
+        for k in self.dirty:
             if k[0] == '_':
                 continue
             if hasattr(s, k):
-                setattr(s, k, v)
+                if k.endswith("_e"):
+                    setattr(s, k, self._fernet.encrypt(self.__dict__[k].encode()))
+                else:
+                    setattr(s, k, self.__dict__[k])
 
         log.debug("_ConfigSQL updating storage")
         self._session.merge(s)
@@ -360,9 +394,11 @@ class _ConfigSQL(object):
             log.error(error)
         log.warning("invalidating configuration")
         self.db_configured = False
-        # self.config_calibre_dir = None
         self.save()
 
+    def get_book_path(self):
+        return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir
+
     def store_calibre_uuid(self, calibre_db, Library_table):
         try:
             calibre_uuid = calibre_db.session.query(Library_table).one_or_none()
@@ -372,7 +408,40 @@ class _ConfigSQL(object):
         except AttributeError:
             pass
 
-def _migrate_table(session, orm_class):
+    def __setattr__(self, attr_name, attr_value):
+        super().__setattr__(attr_name, attr_value)
+        self.__dict__["dirty"].append(attr_name)
+
+
+def _encrypt_fields(session, secret_key):
+    try:
+        session.query(exists().where(_Settings.mail_password_e)).scalar()
+    except OperationalError:
+        with session.bind.connect() as conn:
+            conn.execute(text("ALTER TABLE settings ADD column 'mail_password_e' String"))
+            conn.execute(text("ALTER TABLE settings ADD column 'config_goodreads_api_secret_e' String"))
+            conn.execute(text("ALTER TABLE settings ADD column 'config_ldap_serv_password_e' String"))
+        session.commit()
+        crypter = Fernet(secret_key)
+        settings = session.query(_Settings.mail_password, _Settings.config_goodreads_api_secret,
+                                 _Settings.config_ldap_serv_password).first()
+        if settings.mail_password:
+            session.query(_Settings).update(
+                {_Settings.mail_password_e: crypter.encrypt(settings.mail_password.encode())})
+        if settings.config_goodreads_api_secret:
+            session.query(_Settings).update(
+                {_Settings.config_goodreads_api_secret_e:
+                     crypter.encrypt(settings.config_goodreads_api_secret.encode())})
+        if settings.config_ldap_serv_password:
+            session.query(_Settings).update(
+                {_Settings.config_ldap_serv_password_e:
+                     crypter.encrypt(settings.config_ldap_serv_password.encode())})
+        session.commit()
+
+
+def _migrate_table(session, orm_class, secret_key=None):
+    if secret_key:
+        _encrypt_fields(session, secret_key)
     changed = False
 
     for column_name, column in orm_class.__dict__.items():
@@ -393,9 +462,9 @@ def _migrate_table(session, orm_class):
                 else:
                     column_type = column.type
                 alter_table = text("ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__,
-                                                                        column_name,
-                                                                        column_type,
-                                                                        column_default))
+                                                                             column_name,
+                                                                             column_type,
+                                                                             column_default))
                 log.debug(alter_table)
                 session.execute(alter_table)
                 changed = True
@@ -463,22 +532,20 @@ def autodetect_kepubify_binary():
     return ""
 
 
-def _migrate_database(session):
+def _migrate_database(session, secret_key):
     # make sure the table is created, if it does not exist
     _Base.metadata.create_all(session.bind)
-    _migrate_table(session, _Settings)
+    _migrate_table(session, _Settings, secret_key)
     _migrate_table(session, _Flask_Settings)
 
 
-def load_configuration(conf, session, cli):
-    _migrate_database(session)
-
+def load_configuration(session, secret_key):
+    _migrate_database(session, secret_key)
     if not session.query(_Settings).count():
         session.add(_Settings())
         session.commit()
-    # conf = _ConfigSQL()
-    conf.init_config(session, cli)
-    # return conf
+
+
 
 
 def get_flask_session_key(_session):
@@ -488,3 +555,25 @@ def get_flask_session_key(_session):
         _session.add(flask_settings)
         _session.commit()
     return flask_settings.flask_session_key
+
+
+def get_encryption_key(key_path):
+    key_file = os.path.join(key_path, ".key")
+    generate = True
+    error = ""
+    if os.path.exists(key_file) and os.path.getsize(key_file) > 32:
+        with open(key_file, "rb") as f:
+            key = f.read()
+        try:
+            urlsafe_b64decode(key)
+            generate = False
+        except ValueError:
+            pass
+    if generate:
+        key = Fernet.generate_key()
+        try:
+            with open(key_file, "wb") as f:
+                f.write(key)
+        except PermissionError as e:
+            error = e
+    return key, error

cps/constants.py

@@ -34,6 +34,8 @@ UPDATER_AVAILABLE = True
 
 # Base dir is parent of current file, necessary if called from different folder
 BASE_DIR            = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir))
+# if executable file the files should be placed in the parent dir (parallel to the exe file)
+
 STATIC_DIR          = os.path.join(BASE_DIR, 'cps', 'static')
 TEMPLATES_DIR       = os.path.join(BASE_DIR, 'cps', 'templates')
 TRANSLATIONS_DIR    = os.path.join(BASE_DIR, 'cps', 'translations')
@@ -49,6 +51,9 @@ if HOME_CONFIG:
     CONFIG_DIR = os.environ.get('CALIBRE_DBPATH', home_dir)
 else:
     CONFIG_DIR = os.environ.get('CALIBRE_DBPATH', BASE_DIR)
+    if getattr(sys, 'frozen', False):
+        CONFIG_DIR = os.path.abspath(os.path.join(CONFIG_DIR, os.pardir))
+
 
 DEFAULT_SETTINGS_FILE = "app.db"
 DEFAULT_GDRIVE_FILE = "gdrive.db"
@@ -144,10 +149,10 @@ del env_CALIBRE_PORT
 
 EXTENSIONS_AUDIO = {'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'}
 EXTENSIONS_CONVERT_FROM = ['pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf',
-                           'txt', 'htmlz', 'rtf', 'odt', 'cbz', 'cbr']
+                           'txt', 'htmlz', 'rtf', 'odt', 'cbz', 'cbr', 'prc']
 EXTENSIONS_CONVERT_TO = ['pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2',
                          'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt']
-EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu',
+EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'cb7', 'djvu', 'djv',
                      'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg',
                      'opus', 'wav', 'flac', 'm4a', 'm4b'}
 
@@ -168,13 +173,12 @@ def selected_roles(dictionary):
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
                                   'series_id, languages, publisher, pubdate, identifiers')
 
-STABLE_VERSION = {'version': '0.6.19 Beta'}
+# python build process likes to have x.y.zbw -> b for beta and w a counting number
+STABLE_VERSION = {'version': '0.6.22b'}
 
 NIGHTLY_VERSION = dict()
 NIGHTLY_VERSION[0] = '$Format:%H$'
 NIGHTLY_VERSION[1] = '$Format:%cI$'
-# NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57'
-# NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00'
 
 # CACHE
 CACHE_TYPE_THUMBNAILS    = 'thumbnails'

cps/converter.py

@@ -54,10 +54,9 @@ def get_calibre_version():
 def get_unrar_version():
     unrar_version = _get_command_version(config.config_rarfile_location, r'UNRAR.*\d')
     if unrar_version == "not installed":
-        unrar_version = _get_command_version(config.config_rarfile_location, r'unrar.*\d','-V')
+        unrar_version = _get_command_version(config.config_rarfile_location, r'unrar.*\d', '-V')
     return unrar_version
 
+
 def get_kepubify_version():
-    return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version')
-
-
+    return _get_command_version(config.config_kepubifypath, r'kepubify\s', '--version')

cps/db.py

@@ -19,7 +19,6 @@
 
 import os
 import re
-import ast
 import json
 from datetime import datetime
 from urllib.parse import quote
@@ -112,66 +111,73 @@ class Identifiers(Base):
     def format_type(self):
         format_type = self.type.lower()
         if format_type == 'amazon':
-            return u"Amazon"
+            return "Amazon"
         elif format_type.startswith("amazon_"):
-            return u"Amazon.{0}".format(format_type[7:])
+            return "Amazon.{0}".format(format_type[7:])
         elif format_type == "isbn":
-            return u"ISBN"
+            return "ISBN"
         elif format_type == "doi":
-            return u"DOI"
+            return "DOI"
         elif format_type == "douban":
-            return u"Douban"
+            return "Douban"
         elif format_type == "goodreads":
-            return u"Goodreads"
+            return "Goodreads"
         elif format_type == "babelio":
-            return u"Babelio"
+            return "Babelio"
         elif format_type == "google":
-            return u"Google Books"
+            return "Google Books"
         elif format_type == "kobo":
-            return u"Kobo"
+            return "Kobo"
         elif format_type == "litres":
-            return u"ЛитРес"
+            return "ЛитРес"
         elif format_type == "issn":
-            return u"ISSN"
+            return "ISSN"
         elif format_type == "isfdb":
-            return u"ISFDB"
+            return "ISFDB"
         if format_type == "lubimyczytac":
-            return u"Lubimyczytac"
+            return "Lubimyczytac"
+        if format_type == "databazeknih":
+            return "Databáze knih"
         else:
             return self.type
 
     def __repr__(self):
         format_type = self.type.lower()
         if format_type == "amazon" or format_type == "asin":
-            return u"https://amazon.com/dp/{0}".format(self.val)
+            return "https://amazon.com/dp/{0}".format(self.val)
         elif format_type.startswith('amazon_'):
-            return u"https://amazon.{0}/dp/{1}".format(format_type[7:], self.val)
+            return "https://amazon.{0}/dp/{1}".format(format_type[7:], self.val)
         elif format_type == "isbn":
-            return u"https://www.worldcat.org/isbn/{0}".format(self.val)
+            return "https://www.worldcat.org/isbn/{0}".format(self.val)
         elif format_type == "doi":
-            return u"https://dx.doi.org/{0}".format(self.val)
+            return "https://dx.doi.org/{0}".format(self.val)
         elif format_type == "goodreads":
-            return u"https://www.goodreads.com/book/show/{0}".format(self.val)
+            return "https://www.goodreads.com/book/show/{0}".format(self.val)
         elif format_type == "babelio":
-            return u"https://www.babelio.com/livres/titre/{0}".format(self.val)
+            return "https://www.babelio.com/livres/titre/{0}".format(self.val)
         elif format_type == "douban":
-            return u"https://book.douban.com/subject/{0}".format(self.val)
+            return "https://book.douban.com/subject/{0}".format(self.val)
         elif format_type == "google":
-            return u"https://books.google.com/books?id={0}".format(self.val)
+            return "https://books.google.com/books?id={0}".format(self.val)
         elif format_type == "kobo":
-            return u"https://www.kobo.com/ebook/{0}".format(self.val)
+            return "https://www.kobo.com/ebook/{0}".format(self.val)
         elif format_type == "lubimyczytac":
-            return u"https://lubimyczytac.pl/ksiazka/{0}/ksiazka".format(self.val)
+            return "https://lubimyczytac.pl/ksiazka/{0}/ksiazka".format(self.val)
         elif format_type == "litres":
-            return u"https://www.litres.ru/{0}".format(self.val)
+            return "https://www.litres.ru/{0}".format(self.val)
         elif format_type == "issn":
-            return u"https://portal.issn.org/resource/ISSN/{0}".format(self.val)
+            return "https://portal.issn.org/resource/ISSN/{0}".format(self.val)
         elif format_type == "isfdb":
-            return u"http://www.isfdb.org/cgi-bin/pl.cgi?{0}".format(self.val)
+            return "http://www.isfdb.org/cgi-bin/pl.cgi?{0}".format(self.val)
+        elif format_type == "databazeknih":
+            return "https://www.databazeknih.cz/knihy/{0}".format(self.val)
         elif self.val.lower().startswith("javascript:"):
             return quote(self.val)
+        elif self.val.lower().startswith("data:"):
+            link , __, __ = str.partition(self.val, ",")
+            return link
         else:
-            return u"{0}".format(self.val)
+            return "{0}".format(self.val)
 
 
 class Comments(Base):
@@ -189,7 +195,7 @@ class Comments(Base):
         return self.text
 
     def __repr__(self):
-        return u"<Comments({0})>".format(self.text)
+        return "<Comments({0})>".format(self.text)
 
 
 class Tags(Base):
@@ -204,8 +210,11 @@ class Tags(Base):
     def get(self):
         return self.name
 
+    def __eq__(self, other):
+        return self.name == other
+
     def __repr__(self):
-        return u"<Tags('{0})>".format(self.name)
+        return "<Tags('{0})>".format(self.name)
 
 
 class Authors(Base):
@@ -216,7 +225,7 @@ class Authors(Base):
     sort = Column(String(collation='NOCASE'))
     link = Column(String, nullable=False, default="")
 
-    def __init__(self, name, sort, link):
+    def __init__(self, name, sort, link=""):
         self.name = name
         self.sort = sort
         self.link = link
@@ -224,8 +233,11 @@ class Authors(Base):
     def get(self):
         return self.name
 
+    def __eq__(self, other):
+        return self.name == other
+
     def __repr__(self):
-        return u"<Authors('{0},{1}{2}')>".format(self.name, self.sort, self.link)
+        return "<Authors('{0},{1}{2}')>".format(self.name, self.sort, self.link)
 
 
 class Series(Base):
@@ -242,8 +254,11 @@ class Series(Base):
     def get(self):
         return self.name
 
+    def __eq__(self, other):
+        return self.name == other
+
     def __repr__(self):
-        return u"<Series('{0},{1}')>".format(self.name, self.sort)
+        return "<Series('{0},{1}')>".format(self.name, self.sort)
 
 
 class Ratings(Base):
@@ -258,8 +273,11 @@ class Ratings(Base):
     def get(self):
         return self.rating
 
+    def __eq__(self, other):
+        return self.rating == other
+
     def __repr__(self):
-        return u"<Ratings('{0}')>".format(self.rating)
+        return "<Ratings('{0}')>".format(self.rating)
 
 
 class Languages(Base):
@@ -272,13 +290,16 @@ class Languages(Base):
         self.lang_code = lang_code
 
     def get(self):
-        if self.language_name:
+        if hasattr(self, "language_name"):
             return self.language_name
         else:
             return self.lang_code
 
+    def __eq__(self, other):
+        return self.lang_code == other
+
     def __repr__(self):
-        return u"<Languages('{0}')>".format(self.lang_code)
+        return "<Languages('{0}')>".format(self.lang_code)
 
 
 class Publishers(Base):
@@ -295,8 +316,11 @@ class Publishers(Base):
     def get(self):
         return self.name
 
+    def __eq__(self, other):
+        return self.name == other
+
     def __repr__(self):
-        return u"<Publishers('{0},{1}')>".format(self.name, self.sort)
+        return "<Publishers('{0},{1}')>".format(self.name, self.sort)
 
 
 class Data(Base):
@@ -320,7 +344,16 @@ class Data(Base):
         return self.name
 
     def __repr__(self):
-        return u"<Data('{0},{1}{2}{3}')>".format(self.book, self.format, self.uncompressed_size, self.name)
+        return "<Data('{0},{1}{2}{3}')>".format(self.book, self.format, self.uncompressed_size, self.name)
+
+
+class Metadata_Dirtied(Base):
+    __tablename__ = 'metadata_dirtied'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False, unique=True)
+
+    def __init__(self, book):
+        self.book = book
 
 
 class Books(Base):
@@ -365,7 +398,7 @@ class Books(Base):
         self.has_cover = (has_cover != None)
 
     def __repr__(self):
-        return u"<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
+        return "<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
                                                                  self.timestamp, self.pubdate, self.series_index,
                                                                  self.last_modified, self.path, self.has_cover)
 
@@ -388,9 +421,36 @@ class CustomColumns(Base):
     normalized = Column(Boolean)
 
     def get_display_dict(self):
-        display_dict = ast.literal_eval(self.display)
+        display_dict = json.loads(self.display)
         return display_dict
 
+    def to_json(self, value, extra, sequence):
+        content = dict()
+        content['table'] = "custom_column_" + str(self.id)
+        content['column'] = "value"
+        content['datatype'] = self.datatype
+        content['is_multiple'] = None if not self.is_multiple else "|"
+        content['kind'] = "field"
+        content['name'] = self.name
+        content['search_terms'] = ['#' + self.label]
+        content['label'] = self.label
+        content['colnum'] = self.id
+        content['display'] = self.get_display_dict()
+        content['is_custom'] = True
+        content['is_category'] = self.datatype in ['text', 'rating', 'enumeration', 'series']
+        content['link_column'] = "value"
+        content['category_sort'] = "value"
+        content['is_csp'] = False
+        content['is_editable'] = self.editable
+        content['rec_index'] = sequence + 22     # toDo why ??
+        if isinstance(value, datetime):
+            content['#value#'] = {"__class__": "datetime.datetime", "__value__": value.strftime("%Y-%m-%dT%H:%M:%S+00:00")}
+        else:
+            content['#value#'] = value
+        content['#extra#'] = extra
+        content['is_multiple2'] = {} if not self.is_multiple else {"cache_to_list": "|", "ui_to_list": ",", "list_to_ui": ", "}
+        return json.dumps(content, ensure_ascii=False)
+
 
 class AlchemyEncoder(json.JSONEncoder):
 
@@ -568,12 +628,12 @@ class CalibreDB:
 
         if not config_calibre_dir:
             cls.config.invalidate()
-            return False
+            return None
 
         dbpath = os.path.join(config_calibre_dir, "metadata.db")
         if not os.path.exists(dbpath):
             cls.config.invalidate()
-            return False
+            return None
 
         try:
             cls.engine = create_engine('sqlite://',
@@ -589,7 +649,7 @@ class CalibreDB:
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as ex:
             cls.config.invalidate(ex)
-            return False
+            return None
 
         cls.config.db_configured = True
 
@@ -599,16 +659,15 @@ class CalibreDB:
                 cls.setup_db_cc_classes(cc)
             except OperationalError as e:
                 log.error_or_exception(e)
-                return False
+                return None
 
         cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
-                                                          bind=cls.engine))
+                                                          bind=cls.engine, future=True))
         for inst in cls.instances:
             inst.init_session()
 
         cls._init = True
-        return True
 
     def get_book(self, book_id):
         return self.session.query(Books).filter(Books.id == book_id).first()
@@ -629,7 +688,7 @@ class CalibreDB:
                       .join(read_column, read_column.book == book_id,
                       isouter=True))
             except (KeyError, AttributeError, IndexError):
-                log.error("Custom Column No.{} is not existing in calibre database".format(read_column))
+                log.error("Custom Column No.{} does not exist in calibre database".format(read_column))
                 # Skip linking read column and return None instead of read status
                 bd = self.session.query(Books, None, ub.ArchivedBook.is_archived)
         return (bd.filter(Books.id == book_id)
@@ -643,6 +702,18 @@ class CalibreDB:
     def get_book_format(self, book_id, file_format):
         return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == file_format).first()
 
+    def set_metadata_dirty(self, book_id):
+        if not self.session.query(Metadata_Dirtied).filter(Metadata_Dirtied.book == book_id).one_or_none():
+            self.session.add(Metadata_Dirtied(book_id))
+
+    def delete_dirty_metadata(self, book_id):
+        try:
+            self.session.query(Metadata_Dirtied).filter(Metadata_Dirtied.book == book_id).delete()
+            self.session.commit()
+        except (OperationalError) as e:
+            self.session.rollback()
+            log.error("Database error: {}".format(e))
+
     # Language and content filters for displaying in the UI
     def common_filters(self, allow_show_archived=False, return_all_languages=False):
         if not allow_show_archived:
@@ -676,9 +747,9 @@ class CalibreDB:
             except (KeyError, AttributeError, IndexError):
                 pos_content_cc_filter = false()
                 neg_content_cc_filter = true()
-                log.error("Custom Column No.{} is not existing in calibre database".format(
+                log.error("Custom Column No.{} does not exist in calibre database".format(
                     self.config.config_restricted_column))
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                flash(_("Custom Column No.%(column)d does not exist in calibre database",
                         column=self.config.config_restricted_column),
                       category="error")
 
@@ -701,7 +772,7 @@ class CalibreDB:
                          .select_from(Books)
                          .outerjoin(read_column, read_column.book == Books.id))
             except (KeyError, AttributeError, IndexError):
-                log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column))
+                log.error("Custom Column No.{} does not exist in calibre database".format(config_read_column))
                 # Skip linking read column and return None instead of read status
                 query = self.session.query(database, None, ub.ArchivedBook.is_archived)
         return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id,
@@ -779,8 +850,6 @@ class CalibreDB:
 
     # Orders all Authors in the list according to authors sort
     def order_authors(self, entries, list_return=False, combined=False):
-        # entries_copy = copy.deepcopy(entries)
-        # entries_copy =[]
         for entry in entries:
             if combined:
                 sort_authors = entry.Books.author_sort.split('&')
@@ -945,7 +1014,12 @@ class CalibreDB:
                 title = title[len(prep):] + ', ' + prep
             return title.strip()
 
-        conn = conn or self.session.connection().connection.connection
+        try:
+            # sqlalchemy <1.4.24
+            conn = conn or self.session.connection().connection.driver_connection
+        except AttributeError:
+            # sqlalchemy >1.4.24 and sqlalchemy 2.0
+            conn = conn or self.session.connection().connection.connection
         try:
             conn.create_function("title_sort", 1, _title_sort)
         except sqliteOperationalError:

cps/debug_info.py

@@ -22,6 +22,7 @@ import glob
 import zipfile
 import json
 from io import BytesIO
+from flask_babel.speaklater import LazyString
 
 import os
 
@@ -32,6 +33,13 @@ from .about import collect_stats
 
 log = logger.create()
 
+class lazyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, LazyString):
+            return str(obj)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
+
 def assemble_logfiles(file_name):
     log_list = sorted(glob.glob(file_name + '*'), reverse=True)
     wfd = BytesIO()
@@ -57,8 +65,8 @@ def send_debug():
             file_list.remove(element)
     memory_zip = BytesIO()
     with zipfile.ZipFile(memory_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
-        zf.writestr('settings.txt', json.dumps(config.toDict()))
-        zf.writestr('libs.txt', json.dumps(collect_stats()))
+        zf.writestr('settings.txt', json.dumps(config.to_dict(), sort_keys=True, indent=2))
+        zf.writestr('libs.txt', json.dumps(collect_stats(), sort_keys=True, indent=2, cls=lazyEncoder))
         for fp in file_list:
             zf.write(fp, os.path.basename(fp))
     memory_zip.seek(0)

cps/dep_check.py

@@ -20,7 +20,8 @@ if not importlib:
     except ImportError as e:
         pkgresources = False
 
-def load_dependencys(optional=False):
+
+def load_dependencies(optional=False):
     deps = list()
     if getattr(sys, 'frozen', False):
         pip_installed = os.path.join(BASE_DIR, ".pip_installed")
@@ -41,7 +42,7 @@ def load_dependencys(optional=False):
                         res = re.match(r'(.*?)([<=>\s]+)([\d\.]+),?\s?([<=>\s]+)?([\d\.]+)?', line.strip())
                         try:
                             if getattr(sys, 'frozen', False):
-                                dep_version = exe_deps[res.group(1).lower().replace('_','-')]
+                                dep_version = exe_deps[res.group(1).lower().replace('_', '-')]
                             else:
                                 if importlib:
                                     dep_version = version(res.group(1))
@@ -57,38 +58,38 @@ def load_dependencys(optional=False):
 
 def dependency_check(optional=False):
     d = list()
-    deps = load_dependencys(optional)
+    deps = load_dependencies(optional)
     for dep in deps:
         try:
-            dep_version_int = [int(x) for x in dep[0].split('.')]
+            dep_version_int = [int(x) if x.isnumeric() else 0 for x in dep[0].split('.')]
             low_check = [int(x) for x in dep[3].split('.')]
             high_check = [int(x) for x in dep[5].split('.')]
         except AttributeError:
-            high_check = None
+            high_check = []
         except ValueError:
             d.append({'name': dep[1],
-                     'target': "available",
-                     'found': "Not available"
-                     })
+                      'target': "available",
+                      'found': "Not available"
+                      })
             continue
 
         if dep[2].strip() == "==":
             if dep_version_int != low_check:
                 d.append({'name': dep[1],
-                            'found': dep[0],
-                            "target": dep[2] + dep[3]})
+                          'found': dep[0],
+                          "target": dep[2] + dep[3]})
                 continue
         elif dep[2].strip() == ">=":
             if dep_version_int < low_check:
                 d.append({'name': dep[1],
-                            'found': dep[0],
-                            "target": dep[2] + dep[3]})
+                          'found': dep[0],
+                          "target": dep[2] + dep[3]})
                 continue
         elif dep[2].strip() == ">":
             if dep_version_int <= low_check:
                 d.append({'name': dep[1],
-                            'found': dep[0],
-                            "target": dep[2] + dep[3]})
+                          'found': dep[0],
+                          "target": dep[2] + dep[3]})
                 continue
         if dep[4] and dep[5]:
             if dep[4].strip() == "<":

cps/editbooks.py

@@ -25,20 +25,33 @@ from datetime import datetime
 import json
 from shutil import copyfile
 from uuid import uuid4
-from markupsafe import escape  # dependency of flask
+from markupsafe import escape, Markup  # dependency of flask
 from functools import wraps
+from lxml.etree import ParserError
 
 try:
-    from lxml.html.clean import clean_html
+    # at least bleach 6.0 is needed -> incomplatible change from list arguments to set arguments
+    from bleach import clean_text as clean_html
+    BLEACH = True
 except ImportError:
-    clean_html = None
+    try:
+        from nh3 import clean as clean_html
+        BLEACH = False
+    except ImportError:
+        try:
+            from lxml.html.clean import clean_html
+            BLEACH = False
+        except ImportError:
+            clean_html = None
 
-from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
+from flask import Blueprint, request, flash, redirect, url_for, abort, Response
 from flask_babel import gettext as _
 from flask_babel import lazy_gettext as N_
 from flask_babel import get_locale
 from flask_login import current_user, login_required
-from sqlalchemy.exc import OperationalError, IntegrityError
+from sqlalchemy.exc import OperationalError, IntegrityError, InterfaceError
+from sqlalchemy.orm.exc import StaleDataError
+from sqlalchemy.sql.expression import func
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper, kobo_sync_status
 from . import config, ub, db, calibre_db
@@ -106,7 +119,7 @@ def edit_book(book_id):
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     # Book not found
     if not book:
-        flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"),
+        flash(_("Oops! Selected book is unavailable. File does not exist or is not accessible"),
               category="error")
         return redirect(url_for("web.index"))
 
@@ -124,7 +137,7 @@ def edit_book(book_id):
             edited_books_id = book.id
             modify_date = True
             title_author_error = helper.update_dir_structure(edited_books_id,
-                                                             config.config_calibre_dir,
+                                                             config.get_book_path(),
                                                              input_authors[0],
                                                              renamed_author=renamed)
         if title_author_error:
@@ -150,7 +163,7 @@ def edit_book(book_id):
         if to_save.get("cover_url", None):
             if not current_user.role_upload():
                 edit_error = True
-                flash(_(u"User has no rights to upload cover"), category="error")
+                flash(_("User has no rights to upload cover"), category="error")
             if to_save["cover_url"].endswith('/static/generic_cover.jpg'):
                 book.has_cover = 0
             else:
@@ -202,6 +215,7 @@ def edit_book(book_id):
         if modify_date:
             book.last_modified = datetime.utcnow()
             kobo_sync_status.remove_synced_book(edited_books_id, all=True)
+            calibre_db.set_metadata_dirty(book.id)
 
         calibre_db.session.merge(book)
         calibre_db.session.commit()
@@ -221,10 +235,10 @@ def edit_book(book_id):
         calibre_db.session.rollback()
         flash(str(e), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
-    except (OperationalError, IntegrityError) as e:
+    except (OperationalError, IntegrityError, StaleDataError, InterfaceError) as e:
         log.error_or_exception("Database error: {}".format(e))
         calibre_db.session.rollback()
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig if hasattr(e, "orig") else e), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
     except Exception as ex:
         log.error_or_exception(ex)
@@ -268,7 +282,7 @@ def upload():
                                                   meta.extension.lower())
                 else:
                     error = helper.update_dir_structure(book_id,
-                                                        config.config_calibre_dir,
+                                                        config.get_book_path(),
                                                         input_authors[0],
                                                         meta.file_path,
                                                         title_dir + meta.extension.lower(),
@@ -276,6 +290,8 @@ def upload():
 
                 move_coverfile(meta, db_book)
 
+                if modify_date:
+                    calibre_db.set_metadata_dirty(book_id)
                 # save data to database, reread data
                 calibre_db.session.commit()
 
@@ -284,7 +300,7 @@ def upload():
                 if error:
                     flash(error, category="error")
                 link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book_id), escape(title))
-                upload_text = N_(u"File %(file)s uploaded", file=link)
+                upload_text = N_("File %(file)s uploaded", file=link)
                 WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title)))
                 helper.add_book_to_thumbnail_cache(book_id)
 
@@ -295,10 +311,11 @@ def upload():
                     else:
                         resp = {"location": url_for('web.show_book', book_id=book_id)}
                         return Response(json.dumps(resp), mimetype='application/json')
-            except (OperationalError, IntegrityError) as e:
+            except (OperationalError, IntegrityError, StaleDataError) as e:
                 calibre_db.session.rollback()
                 log.error_or_exception("Database error: {}".format(e))
-                flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+                flash(_("Oops! Database Error: %(error)s.", error=e.orig if hasattr(e, "orig") else e),
+                      category="error")
         return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
 
@@ -311,19 +328,19 @@ def convert_bookformat(book_id):
     book_format_to = request.form.get('book_format_to', None)
 
     if (book_format_from is None) or (book_format_to is None):
-        flash(_(u"Source or destination format for conversion missing"), category="error")
+        flash(_("Source or destination format for conversion missing"), category="error")
         return redirect(url_for('edit-book.show_edit_book', book_id=book_id))
 
     log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to)
-    rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
+    rtn = helper.convert_book_format(book_id, config.get_book_path(), book_format_from.upper(),
                                      book_format_to.upper(), current_user.name)
 
     if rtn is None:
-        flash(_(u"Book successfully queued for converting to %(book_format)s",
+        flash(_("Book successfully queued for converting to %(book_format)s",
                 book_format=book_format_to),
               category="success")
     else:
-        flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
+        flash(_("There was an error converting this book: %(res)s", res=rtn), category="error")
     return redirect(url_for('edit-book.show_edit_book', book_id=book_id))
 
 
@@ -385,7 +402,7 @@ def edit_list_book(param):
         elif param == 'title':
             sort_param = book.sort
             if handle_title_on_edit(book, vals.get('value', "")):
-                rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir)
+                rename_error = helper.update_dir_structure(book.id, config.get_book_path())
                 if not rename_error:
                     ret = Response(json.dumps({'success': True, 'newValue':  book.title}),
                                    mimetype='application/json')
@@ -403,7 +420,7 @@ def edit_list_book(param):
                            mimetype='application/json')
         elif param == 'authors':
             input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true")
-            rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0],
+            rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0],
                                                        renamed_author=renamed)
             if not rename_error:
                 ret = Response(json.dumps({
@@ -443,11 +460,11 @@ def edit_list_book(param):
         if param == 'title' and vals.get('checkT') == "false":
             book.sort = sort_param
             calibre_db.session.commit()
-    except (OperationalError, IntegrityError) as e:
+    except (OperationalError, IntegrityError, StaleDataError) as e:
         calibre_db.session.rollback()
         log.error_or_exception("Database error: {}".format(e))
         ret = Response(json.dumps({'success': False,
-                                   'msg': 'Database error: {}'.format(e.orig)}),
+                                   'msg': 'Database error: {}'.format(e.orig if hasattr(e, "orig") else e)}),
                        mimetype='application/json')
     return ret
 
@@ -465,7 +482,7 @@ def get_sorted_entry(field, bookid):
             if field == 'sort':
                 return json.dumps({'sort': book.title})
             if field == 'author_sort':
-                return json.dumps({'author_sort': book.author})
+                return json.dumps({'authors': " & ".join([a.name for a in calibre_db.order_authors([book])])})
     return ""
 
 
@@ -507,10 +524,10 @@ def merge_list_book():
                     for element in from_book.data:
                         if element.format not in to_file:
                             # create new data entry with: book_id, book_format, uncompressed_size, name
-                            filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir,
+                            filepath_new = os.path.normpath(os.path.join(config.get_book_path(),
                                                                          to_book.path,
                                                                          to_name + "." + element.format.lower()))
-                            filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir,
+                            filepath_old = os.path.normpath(os.path.join(config.get_book_path(),
                                                                          from_book.path,
                                                                          element.name + "." + element.format.lower()))
                             copyfile(filepath_old, filepath_new)
@@ -550,15 +567,16 @@ def table_xchange_author_title():
 
             if edited_books_id:
                 # toDo: Handle error
-                edit_error = helper.update_dir_structure(edited_books_id, config.config_calibre_dir, input_authors[0],
+                edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0],
                                                          renamed_author=renamed)
             if modify_date:
                 book.last_modified = datetime.utcnow()
+                calibre_db.set_metadata_dirty(book.id)
             try:
                 calibre_db.session.commit()
-            except (OperationalError, IntegrityError) as e:
+            except (OperationalError, IntegrityError, StaleDataError) as e:
                 calibre_db.session.rollback()
-                log.error_or_exception("Database error: %s", e)
+                log.error_or_exception("Database error: {}".format(e))
                 return json.dumps({'success': False})
 
             if config.config_use_google_drive:
@@ -568,9 +586,9 @@ def table_xchange_author_title():
 
 
 def merge_metadata(to_save, meta):
-    if to_save.get('author_name', "") == _(u'Unknown'):
+    if to_save.get('author_name', "") == _('Unknown'):
         to_save['author_name'] = ''
-    if to_save.get('book_title', "") == _(u'Unknown'):
+    if to_save.get('book_title', "") == _('Unknown'):
         to_save['book_title'] = ''
     for s_field, m_field in [
             ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'),
@@ -592,6 +610,8 @@ def identifier_list(to_save, book):
         val_key = id_val_prefix + type_key[len(id_type_prefix):]
         if val_key not in to_save.keys():
             continue
+        if to_save[val_key].startswith("data:"):
+            to_save[val_key], __, __ = str.partition(to_save[val_key], ",")
         result.append(db.Identifiers(to_save[val_key], type_value, book.id))
     return result
 
@@ -606,7 +626,7 @@ def prepare_authors(authr):
 
     # we have all author names now
     if input_authors == ['']:
-        input_authors = [_(u'Unknown')]  # prevent empty Author
+        input_authors = [_('Unknown')]  # prevent empty Author
 
     renamed = list()
     for in_aut in input_authors:
@@ -623,11 +643,11 @@ def prepare_authors(authr):
 
 
 def prepare_authors_on_upload(title, authr):
-    if title != _(u'Unknown') and authr != _(u'Unknown'):
+    if title != _('Unknown') and authr != _('Unknown'):
         entry = calibre_db.check_exists_book(authr, title)
         if entry:
             log.info("Uploaded book probably exists in library")
-            flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
+            flash(_("Uploaded book probably exists in the library, consider to change before upload new: ")
                   + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
 
     input_authors, renamed = prepare_authors(authr)
@@ -682,7 +702,7 @@ def create_book_on_upload(modify_date, meta):
     modify_date |= edit_book_languages(meta.languages, db_book, upload_mode=True, invalid=invalid)
     if invalid:
         for lang in invalid:
-            flash(_(u"'%(langname)s' is not a valid language", langname=lang), category="warning")
+            flash(_("'%(langname)s' is not a valid language", langname=lang), category="warning")
 
     # handle tags
     modify_date |= edit_book_tags(meta.tags, db_book)
@@ -732,7 +752,7 @@ def file_handling_on_upload(requested_file):
         meta = uploader.upload(requested_file, config.config_rarfile_location)
     except (IOError, OSError):
         log.error("File %s could not saved to temp dir", requested_file.filename)
-        flash(_(u"File %(filename)s could not saved to temp dir",
+        flash(_("File %(filename)s could not saved to temp dir",
                 filename=requested_file.filename), category="error")
         return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
     return meta, None
@@ -744,7 +764,7 @@ def move_coverfile(meta, db_book):
         cover_file = meta.cover
     else:
         cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
-    new_cover_path = os.path.join(config.config_calibre_dir, db_book.path)
+    new_cover_path = os.path.join(config.get_book_path(), db_book.path)
     try:
         os.makedirs(new_cover_path, exist_ok=True)
         copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg"))
@@ -752,7 +772,7 @@ def move_coverfile(meta, db_book):
             os.unlink(meta.cover)
     except OSError as e:
         log.error("Failed to move cover file %s: %s", new_cover_path, e)
-        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path,
+        flash(_("Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path,
                 error=e),
               category="error")
 
@@ -766,7 +786,7 @@ def delete_whole_book(book_id, book):
 
     # check if only this book links to:
     # author, language, series, tags, custom columns
-    modify_database_object([u''], book.authors, db.Authors, calibre_db.session, 'author')
+    modify_database_object([''], book.authors, db.Authors, calibre_db.session, 'author')
     modify_database_object([u''], book.tags, db.Tags, calibre_db.session, 'tags')
     modify_database_object([u''], book.series, db.Series, calibre_db.session, 'series')
     modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages')
@@ -830,7 +850,7 @@ def delete_book_from_table(book_id, book_format, json_response):
         book = calibre_db.get_book(book_id)
         if book:
             try:
-                result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
+                result, error = helper.delete_book(book, config.get_book_path(), book_format=book_format.upper())
                 if not result:
                     if json_response:
                         return json.dumps([{"location": url_for("edit-book.show_edit_book", book_id=book_id),
@@ -887,7 +907,7 @@ def render_edit_book(book_id):
     cc = calibre_db.session.query(db.CustomColumns).filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)).all()
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     if not book:
-        flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"),
+        flash(_("Oops! Selected book is unavailable. File does not exist or is not accessible"),
               category="error")
         return redirect(url_for("web.index"))
 
@@ -922,7 +942,7 @@ def render_edit_book(book_id):
     if kepub_possible:
         allowed_conversion_formats.append('kepub')
     return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
-                                 title=_(u"edit metadata"), page="editbook",
+                                 title=_("edit metadata"), page="editbook",
                                  conversion_formats=allowed_conversion_formats,
                                  config=config,
                                  source_formats=valid_source_formats)
@@ -983,7 +1003,14 @@ def edit_book_series_index(series_index, book):
 def edit_book_comments(comments, book):
     modify_date = False
     if comments:
-        comments = clean_html(comments)
+        try:
+            if BLEACH:
+                comments = clean_html(comments, tags=set(), attributes=set())
+            else:
+                comments = clean_html(comments)
+        except ParserError as e:
+            log.error("Comments of book {} are corrupted: {}".format(book.id, e))
+            comments = ""
     if len(book.comments):
         if book.comments[0].text != comments:
             book.comments[0].text = comments
@@ -1007,7 +1034,7 @@ def edit_book_languages(languages, book, upload_mode=False, invalid=None):
         if isinstance(invalid, list):
             invalid.append(lang)
         else:
-            raise ValueError(_(u"'%(langname)s' is not a valid language", langname=lang))
+            raise ValueError(_("'%(langname)s' is not a valid language", langname=lang))
     # ToDo: Not working correct
     if upload_mode and len(input_l) == 1:
         # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
@@ -1118,9 +1145,10 @@ def edit_cc_data(book_id, book, to_save, cc):
                 cc_db_value = None
             if to_save[cc_string].strip():
                 if c.datatype in ['int', 'bool', 'float', "datetime", "comments"]:
-                    changed, to_save = edit_cc_data_value(book_id, book, c, to_save, cc_db_value, cc_string)
+                    change, to_save = edit_cc_data_value(book_id, book, c, to_save, cc_db_value, cc_string)
                 else:
-                    changed, to_save = edit_cc_data_string(book, c, to_save, cc_db_value, cc_string)
+                    change, to_save = edit_cc_data_string(book, c, to_save, cc_db_value, cc_string)
+                changed |= change
             else:
                 if cc_db_value is not None:
                     # remove old cc_val
@@ -1149,7 +1177,7 @@ def upload_single_file(file_request, book, book_id):
         # check for empty request
         if requested_file.filename != '':
             if not current_user.role_upload():
-                flash(_(u"User has no rights to upload additional file formats"), category="error")
+                flash(_("User has no rights to upload additional file formats"), category="error")
                 return False
             if '.' in requested_file.filename:
                 file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
@@ -1162,7 +1190,7 @@ def upload_single_file(file_request, book, book_id):
                 return False
 
             file_name = book.path.rsplit('/', 1)[-1]
-            filepath = os.path.normpath(os.path.join(config.config_calibre_dir, book.path))
+            filepath = os.path.normpath(os.path.join(config.get_book_path(), book.path))
             saved_filename = os.path.join(filepath, file_name + '.' + file_ext)
 
             # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
@@ -1170,12 +1198,12 @@ def upload_single_file(file_request, book, book_id):
                 try:
                     os.makedirs(filepath)
                 except OSError:
-                    flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
+                    flash(_("Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
                     return False
             try:
                 requested_file.save(saved_filename)
             except OSError:
-                flash(_(u"Failed to store file %(file)s.", file=saved_filename), category="error")
+                flash(_("Failed to store file %(file)s.", file=saved_filename), category="error")
                 return False
 
             file_size = os.path.getsize(saved_filename)
@@ -1190,20 +1218,21 @@ def upload_single_file(file_request, book, book_id):
                     calibre_db.session.add(db_format)
                     calibre_db.session.commit()
                     calibre_db.update_title_sort(config)
-                except (OperationalError, IntegrityError) as e:
+                except (OperationalError, IntegrityError, StaleDataError) as e:
                     calibre_db.session.rollback()
                     log.error_or_exception("Database error: {}".format(e))
-                    flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+                    flash(_("Oops! Database Error: %(error)s.", error=e.orig if hasattr(e, "orig") else e),
+                          category="error")
                     return False  # return redirect(url_for('web.show_book', book_id=book.id))
 
             # Queue uploader info
             link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book.id), escape(book.title))
-            upload_text = N_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link)
+            upload_text = N_("File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link)
             WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(book.title)))
 
             return uploader.process(
                 saved_filename, *os.path.splitext(requested_file.filename),
-                rarExecutable=config.config_rarfile_location)
+                rar_executable=config.config_rarfile_location)
     return None
 
 
@@ -1213,7 +1242,7 @@ def upload_cover(cover_request, book):
         # check for empty request
         if requested_file.filename != '':
             if not current_user.role_upload():
-                flash(_(u"User has no rights to upload cover"), category="error")
+                flash(_("User has no rights to upload cover"), category="error")
                 return False
             ret, message = helper.save_cover(requested_file, book.path)
             if ret is True:
@@ -1237,18 +1266,18 @@ def handle_title_on_edit(book, book_title):
 
 
 def handle_author_on_edit(book, author_name, update_stored=True):
+    change = False
     # handle author(s)
     input_authors, renamed = prepare_authors(author_name)
 
-    change = modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
-
+    # change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
     # Search for each author if author is in database, if not, author name and sorted author name is generated new
     # everything then is assembled for sorted author field in database
     sort_authors_list = list()
     for inp in input_authors:
         stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
         if not stored_author:
-            stored_author = helper.get_sorted_author(inp)
+            stored_author = helper.get_sorted_author(inp.replace('|', ','))
         else:
             stored_author = stored_author.sort
         sort_authors_list.append(helper.get_sorted_author(stored_author))
@@ -1256,6 +1285,9 @@ def handle_author_on_edit(book, author_name, update_stored=True):
     if book.author_sort != sort_authors and update_stored:
         book.author_sort = sort_authors
         change = True
+
+    change |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
+
     return input_authors, change, renamed
 
 
@@ -1263,14 +1295,15 @@ def search_objects_remove(db_book_object, db_type, input_elements):
     del_elements = []
     for c_elements in db_book_object:
         found = False
-        if db_type == 'languages':
-            type_elements = c_elements.lang_code
-        elif db_type == 'custom':
+        #if db_type == 'languages':
+        #    type_elements = c_elements.lang_code
+        if db_type == 'custom':
             type_elements = c_elements.value
         else:
-            type_elements = c_elements.name
+            # type_elements = c_elements.name
+            type_elements = c_elements
         for inp_element in input_elements:
-            if inp_element.lower() == type_elements.lower():
+            if type_elements == inp_element:
                 found = True
                 break
         # if the element was not found in the new list, add it to remove list
@@ -1284,13 +1317,11 @@ def search_objects_add(db_book_object, db_type, input_elements):
     for inp_element in input_elements:
         found = False
         for c_elements in db_book_object:
-            if db_type == 'languages':
-                type_elements = c_elements.lang_code
-            elif db_type == 'custom':
+            if db_type == 'custom':
                 type_elements = c_elements.value
             else:
-                type_elements = c_elements.name
-            if inp_element == type_elements:
+                type_elements = c_elements
+            if type_elements == inp_element:
                 found = True
                 break
         if not found:
@@ -1306,6 +1337,7 @@ def remove_objects(db_book_object, db_session, del_elements):
             changed = True
             if len(del_element.books) == 0:
                 db_session.delete(del_element)
+                db_session.flush()
     return changed
 
 
@@ -1319,27 +1351,34 @@ def add_objects(db_book_object, db_object, db_session, db_type, add_elements):
         db_filter = db_object.name
     for add_element in add_elements:
         # check if an element with that name exists
-        db_element = db_session.query(db_object).filter(db_filter == add_element).first()
+        changed = True
+        # db_session.query(db.Tags).filter((func.lower(db.Tags.name).ilike("GênOt"))).all()
+        db_element = db_session.query(db_object).filter((func.lower(db_filter).ilike(add_element))).first()
+        # db_element = db_session.query(db_object).filter(func.lower(db_filter) == add_element.lower()).first()
         # if no element is found add it
-        if db_type == 'author':
-            new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')), "")
-        elif db_type == 'series':
-            new_element = db_object(add_element, add_element)
-        elif db_type == 'custom':
-            new_element = db_object(value=add_element)
-        elif db_type == 'publisher':
-            new_element = db_object(add_element, None)
-        else:  # db_type should be tag or language
-            new_element = db_object(add_element)
         if db_element is None:
-            changed = True
+            if db_type == 'author':
+                new_element = db_object(add_element, helper.get_sorted_author(add_element.replace('|', ',')))
+            elif db_type == 'series':
+                new_element = db_object(add_element, add_element)
+            elif db_type == 'custom':
+                new_element = db_object(value=add_element)
+            elif db_type == 'publisher':
+                new_element = db_object(add_element, None)
+            else:  # db_type should be tag or language
+                new_element = db_object(add_element)
             db_session.add(new_element)
             db_book_object.append(new_element)
         else:
-            db_element = create_objects_for_addition(db_element, add_element, db_type)
+            db_no_case = db_session.query(db_object).filter(db_filter == add_element).first()
+            if db_no_case:
+                # check for new case of element
+                db_element = create_objects_for_addition(db_element, add_element, db_type)
+            else:
+                db_element = create_objects_for_addition(db_element, add_element, db_type)
             # add element to book
-            changed = True
             db_book_object.append(db_element)
+
     return changed
 
 
@@ -1374,13 +1413,24 @@ def modify_database_object(input_elements, db_book_object, db_object, db_session
     if not isinstance(input_elements, list):
         raise TypeError(str(input_elements) + " should be passed as a list")
     input_elements = [x for x in input_elements if x != '']
-    # we have all input element (authors, series, tags) names now
+
+    changed = False
+    # If elements are renamed (upper lower case), rename it
+    for rec_a, rec_b in zip(db_book_object, input_elements):
+        if db_type == "custom":
+            if rec_a.value.casefold() == rec_b.casefold() and rec_a.value != rec_b:
+                create_objects_for_addition(rec_a, rec_b, db_type)
+        else:
+            if rec_a.get().casefold() == rec_b.casefold() and rec_a.get() != rec_b:
+                create_objects_for_addition(rec_a, rec_b, db_type)
+        # we have all input element (authors, series, tags) names now
     # 1. search for elements to remove
     del_elements = search_objects_remove(db_book_object, db_type, input_elements)
     # 2. search for elements that need to be added
     add_elements = search_objects_add(db_book_object, db_type, input_elements)
+
     # if there are elements to remove, we remove them now
-    changed = remove_objects(db_book_object, db_session, del_elements)
+    changed |= remove_objects(db_book_object, db_session, del_elements)
     # if there are elements to add, we add them now!
     if len(add_elements) > 0:
         changed |= add_objects(db_book_object, db_object, db_session, db_type, add_elements)

cps/epub.py

@@ -21,25 +21,47 @@ import zipfile
 from lxml import etree
 
 from . import isoLanguages, cover
+from . import config, logger
 from .helper import split_authors
+from .epub_helper import get_content_opf, default_ns
 from .constants import BookMeta
 
+log = logger.create()
+
 
 def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name):
     if cover_file is None:
         return None
-    else:
-        cf = extension = None
-        zip_cover_path = os.path.join(cover_path, cover_file).replace('\\', '/')
 
-        prefix = os.path.splitext(tmp_file_name)[0]
-        tmp_cover_name = prefix + '.' + os.path.basename(zip_cover_path)
-        ext = os.path.splitext(tmp_cover_name)
-        if len(ext) > 1:
-            extension = ext[1].lower()
-        if extension in cover.COVER_EXTENSIONS:
-            cf = zip_file.read(zip_cover_path)
-        return cover.cover_processing(tmp_file_name, cf, extension)
+    cf = extension = None
+    zip_cover_path = os.path.join(cover_path, cover_file).replace('\\', '/')
+
+    prefix = os.path.splitext(tmp_file_name)[0]
+    tmp_cover_name = prefix + '.' + os.path.basename(zip_cover_path)
+    ext = os.path.splitext(tmp_cover_name)
+    if len(ext) > 1:
+        extension = ext[1].lower()
+    if extension in cover.COVER_EXTENSIONS:
+        cf = zip_file.read(zip_cover_path)
+    return cover.cover_processing(tmp_file_name, cf, extension)
+
+def get_epub_layout(book, book_data):
+    file_path = os.path.normpath(os.path.join(config.get_book_path(),
+                                              book.path, book_data.name + "." + book_data.format.lower()))
+
+    try:
+        tree, __ = get_content_opf(file_path, default_ns)
+        p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
+
+        layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns)
+    except (etree.XMLSyntaxError, KeyError, IndexError) as e:
+        log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e))
+        layout = []
+
+    if len(layout) == 0:
+        return None
+    else:
+        return layout[0]
 
 
 def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
@@ -49,13 +71,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
         'dc': 'http://purl.org/dc/elements/1.1/'
     }
 
-    epub_zip = zipfile.ZipFile(tmp_file_path)
-
-    txt = epub_zip.read('META-INF/container.xml')
-    tree = etree.fromstring(txt)
-    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
-    cf = epub_zip.read(cf_name)
-    tree = etree.fromstring(cf)
+    tree, cf_name = get_content_opf(tmp_file_path, ns)
 
     cover_path = os.path.dirname(cf_name)
 
@@ -73,20 +89,20 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
             elif s == 'date':
                 epub_metadata[s] = tmp[0][:10]
             else:
-                epub_metadata[s] = tmp[0]
+                epub_metadata[s] = tmp[0].strip()
         else:
             epub_metadata[s] = 'Unknown'
 
     if epub_metadata['subject'] == 'Unknown':
         epub_metadata['subject'] = ''
 
-    if epub_metadata['publisher'] == u'Unknown':
+    if epub_metadata['publisher'] == 'Unknown':
         epub_metadata['publisher'] = ''
 
-    if epub_metadata['date'] == u'Unknown':
+    if epub_metadata['date'] == 'Unknown':
         epub_metadata['date'] = ''
 
-    if epub_metadata['description'] == u'Unknown':
+    if epub_metadata['description'] == 'Unknown':
         description = tree.xpath("//*[local-name() = 'description']/text()")
         if len(description) > 0:
             epub_metadata['description'] = description
@@ -98,15 +114,19 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     epub_metadata = parse_epub_series(ns, tree, epub_metadata)
 
+    epub_zip = zipfile.ZipFile(tmp_file_path)
     cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)
 
     identifiers = []
     for node in p.xpath('dc:identifier', namespaces=ns):
-        identifier_name=node.attrib.values()[-1];
-        identifier_value=node.text;
-        if identifier_name in ('uuid','calibre'):
-            continue;
-        identifiers.append( [identifier_name, identifier_value] )
+        try:
+            identifier_name = node.attrib.values()[-1]
+        except IndexError:
+            continue
+        identifier_value = node.text
+        if identifier_name in ('uuid', 'calibre') or identifier_value is None:
+            continue
+        identifiers.append([identifier_name, identifier_value])
 
     if not epub_metadata['title']:
         title = original_file_name
@@ -131,40 +151,40 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
 def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):
     cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
-    cover_file = None
-    # if len(cover_section) > 0:
     for cs in cover_section:
         cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
         if cover_file:
-            break
-    if not cover_file:
-        meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
-        if len(meta_cover) > 0:
+            return cover_file
+
+    meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
+    if len(meta_cover) > 0:
+        cover_section = tree.xpath(
+            "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
+        if not cover_section:
             cover_section = tree.xpath(
-                "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
-            if not cover_section:
-                cover_section = tree.xpath(
-                    "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns)
+                "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns)
+    else:
+        cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns)
+
+    cover_file = None
+    for cs in cover_section:
+        if cs.endswith('.xhtml') or cs.endswith('.html'):
+            markup = epub_zip.read(os.path.join(cover_path, cs))
+            markup_tree = etree.fromstring(markup)
+            # no matter xhtml or html with no namespace
+            img_src = markup_tree.xpath("//*[local-name() = 'img']/@src")
+            # Alternative image source
+            if not len(img_src):
+                img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]")
+            if len(img_src):
+                # img_src maybe start with "../"" so fullpath join then relpath to cwd
+                filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])),
+                                                        img_src[0]))
+                cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
         else:
-            cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns)
-        for cs in cover_section:
-            filetype = cs.rsplit('.', 1)[-1]
-            if filetype == "xhtml" or filetype == "html":  # if cover is (x)html format
-                markup = epub_zip.read(os.path.join(cover_path, cs))
-                markup_tree = etree.fromstring(markup)
-                # no matter xhtml or html with no namespace
-                img_src = markup_tree.xpath("//*[local-name() = 'img']/@src")
-                # Alternative image source
-                if not len(img_src):
-                    img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]")
-                if len(img_src):
-                    # img_src maybe start with "../"" so fullpath join then relpath to cwd
-                    filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])),
-                                                            img_src[0]))
-                    cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path)
-            else:
-                cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
-            if cover_file: break
+            cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path)
+        if cover_file:
+            break
     return cover_file
 
 

cps/epub_helper.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018 lemmsh, Kennyl, Kyosfonica, matthazinski
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import zipfile
+from lxml import etree
+
+from . import isoLanguages
+
+default_ns = {
+    'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
+    'pkg': 'http://www.idpf.org/2007/opf',
+}
+
+OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
+PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
+
+OPF = "{%s}" % OPF_NAMESPACE
+PURL = "{%s}" % PURL_NAMESPACE
+
+etree.register_namespace("opf", OPF_NAMESPACE)
+etree.register_namespace("dc", PURL_NAMESPACE)
+
+OPF_NS = {None: OPF_NAMESPACE}  # the default namespace (no prefix)
+NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
+
+
+def updateEpub(src, dest, filename, data, ):
+    # create a temp copy of the archive without filename
+    with zipfile.ZipFile(src, 'r') as zin:
+        with zipfile.ZipFile(dest, 'w') as zout:
+            zout.comment = zin.comment # preserve the comment
+            for item in zin.infolist():
+                if item.filename != filename:
+                    zout.writestr(item, zin.read(item.filename))
+
+    # now add filename with its new data
+    with zipfile.ZipFile(dest, mode='a', compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr(filename, data)
+
+
+def get_content_opf(file_path, ns=default_ns):
+    epubZip = zipfile.ZipFile(file_path)
+    txt = epubZip.read('META-INF/container.xml')
+    tree = etree.fromstring(txt)
+    cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0]
+    cf = epubZip.read(cf_name)
+
+    return etree.fromstring(cf), cf_name
+
+
+def create_new_metadata_backup(book,  custom_columns, export_language, translated_cover_name, lang_type=3):
+    # generate root package element
+    package = etree.Element(OPF + "package", nsmap=OPF_NS)
+    package.set("unique-identifier", "uuid_id")
+    package.set("version", "2.0")
+
+    # generate metadata element and all sub elements of it
+    metadata = etree.SubElement(package, "metadata", nsmap=NSMAP)
+    identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP)
+    identifier.set(OPF + "scheme", "calibre")
+    identifier.text = str(book.id)
+    identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP)
+    identifier2.set(OPF + "scheme", "uuid")
+    identifier2.text = book.uuid
+    for i in book.identifiers:
+        identifier = etree.SubElement(metadata, PURL + "identifier", nsmap=NSMAP)
+        identifier.set(OPF + "scheme", i.format_type())
+        identifier.text = str(i.val)
+    title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP)
+    title.text = book.title
+    for author in book.authors:
+        creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP)
+        creator.text = str(author.name)
+        creator.set(OPF + "file-as", book.author_sort)     # ToDo Check
+        creator.set(OPF + "role", "aut")
+    contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP)
+    contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]"
+    contributor.set(OPF + "file-as", "calibre")     # ToDo Check
+    contributor.set(OPF + "role", "bkp")
+
+    date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP)
+    date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate)
+    if book.comments and book.comments[0].text:
+        for b in book.comments:
+            description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
+            description.text = b.text
+    for b in book.publishers:
+        publisher = etree.SubElement(metadata, PURL + "publisher", nsmap=NSMAP)
+        publisher.text = str(b.name)
+    if not book.languages:
+        language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
+        language.text = export_language
+    else:
+        for b in book.languages:
+            language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
+            language.text = str(b.lang_code) if lang_type == 3 else isoLanguages.get(part3=b.lang_code).part1
+    for b in book.tags:
+        subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP)
+        subject.text = str(b.name)
+    etree.SubElement(metadata, "meta", name="calibre:author_link_map",
+                     content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}",
+                     nsmap=NSMAP)
+    for b in book.series:
+        etree.SubElement(metadata, "meta", name="calibre:series",
+                         content=str(str(b.name)),
+                         nsmap=NSMAP)
+    if book.series:
+        etree.SubElement(metadata, "meta", name="calibre:series_index",
+                         content=str(book.series_index),
+                         nsmap=NSMAP)
+    if len(book.ratings) and book.ratings[0].rating > 0:
+        etree.SubElement(metadata, "meta", name="calibre:rating",
+                         content=str(book.ratings[0].rating),
+                         nsmap=NSMAP)
+    etree.SubElement(metadata, "meta", name="calibre:timestamp",
+                     content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(
+                         d=book.timestamp),
+                     nsmap=NSMAP)
+    etree.SubElement(metadata, "meta", name="calibre:title_sort",
+                     content=book.sort,
+                     nsmap=NSMAP)
+    sequence = 0
+    for cc in custom_columns:
+        value = None
+        extra = None
+        cc_entry = getattr(book, "custom_column_" + str(cc.id))
+        if cc_entry.__len__():
+            value = [c.value for c in cc_entry] if cc.is_multiple else cc_entry[0].value
+            extra = cc_entry[0].extra if hasattr(cc_entry[0], "extra") else None
+        etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label),
+                         content=cc.to_json(value, extra, sequence),
+                         nsmap=NSMAP)
+        sequence += 1
+
+    # generate guide element and all sub elements of it
+    # Title is translated from default export language
+    guide = etree.SubElement(package, "guide")
+    etree.SubElement(guide, "reference", type="cover", title=translated_cover_name, href="cover.jpg")
+
+    return package
+
+def replace_metadata(tree, package):
+    rep_element = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
+    new_element = package.xpath('//metadata', namespaces=default_ns)[0]
+    tree.replace(rep_element, new_element)
+    return etree.tostring(tree,
+                   xml_declaration=True,
+                   encoding='utf-8',
+                   pretty_print=True).decode('utf-8')
+
+

cps/fb2.py

@@ -38,19 +38,19 @@ def get_fb2_info(tmp_file_path, original_file_extension):
         if len(last_name):
             last_name = last_name[0]
         else:
-            last_name = u''
+            last_name = ''
         middle_name = element.xpath('fb:middle-name/text()', namespaces=ns)
         if len(middle_name):
             middle_name = middle_name[0]
         else:
-            middle_name = u''
+            middle_name = ''
         first_name = element.xpath('fb:first-name/text()', namespaces=ns)
         if len(first_name):
             first_name = first_name[0]
         else:
-            first_name = u''
-        return (first_name + u' '
-                + middle_name + u' '
+            first_name = ''
+        return (first_name + ' '
+                + middle_name + ' '
                 + last_name)
 
     author = str(", ".join(map(get_author, authors)))
@@ -59,12 +59,12 @@ def get_fb2_info(tmp_file_path, original_file_extension):
     if len(title):
         title = str(title[0])
     else:
-        title = u''
+        title = ''
     description = tree.xpath('/fb:FictionBook/fb:description/fb:publish-info/fb:book-name/text()', namespaces=ns)
     if len(description):
         description = str(description[0])
     else:
-        description = u''
+        description = ''
 
     return BookMeta(
         file_path=tmp_file_path,

cps/file_helper.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2023 OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from tempfile import gettempdir
+import os
+import shutil
+
+def get_temp_dir():
+    tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+    if not os.path.isdir(tmp_dir):
+        os.mkdir(tmp_dir)
+    return tmp_dir
+
+
+def del_temp_dir():
+    tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+    shutil.rmtree(tmp_dir)

cps/gdrive.py

@@ -23,7 +23,6 @@
 import os
 import hashlib
 import json
-import tempfile
 from uuid import uuid4
 from time import time
 from shutil import move, copyfile
@@ -34,6 +33,7 @@ from flask_login import login_required
 
 from . import logger, gdriveutils, config, ub, calibre_db, csrf
 from .admin import admin_required
+from .file_helper import get_temp_dir
 
 gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive')
 log = logger.create()
@@ -55,7 +55,7 @@ def authenticate_google_drive():
     try:
         authUrl = gdriveutils.Gauth.Instance().auth.GetAuthUrl()
     except gdriveutils.InvalidConfigError:
-        flash(_(u'Google Drive setup not completed, try to deactivate and activate Google Drive again'),
+        flash(_('Google Drive setup not completed, try to deactivate and activate Google Drive again'),
               category="error")
         return redirect(url_for('web.index'))
     return redirect(authUrl)
@@ -91,9 +91,9 @@ def watch_gdrive():
             config.save()
         except HttpError as e:
             reason=json.loads(e.content)['error']['errors'][0]
-            if reason['reason'] == u'push.webhookUrlUnauthorized':
-                flash(_(u'Callback domain is not verified, '
-                        u'please follow steps to verify domain in google developer console'), category="error")
+            if reason['reason'] == 'push.webhookUrlUnauthorized':
+                flash(_('Callback domain is not verified, '
+                        'please follow steps to verify domain in google developer console'), category="error")
             else:
                 flash(reason['message'], category="error")
 
@@ -139,9 +139,7 @@ try:
                 dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
                 if not response['deleted'] and response['file']['title'] == 'metadata.db' \
                     and response['file']['md5Checksum'] != hashlib.md5(dbpath):  # nosec
-                    tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web')
-                    if not os.path.isdir(tmp_dir):
-                        os.mkdir(tmp_dir)
+                    tmp_dir = get_temp_dir()
 
                     log.info('Database file updated')
                     copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time())))

cps/gdriveutils.py

@@ -33,7 +33,7 @@ try:
 except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError
-from sqlalchemy.sql.expression import text
+from sqlalchemy.orm.exc import StaleDataError
 
 try:
     from httplib2 import __version__ as httplib2_version
@@ -146,7 +146,7 @@ engine = create_engine('sqlite:///{0}'.format(cli_param.gd_path), echo=False)
 Base = declarative_base()
 
 # Open session for database connection
-Session = sessionmaker()
+Session = sessionmaker(autoflush=False)
 Session.configure(bind=engine)
 session = scoped_session(Session)
 
@@ -173,30 +173,12 @@ class PermissionAdded(Base):
         return str(self.gdrive_id)
 
 
-def migrate():
-    if not engine.dialect.has_table(engine.connect(), "permissions_added"):
-        PermissionAdded.__table__.create(bind = engine)
-    for sql in session.execute(text("select sql from sqlite_master where type='table'")):
-        if 'CREATE TABLE gdrive_ids' in sql[0]:
-            currUniqueConstraint = 'UNIQUE (gdrive_id)'
-            if currUniqueConstraint in sql[0]:
-                sql=sql[0].replace(currUniqueConstraint, 'UNIQUE (gdrive_id, path)')
-                sql=sql.replace(GdriveId.__tablename__, GdriveId.__tablename__ + '2')
-                session.execute(sql)
-                session.execute("INSERT INTO gdrive_ids2 (id, gdrive_id, path) SELECT id, "
-                                "gdrive_id, path FROM gdrive_ids;")
-                session.commit()
-                session.execute('DROP TABLE %s' % 'gdrive_ids')
-                session.execute('ALTER TABLE gdrive_ids2 RENAME to gdrive_ids')
-            break
-
 if not os.path.exists(cli_param.gd_path):
     try:
         Base.metadata.create_all(engine)
     except Exception as ex:
         log.error("Error connect to database: {} - {}".format(cli_param.gd_path, ex))
         raise
-migrate()
 
 
 def getDrive(drive=None, gauth=None):
@@ -318,7 +300,7 @@ def getFolderId(path, drive):
                 session.commit()
         else:
             currentFolderId = storedPathName.gdrive_id
-    except (OperationalError, IntegrityError) as ex:
+    except (OperationalError, IntegrityError, StaleDataError) as ex:
         log.error_or_exception('Database error: {}'.format(ex))
         session.rollback()
     except ApiRequestError as ex:
@@ -343,7 +325,7 @@ def getFileFromEbooksFolder(path, fileName):
 
 
 def moveGdriveFileRemote(origin_file_id, new_title):
-    origin_file_id['title']= new_title
+    origin_file_id['title'] = new_title
     origin_file_id.Upload()
 
 
@@ -421,7 +403,7 @@ def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
             driveFile.Upload()
 
 
-def uploadFileToEbooksFolder(destFile, f):
+def uploadFileToEbooksFolder(destFile, f, string=False):
     drive = getDrive(Gdrive.Instance().drive)
     parent = getEbooksFolder(drive)
     splitDir = destFile.split('/')
@@ -434,7 +416,10 @@ def uploadFileToEbooksFolder(destFile, f):
             else:
                 driveFile = drive.CreateFile({'title': x,
                                               'parents': [{"kind": "drive#fileLink", 'id': parent['id']}], })
-            driveFile.SetContentFile(f)
+            if not string:
+                driveFile.SetContentFile(f)
+            else:
+                driveFile.SetContentString(f)
             driveFile.Upload()
         else:
             existing_Folder = drive.ListFile({'q': "title = '%s' and '%s' in parents and trashed = false" %
@@ -555,7 +540,7 @@ def updateGdriveCalibreFromLocal():
 
 # update gdrive.db on edit of books title
 def updateDatabaseOnEdit(ID,newPath):
-    sqlCheckPath = newPath if newPath[-1] == '/' else newPath + u'/'
+    sqlCheckPath = newPath if newPath[-1] == '/' else newPath + '/'
     storedPathName = session.query(GdriveId).filter(GdriveId.gdrive_id == ID).first()
     if storedPathName:
         storedPathName.path = sqlCheckPath
@@ -577,6 +562,7 @@ def deleteDatabaseEntry(ID):
 
 
 # Gets cover file from gdrive
+# ToDo: Check is this right everyone get read permissions on cover files?
 def get_cover_via_gdrive(cover_path):
     df = getFileFromEbooksFolder(cover_path, 'cover.jpg')
     if df:
@@ -599,6 +585,29 @@ def get_cover_via_gdrive(cover_path):
     else:
         return None
 
+# Gets cover file from gdrive
+def get_metadata_backup_via_gdrive(metadata_path):
+    df = getFileFromEbooksFolder(metadata_path, 'metadata.opf')
+    if df:
+        if not session.query(PermissionAdded).filter(PermissionAdded.gdrive_id == df['id']).first():
+            df.GetPermissions()
+            df.InsertPermission({
+                            'type': 'anyone',
+                            'value': 'anyone',
+                            'role': 'writer',       # ToDo needs write access
+                            'withLink': True})
+            permissionAdded = PermissionAdded()
+            permissionAdded.gdrive_id = df['id']
+            session.add(permissionAdded)
+            try:
+                session.commit()
+            except OperationalError as ex:
+                log.error_or_exception('Database error: {}'.format(ex))
+                session.rollback()
+        return df.metadata.get('webContentLink')
+    else:
+        return None
+
 # Creates chunks for downloading big files
 def partial(total_byte_len, part_size_limit):
     s = []

cps/helper.py

@@ -18,20 +18,22 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import random
 import io
-import sys
 import mimetypes
 import re
 import shutil
 import socket
 from datetime import datetime, timedelta
-from tempfile import gettempdir
 import requests
 import unidecode
+from uuid import uuid4
+from lxml import etree
 
 from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
 from flask_babel import lazy_gettext as N_
+from flask_babel import get_locale
 from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, or_, text, func
 from sqlalchemy.exc import InvalidRequestError, OperationalError
@@ -54,10 +56,13 @@ from .tasks.convert import TaskConvert
 from . import logger, config, db, ub, fs
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES
-from .subproc_wrapper import process_wait
+from .subproc_wrapper import process_wait, process_open
 from .services.worker import WorkerThread
 from .tasks.mail import TaskEmail
 from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
+from .tasks.metadata_backup import TaskBackupMetadata
+from .file_helper import get_temp_dir
+from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub, replace_metadata
 
 log = logger.create()
 
@@ -72,93 +77,93 @@ except (ImportError, RuntimeError) as e:
 
 
 # Convert existing book entry to new format
-def convert_book_format(book_id, calibre_path, old_book_format, new_book_format, user_id, kindle_mail=None):
+def convert_book_format(book_id, calibre_path, old_book_format, new_book_format, user_id, ereader_mail=None):
     book = calibre_db.get_book(book_id)
     data = calibre_db.get_book_format(book.id, old_book_format)
-    file_path = os.path.join(calibre_path, book.path, data.name)
     if not data:
-        error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
+        error_message = _("%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
         return error_message
+    file_path = os.path.join(calibre_path, book.path, data.name)
     if config.config_use_google_drive:
         if not gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()):
-            error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
+            error_message = _("%(format)s not found on Google Drive: %(fn)s",
                               format=old_book_format, fn=data.name + "." + old_book_format.lower())
             return error_message
     else:
         if not os.path.exists(file_path + "." + old_book_format.lower()):
-            error_message = _(u"%(format)s not found: %(fn)s",
+            error_message = _("%(format)s not found: %(fn)s",
                               format=old_book_format, fn=data.name + "." + old_book_format.lower())
             return error_message
     # read settings and append converter task to queue
-    if kindle_mail:
+    if ereader_mail:
         settings = config.get_mail_settings()
-        settings['subject'] = _('Send to Kindle')  # pretranslate Subject for e-mail
-        settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
+        settings['subject'] = _('Send to eReader')  # pretranslate Subject for Email
+        settings['body'] = _('This Email has been sent via Calibre-Web.')
     else:
         settings = dict()
     link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book.id), escape(book.title))  # prevent xss
-    txt = u"{} -> {}: {}".format(
+    txt = "{} -> {}: {}".format(
            old_book_format.upper(),
            new_book_format.upper(),
            link)
     settings['old_book_format'] = old_book_format
     settings['new_book_format'] = new_book_format
-    WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
+    WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, ereader_mail, user_id))
     return None
 
 
 # Texts are not lazy translated as they are supposed to get send out as is
-def send_test_mail(kindle_mail, user_name):
-    WorkerThread.add(user_name, TaskEmail(_(u'Calibre-Web test e-mail'), None, None,
-                     config.get_mail_settings(), kindle_mail, N_(u"Test e-mail"),
-                                          _(u'This e-mail has been sent via Calibre-Web.')))
+def send_test_mail(ereader_mail, user_name):
+    WorkerThread.add(user_name, TaskEmail(_('Calibre-Web Test Email'), None, None,
+                     config.get_mail_settings(), ereader_mail, N_("Test Email"),
+                                          _('This Email has been sent via Calibre-Web.')))
     return
 
 
 # Send registration email or password reset email, depending on parameter resend (False means welcome email)
 def send_registration_mail(e_mail, user_name, default_password, resend=False):
-    txt = "Hello %s!\r\n" % user_name
+    txt = "Hi %s!\r\n" % user_name
     if not resend:
-        txt += "Your new account at Calibre-Web has been created. Thanks for joining us!\r\n"
-    txt += "Please log in to your account using the following informations:\r\n"
-    txt += "User name: %s\r\n" % user_name
+        txt += "Your account at Calibre-Web has been created.\r\n"
+    txt += "Please log in using the following information:\r\n"
+    txt += "Username: %s\r\n" % user_name
     txt += "Password: %s\r\n" % default_password
-    txt += "Don't forget to change your password after first login.\r\n"
-    txt += "Sincerely\r\n\r\n"
-    txt += "Your Calibre-Web team"
+    txt += "Don't forget to change your password after your first login.\r\n"
+    txt += "Regards,\r\n\r\n"
+    txt += "Calibre-Web"
     WorkerThread.add(None, TaskEmail(
-        subject=_(u'Get Started with Calibre-Web'),
+        subject=_('Get Started with Calibre-Web'),
         filepath=None,
         attachment=None,
         settings=config.get_mail_settings(),
         recipient=e_mail,
-        task_message=N_(u"Registration e-mail for user: %(name)s", name=user_name),
+        task_message=N_("Registration Email for user: %(name)s", name=user_name),
         text=txt
     ))
     return
 
 
-def check_send_to_kindle_with_converter(formats):
+def check_send_to_ereader_with_converter(formats):
     book_formats = list()
-    if 'EPUB' in formats and 'MOBI' not in formats:
-        book_formats.append({'format': 'Mobi',
+    if 'MOBI' in formats and 'EPUB' not in formats:
+        book_formats.append({'format': 'Epub',
                              'convert': 1,
-                             'text': _('Convert %(orig)s to %(format)s and send to Kindle',
-                                       orig='Epub',
-                                       format='Mobi')})
-    if 'AZW3' in formats and 'MOBI' not in formats:
-        book_formats.append({'format': 'Mobi',
+                             'text': _('Convert %(orig)s to %(format)s and send to eReader',
+                                       orig='Mobi',
+                                       format='Epub')})
+    if 'AZW3' in formats and 'EPUB' not in formats:
+        book_formats.append({'format': 'Epub',
                              'convert': 2,
-                             'text': _('Convert %(orig)s to %(format)s and send to Kindle',
+                             'text': _('Convert %(orig)s to %(format)s and send to eReader',
                                        orig='Azw3',
-                                       format='Mobi')})
+                                       format='Epub')})
     return book_formats
 
 
-def check_send_to_kindle(entry):
+def check_send_to_ereader(entry):
     """
-        returns all available book formats for sending to Kindle
+        returns all available book formats for sending to eReader
     """
     formats = list()
     book_formats = list()
@@ -166,30 +171,30 @@ def check_send_to_kindle(entry):
         for ele in iter(entry.data):
             if ele.uncompressed_size < config.mail_size:
                 formats.append(ele.format)
-        if 'MOBI' in formats:
-            book_formats.append({'format': 'Mobi',
+        if 'EPUB' in formats:
+            book_formats.append({'format': 'Epub',
                                  'convert': 0,
-                                 'text': _('Send %(format)s to Kindle', format='Mobi')})
+                                 'text': _('Send %(format)s to eReader', format='Epub')})
         if 'PDF' in formats:
             book_formats.append({'format': 'Pdf',
                                  'convert': 0,
-                                 'text': _('Send %(format)s to Kindle', format='Pdf')})
+                                 'text': _('Send %(format)s to eReader', format='Pdf')})
         if 'AZW' in formats:
             book_formats.append({'format': 'Azw',
                                  'convert': 0,
-                                 'text': _('Send %(format)s to Kindle', format='Azw')})
+                                 'text': _('Send %(format)s to eReader', format='Azw')})
         if config.config_converterpath:
-            book_formats.extend(check_send_to_kindle_with_converter(formats))
+            book_formats.extend(check_send_to_ereader_with_converter(formats))
         return book_formats
     else:
-        log.error(u'Cannot find book entry %d', entry.id)
+        log.error('Cannot find book entry %d', entry.id)
         return None
 
 
 # Check if a reader is existing for any of the book formats, if not, return empty list, otherwise return
 # list with supported formats
 def check_read_formats(entry):
-    extensions_reader = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR', 'DJVU'}
+    extensions_reader = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR', 'DJVU', 'DJV'}
     book_formats = list()
     if len(entry.data):
         for ele in iter(entry.data):
@@ -199,30 +204,30 @@ def check_read_formats(entry):
 
 
 # Files are processed in the following order/priority:
-# 1: If Mobi file is existing, it's directly send to kindle email,
-# 2: If Epub file is existing, it's converted and send to kindle email,
-# 3: If Pdf file is existing, it's directly send to kindle email
-def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
+# 1: If epub file is existing, it's directly send to eReader email,
+# 2: If mobi file is existing, it's converted and send to eReader email,
+# 3: If Pdf file is existing, it's directly send to eReader email
+def send_mail(book_id, book_format, convert, ereader_mail, calibrepath, user_id):
     """Send email with attachments"""
     book = calibre_db.get_book(book_id)
 
     if convert == 1:
         # returns None if success, otherwise errormessage
-        return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail)
+        return convert_book_format(book_id, calibrepath, 'mobi', book_format.lower(), user_id, ereader_mail)
     if convert == 2:
         # returns None if success, otherwise errormessage
-        return convert_book_format(book_id, calibrepath, u'azw3', book_format.lower(), user_id, kindle_mail)
+        return convert_book_format(book_id, calibrepath, 'azw3', book_format.lower(), user_id, ereader_mail)
 
     for entry in iter(book.data):
         if entry.format.upper() == book_format.upper():
             converted_file_name = entry.name + '.' + book_format.lower()
             link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book_id), escape(book.title))
-            email_text = N_(u"%(book)s send to Kindle", book=link)
-            WorkerThread.add(user_id, TaskEmail(_(u"Send to Kindle"), book.path, converted_file_name,
-                             config.get_mail_settings(), kindle_mail,
-                             email_text, _(u'This e-mail has been sent via Calibre-Web.')))
+            email_text = N_("%(book)s send to eReader", book=link)
+            WorkerThread.add(user_id, TaskEmail(_("Send to eReader"), book.path, converted_file_name,
+                             config.get_mail_settings(), ereader_mail,
+                             email_text, _('This Email has been sent via Calibre-Web.')))
             return
-    return _(u"The requested file could not be read. Maybe wrong permissions?")
+    return _("The requested file could not be read. Maybe wrong permissions?")
 
 
 def get_valid_filename(value, replace_whitespace=True, chars=128):
@@ -230,19 +235,18 @@ def get_valid_filename(value, replace_whitespace=True, chars=128):
     Returns the given string converted to a string that can be used for a clean
     filename. Limits num characters to 128 max.
     """
-    if value[-1:] == u'.':
-        value = value[:-1]+u'_'
+    if value[-1:] == '.':
+        value = value[:-1]+'_'
     value = value.replace("/", "_").replace(":", "_").strip('\0')
     if config.config_unicode_filename:
         value = (unidecode.unidecode(value))
     if replace_whitespace:
         #  *+:\"/<>? are replaced by _
-        value = re.sub(r'[*+:\\\"/<>?]+', u'_', value, flags=re.U)
+        value = re.sub(r'[*+:\\\"/<>?]+', '_', value, flags=re.U)
         # pipe has to be replaced with comma
-        value = re.sub(r'[|]+', u',', value, flags=re.U)
+        value = re.sub(r'[|]+', ',', value, flags=re.U)
 
-    filename_encoding_for_length = 'utf-16' if sys.platform == "win32" or sys.platform == "darwin" else 'utf-8'
-    value = value.encode(filename_encoding_for_length)[:chars].decode('utf-8', errors='ignore').strip()
+    value = value.encode('utf-8')[:chars].decode('utf-8', errors='ignore').strip()
 
     if not value:
         raise ValueError("Filename cannot be empty")
@@ -319,12 +323,12 @@ def edit_book_read_status(book_id, read_status=None):
         try:
             calibre_db.update_title_sort(config)
             book = calibre_db.get_filtered_book(book_id)
-            read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
-            if len(read_status):
+            book_read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
+            if len(book_read_status):
                 if read_status is None:
-                    read_status[0].value = not read_status[0].value
+                    book_read_status[0].value = not book_read_status[0].value
                 else:
-                    read_status[0].value = read_status is True
+                    book_read_status[0].value = read_status is True
                 calibre_db.session.commit()
             else:
                 cc_class = db.cc_classes[config.config_read_column]
@@ -333,11 +337,11 @@ def edit_book_read_status(book_id, read_status=None):
                 calibre_db.session.commit()
         except (KeyError, AttributeError, IndexError):
             log.error(
-                "Custom Column No.{} is not existing in calibre database".format(config.config_read_column))
-            return "Custom Column No.{} is not existing in calibre database".format(config.config_read_column)
+                "Custom Column No.{} does not exist in calibre database".format(config.config_read_column))
+            return "Custom Column No.{} does not exist in calibre database".format(config.config_read_column)
         except (OperationalError, InvalidRequestError) as ex:
             calibre_db.session.rollback()
-            log.error(u"Read status could not set: {}".format(ex))
+            log.error("Read status could not set: {}".format(ex))
             return _("Read status could not set: {}".format(ex.orig))
     return ""
 
@@ -412,8 +416,8 @@ def clean_author_database(renamed_author, calibre_path="", local_book=None, gdri
                         g_file = gd.getFileFromEbooksFolder(all_new_path,
                                                             file_format.name + '.' + file_format.format.lower())
                         if g_file:
-                            gd.moveGdriveFileRemote(g_file, all_new_name + u'.' + file_format.format.lower())
-                            gd.updateDatabaseOnEdit(g_file['id'], all_new_name + u'.' + file_format.format.lower())
+                            gd.moveGdriveFileRemote(g_file, all_new_name + '.' + file_format.format.lower())
+                            gd.updateDatabaseOnEdit(g_file['id'], all_new_name + '.' + file_format.format.lower())
                         else:
                             log.error("File {} not found on gdrive"
                                       .format(all_new_path, file_format.name + '.' + file_format.format.lower()))
@@ -506,25 +510,25 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author):
     authordir = book.path.split('/')[0]
     titledir = book.path.split('/')[1]
     new_authordir = rename_all_authors(first_author, renamed_author, gdrive=True)
-    new_titledir = get_valid_filename(book.title, chars=96) + u" (" + str(book_id) + u")"
+    new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")"
 
     if titledir != new_titledir:
         g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir)
         if g_file:
             gd.moveGdriveFileRemote(g_file, new_titledir)
-            book.path = book.path.split('/')[0] + u'/' + new_titledir
+            book.path = book.path.split('/')[0] + '/' + new_titledir
             gd.updateDatabaseOnEdit(g_file['id'], book.path)     # only child folder affected
         else:
-            return _(u'File %(file)s not found on Google Drive', file=book.path)  # file not found
+            return _('File %(file)s not found on Google Drive', file=book.path)  # file not found
 
     if authordir != new_authordir and authordir not in renamed_author:
         g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir)
         if g_file:
             gd.moveGdriveFolderRemote(g_file, new_authordir)
-            book.path = new_authordir + u'/' + book.path.split('/')[1]
+            book.path = new_authordir + '/' + book.path.split('/')[1]
             gd.updateDatabaseOnEdit(g_file['id'], book.path)
         else:
-            return _(u'File %(file)s not found on Google Drive', file=authordir)  # file not found
+            return _('File %(file)s not found on Google Drive', file=authordir)  # file not found
 
     # change location in database to new author/title path
     book.path = os.path.join(new_authordir, new_titledir).replace('\\', '/')
@@ -596,7 +600,7 @@ def delete_book_gdrive(book, book_format):
         gd.deleteDatabaseEntry(g_file['id'])
         g_file.Trash()
     else:
-        error = _(u'Book path %(path)s not found on Google Drive', path=book.path)  # file not found
+        error = _('Book path %(path)s not found on Google Drive', path=book.path)  # file not found
 
     return error is None, error
 
@@ -608,7 +612,7 @@ def reset_password(user_id):
     if not config.get_mail_server_configured():
         return 2, None
     try:
-        password = generate_random_password()
+        password = generate_random_password(config.config_password_min_length)
         existing_user.password = generate_password_hash(password)
         ub.session.commit()
         send_registration_mail(existing_user.email, existing_user.name, password, True)
@@ -617,11 +621,35 @@ def reset_password(user_id):
         ub.session.rollback()
         return 0, None
 
+def generate_random_password(min_length):
+    min_length = max(8, min_length) - 4
+    random_source = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?"
+    # select 1 lowercase
+    s = "abcdefghijklmnopqrstuvwxyz"
+    password = [s[c % len(s)] for c in os.urandom(1)]
+    # select 1 uppercase
+    s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    password.extend([s[c % len(s)] for c in os.urandom(1)])
+    # select 1 digit
+    s = "01234567890"
+    password.extend([s[c % len(s)] for c in os.urandom(1)])
+    # select 1 special symbol
+    s = "!@#$%&*()?"
+    password.extend([s[c % len(s)] for c in os.urandom(1)])
 
-def generate_random_password():
+    # generate other characters
+    password.extend([random_source[c % len(random_source)] for c in os.urandom(min_length)])
+
+    # password_list = list(password)
+    # shuffle all characters
+    random.SystemRandom().shuffle(password)
+    return ''.join(password)
+
+
+'''def generate_random_password(min_length):
     s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?"
-    passlen = 8
-    return "".join(s[c % len(s)] for c in os.urandom(passlen))
+    passlen = min_length
+    return "".join(s[c % len(s)] for c in os.urandom(passlen))'''
 
 
 def uniq(inpt):
@@ -636,28 +664,47 @@ def uniq(inpt):
 def check_email(email):
     email = valid_email(email)
     if ub.session.query(ub.User).filter(func.lower(ub.User.email) == email.lower()).first():
-        log.error(u"Found an existing account for this e-mail address")
-        raise Exception(_(u"Found an existing account for this e-mail address"))
+        log.error("Found an existing account for this Email address")
+        raise Exception(_("Found an existing account for this Email address"))
     return email
 
 
 def check_username(username):
     username = username.strip()
     if ub.session.query(ub.User).filter(func.lower(ub.User.name) == username.lower()).scalar():
-        log.error(u"This username is already taken")
-        raise Exception(_(u"This username is already taken"))
+        log.error("This username is already taken")
+        raise Exception(_("This username is already taken"))
     return username
 
 
 def valid_email(email):
     email = email.strip()
-    # Regex according to https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/email#validation
-    if not re.search(r"^[\w.!#$%&'*+\\/=?^_`{|}~-]+@[\w](?:[\w-]{0,61}[\w])?(?:\.[\w](?:[\w-]{0,61}[\w])?)*$",
-                     email):
-        log.error(u"Invalid e-mail address format")
-        raise Exception(_(u"Invalid e-mail address format"))
+    # if email is not deleted
+    if email:
+        # Regex according to https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/email#validation
+        if not re.search(r"^[\w.!#$%&'*+\\/=?^_`{|}~-]+@[\w](?:[\w-]{0,61}[\w])?(?:\.[\w](?:[\w-]{0,61}[\w])?)*$",
+                         email):
+            log.error("Invalid Email address format")
+            raise Exception(_("Invalid Email address format"))
     return email
 
+def valid_password(check_password):
+    if config.config_password_policy:
+        verify = ""
+        if config.config_password_min_length > 0:
+            verify += "^(?=.{" + str(config.config_password_min_length) + ",}$)"
+        if config.config_password_number:
+            verify += "(?=.*?\d)"
+        if config.config_password_lower:
+            verify += "(?=.*?[a-z])"
+        if config.config_password_upper:
+            verify += "(?=.*?[A-Z])"
+        if config.config_password_special:
+            verify += "(?=.*?[^A-Za-z\s0-9])"
+        match = re.match(verify, check_password)
+        if not match:
+            raise Exception(_("Password doesn't comply with password validation rules"))
+    return check_password
 # ################################# External interface #################################
 
 
@@ -680,35 +727,35 @@ def update_dir_structure(book_id,
 
 def delete_book(book, calibrepath, book_format):
     if not book_format:
-        clear_cover_thumbnail_cache(book.id)        ## here it breaks
+        clear_cover_thumbnail_cache(book.id) ## here it breaks
+        calibre_db.delete_dirty_metadata(book.id)
     if config.config_use_google_drive:
         return delete_book_gdrive(book, book_format)
     else:
         return delete_book_file(book, calibrepath, book_format)
 
 
-def get_cover_on_failure(use_generic_cover):
-    if use_generic_cover:
-        try:
-            return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
-        except PermissionError:
-            log.error("No permission to access generic_cover.jpg file.")
-            abort(403)
-    abort(404)
+def get_cover_on_failure():
+    try:
+        return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
+    except PermissionError:
+        log.error("No permission to access generic_cover.jpg file.")
+        abort(403)
 
 
 def get_book_cover(book_id, resolution=None):
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
-    return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
+    return get_book_cover_internal(book, resolution=resolution)
 
 
-# Called only by kobo sync -> cover not found should be answered with 404 and not with default cover
 def get_book_cover_with_uuid(book_uuid, resolution=None):
     book = calibre_db.get_book_by_uuid(book_uuid)
-    return get_book_cover_internal(book, use_generic_cover_on_failure=False, resolution=resolution)
+    if not book:
+        return  # allows kobo.HandleCoverImageRequest to proxy request
+    return get_book_cover_internal(book, resolution=resolution)
 
 
-def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None):
+def get_book_cover_internal(book, resolution=None):
     if book and book.has_cover:
 
         # Send the book cover thumbnail if it exists in cache
@@ -724,26 +771,26 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None)
         if config.config_use_google_drive:
             try:
                 if not gd.is_gdrive_ready():
-                    return get_cover_on_failure(use_generic_cover_on_failure)
+                    return get_cover_on_failure()
                 path = gd.get_cover_via_gdrive(book.path)
                 if path:
                     return redirect(path)
                 else:
                     log.error('{}/cover.jpg not found on Google Drive'.format(book.path))
-                    return get_cover_on_failure(use_generic_cover_on_failure)
+                    return get_cover_on_failure()
             except Exception as ex:
                 log.error_or_exception(ex)
-                return get_cover_on_failure(use_generic_cover_on_failure)
+                return get_cover_on_failure()
 
         # Send the book cover from the Calibre directory
         else:
-            cover_file_path = os.path.join(config.config_calibre_dir, book.path)
+            cover_file_path = os.path.join(config.get_book_path(), book.path)
             if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")):
                 return send_from_directory(cover_file_path, "cover.jpg")
             else:
-                return get_cover_on_failure(use_generic_cover_on_failure)
+                return get_cover_on_failure()
     else:
-        return get_cover_on_failure(use_generic_cover_on_failure)
+        return get_cover_on_failure()
 
 
 def get_book_cover_thumbnail(book, resolution):
@@ -766,7 +813,7 @@ def get_series_thumbnail_on_failure(series_id, resolution):
         .filter(db.Books.has_cover == 1) \
         .first()
 
-    return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
+    return get_book_cover_internal(book, resolution=resolution)
 
 
 def get_series_cover_thumbnail(series_id, resolution=None):
@@ -805,7 +852,7 @@ def save_cover_from_url(url, book_path):
             img = advocate.get(url, timeout=(10, 200), allow_redirects=False)      # ToDo: Error Handling
         else:
             log.error("python module advocate is not installed but is needed")
-            return False, _("Python module 'advocate' is not installed but is needed for cover downloads")
+            return False, _("Python module 'advocate' is not installed but is needed for cover uploads")
         img.raise_for_status()
         return save_cover(img, book_path)
     except (socket.gaierror,
@@ -830,8 +877,8 @@ def save_cover_from_filestorage(filepath, saved_filename, img):
         try:
             os.makedirs(filepath)
         except OSError:
-            log.error(u"Failed to create path for cover")
-            return False, _(u"Failed to create path for cover")
+            log.error("Failed to create path for cover")
+            return False, _("Failed to create path for cover")
     try:
         # upload of jgp file without wand
         if isinstance(img, requests.Response):
@@ -846,8 +893,8 @@ def save_cover_from_filestorage(filepath, saved_filename, img):
                 # upload of jpg/png... from hdd
                 img.save(os.path.join(filepath, saved_filename))
     except (IOError, OSError):
-        log.error(u"Cover-file is not a valid image file, or could not be stored")
-        return False, _(u"Cover-file is not a valid image file, or could not be stored")
+        log.error("Cover-file is not a valid image file, or could not be stored")
+        return False, _("Cover-file is not a valid image file, or could not be stored")
     return True, None
 
 
@@ -877,10 +924,7 @@ def save_cover(img, book_path):
             return False, _("Only jpg/jpeg files are supported as coverfile")
 
     if config.config_use_google_drive:
-        tmp_dir = os.path.join(gettempdir(), 'calibre_web')
-
-        if not os.path.isdir(tmp_dir):
-            os.mkdir(tmp_dir)
+        tmp_dir = get_temp_dir()
         ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img)
         if ret is True:
             gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"),
@@ -890,33 +934,92 @@ def save_cover(img, book_path):
         else:
             return False, message
     else:
-        return save_cover_from_filestorage(os.path.join(config.config_calibre_dir, book_path), "cover.jpg", img)
+        return save_cover_from_filestorage(os.path.join(config.get_book_path(), book_path), "cover.jpg", img)
 
 
 def do_download_file(book, book_format, client, data, headers):
+    book_name = data.name
     if config.config_use_google_drive:
         # startTime = time.time()
-        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
+        df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format)
         # log.debug('%s', time.time() - startTime)
         if df:
-            return gd.do_gdrive_download(df, headers)
+            if config.config_embed_metadata and (
+                 (book_format == "kepub" and config.config_kepubifypath ) or
+                 (book_format != "kepub" and config.config_binariesdir)):
+                output_path = os.path.join(config.config_calibre_dir, book.path)
+                if not os.path.exists(output_path):
+                    os.makedirs(output_path)
+                output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format)
+                gd.downloadFile(book.path, book_name + "." + book_format, output)
+                if book_format == "kepub" and config.config_kepubifypath:
+                    filename, download_name = do_kepubify_metadata_replace(book, output)
+                elif book_format != "kepub" and config.config_binariesdir:
+                    filename, download_name = do_calibre_export(book.id, book_format)
+            else:
+                return gd.do_gdrive_download(df, headers)
         else:
             abort(404)
     else:
-        filename = os.path.join(config.config_calibre_dir, book.path)
-        if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
+        filename = os.path.join(config.get_book_path(), book.path)
+        if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)):
             # ToDo: improve error handling
-            log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
+            log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format))
 
         if client == "kobo" and book_format == "kepub":
             headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
 
-        response = make_response(send_from_directory(filename, data.name + "." + book_format))
-        # ToDo Check headers parameter
-        for element in headers:
-            response.headers[element[0]] = element[1]
-        log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format)))
-        return response
+        if book_format == "kepub" and config.config_kepubifypath and config.config_embed_metadata:
+            filename, download_name = do_kepubify_metadata_replace(book, os.path.join(filename,
+                                                                                      book_name + "." + book_format))
+        elif book_format != "kepub" and config.config_binariesdir and config.config_embed_metadata:
+                filename, download_name = do_calibre_export(book.id, book_format)
+        else:
+            download_name = book_name
+
+    response = make_response(send_from_directory(filename, download_name + "." + book_format))
+    # ToDo Check headers parameter
+    for element in headers:
+        response.headers[element[0]] = element[1]
+    log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format)))
+    return response
+
+
+def do_kepubify_metadata_replace(book, file_path):
+    custom_columns = (calibre_db.session.query(db.CustomColumns)
+                      .filter(db.CustomColumns.mark_for_delete == 0)
+                      .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
+                      .order_by(db.CustomColumns.label).all())
+
+    tree, cf_name = get_content_opf(file_path)
+    package = create_new_metadata_backup(book, custom_columns, current_user.locale, _("Cover"), lang_type=2)
+    content = replace_metadata(tree, package)
+    tmp_dir = get_temp_dir()
+    temp_file_name = str(uuid4())
+    # open zipfile and replace metadata block in content.opf
+    updateEpub(file_path, os.path.join(tmp_dir, temp_file_name + ".kepub"), cf_name, content)
+    return tmp_dir, temp_file_name
+
+
+def do_calibre_export(book_id, book_format, ):
+    try:
+        quotes = [3, 5, 7, 9]
+        tmp_dir = get_temp_dir()
+        calibredb_binarypath = get_calibre_binarypath("calibredb")
+        temp_file_name = str(uuid4())
+        opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir,
+                       '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name),
+                       str(book_id)]
+        p = process_open(opf_command, quotes)
+        _, err = p.communicate()
+        if err:
+            log.error('Metadata embedder encountered an error: %s', err)
+        return tmp_dir, temp_file_name
+    except OSError as ex:
+        # ToDo real error handling
+        log.error_or_exception(ex)
+        return None, None
+
 
 ##################################
 
@@ -937,7 +1040,48 @@ def check_unrar(unrar_location):
 
     except (OSError, UnicodeDecodeError) as err:
         log.error_or_exception(err)
-        return _('Error excecuting UnRar')
+        return _('Error executing UnRar')
+
+
+def check_calibre(calibre_location):
+    if not calibre_location:
+        return
+
+    if not os.path.exists(calibre_location):
+        return _('Could not find the specified directory')
+
+    if not os.path.isdir(calibre_location):
+        return _('Please specify a directory, not a file')
+
+    try:
+        supported_binary_paths = [os.path.join(calibre_location, binary)
+                                  for binary in SUPPORTED_CALIBRE_BINARIES.values()]
+        binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths]
+        binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths]
+        if all(binaries_available) and all(binaries_executable):
+            values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)')
+                      for binary_path in supported_binary_paths]
+            if all(values):
+                version = values[0].group(1)
+                log.debug("calibre version %s", version)
+            else:
+                return _('Calibre binaries not viable')
+        else:
+            ret_val = []
+            missing_binaries=[path for path, available in
+                              zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available]
+
+            missing_perms=[path for path, available in
+                           zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available]
+            if missing_binaries:
+                ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)))
+            if missing_perms:
+                ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms)))
+            return ", ".join(ret_val)
+
+    except (OSError, UnicodeDecodeError) as err:
+        log.error_or_exception(err)
+        return _('Error excecuting Calibre')
 
 
 def check_calibre(calibre_location):
@@ -993,43 +1137,49 @@ def tags_filters():
 
 
 # checks if domain is in database (including wildcards)
-# example SELECT * FROM @TABLE WHERE  'abcdefg' LIKE Name;
+# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name;
 # from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/
 # in all calls the email address is checked for validity
 def check_valid_domain(domain_text):
     sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 1);"
-    result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()
-    if not len(result):
+    if not len(ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()):
         return False
     sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 0);"
-    result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()
-    return not len(result)
+    return not len(ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all())
 
 
 def get_download_link(book_id, book_format, client):
     book_format = book_format.split(".")[0]
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
-    data1= ""
     if book:
         data1 = calibre_db.get_book_format(book.id, book_format.upper())
+        if data1:
+            # collect downloaded books only for registered user and not for anonymous user
+            if current_user.is_authenticated:
+                ub.update_download(book_id, int(current_user.id))
+            file_name = book.title
+            if len(book.authors) > 0:
+                file_name = file_name + ' - ' + book.authors[0].name
+            file_name = get_valid_filename(file_name, replace_whitespace=False)
+            headers = Headers()
+            headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
+            headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % (
+                quote(file_name), book_format, quote(file_name), book_format)
+            return do_download_file(book, book_format, client, data1, headers)
     else:
         log.error("Book id {} not found for downloading".format(book_id))
-        abort(404)
-    if data1:
-        # collect downloaded books only for registered user and not for anonymous user
-        if current_user.is_authenticated:
-            ub.update_download(book_id, int(current_user.id))
-        file_name = book.title
-        if len(book.authors) > 0:
-            file_name = file_name + ' - ' + book.authors[0].name
-        file_name = get_valid_filename(file_name, replace_whitespace=False)
-        headers = Headers()
-        headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
-        headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % (
-            quote(file_name.encode('utf-8')), book_format, quote(file_name.encode('utf-8')), book_format)
-        return do_download_file(book, book_format, client, data1, headers)
-    else:
-        abort(404)
+    abort(404)
+
+
+def get_calibre_binarypath(binary):
+        binariesdir = config.config_binariesdir
+        if binariesdir:
+            try:
+                return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary])
+            except KeyError as ex:
+                log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary])
+                pass
+        return ""
 
 
 def get_calibre_binarypath(binary):
@@ -1066,3 +1216,11 @@ def add_book_to_thumbnail_cache(book_id):
 def update_thumbnail_cache():
     if config.schedule_generate_book_covers:
         WorkerThread.add(None, TaskGenerateCoverThumbnails())
+
+
+def set_all_metadata_dirty():
+    WorkerThread.add(None, TaskBackupMetadata(export_language=get_locale(),
+                                              translated_title=_("Cover"),
+                                              set_dirty=True,
+                                              task_message=N_("Queue all books for metadata backup")),
+                     hidden=False)

cps/kobo.py

@@ -21,6 +21,7 @@ import base64
 import datetime
 import os
 import uuid
+import zipfile
 from time import gmtime, strftime
 import json
 from urllib.parse import unquote
@@ -45,7 +46,9 @@ import requests
 
 
 from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub, csrf, kobo_sync_status
-from .constants import sqlalchemy_version2, COVER_THUMBNAIL_SMALL
+from . import isoLanguages
+from .epub import get_epub_layout
+from .constants import COVER_THUMBNAIL_SMALL #, sqlalchemy_version2
 from .helper import get_download_link
 from .services import SyncToken as SyncToken
 from .web import download_required
@@ -53,7 +56,7 @@ from .kobo_auth import requires_kobo_auth, get_auth_token
 
 KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]}
 KOBO_STOREAPI_URL = "https://storeapi.kobo.com"
-KOBO_IMAGEHOST_URL = "https://kbimages1-a.akamaihd.net"
+KOBO_IMAGEHOST_URL = "https://cdn.kobo.com/book-images"
 
 SYNC_ITEM_LIMIT = 100
 
@@ -134,11 +137,15 @@ def convert_to_kobo_timestamp_string(timestamp):
 
 @kobo.route("/v1/library/sync")
 @requires_kobo_auth
-@download_required
+# @download_required
 def HandleSyncRequest():
+    if not current_user.role_download():
+        log.info("Users need download permissions for syncing library to Kobo reader")
+        return abort(403)
     sync_token = SyncToken.SyncToken.from_headers(request.headers)
-    log.info("Kobo library sync request received.")
+    log.info("Kobo library sync request received")
     log.debug("SyncToken: {}".format(sync_token))
+    log.debug("Download link format {}".format(get_download_url_for_book('[bookid]','[bookformat]')))
     if not current_app.wsgi_app.is_proxied:
         log.debug('Kobo: Received unproxied request, changed request port to external server port')
 
@@ -155,23 +162,17 @@ def HandleSyncRequest():
     new_archived_last_modified = datetime.datetime.min
     sync_results = []
 
-    # We reload the book database so that the user get's a fresh view of the library
+    # We reload the book database so that the user gets a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
     calibre_db.reconnect_db(config, ub.app_DB_path)
 
     only_kobo_shelves = current_user.kobo_only_shelves_sync
 
     if only_kobo_shelves:
-        if sqlalchemy_version2:
-            changed_entries = select(db.Books,
-                                     ub.ArchivedBook.last_modified,
-                                     ub.BookShelf.date_added,
-                                     ub.ArchivedBook.is_archived)
-        else:
-            changed_entries = calibre_db.session.query(db.Books,
-                                                       ub.ArchivedBook.last_modified,
-                                                       ub.BookShelf.date_added,
-                                                       ub.ArchivedBook.is_archived)
+        changed_entries = calibre_db.session.query(db.Books,
+                                                   ub.ArchivedBook.last_modified,
+                                                   ub.BookShelf.date_added,
+                                                   ub.ArchivedBook.is_archived)
         changed_entries = (changed_entries
                            .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
                                                                           ub.ArchivedBook.user_id == current_user.id))
@@ -188,12 +189,9 @@ def HandleSyncRequest():
                            .filter(ub.Shelf.kobo_sync)
                            .distinct())
     else:
-        if sqlalchemy_version2:
-            changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
-        else:
-            changed_entries = calibre_db.session.query(db.Books,
-                                                       ub.ArchivedBook.last_modified,
-                                                       ub.ArchivedBook.is_archived)
+        changed_entries = calibre_db.session.query(db.Books,
+                                                   ub.ArchivedBook.last_modified,
+                                                   ub.ArchivedBook.is_archived)
         changed_entries = (changed_entries
                            .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
                                                                           ub.ArchivedBook.user_id == current_user.id))
@@ -205,15 +203,12 @@ def HandleSyncRequest():
                            .order_by(db.Books.id))
 
     reading_states_in_new_entitlements = []
-    if sqlalchemy_version2:
-        books = calibre_db.session.execute(changed_entries.limit(SYNC_ITEM_LIMIT))
-    else:
-        books = changed_entries.limit(SYNC_ITEM_LIMIT)
+    books = changed_entries.limit(SYNC_ITEM_LIMIT)
     log.debug("Books to Sync: {}".format(len(books.all())))
     for book in books:
         formats = [data.format for data in book.Books.data]
         if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats:
-            helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
+            helper.convert_book_format(book.Books.id, config.get_book_path(), 'EPUB', 'KEPUB', current_user.name)
 
         kobo_reading_state = get_or_create_reading_state(book.Books.id)
         entitlement = {
@@ -226,7 +221,7 @@ def HandleSyncRequest():
             new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
             reading_states_in_new_entitlements.append(book.Books.id)
 
-        ts_created = book.Books.timestamp
+        ts_created = book.Books.timestamp.replace(tzinfo=None)
 
         try:
             ts_created = max(ts_created, book.date_added)
@@ -239,7 +234,7 @@ def HandleSyncRequest():
             sync_results.append({"ChangedEntitlement": entitlement})
 
         new_books_last_modified = max(
-            book.Books.last_modified, new_books_last_modified
+            book.Books.last_modified.replace(tzinfo=None), new_books_last_modified
         )
         try:
             new_books_last_modified = max(
@@ -251,27 +246,16 @@ def HandleSyncRequest():
         new_books_last_created = max(ts_created, new_books_last_created)
         kobo_sync_status.add_synced_books(book.Books.id)
 
-    if sqlalchemy_version2:
-        max_change = calibre_db.session.execute(changed_entries
-                                                .filter(ub.ArchivedBook.is_archived)
-                                                .filter(ub.ArchivedBook.user_id == current_user.id)
-                                                .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()))\
-            .columns(db.Books).first()
-    else:
-        max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived)\
-            .filter(ub.ArchivedBook.user_id == current_user.id) \
-            .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first()
+    max_change = changed_entries.filter(ub.ArchivedBook.is_archived)\
+        .filter(ub.ArchivedBook.user_id == current_user.id) \
+        .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first()
 
     max_change = max_change.last_modified if max_change else new_archived_last_modified
 
     new_archived_last_modified = max(new_archived_last_modified, max_change)
 
     # no. of books returned
-    if sqlalchemy_version2:
-        entries = calibre_db.session.execute(changed_entries).all()
-        book_count = len(entries)
-    else:
-        book_count = changed_entries.count()
+    book_count = changed_entries.count()
     # last entry:
     cont_sync = bool(book_count)
     log.debug("Remaining books to Sync: {}".format(book_count))
@@ -334,7 +318,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False):
             extra_headers["x-kobo-recent-reads"] = store_response.headers.get("x-kobo-recent-reads")
 
         except Exception as ex:
-            log.error("Failed to receive or parse response from Kobo's sync endpoint: {}".format(ex))
+            log.error_or_exception("Failed to receive or parse response from Kobo's sync endpoint: {}".format(ex))
     if set_cont:
         extra_headers["x-kobo-sync"] = "continue"
     sync_token.to_headers(extra_headers)
@@ -355,7 +339,7 @@ def HandleMetadataRequest(book_uuid):
     log.info("Kobo library metadata request received for book %s" % book_uuid)
     book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
-        log.info(u"Book %s not found in database", book_uuid)
+        log.info("Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
 
     metadata = get_metadata(book)
@@ -364,7 +348,7 @@ def HandleMetadataRequest(book_uuid):
     return response
 
 
-def get_download_url_for_book(book, book_format):
+def get_download_url_for_book(book_id, book_format):
     if not current_app.wsgi_app.is_proxied:
         if ':' in request.host and not request.host.endswith(']'):
             host = "".join(request.host.split(':')[:-1])
@@ -376,13 +360,13 @@ def get_download_url_for_book(book, book_format):
             url_base=host,
             url_port=config.config_external_port,
             auth_token=get_auth_token(),
-            book_id=book.id,
+            book_id=book_id,
             book_format=book_format.lower()
         )
     return url_for(
         "kobo.download_book",
         auth_token=kobo_auth.get_auth_token(),
-        book_id=book.id,
+        book_id=book_id,
         book_format=book_format.lower(),
         _external=True,
     )
@@ -443,6 +427,12 @@ def get_seriesindex(book):
     return book.series_index or 1
 
 
+def get_language(book):
+    if not book.languages:
+        return 'en'
+    return isoLanguages.get(part3=book.languages[0].lang_code).part1
+
+
 def get_metadata(book):
     download_urls = []
     kepub = [data for data in book.data if data.format == 'KEPUB']
@@ -452,16 +442,21 @@ def get_metadata(book):
             continue
         for kobo_format in KOBO_FORMATS[book_data.format]:
             # log.debug('Id: %s, Format: %s' % (book.id, kobo_format))
-            download_urls.append(
-                {
-                    "Format": kobo_format,
-                    "Size": book_data.uncompressed_size,
-                    "Url": get_download_url_for_book(book, book_data.format),
-                    # The Kobo forma accepts platforms: (Generic, Android)
-                    "Platform": "Generic",
-                    # "DrmType": "None", # Not required
-                }
-            )
+            try:
+                if get_epub_layout(book, book_data) == 'pre-paginated':
+                    kobo_format = 'EPUB3FL'
+                download_urls.append(
+                    {
+                        "Format": kobo_format,
+                        "Size": book_data.uncompressed_size,
+                        "Url": get_download_url_for_book(book.id, book_data.format),
+                        # The Kobo forma accepts platforms: (Generic, Android)
+                        "Platform": "Generic",
+                        # "DrmType": "None", # Not required
+                    }
+                )
+            except (zipfile.BadZipfile, FileNotFoundError) as e:
+                log.error(e)
 
     book_uuid = book.uuid
     metadata = {
@@ -480,7 +475,7 @@ def get_metadata(book):
         "IsInternetArchive": False,
         "IsPreOrder": False,
         "IsSocialEnabled": True,
-        "Language": "en",
+        "Language": get_language(book),
         "PhoneticPronunciations": {},
         "PublicationDate": convert_to_kobo_timestamp_string(book.pubdate),
         "Publisher": {"Imprint": "", "Name": get_publisher(book), },
@@ -508,7 +503,7 @@ def get_metadata(book):
 @requires_kobo_auth
 # Creates a Shelf with the given items, and returns the shelf's uuid.
 def HandleTagCreate():
-    # catch delete requests, otherwise the are handeld in the book delete handler
+    # catch delete requests, otherwise they are handled in the book delete handler
     if request.method == "DELETE":
         abort(405)
     name, items = None, None
@@ -702,20 +697,12 @@ def sync_shelves(sync_token, sync_results, only_kobo_shelves=False):
             })
         extra_filters.append(ub.Shelf.kobo_sync)
 
-    if sqlalchemy_version2:
-        shelflist = ub.session.execute(select(ub.Shelf).outerjoin(ub.BookShelf).filter(
-            or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
-                func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified),
-            ub.Shelf.user_id == current_user.id,
-            *extra_filters
-        ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())).columns(ub.Shelf)
-    else:
-        shelflist = ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter(
-            or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
-                func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified),
-            ub.Shelf.user_id == current_user.id,
-            *extra_filters
-        ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())
+    shelflist = ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter(
+        or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
+            func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified),
+        ub.Shelf.user_id == current_user.id,
+        *extra_filters
+    ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())
 
     for shelf in shelflist:
         if not shelf_lib.check_shelf_view_permissions(shelf):
@@ -752,7 +739,7 @@ def create_kobo_tag(shelf):
     for book_shelf in shelf.books:
         book = calibre_db.get_book(book_shelf.book_id)
         if not book:
-            log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database",  book_shelf.book_id, shelf.id)
+            log.info("Book (id: %s) in BookShelf (id: %s) not found in book database",  book_shelf.book_id, shelf.id)
             continue
         tag["Items"].append(
             {
@@ -769,7 +756,7 @@ def create_kobo_tag(shelf):
 def HandleStateRequest(book_uuid):
     book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
-        log.info(u"Book %s not found in database", book_uuid)
+        log.info("Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
 
     kobo_reading_state = get_or_create_reading_state(book.id)
@@ -916,20 +903,26 @@ def get_current_bookmark_response(current_bookmark):
 @kobo.route("/<book_uuid>/<width>/<height>/<Quality>/<isGreyscale>/image.jpg")
 @requires_kobo_auth
 def HandleCoverImageRequest(book_uuid, width, height, Quality, isGreyscale):
-    book_cover = helper.get_book_cover_with_uuid(book_uuid, resolution=COVER_THUMBNAIL_SMALL)
-    if not book_cover:
-        if config.config_kobo_proxy:
-            log.debug("Cover for unknown book: %s proxied to kobo" % book_uuid)
-            return redirect(KOBO_IMAGEHOST_URL +
-                            "/{book_uuid}/{width}/{height}/false/image.jpg".format(book_uuid=book_uuid,
-                                                                                   width=width,
-                                                                                   height=height), 307)
-        else:
-            log.debug("Cover for unknown book: %s requested" % book_uuid)
-            # additional proxy request make no sense, -> direct return
-            return make_response(jsonify({}))
-    log.debug("Cover request received for book %s" % book_uuid)
-    return book_cover
+    try:
+        resolution = None if int(height) > 1000 else COVER_THUMBNAIL_SMALL
+    except ValueError:
+        log.error("Requested height %s of book %s is invalid" % (book_uuid, height))
+        resolution = COVER_THUMBNAIL_SMALL
+    book_cover = helper.get_book_cover_with_uuid(book_uuid, resolution=resolution)
+    if book_cover:
+        log.debug("Serving local cover image of book %s" % book_uuid)
+        return book_cover
+
+    if not config.config_kobo_proxy:
+        log.debug("Returning 404 for cover image of unknown book %s" % book_uuid)
+        # additional proxy request make no sense, -> direct return
+        return abort(404)
+
+    log.debug("Redirecting request for cover image of unknown book %s to Kobo" % book_uuid)
+    return redirect(KOBO_IMAGEHOST_URL +
+                    "/{book_uuid}/{width}/{height}/false/image.jpg".format(book_uuid=book_uuid,
+                                                                            width=width,
+                                                                            height=height), 307)
 
 
 @kobo.route("")
@@ -944,7 +937,7 @@ def HandleBookDeletionRequest(book_uuid):
     log.info("Kobo book delete request received for book %s" % book_uuid)
     book = calibre_db.get_book_by_uuid(book_uuid)
     if not book:
-        log.info(u"Book %s not found in database", book_uuid)
+        log.info("Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
 
     book_id = book.id
@@ -958,7 +951,7 @@ def HandleBookDeletionRequest(book_uuid):
 @csrf.exempt
 @kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
 def HandleUnimplementedRequest(dummy=None):
-    log.debug("Unimplemented Library Request received: %s", request.base_url)
+    log.debug("Unimplemented Library Request received: %s (request is forwarded to kobo if configured)", request.base_url)
     return redirect_or_proxy_request()
 
 
@@ -969,8 +962,9 @@ def HandleUnimplementedRequest(dummy=None):
 @kobo.route("/v1/user/wishlist", methods=["GET", "POST"])
 @kobo.route("/v1/user/recommendations", methods=["GET", "POST"])
 @kobo.route("/v1/analytics/<dummy>", methods=["GET", "POST"])
+@kobo.route("/v1/assets", methods=["GET"])
 def HandleUserRequest(dummy=None):
-    log.debug("Unimplemented User Request received: %s", request.base_url)
+    log.debug("Unimplemented User Request received: %s (request is forwarded to kobo if configured)", request.base_url)
     return redirect_or_proxy_request()
 
 
@@ -1010,7 +1004,7 @@ def handle_getests():
 @kobo.route("/v1/affiliate", methods=["GET", "POST"])
 @kobo.route("/v1/deals", methods=["GET", "POST"])
 def HandleProductsRequest(dummy=None):
-    log.debug("Unimplemented Products Request received: %s", request.base_url)
+    log.debug("Unimplemented Products Request received: %s (request is forwarded to kobo if configured)", request.base_url)
     return redirect_or_proxy_request()
 
 
@@ -1027,7 +1021,7 @@ def make_calibre_web_auth_response():
                 "RefreshToken": RefreshToken,
                 "TokenType": "Bearer",
                 "TrackingId": str(uuid.uuid4()),
-                "UserKey": content['UserKey'],
+                "UserKey": content.get('UserKey',""),
             }
         )
     )

cps/kobo_auth.py

@@ -64,11 +64,12 @@ from datetime import datetime
 from os import urandom
 from functools import wraps
 
-from flask import g, Blueprint, url_for, abort, request
+from flask import g, Blueprint, abort, request
 from flask_login import login_user, current_user, login_required
 from flask_babel import gettext as _
+from flask_limiter import RateLimitExceeded
 
-from . import logger, config, calibre_db, db, helper, ub, lm
+from . import logger, config, calibre_db, db, helper, ub, lm, limiter
 from .render_template import render_title_template
 
 log = logger.create()
@@ -107,12 +108,12 @@ def generate_auth_token(user_id):
 
     for book in books:
         formats = [data.format for data in book.data]
-        if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
+        if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats:
             helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
 
     return render_title_template(
         "generate_kobo_auth_url.html",
-        title=_(u"Kobo Setup"),
+        title=_("Kobo Setup"),
         auth_token=auth_token.auth_token,
         warning = warning
     )
@@ -151,6 +152,10 @@ def requires_kobo_auth(f):
     def inner(*args, **kwargs):
         auth_token = get_auth_token()
         if auth_token is not None:
+            try:
+                limiter.check()
+            except RateLimitExceeded:
+                return abort(429)
             user = (
                 ub.session.query(ub.User)
                 .join(ub.RemoteAuthToken)
@@ -159,7 +164,8 @@ def requires_kobo_auth(f):
             )
             if user is not None:
                 login_user(user)
+                [limiter.limiter.storage.clear(k.key) for k in limiter.current_limits]
                 return f(*args, **kwargs)
-            log.debug("Received Kobo request without a recognizable auth token.")
-            return abort(401)
+        log.debug("Received Kobo request without a recognizable auth token.")
+        return abort(401)
     return inner

cps/logger.py

@@ -54,7 +54,6 @@ class _Logger(logging.Logger):
             else:
                 self.error(message, *args, **kwargs)
 
-
     def debug_no_auth(self, message, *args, **kwargs):
         message = message.strip("\r\n")
         if message.startswith("send: AUTH"):
@@ -66,6 +65,7 @@ class _Logger(logging.Logger):
 def get(name=None):
     return logging.getLogger(name)
 
+
 def create():
     parent_frame = inspect.stack(0)[1]
     if hasattr(parent_frame, 'frame'):
@@ -75,9 +75,11 @@ def create():
     parent_module = inspect.getmodule(parent_frame)
     return get(parent_module.__name__)
 
+
 def is_debug_enabled():
     return logging.root.level <= logging.DEBUG
 
+
 def is_info_enabled(logger):
     return logging.getLogger(logger).level <= logging.INFO
 
@@ -114,10 +116,10 @@ def get_accesslogfile(log_file):
 
 
 def setup(log_file, log_level=None):
-    '''
+    """
     Configure the logging output.
     May be called multiple times.
-    '''
+    """
     log_level = log_level or DEFAULT_LOG_LEVEL
     logging.setLoggerClass(_Logger)
     logging.getLogger(__package__).setLevel(log_level)
@@ -127,7 +129,7 @@ def setup(log_file, log_level=None):
         # avoid spamming the log with debug messages from libraries
         r.setLevel(log_level)
 
-    # Otherwise name get's destroyed on windows
+    # Otherwise, name gets destroyed on Windows
     if log_file != LOG_TO_STDERR and log_file != LOG_TO_STDOUT:
         log_file = _absolute_log_file(log_file, DEFAULT_LOG_FILE)
 
@@ -148,7 +150,7 @@ def setup(log_file, log_level=None):
     else:
         try:
             file_handler = RotatingFileHandler(log_file, maxBytes=100000, backupCount=2, encoding='utf-8')
-        except IOError:
+        except (IOError, PermissionError):
             if log_file == DEFAULT_LOG_FILE:
                 raise
             file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=100000, backupCount=2, encoding='utf-8')
@@ -159,13 +161,14 @@ def setup(log_file, log_level=None):
         r.removeHandler(h)
         h.close()
     r.addHandler(file_handler)
+    logging.captureWarnings(True)
     return "" if log_file == DEFAULT_LOG_FILE else log_file
 
 
 def create_access_log(log_file, log_name, formatter):
-    '''
+    """
     One-time configuration for the web server's access log.
-    '''
+    """
     log_file = _absolute_log_file(log_file, DEFAULT_ACCESS_LOG)
     logging.debug("access log: %s", log_file)
 
@@ -174,7 +177,7 @@ def create_access_log(log_file, log_name, formatter):
     access_log.setLevel(logging.INFO)
     try:
         file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2, encoding='utf-8')
-    except IOError:
+    except (IOError, PermissionError):
         if log_file == DEFAULT_ACCESS_LOG:
             raise
         file_handler = RotatingFileHandler(DEFAULT_ACCESS_LOG, maxBytes=50000, backupCount=2, encoding='utf-8')
@@ -182,8 +185,7 @@ def create_access_log(log_file, log_name, formatter):
 
     file_handler.setFormatter(formatter)
     access_log.addHandler(file_handler)
-    return access_log, \
-           "" if _absolute_log_file(log_file, DEFAULT_ACCESS_LOG) == DEFAULT_ACCESS_LOG else log_file
+    return access_log, "" if _absolute_log_file(log_file, DEFAULT_ACCESS_LOG) == DEFAULT_ACCESS_LOG else log_file
 
 
 # Enable logging of smtp lib debug output

cps/main.py

@@ -18,9 +18,14 @@
 
 import sys
 
-from . import create_app
+from . import create_app, limiter
 from .jinjia import jinjia
 from .remotelogin import remotelogin
+from flask import request
+
+
+def request_username():
+    return request.authorization.username
 
 def main():
     app = create_app()
@@ -39,6 +44,7 @@ def main():
     try:
         from .kobo import kobo, get_kobo_activated
         from .kobo_auth import kobo_auth
+        from flask_limiter.util import get_remote_address
         kobo_available = get_kobo_activated()
     except (ImportError, AttributeError):  # Catch also error for not installed flask-WTF (missing csrf decorator)
         kobo_available = False
@@ -56,6 +62,7 @@ def main():
     app.register_blueprint(tasks)
     app.register_blueprint(web)
     app.register_blueprint(opds)
+    limiter.limit("3/minute",key_func=request_username)(opds)
     app.register_blueprint(jinjia)
     app.register_blueprint(about)
     app.register_blueprint(shelf)
@@ -67,6 +74,7 @@ def main():
     if kobo_available:
         app.register_blueprint(kobo)
         app.register_blueprint(kobo_auth)
+        limiter.limit("3/minute", key_func=get_remote_address)(kobo)
     if oauth_available:
         app.register_blueprint(oauth)
     success = web_server.start()

cps/metadata_provider/amazon.py

@@ -63,11 +63,11 @@ class Amazon(Metadata):
                     r.raise_for_status()
                 except Exception as ex:
                     log.warning(ex)
-                    return
+                    return None
                 long_soup = BS(r.text, "lxml")  #~4sec :/
                 soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
                 if soup2 is None:
-                    return
+                    return None
                 try:
                     match = MetaRecord(
                         title = "",
@@ -98,7 +98,7 @@ class Amazon(Metadata):
                     try:
                         match.authors = [next(
                             filter(lambda i: i != " " and i != "\n" and not i.startswith("{"),
-                                   x.findAll(text=True))).strip()
+                                   x.findAll(string=True))).strip()
                                         for x in soup2.findAll("span", attrs={"class": "author"})]
                     except (AttributeError, TypeError, StopIteration):
                         match.authors = ""
@@ -115,7 +115,7 @@ class Amazon(Metadata):
                     return match, index
                 except Exception as e:
                     log.error_or_exception(e)
-                    return
+                    return None
 
         val = list()
         if self.active:
@@ -127,10 +127,10 @@ class Amazon(Metadata):
                 results.raise_for_status()
             except requests.exceptions.HTTPError as e:
                 log.error_or_exception(e)
-                return None
+                return []
             except Exception as e:
                 log.warning(e)
-                return None
+                return []
             soup = BS(results.text, 'html.parser')
             links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
                           soup.findAll("div", attrs={"data-component-type": "s-search-result"})]

cps/metadata_provider/douban.py

@@ -43,7 +43,8 @@ class Douban(Metadata):
     __id__ = "douban"
     DESCRIPTION = "豆瓣"
     META_URL = "https://book.douban.com/"
-    SEARCH_URL = "https://www.douban.com/j/search"
+    SEARCH_JSON_URL = "https://www.douban.com/j/search"
+    SEARCH_URL = "https://www.douban.com/search"
 
     ID_PATTERN = re.compile(r"sid: (?P<id>\d+),")
     AUTHORS_PATTERN = re.compile(r"作者|译者")
@@ -52,6 +53,7 @@ class Douban(Metadata):
     PUBLISHED_DATE_PATTERN = re.compile(r"出版年")
     SERIES_PATTERN = re.compile(r"丛书")
     IDENTIFIERS_PATTERN = re.compile(r"ISBN|统一书号")
+    CRITERIA_PATTERN = re.compile("criteria = '(.+)'")
 
     TITTLE_XPATH = "//span[@property='v:itemreviewed']"
     COVER_XPATH = "//a[@class='nbg']"
@@ -63,56 +65,90 @@ class Douban(Metadata):
     session = requests.Session()
     session.headers = {
         'user-agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',
     }
 
-    def search(
-        self, query: str, generic_cover: str = "", locale: str = "en"
-    ) -> Optional[List[MetaRecord]]:
+    def search(self,
+               query: str,
+               generic_cover: str = "",
+               locale: str = "en") -> List[MetaRecord]:
+        val = []
         if self.active:
-            log.debug(f"starting search {query} on douban")
+            log.debug(f"start searching {query} on douban")
             if title_tokens := list(
-                self.get_title_tokens(query, strip_joiners=False)
-            ):
+                    self.get_title_tokens(query, strip_joiners=False)):
                 query = "+".join(title_tokens)
 
-            try:
-                r = self.session.get(
-                    self.SEARCH_URL, params={"cat": 1001, "q": query}
-                )
-                r.raise_for_status()
+            book_id_list = self._get_book_id_list_from_html(query)
 
-            except Exception as e:
-                log.warning(e)
-                return None
-
-            results = r.json()
-            if results["total"] == 0:
+            if not book_id_list:
+                log.debug("No search results in Douban")
                 return []
 
-            book_id_list = [
-                self.ID_PATTERN.search(item).group("id")
-                for item in results["items"][:10] if self.ID_PATTERN.search(item)
-            ]
-
-            with futures.ThreadPoolExecutor(max_workers=5) as executor:
+            with futures.ThreadPoolExecutor(
+                    max_workers=5, thread_name_prefix='douban') as executor:
 
                 fut = [
-                    executor.submit(self._parse_single_book, book_id, generic_cover)
-                    for book_id in book_id_list
+                    executor.submit(self._parse_single_book, book_id,
+                                    generic_cover) for book_id in book_id_list
                 ]
-                
+
                 val = [
-                    future.result() 
-                    for future in futures.as_completed(fut) if future.result()
+                    future.result() for future in futures.as_completed(fut)
+                    if future.result()
                 ]
 
         return val
 
-    def _parse_single_book(
-        self, id: str, generic_cover: str = ""
-    ) -> Optional[MetaRecord]:
+    def _get_book_id_list_from_html(self, query: str) -> List[str]:
+        try:
+            r = self.session.get(self.SEARCH_URL,
+                                 params={
+                                     "cat": 1001,
+                                     "q": query
+                                 })
+            r.raise_for_status()
+
+        except Exception as e:
+            log.warning(e)
+            return []
+
+        html = etree.HTML(r.content.decode("utf8"))
+        result_list = html.xpath(self.COVER_XPATH)
+
+        return [
+            self.ID_PATTERN.search(item.get("onclick")).group("id")
+            for item in result_list[:10]
+            if self.ID_PATTERN.search(item.get("onclick"))
+        ]
+
+    def _get_book_id_list_from_json(self, query: str) -> List[str]:
+        try:
+            r = self.session.get(self.SEARCH_JSON_URL,
+                                 params={
+                                     "cat": 1001,
+                                     "q": query
+                                 })
+            r.raise_for_status()
+
+        except Exception as e:
+            log.warning(e)
+            return []
+
+        results = r.json()
+        if results["total"] == 0:
+            return []
+
+        return [
+            self.ID_PATTERN.search(item).group("id")
+            for item in results["items"][:10] if self.ID_PATTERN.search(item)
+        ]
+
+    def _parse_single_book(self,
+                           id: str,
+                           generic_cover: str = "") -> Optional[MetaRecord]:
         url = f"https://book.douban.com/subject/{id}/"
+        log.debug(f"start parsing {url}")
 
         try:
             r = self.session.get(url)
@@ -133,10 +169,12 @@ class Douban(Metadata):
             ),
         )
 
-        html = etree.HTML(r.content.decode("utf8"))
+        decode_content = r.content.decode("utf8")
+        html = etree.HTML(decode_content)
 
         match.title = html.xpath(self.TITTLE_XPATH)[0].text
-        match.cover = html.xpath(self.COVER_XPATH)[0].attrib["href"] or generic_cover
+        match.cover = html.xpath(
+            self.COVER_XPATH)[0].attrib["href"] or generic_cover
         try:
             rating_num = float(html.xpath(self.RATING_XPATH)[0].text.strip())
         except Exception:
@@ -146,35 +184,39 @@ class Douban(Metadata):
         tag_elements = html.xpath(self.TAGS_XPATH)
         if len(tag_elements):
             match.tags = [tag_element.text for tag_element in tag_elements]
+        else:
+            match.tags = self._get_tags(decode_content)
 
         description_element = html.xpath(self.DESCRIPTION_XPATH)
         if len(description_element):
-            match.description = html2text(etree.tostring(
-                description_element[-1], encoding="utf8").decode("utf8"))
+            match.description = html2text(
+                etree.tostring(description_element[-1]).decode("utf8"))
 
         info = html.xpath(self.INFO_XPATH)
 
         for element in info:
             text = element.text
             if self.AUTHORS_PATTERN.search(text):
-                next = element.getnext()
-                while next is not None and next.tag != "br":
-                    match.authors.append(next.text)
-                    next = next.getnext()
+                next_element = element.getnext()
+                while next_element is not None and next_element.tag != "br":
+                    match.authors.append(next_element.text)
+                    next_element = next_element.getnext()
             elif self.PUBLISHER_PATTERN.search(text):
-                match.publisher = element.tail.strip()
+                if publisher := element.tail.strip():
+                    match.publisher = publisher
+                else:
+                    match.publisher = element.getnext().text
             elif self.SUBTITLE_PATTERN.search(text):
-                match.title = f'{match.title}:' + element.tail.strip()
+                match.title = f'{match.title}:{element.tail.strip()}'
             elif self.PUBLISHED_DATE_PATTERN.search(text):
                 match.publishedDate = self._clean_date(element.tail.strip())
-            elif self.SUBTITLE_PATTERN.search(text):
+            elif self.SERIES_PATTERN.search(text):
                 match.series = element.getnext().text
             elif i_type := self.IDENTIFIERS_PATTERN.search(text):
                 match.identifiers[i_type.group()] = element.tail.strip()
 
         return match
 
-    
     def _clean_date(self, date: str) -> str:
         """
         Clean up the date string to be in the format YYYY-MM-DD
@@ -194,13 +236,24 @@ class Douban(Metadata):
                 if date[i].isdigit():
                     digit.append(date[i])
                 elif digit:
-                    ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+                    ls.append("".join(digit) if len(digit) ==
+                              2 else f"0{digit[0]}")
                     digit = []
             if digit:
-                ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+                ls.append("".join(digit) if len(digit) ==
+                          2 else f"0{digit[0]}")
 
             moon = ls[0]
-            if len(ls)>1:
-                day = ls[1]     
+            if len(ls) > 1:
+                day = ls[1]
 
         return f"{year}-{moon}-{day}"
+
+    def _get_tags(self, text: str) -> List[str]:
+        tags = []
+        if criteria := self.CRITERIA_PATTERN.search(text):
+            tags.extend(
+                item.replace('7:', '') for item in criteria.group().split('|')
+                if item.startswith('7:'))
+
+        return tags

cps/metadata_provider/google.py

@@ -19,6 +19,7 @@
 # Google Books api document: https://developers.google.com/books/docs/v1/using
 from typing import Dict, List, Optional
 from urllib.parse import quote
+from datetime import datetime
 
 import requests
 
@@ -81,7 +82,11 @@ class Google(Metadata):
         match.description = result["volumeInfo"].get("description", "")
         match.languages = self._parse_languages(result=result, locale=locale)
         match.publisher = result["volumeInfo"].get("publisher", "")
-        match.publishedDate = result["volumeInfo"].get("publishedDate", "")
+        try:
+            datetime.strptime(result["volumeInfo"].get("publishedDate", ""), "%Y-%m-%d")
+            match.publishedDate = result["volumeInfo"].get("publishedDate", "")
+        except ValueError:
+            match.publishedDate = ""
         match.rating = result["volumeInfo"].get("averageRating", 0)
         match.series, match.series_index = "", 1
         match.tags = result["volumeInfo"].get("categories", [])
@@ -103,6 +108,13 @@ class Google(Metadata):
     def _parse_cover(result: Dict, generic_cover: str) -> str:
         if result["volumeInfo"].get("imageLinks"):
             cover_url = result["volumeInfo"]["imageLinks"]["thumbnail"]
+            
+            # strip curl in cover
+            cover_url = cover_url.replace("&edge=curl", "")
+            
+            # request 800x900 cover image (higher resolution)
+            cover_url += "&fife=w800-h900"
+            
             return cover_url.replace("http://", "https://")
         return generic_cover
 

cps/metadata_provider/lubimyczytac.py

@@ -102,7 +102,7 @@ class LubimyCzytac(Metadata):
     PUBLISH_DATE = "//dt[contains(@title,'Data pierwszego wydania"
     FIRST_PUBLISH_DATE = f"{DETAILS}{PUBLISH_DATE} oryginalnego')]{SIBLINGS}[1]/text()"
     FIRST_PUBLISH_DATE_PL = f"{DETAILS}{PUBLISH_DATE} polskiego')]{SIBLINGS}[1]/text()"
-    TAGS = "//nav[@aria-label='breadcrumb']//a[contains(@href,'/ksiazki/k/')]/text()"
+    TAGS = "//nav[@aria-label='breadcrumbs']//a[contains(@href,'/ksiazki/k/')]/span/text()"
 
     RATING = "//meta[@property='books:rating:value']/@content"
     COVER = "//meta[@property='og:image']/@content"

cps/metadata_provider/scholar.py

@@ -49,10 +49,12 @@ class scholar(Metadata):
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = " ".join(tokens)
             try:
+                scholarly.set_timeout(20)
+                scholarly.set_retries(2)
                 scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
             except Exception as e:
                 log.warning(e)
-                return None
+                return list()
             for result in scholar_gen:
                 match = self._parse_search_result(
                     result=result, generic_cover="", locale=locale

cps/oauth_bb.py

@@ -74,7 +74,7 @@ def register_user_with_oauth(user=None):
     if len(all_oauth.keys()) == 0:
         return
     if user is None:
-        flash(_(u"Register with %(provider)s", provider=", ".join(list(all_oauth.values()))), category="success")
+        flash(_("Register with %(provider)s", provider=", ".join(list(all_oauth.values()))), category="success")
     else:
         for oauth_key in all_oauth.keys():
             # Find this OAuth token in the database, or create it
@@ -134,8 +134,8 @@ def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider
         # already bind with user, just login
         if oauth_entry.user:
             login_user(oauth_entry.user)
-            log.debug(u"You are now logged in as: '%s'", oauth_entry.user.name)
-            flash(_(u"you are now logged in as: '%(nickname)s'", nickname= oauth_entry.user.name),
+            log.debug("You are now logged in as: '%s'", oauth_entry.user.name)
+            flash(_("Success! You are now logged in as: %(nickname)s", nickname= oauth_entry.user.name),
                   category="success")
             return redirect(url_for('web.index'))
         else:
@@ -145,21 +145,21 @@ def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider
                 try:
                     ub.session.add(oauth_entry)
                     ub.session.commit()
-                    flash(_(u"Link to %(oauth)s Succeeded", oauth=provider_name), category="success")
+                    flash(_("Link to %(oauth)s Succeeded", oauth=provider_name), category="success")
                     log.info("Link to {} Succeeded".format(provider_name))
                     return redirect(url_for('web.profile'))
                 except Exception as ex:
                     log.error_or_exception(ex)
                     ub.session.rollback()
             else:
-                flash(_(u"Login failed, No User Linked With OAuth Account"), category="error")
+                flash(_("Login failed, No User Linked With OAuth Account"), category="error")
             log.info('Login failed, No User Linked With OAuth Account')
             return redirect(url_for('web.login'))
             # return redirect(url_for('web.login'))
             # if config.config_public_reg:
             #   return redirect(url_for('web.register'))
             # else:
-            #    flash(_(u"Public registration is not enabled"), category="error")
+            #    flash(_("Public registration is not enabled"), category="error")
             #    return redirect(url_for(redirect_url))
     except (NoResultFound, AttributeError):
         return redirect(url_for(redirect_url))
@@ -194,15 +194,15 @@ def unlink_oauth(provider):
                 ub.session.delete(oauth_entry)
                 ub.session.commit()
                 logout_oauth_user()
-                flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success")
+                flash(_("Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success")
                 log.info("Unlink to {} Succeeded".format(oauth_check[provider]))
             except Exception as ex:
                 log.error_or_exception(ex)
                 ub.session.rollback()
-                flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error")
+                flash(_("Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error")
     except NoResultFound:
         log.warning("oauth %s for user %d not found", provider, current_user.id)
-        flash(_(u"Not Linked to %(oauth)s", oauth=provider), category="error")
+        flash(_("Not Linked to %(oauth)s", oauth=provider), category="error")
     return redirect(url_for('web.profile'))
 
 def generate_oauth_blueprints():
@@ -258,13 +258,13 @@ if ub.oauth_support:
     @oauth_authorized.connect_via(oauthblueprints[0]['blueprint'])
     def github_logged_in(blueprint, token):
         if not token:
-            flash(_(u"Failed to log in with GitHub."), category="error")
+            flash(_("Failed to log in with GitHub."), category="error")
             log.error("Failed to log in with GitHub")
             return False
 
         resp = blueprint.session.get("/user")
         if not resp.ok:
-            flash(_(u"Failed to fetch user info from GitHub."), category="error")
+            flash(_("Failed to fetch user info from GitHub."), category="error")
             log.error("Failed to fetch user info from GitHub")
             return False
 
@@ -276,13 +276,13 @@ if ub.oauth_support:
     @oauth_authorized.connect_via(oauthblueprints[1]['blueprint'])
     def google_logged_in(blueprint, token):
         if not token:
-            flash(_(u"Failed to log in with Google."), category="error")
+            flash(_("Failed to log in with Google."), category="error")
             log.error("Failed to log in with Google")
             return False
 
         resp = blueprint.session.get("/oauth2/v2/userinfo")
         if not resp.ok:
-            flash(_(u"Failed to fetch user info from Google."), category="error")
+            flash(_("Failed to fetch user info from Google."), category="error")
             log.error("Failed to fetch user info from Google")
             return False
 
@@ -295,8 +295,8 @@ if ub.oauth_support:
     @oauth_error.connect_via(oauthblueprints[0]['blueprint'])
     def github_error(blueprint, error, error_description=None, error_uri=None):
         msg = (
-            u"OAuth error from {name}! "
-            u"error={error} description={description} uri={uri}"
+            "OAuth error from {name}! "
+            "error={error} description={description} uri={uri}"
         ).format(
             name=blueprint.name,
             error=error,
@@ -308,8 +308,8 @@ if ub.oauth_support:
     @oauth_error.connect_via(oauthblueprints[1]['blueprint'])
     def google_error(blueprint, error, error_description=None, error_uri=None):
         msg = (
-            u"OAuth error from {name}! "
-            u"error={error} description={description} uri={uri}"
+            "OAuth error from {name}! "
+            "error={error} description={description} uri={uri}"
         ).format(
             name=blueprint.name,
             error=error,
@@ -329,10 +329,10 @@ def github_login():
         if account_info.ok:
             account_info_json = account_info.json()
             return bind_oauth_or_register(oauthblueprints[0]['id'], account_info_json['id'], 'github.login', 'github')
-        flash(_(u"GitHub Oauth error, please retry later."), category="error")
+        flash(_("GitHub Oauth error, please retry later."), category="error")
         log.error("GitHub Oauth error, please retry later")
     except (InvalidGrantError, TokenExpiredError) as e:
-        flash(_(u"GitHub Oauth error: {}").format(e), category="error")
+        flash(_("GitHub Oauth error: {}").format(e), category="error")
         log.error(e)
     return redirect(url_for('web.login'))
 
@@ -353,10 +353,10 @@ def google_login():
         if resp.ok:
             account_info_json = resp.json()
             return bind_oauth_or_register(oauthblueprints[1]['id'], account_info_json['id'], 'google.login', 'google')
-        flash(_(u"Google Oauth error, please retry later."), category="error")
+        flash(_("Google Oauth error, please retry later."), category="error")
         log.error("Google Oauth error, please retry later")
     except (InvalidGrantError, TokenExpiredError) as e:
-        flash(_(u"Google Oauth error: {}").format(e), category="error")
+        flash(_("Google Oauth error: {}").format(e), category="error")
         log.error(e)
     return redirect(url_for('web.login'))
 

cps/opds.py

@@ -21,41 +21,28 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 import datetime
+import json
 from urllib.parse import unquote_plus
-from functools import wraps
 
-from flask import Blueprint, request, render_template, Response, g, make_response, abort
+from flask import Blueprint, request, render_template, make_response, abort, Response
 from flask_login import current_user
 from flask_babel import get_locale
+from flask_babel import gettext as _
 from sqlalchemy.sql.expression import func, text, or_, and_, true
 from sqlalchemy.exc import InvalidRequestError, OperationalError
-from werkzeug.security import check_password_hash
 
-from . import constants, logger, config, db, calibre_db, ub, services, isoLanguages
+from . import logger, config, db, calibre_db, ub, isoLanguages
+from .usermanagement import requires_basic_auth_if_no_ano
 from .helper import get_download_link, get_book_cover
 from .pagination import Pagination
 from .web import render_read_books
-from .usermanagement import load_user_from_request
-from flask_babel import gettext as _
+
 
 opds = Blueprint('opds', __name__)
 
 log = logger.create()
 
 
-def requires_basic_auth_if_no_ano(f):
-    @wraps(f)
-    def decorated(*args, **kwargs):
-        auth = request.authorization
-        if config.config_anonbrowse != 1:
-            if not auth or auth.type != 'basic' or not check_auth(auth.username, auth.password):
-                return authenticate()
-        return f(*args, **kwargs)
-    if config.config_login_type == constants.LOGIN_LDAP and services.ldap and config.config_anonbrowse != 1:
-        return services.ldap.basic_auth_required(f)
-    return decorated
-
-
 @opds.route("/opds/")
 @opds.route("/opds")
 @requires_basic_auth_if_no_ano
@@ -69,7 +56,7 @@ def feed_osd():
     return render_xml_template('osd.xml', lang='en-EN')
 
 
-@opds.route("/opds/search", defaults={'query': ""})
+# @opds.route("/opds/search", defaults={'query': ""})
 @opds.route("/opds/search/<path:query>")
 @requires_basic_auth_if_no_ano
 def feed_cc_search(query):
@@ -328,7 +315,7 @@ def feed_format(book_id):
 @requires_basic_auth_if_no_ano
 def feed_languagesindex():
     off = request.args.get("offset") or 0
-    if current_user.filter_language() == u"all":
+    if current_user.filter_language() == "all":
         languages = calibre_db.speaking_language()
     else:
         languages = calibre_db.session.query(db.Languages).filter(
@@ -355,7 +342,8 @@ def feed_languages(book_id):
 @requires_basic_auth_if_no_ano
 def feed_shelfindex():
     off = request.args.get("offset") or 0
-    shelf = g.shelves_access
+    shelf = ub.session.query(ub.Shelf).filter(
+        or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == current_user.id)).order_by(ub.Shelf.name).all()
     number = len(shelf)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             number)
@@ -402,11 +390,7 @@ def feed_shelf(book_id):
 @opds.route("/opds/download/<book_id>/<book_format>/")
 @requires_basic_auth_if_no_ano
 def opds_download_link(book_id, book_format):
-    # I gave up with this: With enabled ldap login, the user doesn't get logged in, therefore it's always guest
-    # workaround, loading the user from the request and checking its download rights here
-    # in case of anonymous browsing user is None
-    user = load_user_from_request(request) or current_user
-    if not user.role_download():
+    if not current_user.role_download():
         return abort(403)
     if "Kobo" in request.headers.get('User-Agent'):
         client = "kobo"
@@ -429,6 +413,17 @@ def get_metadata_calibre_companion(uuid, library):
         return ""
 
 
+@opds.route("/opds/stats")
+@requires_basic_auth_if_no_ano
+def get_database_stats():
+    stat = dict()
+    stat['books'] = calibre_db.session.query(db.Books).count()
+    stat['authors'] = calibre_db.session.query(db.Authors).count()
+    stat['categories'] = calibre_db.session.query(db.Tags).count()
+    stat['series'] = calibre_db.session.query(db.Series).count()
+    return Response(json.dumps(stat), mimetype="application/json")
+
+
 @opds.route("/opds/thumb_240_240/<book_id>")
 @opds.route("/opds/cover_240_240/<book_id>")
 @opds.route("/opds/cover_90_90/<book_id>")
@@ -478,27 +473,6 @@ def feed_search(term):
         return render_xml_template('feed.xml', searchterm="")
 
 
-def check_auth(username, password):
-    try:
-        username = username.encode('windows-1252')
-    except UnicodeEncodeError:
-        username = username.encode('utf-8')
-    user = ub.session.query(ub.User).filter(func.lower(ub.User.name) ==
-                                            username.decode('utf-8').lower()).first()
-    if bool(user and check_password_hash(str(user.password), password)):
-        return True
-    else:
-        ip_address = request.headers.get('X-Forwarded-For', request.remote_addr)
-        log.warning('OPDS Login failed for user "%s" IP-address: %s', username.decode('utf-8'), ip_address)
-        return False
-
-
-def authenticate():
-    return Response(
-        'Could not verify your access level for that URL.\n'
-        'You have to login with proper credentials', 401,
-        {'WWW-Authenticate': 'Basic realm="Login Required"'})
-
 
 def render_xml_template(*args, **kwargs):
     # ToDo: return time in current timezone similar to %z
@@ -528,7 +502,7 @@ def render_element_index(database_column, linked_table, folder):
         entries = entries.join(linked_table).join(db.Books)
     entries = entries.filter(calibre_db.common_filters()).group_by(func.upper(func.substr(database_column, 1, 1))).all()
     elements = []
-    if off == 0:
+    if off == 0 and entries:
         elements.append({'id': "00", 'name': _("All")})
         shift = 1
     for entry in entries[

cps/redirect.py

@@ -25,7 +25,7 @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# http://flask.pocoo.org/snippets/62/
+# https://web.archive.org/web/20120517003641/http://flask.pocoo.org/snippets/62/
 
 from urllib.parse import urlparse, urljoin
 

cps/remotelogin.py

@@ -58,8 +58,8 @@ def remote_login():
     ub.session.add(auth_token)
     ub.session_commit()
     verify_url = url_for('remotelogin.verify_token', token=auth_token.auth_token, _external=true)
-    log.debug(u"Remot Login request with token: %s", auth_token.auth_token)
-    return render_title_template('remote_login.html', title=_(u"Login"), token=auth_token.auth_token,
+    log.debug("Remot Login request with token: %s", auth_token.auth_token)
+    return render_title_template('remote_login.html', title=_("Login"), token=auth_token.auth_token,
                                  verify_url=verify_url, page="remotelogin")
 
 
@@ -71,8 +71,8 @@ def verify_token(token):
 
     # Token not found
     if auth_token is None:
-        flash(_(u"Token not found"), category="error")
-        log.error(u"Remote Login token not found")
+        flash(_("Token not found"), category="error")
+        log.error("Remote Login token not found")
         return redirect(url_for('web.index'))
 
     # Token expired
@@ -80,8 +80,8 @@ def verify_token(token):
         ub.session.delete(auth_token)
         ub.session_commit()
 
-        flash(_(u"Token has expired"), category="error")
-        log.error(u"Remote Login token expired")
+        flash(_("Token has expired"), category="error")
+        log.error("Remote Login token expired")
         return redirect(url_for('web.index'))
 
     # Update token with user information
@@ -89,8 +89,8 @@ def verify_token(token):
     auth_token.verified = True
     ub.session_commit()
 
-    flash(_(u"Success! Please return to your device"), category="success")
-    log.debug(u"Remote Login token for userid %s verified", auth_token.user_id)
+    flash(_("Success! Please return to your device"), category="success")
+    log.debug("Remote Login token for userid %s verified", auth_token.user_id)
     return redirect(url_for('web.index'))
 
 
@@ -105,7 +105,7 @@ def token_verified():
     # Token not found
     if auth_token is None:
         data['status'] = 'error'
-        data['message'] = _(u"Token not found")
+        data['message'] = _("Token not found")
 
     # Token expired
     elif datetime.now() > auth_token.expiration:
@@ -113,7 +113,7 @@ def token_verified():
         ub.session_commit()
 
         data['status'] = 'error'
-        data['message'] = _(u"Token has expired")
+        data['message'] = _("Token has expired")
 
     elif not auth_token.verified:
         data['status'] = 'not_verified'
@@ -126,8 +126,8 @@ def token_verified():
         ub.session_commit("User {} logged in via remotelogin, token deleted".format(user.name))
 
         data['status'] = 'success'
-        log.debug(u"Remote Login for userid %s succeded", user.id)
-        flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.name), category="success")
+        log.debug("Remote Login for userid %s succeeded", user.id)
+        flash(_("Success! You are now logged in as: %(nickname)s", nickname=user.name), category="success")
 
     response = make_response(json.dumps(data, ensure_ascii=False))
     response.headers["Content-Type"] = "application/json; charset=utf-8"

cps/render_template.py

@@ -20,11 +20,13 @@ from flask import render_template, g, abort, request
 from flask_babel import gettext as _
 from werkzeug.local import LocalProxy
 from flask_login import current_user
+from sqlalchemy.sql.expression import or_
 
-from . import config, constants, logger
+from . import config, constants, logger, ub
 from .ub import User
 
 
+
 log = logger.create()
 
 def get_sidebar_config(kwargs=None):
@@ -45,12 +47,12 @@ def get_sidebar_config(kwargs=None):
                     "show_text": _('Show Hot Books'), "config_show": True})
     if current_user.role_admin():
         sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.download_list',
-                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not current_user.is_anonymous),
                         "page": "download", "show_text": _('Show Downloaded Books'),
                         "config_show": content})
     else:
         sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.books_list',
-                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                        "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not current_user.is_anonymous),
                         "page": "download", "show_text": _('Show Downloaded Books'),
                         "config_show": content})
     sidebar.append(
@@ -58,62 +60,52 @@ def get_sidebar_config(kwargs=None):
          "visibility": constants.SIDEBAR_BEST_RATED, 'public': True, "page": "rated",
          "show_text": _('Show Top Rated Books'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-eye-open", "text": _('Read Books'), "link": 'web.books_list', "id": "read",
-                    "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous),
-                    "page": "read", "show_text": _('Show read and unread'), "config_show": content})
+                    "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not current_user.is_anonymous),
+                    "page": "read", "show_text": _('Show Read and Unread'), "config_show": content})
     sidebar.append(
         {"glyph": "glyphicon-eye-close", "text": _('Unread Books'), "link": 'web.books_list', "id": "unread",
-         "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous), "page": "unread",
+         "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not current_user.is_anonymous), "page": "unread",
          "show_text": _('Show unread'), "config_show": False})
     sidebar.append({"glyph": "glyphicon-random", "text": _('Discover'), "link": 'web.books_list', "id": "rand",
                     "visibility": constants.SIDEBAR_RANDOM, 'public': True, "page": "discover",
                     "show_text": _('Show Random Books'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-inbox", "text": _('Categories'), "link": 'web.category_list', "id": "cat",
                     "visibility": constants.SIDEBAR_CATEGORY, 'public': True, "page": "category",
-                    "show_text": _('Show category selection'), "config_show": True})
+                    "show_text": _('Show Category Section'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-bookmark", "text": _('Series'), "link": 'web.series_list', "id": "serie",
                     "visibility": constants.SIDEBAR_SERIES, 'public': True, "page": "series",
-                    "show_text": _('Show series selection'), "config_show": True})
+                    "show_text": _('Show Series Section'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-user", "text": _('Authors'), "link": 'web.author_list', "id": "author",
                     "visibility": constants.SIDEBAR_AUTHOR, 'public': True, "page": "author",
-                    "show_text": _('Show author selection'), "config_show": True})
+                    "show_text": _('Show Author Section'), "config_show": True})
     sidebar.append(
         {"glyph": "glyphicon-text-size", "text": _('Publishers'), "link": 'web.publisher_list', "id": "publisher",
          "visibility": constants.SIDEBAR_PUBLISHER, 'public': True, "page": "publisher",
-         "show_text": _('Show publisher selection'), "config_show":True})
+         "show_text": _('Show Publisher Section'), "config_show":True})
     sidebar.append({"glyph": "glyphicon-flag", "text": _('Languages'), "link": 'web.language_overview', "id": "lang",
-                    "visibility": constants.SIDEBAR_LANGUAGE, 'public': (g.user.filter_language() == 'all'),
+                    "visibility": constants.SIDEBAR_LANGUAGE, 'public': (current_user.filter_language() == 'all'),
                     "page": "language",
-                    "show_text": _('Show language selection'), "config_show": True})
+                    "show_text": _('Show Language Section'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-star-empty", "text": _('Ratings'), "link": 'web.ratings_list', "id": "rate",
                     "visibility": constants.SIDEBAR_RATING, 'public': True,
-                    "page": "rating", "show_text": _('Show ratings selection'), "config_show": True})
+                    "page": "rating", "show_text": _('Show Ratings Section'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-file", "text": _('File formats'), "link": 'web.formats_list', "id": "format",
                     "visibility": constants.SIDEBAR_FORMAT, 'public': True,
-                    "page": "format", "show_text": _('Show file formats selection'), "config_show": True})
+                    "page": "format", "show_text": _('Show File Formats Section'), "config_show": True})
     sidebar.append(
         {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
-         "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
-         "show_text": _('Show archived books'), "config_show": content})
+         "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not current_user.is_anonymous), "page": "archived",
+         "show_text": _('Show Archived Books'), "config_show": content})
     if not simple:
         sidebar.append(
             {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_table', "id": "list",
-             "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list",
+             "visibility": constants.SIDEBAR_LIST, 'public': (not current_user.is_anonymous), "page": "list",
              "show_text": _('Show Books List'), "config_show": content})
+    g.shelves_access = ub.session.query(ub.Shelf).filter(
+        or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == current_user.id)).order_by(ub.Shelf.name).all()
+
     return sidebar, simple
 
-'''def get_readbooks_ids():
-    if not config.config_read_column:
-        readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
-            .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
-        return frozenset([x.book_id for x in readBooks])
-    else:
-        try:
-            readBooks = calibre_db.session.query(db.cc_classes[config.config_read_column])\
-                .filter(db.cc_classes[config.config_read_column].value == True).all()
-            return frozenset([x.book for x in readBooks])
-        except (KeyError, AttributeError, IndexError):
-            log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column))
-            return []'''
 
 # Returns the template for rendering and includes the instance name
 def render_title_template(*args, **kwargs):

cps/schedule.py

@@ -19,19 +19,26 @@
 import datetime
 
 from . import config, constants
-from .services.background_scheduler import BackgroundScheduler, use_APScheduler
+from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler
 from .tasks.database import TaskReconnectDatabase
+from .tasks.tempFolder import TaskDeleteTempFolder
 from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache
 from .services.worker import WorkerThread
-
+from .tasks.metadata_backup import TaskBackupMetadata
 
 def get_scheduled_tasks(reconnect=True):
     tasks = list()
-    # config.schedule_reconnect or
-    # Reconnect Calibre database (metadata.db)
+    # Reconnect Calibre database (metadata.db) based on config.schedule_reconnect
     if reconnect:
         tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False])
 
+    # Delete temp folder
+    tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True])
+
+    # Generate metadata.opf file for each changed book
+    if config.schedule_metadata_backup:
+        tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False])
+
     # Generate all missing book cover thumbnails
     if config.schedule_generate_book_covers:
         tasks.append([lambda: TaskClearCoverThumbnailCache(0), 'delete superfluous book covers', True])
@@ -62,10 +69,10 @@ def register_scheduled_tasks(reconnect=True):
         duration = config.schedule_duration
 
         # Register scheduled tasks
-        scheduler.schedule_tasks(tasks=get_scheduled_tasks(reconnect), trigger='cron', hour=start)
+        scheduler.schedule_tasks(tasks=get_scheduled_tasks(reconnect), trigger=CronTrigger(hour=start))
         end_time = calclulate_end_time(start, duration)
-        scheduler.schedule(func=end_scheduled_tasks, trigger='cron', name="end scheduled task", hour=end_time.hour,
-                           minute=end_time.minute)
+        scheduler.schedule(func=end_scheduled_tasks, trigger=CronTrigger(hour=end_time.hour, minute=end_time.minute),
+                           name="end scheduled task")
 
         # Kick-off tasks, if they should currently be running
         if should_task_be_running(start, duration):
@@ -83,6 +90,8 @@ def register_startup_tasks():
         # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks
         if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration):
             scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False))
+        else:
+            scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]])
 
 
 def should_task_be_running(start, duration):
@@ -91,6 +100,7 @@ def should_task_be_running(start, duration):
     end_time = start_time + datetime.timedelta(hours=duration // 60, minutes=duration % 60)
     return start_time < now < end_time
 
+
 def calclulate_end_time(start, duration):
     start_time = datetime.datetime.now().replace(hour=start, minute=0)
     return start_time + datetime.timedelta(hours=duration // 60, minutes=duration % 60)

cps/search.py

@@ -22,7 +22,7 @@ from flask import session as flask_session
 from flask_login import current_user
 from flask_babel import format_date
 from flask_babel import gettext as _
-from sqlalchemy.sql.expression import func, not_, and_, or_, text
+from sqlalchemy.sql.expression import func, not_, and_, or_, text, true
 from sqlalchemy.sql.functions import coalesce
 
 from . import logger, db, calibre_db, config, ub
@@ -45,7 +45,7 @@ def simple_search():
         return render_title_template('search.html',
                                      searchterm="",
                                      result_count=0,
-                                     title=_(u"Search"),
+                                     title=_("Search"),
                                      page="search")
 
 
@@ -119,32 +119,26 @@ def adv_search_ratings(q, rating_high, rating_low):
     return q
 
 
-def adv_search_read_status(q, read_status):
-    if read_status:
-        if config.config_read_column:
-            try:
-                if read_status == "True":
-                    q = q.join(db.cc_classes[config.config_read_column], isouter=True) \
-                        .filter(db.cc_classes[config.config_read_column].value == True)
-                else:
-                    q = q.join(db.cc_classes[config.config_read_column], isouter=True) \
-                        .filter(coalesce(db.cc_classes[config.config_read_column].value, False) != True)
-            except (KeyError, AttributeError):
-                log.error(u"Custom Column No.%d is not existing in calibre database", config.config_read_column)
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
-                        column=config.config_read_column),
-                      category="error")
-                return q
+def adv_search_read_status(read_status):
+    if not config.config_read_column:
+        if read_status == "True":
+            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
+                             ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
         else:
+            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
+    else:
+        try:
             if read_status == "True":
-                q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \
-                    .filter(ub.ReadBook.user_id == int(current_user.id),
-                            ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
+                db_filter = db.cc_classes[config.config_read_column].value == True
             else:
-                q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \
-                    .filter(ub.ReadBook.user_id == int(current_user.id),
-                            coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED)
-    return q
+                db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
+        except (KeyError, AttributeError, IndexError):
+            log.error("Custom Column No.{} does not exist in calibre database".format(config.config_read_column))
+            flash(_("Custom Column No.%(column)d does not exist in calibre database",
+                    column=config.config_read_column),
+                  category="error")
+            return true()
+    return db_filter
 
 
 def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs):
@@ -191,18 +185,18 @@ def extend_search_term(searchterm,
     searchterm.extend((author_name.replace('|', ','), book_title, publisher))
     if pub_start:
         try:
-            searchterm.extend([_(u"Published after ") +
+            searchterm.extend([_("Published after ") +
                                format_date(datetime.strptime(pub_start, "%Y-%m-%d"),
                                            format='medium')])
         except ValueError:
-            pub_start = u""
+            pub_start = ""
     if pub_end:
         try:
-            searchterm.extend([_(u"Published before ") +
+            searchterm.extend([_("Published before ") +
                                format_date(datetime.strptime(pub_end, "%Y-%m-%d"),
                                            format='medium')])
         except ValueError:
-            pub_end = u""
+            pub_end = ""
     elements = {'tag': db.Tags, 'serie':db.Series, 'shelf':ub.Shelf}
     for key, db_element in elements.items():
         tag_names = calibre_db.session.query(db_element).filter(db_element.id.in_(tags['include_' + key])).all()
@@ -220,11 +214,11 @@ def extend_search_term(searchterm,
         language_names = calibre_db.speaking_language(language_names)
     searchterm.extend(language.name for language in language_names)
     if rating_high:
-        searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)])
+        searchterm.extend([_("Rating <= %(rating)s", rating=rating_high)])
     if rating_low:
-        searchterm.extend([_(u"Rating >= %(rating)s", rating=rating_low)])
-    if read_status:
-        searchterm.extend([_(u"Read Status = %(status)s", status=read_status)])
+        searchterm.extend([_("Rating >= %(rating)s", rating=rating_low)])
+    if read_status != "Any":
+        searchterm.extend([_("Read Status = '%(status)s'", status=read_status)])
     searchterm.extend(ext for ext in tags['include_extension'])
     searchterm.extend(ext for ext in tags['exclude_extension'])
     # handle custom columns
@@ -238,23 +232,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
 
     cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True)
     calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
-    if not config.config_read_column:
-        query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books)
-                 .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id,
-                                              int(current_user.id) == ub.ReadBook.user_id)))
-    else:
-        try:
-            read_column = cc[config.config_read_column]
-            query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value)
-                     .select_from(db.Books)
-                     .outerjoin(read_column, read_column.book == db.Books.id))
-        except (KeyError, AttributeError):
-            log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
-            # Skip linking read column
-            query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None)
-    query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
-                                                  int(current_user.id) == ub.ArchivedBook.user_id))
-
+    query = calibre_db.generate_linked_query(config.config_read_column, db.Books)
     q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book)\
         .outerjoin(db.Series)\
         .filter(calibre_db.common_filters(True))
@@ -289,23 +267,23 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
             column_start = term.get('custom_column_' + str(c.id) + '_start')
             column_end = term.get('custom_column_' + str(c.id) + '_end')
             if column_start:
-                search_term.extend([u"{} >= {}".format(c.name,
+                search_term.extend(["{} >= {}".format(c.name,
                                                        format_date(datetime.strptime(column_start, "%Y-%m-%d").date(),
                                                                    format='medium')
                                                        )])
                 cc_present = True
             if column_end:
-                search_term.extend([u"{} <= {}".format(c.name,
+                search_term.extend(["{} <= {}".format(c.name,
                                                        format_date(datetime.strptime(column_end, "%Y-%m-%d").date(),
                                                                    format='medium')
                                                        )])
                 cc_present = True
         elif term.get('custom_column_' + str(c.id)):
-            search_term.extend([(u"{}: {}".format(c.name, term.get('custom_column_' + str(c.id))))])
+            search_term.extend([("{}: {}".format(c.name, term.get('custom_column_' + str(c.id))))])
             cc_present = True
 
     if any(tags.values()) or author_name or book_title or publisher or pub_start or pub_end or rating_low \
-       or rating_high or description or cc_present or read_status:
+       or rating_high or description or cc_present or read_status != "Any":
         search_term, pub_start, pub_end = extend_search_term(search_term,
                                                              author_name,
                                                              book_title,
@@ -324,7 +302,8 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
             q = q.filter(func.datetime(db.Books.pubdate) > func.datetime(pub_start))
         if pub_end:
             q = q.filter(func.datetime(db.Books.pubdate) < func.datetime(pub_end))
-        q = adv_search_read_status(q, read_status)
+        if read_status != "Any":
+            q = q.filter(adv_search_read_status(read_status))
         if publisher:
             q = q.filter(db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + publisher + "%")))
         q = adv_search_tag(q, tags['include_tag'], tags['exclude_tag'])
@@ -361,7 +340,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
                                  pagination=pagination,
                                  entries=entries,
                                  result_count=result_count,
-                                 title=_(u"Advanced Search"), page="advsearch",
+                                 title=_("Advanced Search"), page="advsearch",
                                  order=order[1])
 
 
@@ -388,22 +367,28 @@ def render_prepare_search_form(cc):
         .filter(calibre_db.common_filters()) \
         .group_by(db.Data.format)\
         .order_by(db.Data.format).all()
-    if current_user.filter_language() == u"all":
+    if current_user.filter_language() == "all":
         languages = calibre_db.speaking_language()
     else:
         languages = None
     return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
-                                 series=series,shelves=shelves, title=_(u"Advanced Search"), cc=cc, page="advsearch")
+                                 series=series,shelves=shelves, title=_("Advanced Search"), cc=cc, page="advsearch")
 
 
 def render_search_results(term, offset=None, order=None, limit=None):
-    join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series
-    entries, result_count, pagination = calibre_db.get_search_results(term,
-                                                                      config,
-                                                                      offset,
-                                                                      order,
-                                                                      limit,
-                                                                      *join)
+    if term:
+        join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series
+        entries, result_count, pagination = calibre_db.get_search_results(term,
+                                                                          config,
+                                                                          offset,
+                                                                          order,
+                                                                          limit,
+                                                                          *join)
+    else:
+        entries = list()
+        order = [None, None]
+        pagination = result_count = None
+
     return render_title_template('search.html',
                                  searchterm=term,
                                  pagination=pagination,
@@ -411,7 +396,7 @@ def render_search_results(term, offset=None, order=None, limit=None):
                                  adv_searchterm=term,
                                  entries=entries,
                                  result_count=result_count,
-                                 title=_(u"Search"),
+                                 title=_("Search"),
                                  page="search",
                                  order=order[1])
 

cps/server.py

@@ -21,12 +21,13 @@ import os
 import errno
 import signal
 import socket
-import subprocess  # nosec
+import asyncio
 
 try:
-    from gevent.pywsgi import WSGIServer
+    from gevent_.pywsgi import WSGIServer
     from .gevent_wsgi import MyWSGIHandler
     from gevent.pool import Pool
+    from gevent.socket import socket as GeventSocket
     from gevent import __version__ as _version
     from greenlet import GreenletExit
     import ssl
@@ -36,6 +37,7 @@ except ImportError:
     from .tornado_wsgi import MyWSGIContainer
     from tornado.httpserver import HTTPServer
     from tornado.ioloop import IOLoop
+    from tornado import netutil
     from tornado import version as _version
     VERSION = 'Tornado ' + _version
     _GEVENT = False
@@ -95,7 +97,12 @@ class WebServer(object):
                 log.warning('Cert path: %s', certfile_path)
                 log.warning('Key path:  %s', keyfile_path)
 
-    def _make_gevent_unix_socket(self, socket_file):
+    def _make_gevent_socket_activated(self):
+        # Reuse an already open socket on fd=SD_LISTEN_FDS_START
+        SD_LISTEN_FDS_START = 3
+        return GeventSocket(fileno=SD_LISTEN_FDS_START)
+
+    def _prepare_unix_socket(self, socket_file):
         # the socket file must not exist prior to bind()
         if os.path.exists(socket_file):
             # avoid nuking regular files and symbolic links (could be a mistype or security issue)
@@ -103,35 +110,41 @@ class WebServer(object):
                 raise OSError(errno.EEXIST, os.strerror(errno.EEXIST), socket_file)
             os.remove(socket_file)
 
-        unix_sock = WSGIServer.get_listener(socket_file, family=socket.AF_UNIX)
         self.unix_socket_file = socket_file
 
-        # ensure current user and group have r/w permissions, no permissions for other users
-        # this way the socket can be shared in a semi-secure manner
-        # between the user running calibre-web and the user running the fronting webserver
-        os.chmod(socket_file, 0o660)
-
-        return unix_sock
-
-    def _make_gevent_socket(self):
+    def _make_gevent_listener(self):
         if os.name != 'nt':
+            socket_activated = os.environ.get("LISTEN_FDS")
+            if socket_activated:
+                sock = self._make_gevent_socket_activated()
+                sock_info = sock.getsockname()
+                return sock, "systemd-socket:" + _readable_listen_address(sock_info[0], sock_info[1])
             unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
             if unix_socket_file:
-                return self._make_gevent_unix_socket(unix_socket_file), "unix:" + unix_socket_file
+                self._prepare_unix_socket(unix_socket_file)
+                unix_sock = WSGIServer.get_listener(unix_socket_file, family=socket.AF_UNIX)
+                # ensure current user and group have r/w permissions, no permissions for other users
+                # this way the socket can be shared in a semi-secure manner
+                # between the user running calibre-web and the user running the fronting webserver
+                os.chmod(unix_socket_file, 0o660)
+
+                return unix_sock, "unix:" + unix_socket_file
 
         if self.listen_address:
-            return (self.listen_address, self.listen_port), None
+            return ((self.listen_address, self.listen_port),
+                    _readable_listen_address(self.listen_address, self.listen_port))
 
         if os.name == 'nt':
             self.listen_address = '0.0.0.0'
-            return (self.listen_address, self.listen_port), None
+            return ((self.listen_address, self.listen_port),
+                    _readable_listen_address(self.listen_address, self.listen_port))
 
         try:
             address = ('::', self.listen_port)
             sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
         except socket.error as ex:
             log.error('%s', ex)
-            log.warning('Unable to listen on "", trying on IPv4 only...')
+            log.warning('Unable to listen on {}, trying on IPv4 only...'.format(address))
             address = ('', self.listen_port)
             sock = WSGIServer.get_listener(address, family=socket.AF_INET)
 
@@ -152,7 +165,7 @@ class WebServer(object):
         # The value of __package__ indicates how Python was called. It may
         # not exist if a setuptools script is installed as an egg. It may be
         # set incorrectly for entry points created with pip on Windows.
-        if getattr(__main__, "__package__", None) is None or (
+        if getattr(__main__, "__package__", "") in ["", None] or (
             os.name == "nt"
             and __main__.__package__ == ""
             and not os.path.exists(py_script)
@@ -193,17 +206,18 @@ class WebServer(object):
                 rv.extend(("-m", py_module.lstrip(".")))
 
         rv.extend(args)
+        if os.name == 'nt':
+            rv = ['"{}"'.format(a) for a in rv]
         return rv
 
     def _start_gevent(self):
         ssl_args = self.ssl_args or {}
 
         try:
-            sock, output = self._make_gevent_socket()
-            if output is None:
-                output = _readable_listen_address(self.listen_address, self.listen_port)
+            sock, output = self._make_gevent_listener()
             log.info('Starting Gevent server on %s', output)
             self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, handler_class=MyWSGIHandler,
+                                         error_log=log,
                                          spawn=Pool(), **ssl_args)
             if ssl_args:
                 wrap_socket = self.wsgiserver.wrap_socket
@@ -225,17 +239,42 @@ class WebServer(object):
         if os.name == 'nt' and sys.version_info > (3, 7):
             import asyncio
             asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-        log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port))
+        try:
+            # Max Buffersize set to 200MB
+            http_server = HTTPServer(MyWSGIContainer(self.app),
+                                     max_buffer_size=209700000,
+                                     ssl_options=self.ssl_args)
 
-        # Max Buffersize set to 200MB
-        http_server = HTTPServer(MyWSGIContainer(self.app),
-                                 max_buffer_size=209700000,
-                                 ssl_options=self.ssl_args)
-        http_server.listen(self.listen_port, self.listen_address)
-        self.wsgiserver = IOLoop.current()
-        self.wsgiserver.start()
-        # wait for stop signal
-        self.wsgiserver.close(True)
+            unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
+            if os.environ.get("LISTEN_FDS") and os.name != 'nt':
+                SD_LISTEN_FDS_START = 3
+                sock = socket.socket(fileno=SD_LISTEN_FDS_START)
+                http_server.add_socket(sock)
+                sock.setblocking(0)
+                socket_name =sock.getsockname()
+                output = "systemd-socket:" + _readable_listen_address(socket_name[0], socket_name[1])
+            elif unix_socket_file and os.name != 'nt':
+                self._prepare_unix_socket(unix_socket_file)
+                output = "unix:" + unix_socket_file
+                unix_socket = netutil.bind_unix_socket(self.unix_socket_file)
+                http_server.add_socket(unix_socket)
+                # ensure current user and group have r/w permissions, no permissions for other users
+                # this way the socket can be shared in a semi-secure manner
+                # between the user running calibre-web and the user running the fronting webserver
+                os.chmod(self.unix_socket_file, 0o660)
+            else:
+                output = _readable_listen_address(self.listen_address, self.listen_port)
+                http_server.listen(self.listen_port, self.listen_address)
+            log.info('Starting Tornado server on %s', output)
+
+            self.wsgiserver = IOLoop.current()
+            self.wsgiserver.start()
+            # wait for stop signal
+            self.wsgiserver.close(True)
+        finally:
+            if self.unix_socket_file:
+                os.remove(self.unix_socket_file)
+                self.unix_socket_file = None
 
     def start(self):
         try:
@@ -261,9 +300,16 @@ class WebServer(object):
 
         log.info("Performing restart of Calibre-Web")
         args = self._get_args_for_reloading()
-        subprocess.call(args, close_fds=True)  # nosec
+        os.execv(args[0].lstrip('"').rstrip('"'), args)
         return True
 
+    @staticmethod
+    def shutdown_scheduler():
+        from .services.background_scheduler import BackgroundScheduler
+        scheduler = BackgroundScheduler()
+        if scheduler:
+            scheduler.scheduler.shutdown()
+
     def _killServer(self, __, ___):
         self.stop()
 
@@ -272,9 +318,14 @@ class WebServer(object):
         updater_thread.stop()
 
         log.info("webserver stop (restart=%s)", restart)
+        self.shutdown_scheduler()
         self.restart = restart
         if self.wsgiserver:
             if _GEVENT:
                 self.wsgiserver.close()
             else:
-                self.wsgiserver.add_callback_from_signal(self.wsgiserver.stop)
+                if restart:
+                    self.wsgiserver.call_later(1.0, self.wsgiserver.stop)
+                else:
+                    self.wsgiserver.asyncio_loop.call_soon_threadsafe(self.wsgiserver.stop)
+

cps/services/SyncToken.py

@@ -19,11 +19,9 @@
 
 import sys
 from base64 import b64decode, b64encode
-from jsonschema import validate, exceptions, __version__
+from jsonschema import validate, exceptions
 from datetime import datetime
 
-from urllib.parse import unquote
-
 from flask import json
 from .. import logger
 

cps/services/background_scheduler.py

@@ -23,6 +23,8 @@ from .worker import WorkerThread
 
 try:
     from apscheduler.schedulers.background import BackgroundScheduler as BScheduler
+    from apscheduler.triggers.cron import CronTrigger
+    from apscheduler.triggers.date import DateTrigger
     use_APScheduler = True
 except (ImportError, RuntimeError) as e:
     use_APScheduler = False
@@ -43,35 +45,33 @@ class BackgroundScheduler:
             cls.scheduler = BScheduler()
             cls.scheduler.start()
 
-            atexit.register(lambda: cls.scheduler.shutdown())
-
         return cls._instance
 
-    def schedule(self, func, trigger, name=None, **trigger_args):
+    def schedule(self, func, trigger, name=None):
         if use_APScheduler:
-            return self.scheduler.add_job(func=func, trigger=trigger, name=name, **trigger_args)
+            return self.scheduler.add_job(func=func, trigger=trigger, name=name)
 
     # Expects a lambda expression for the task
-    def schedule_task(self, task, user=None, name=None, hidden=False, trigger='cron', **trigger_args):
+    def schedule_task(self, task, user=None, name=None, hidden=False, trigger=None):
         if use_APScheduler:
             def scheduled_task():
                 worker_task = task()
                 worker_task.scheduled = True
                 WorkerThread.add(user, worker_task, hidden=hidden)
-            return self.schedule(func=scheduled_task, trigger=trigger, name=name, **trigger_args)
+            return self.schedule(func=scheduled_task, trigger=trigger, name=name)
 
     # Expects a list of lambda expressions for the tasks
-    def schedule_tasks(self, tasks, user=None, trigger='cron', **trigger_args):
+    def schedule_tasks(self, tasks, user=None, trigger=None):
         if use_APScheduler:
             for task in tasks:
-                self.schedule_task(task[0], user=user, trigger=trigger, name=task[1], hidden=task[2], **trigger_args)
+                self.schedule_task(task[0], user=user, trigger=trigger, name=task[1], hidden=task[2])
 
     # Expects a lambda expression for the task
     def schedule_task_immediately(self, task, user=None, name=None, hidden=False):
         if use_APScheduler:
             def immediate_task():
                 WorkerThread.add(user, task(), hidden)
-            return self.schedule(func=immediate_task, trigger='date', name=name)
+            return self.schedule(func=immediate_task, trigger=DateTrigger(), name=name)
 
     # Expects a list of lambda expressions for the tasks
     def schedule_tasks_immediately(self, tasks, user=None):

cps/services/simpleldap.py

@@ -20,6 +20,7 @@ import base64
 
 from flask_simpleldap import LDAP, LDAPException
 from flask_simpleldap import ldap as pyLDAP
+from flask import current_app
 from .. import constants, logger
 
 try:
@@ -28,8 +29,47 @@ except ImportError:
     pass
 
 log = logger.create()
-_ldap = LDAP()
 
+class LDAPLogger(object):
+
+    def write(self, message):
+        try:
+            log.debug(message.strip("\n").replace("\n", ""))
+        except Exception:
+            log.debug("Logging Error")
+
+
+class mySimpleLDap(LDAP):
+
+    @staticmethod
+    def init_app(app):
+        super(mySimpleLDap, mySimpleLDap).init_app(app)
+        app.config.setdefault('LDAP_LOGLEVEL', 0)
+
+    @property
+    def initialize(self):
+        """Initialize a connection to the LDAP server.
+
+        :return: LDAP connection object.
+        """
+        try:
+            log_level = 2 if current_app.config['LDAP_LOGLEVEL'] == logger.logging.DEBUG else 0
+            conn = pyLDAP.initialize('{0}://{1}:{2}'.format(
+                current_app.config['LDAP_SCHEMA'],
+                current_app.config['LDAP_HOST'],
+                current_app.config['LDAP_PORT']), trace_level=log_level, trace_file=LDAPLogger())
+            conn.set_option(pyLDAP.OPT_NETWORK_TIMEOUT,
+                            current_app.config['LDAP_TIMEOUT'])
+            conn = self._set_custom_options(conn)
+            conn.protocol_version = pyLDAP.VERSION3
+            if current_app.config['LDAP_USE_TLS']:
+                conn.start_tls_s()
+            return conn
+        except pyLDAP.LDAPError as e:
+            raise LDAPException(self.error(e.args))
+
+
+_ldap = mySimpleLDap()
 
 def init_app(app, config):
     if config.config_login_type != constants.LOGIN_LDAP:
@@ -44,15 +84,15 @@ def init_app(app, config):
         app.config['LDAP_SCHEMA'] = 'ldap'
     if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
         if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
-            if config.config_ldap_serv_password is None:
-                config.config_ldap_serv_password = ''
-            app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password)
+            if config.config_ldap_serv_password_e is None:
+                config.config_ldap_serv_password_e = ''
+            app.config['LDAP_PASSWORD'] = config.config_ldap_serv_password_e
         else:
-            app.config['LDAP_PASSWORD'] = base64.b64decode("")
+            app.config['LDAP_PASSWORD'] = ""
         app.config['LDAP_USERNAME'] = config.config_ldap_serv_username
     else:
         app.config['LDAP_USERNAME'] = ""
-        app.config['LDAP_PASSWORD'] = base64.b64decode("")
+        app.config['LDAP_PASSWORD'] = ""
     if bool(config.config_ldap_cert_path):
         app.config['LDAP_CUSTOM_OPTIONS'].update({
             pyLDAP.OPT_X_TLS_REQUIRE_CERT: pyLDAP.OPT_X_TLS_DEMAND,
@@ -70,7 +110,7 @@ def init_app(app, config):
     app.config['LDAP_OPENLDAP'] = bool(config.config_ldap_openldap)
     app.config['LDAP_GROUP_OBJECT_FILTER'] = config.config_ldap_group_object_filter
     app.config['LDAP_GROUP_MEMBERS_FIELD'] = config.config_ldap_group_members_field
-
+    app.config['LDAP_LOGLEVEL'] = config.config_log_level
     try:
         _ldap.init_app(app)
     except ValueError:

cps/shelf.py

@@ -46,13 +46,13 @@ def add_to_shelf(shelf_id, book_id):
     if shelf is None:
         log.error("Invalid shelf specified: %s", shelf_id)
         if not xhr:
-            flash(_(u"Invalid shelf specified"), category="error")
+            flash(_("Invalid shelf specified"), category="error")
             return redirect(url_for('web.index'))
         return "Invalid shelf specified", 400
 
     if not check_shelf_edit_permissions(shelf):
         if not xhr:
-            flash(_(u"Sorry you are not allowed to add a book to that shelf"), category="error")
+            flash(_("Sorry you are not allowed to add a book to that shelf"), category="error")
             return redirect(url_for('web.index'))
         return "Sorry you are not allowed to add a book to the that shelf", 403
 
@@ -61,7 +61,7 @@ def add_to_shelf(shelf_id, book_id):
     if book_in_shelf:
         log.error("Book %s is already part of %s", book_id, shelf)
         if not xhr:
-            flash(_(u"Book is already part of the shelf: %(shelfname)s", shelfname=shelf.name), category="error")
+            flash(_("Book is already part of the shelf: %(shelfname)s", shelfname=shelf.name), category="error")
             return redirect(url_for('web.index'))
         return "Book is already part of the shelf: %s" % shelf.name, 400
 
@@ -71,6 +71,14 @@ def add_to_shelf(shelf_id, book_id):
     else:
         maxOrder = maxOrder[0]
 
+    if not calibre_db.session.query(db.Books).filter(db.Books.id == book_id).one_or_none():
+        log.error("Invalid Book Id: %s. Could not be added to shelf %s", book_id, shelf.name)
+        if not xhr:
+            flash(_("%(book_id)s is a invalid Book Id. Could not be added to Shelf", book_id=book_id),
+                  category="error")
+            return redirect(url_for('web.index'))
+        return "%s is a invalid Book Id. Could not be added to Shelf" % book_id, 400
+
     shelf.books.append(ub.BookShelf(shelf=shelf.id, book_id=book_id, order=maxOrder + 1))
     shelf.last_modified = datetime.utcnow()
     try:
@@ -79,14 +87,14 @@ def add_to_shelf(shelf_id, book_id):
     except (OperationalError, InvalidRequestError) as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
         if "HTTP_REFERER" in request.environ:
             return redirect(request.environ["HTTP_REFERER"])
         else:
             return redirect(url_for('web.index'))
     if not xhr:
         log.debug("Book has been added to shelf: {}".format(shelf.name))
-        flash(_(u"Book has been added to shelf: %(sname)s", sname=shelf.name), category="success")
+        flash(_("Book has been added to shelf: %(sname)s", sname=shelf.name), category="success")
         if "HTTP_REFERER" in request.environ:
             return redirect(request.environ["HTTP_REFERER"])
         else:
@@ -100,12 +108,12 @@ def search_to_shelf(shelf_id):
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     if shelf is None:
         log.error("Invalid shelf specified: {}".format(shelf_id))
-        flash(_(u"Invalid shelf specified"), category="error")
+        flash(_("Invalid shelf specified"), category="error")
         return redirect(url_for('web.index'))
 
     if not check_shelf_edit_permissions(shelf):
         log.warning("You are not allowed to add a book to the shelf".format(shelf.name))
-        flash(_(u"You are not allowed to add a book to the shelf"), category="error")
+        flash(_("You are not allowed to add a book to the shelf"), category="error")
         return redirect(url_for('web.index'))
 
     if current_user.id in ub.searched_ids and ub.searched_ids[current_user.id]:
@@ -123,7 +131,7 @@ def search_to_shelf(shelf_id):
 
         if not books_for_shelf:
             log.error("Books are already part of {}".format(shelf.name))
-            flash(_(u"Books are already part of the shelf: %(name)s", name=shelf.name), category="error")
+            flash(_("Books are already part of the shelf: %(name)s", name=shelf.name), category="error")
             return redirect(url_for('web.index'))
 
         maxOrder = ub.session.query(func.max(ub.BookShelf.order)).filter(ub.BookShelf.shelf == shelf_id).first()[0] or 0
@@ -135,14 +143,14 @@ def search_to_shelf(shelf_id):
         try:
             ub.session.merge(shelf)
             ub.session.commit()
-            flash(_(u"Books have been added to shelf: %(sname)s", sname=shelf.name), category="success")
+            flash(_("Books have been added to shelf: %(sname)s", sname=shelf.name), category="success")
         except (OperationalError, InvalidRequestError) as e:
             ub.session.rollback()
             log.error_or_exception("Settings Database error: {}".format(e))
-            flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+            flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
     else:
         log.error("Could not add books to shelf: {}".format(shelf.name))
-        flash(_(u"Could not add books to shelf: %(sname)s", sname=shelf.name), category="error")
+        flash(_("Could not add books to shelf: %(sname)s", sname=shelf.name), category="error")
     return redirect(url_for('web.index'))
 
 
@@ -182,13 +190,13 @@ def remove_from_shelf(shelf_id, book_id):
         except (OperationalError, InvalidRequestError) as e:
             ub.session.rollback()
             log.error_or_exception("Settings Database error: {}".format(e))
-            flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+            flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
             if "HTTP_REFERER" in request.environ:
                 return redirect(request.environ["HTTP_REFERER"])
             else:
                 return redirect(url_for('web.index'))
         if not xhr:
-            flash(_(u"Book has been removed from shelf: %(sname)s", sname=shelf.name), category="success")
+            flash(_("Book has been removed from shelf: %(sname)s", sname=shelf.name), category="success")
             if "HTTP_REFERER" in request.environ:
                 return redirect(request.environ["HTTP_REFERER"])
             else:
@@ -197,7 +205,7 @@ def remove_from_shelf(shelf_id, book_id):
     else:
         if not xhr:
             log.warning("You are not allowed to remove a book from shelf: {}".format(shelf.name))
-            flash(_(u"Sorry you are not allowed to remove a book from this shelf"),
+            flash(_("Sorry you are not allowed to remove a book from this shelf"),
                   category="error")
             return redirect(url_for('web.index'))
         return "Sorry you are not allowed to remove a book from this shelf", 403
@@ -207,7 +215,7 @@ def remove_from_shelf(shelf_id, book_id):
 @login_required
 def create_shelf():
     shelf = ub.Shelf()
-    return create_edit_shelf(shelf, page_title=_(u"Create a Shelf"), page="shelfcreate")
+    return create_edit_shelf(shelf, page_title=_("Create a Shelf"), page="shelfcreate")
 
 
 @shelf.route("/shelf/edit/<int:shelf_id>", methods=["GET", "POST"])
@@ -215,9 +223,9 @@ def create_shelf():
 def edit_shelf(shelf_id):
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     if not check_shelf_edit_permissions(shelf):
-        flash(_(u"Sorry you are not allowed to edit this shelf"), category="error")
+        flash(_("Sorry you are not allowed to edit this shelf"), category="error")
         return redirect(url_for('web.index'))
-    return create_edit_shelf(shelf, page_title=_(u"Edit a shelf"), page="shelfedit", shelf_id=shelf_id)
+    return create_edit_shelf(shelf, page_title=_("Edit a shelf"), page="shelfedit", shelf_id=shelf_id)
 
 
 @shelf.route("/shelf/delete/<int:shelf_id>", methods=["POST"])
@@ -232,7 +240,7 @@ def delete_shelf(shelf_id):
     except InvalidRequestError as e:
         ub.session.rollback()
         log.error_or_exception("Settings Database error: {}".format(e))
-        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
     return redirect(url_for('web.index'))
 
 
@@ -263,13 +271,13 @@ def order_shelf(shelf_id):
             for book in books_in_shelf:
                 setattr(book, 'order', to_save[str(book.book_id)])
                 counter += 1
-                # if order diffrent from before -> shelf.last_modified = datetime.utcnow()
+                # if order different from before -> shelf.last_modified = datetime.utcnow()
             try:
                 ub.session.commit()
             except (OperationalError, InvalidRequestError) as e:
                 ub.session.rollback()
                 log.error_or_exception("Settings Database error: {}".format(e))
-                flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+                flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
 
         result = list()
         if shelf:
@@ -278,7 +286,7 @@ def order_shelf(shelf_id):
                 .add_columns(calibre_db.common_filters().label("visible")) \
                 .filter(ub.BookShelf.shelf == shelf_id).order_by(ub.BookShelf.order.asc()).all()
         return render_title_template('shelf_order.html', entries=result,
-                                     title=_(u"Change order of Shelf: '%(name)s'", name=shelf.name),
+                                     title=_("Change order of Shelf: '%(name)s'", name=shelf.name),
                                      shelf=shelf, page="shelforder")
     else:
         abort(404)
@@ -295,11 +303,14 @@ def check_shelf_edit_permissions(cur_shelf):
 
 
 def check_shelf_view_permissions(cur_shelf):
-    if cur_shelf.is_public:
-        return True
-    if current_user.is_anonymous or cur_shelf.user_id != current_user.id:
-        log.error("User is unauthorized to view non-public shelf: {}".format(cur_shelf.name))
-        return False
+    try:
+        if cur_shelf.is_public:
+            return True
+        if current_user.is_anonymous or cur_shelf.user_id != current_user.id:
+            log.error("User is unauthorized to view non-public shelf: {}".format(cur_shelf.name))
+            return False
+    except Exception as e:
+        log.error(e)
     return True
 
 
@@ -310,7 +321,7 @@ def create_edit_shelf(shelf, page_title, page, shelf_id=False):
     if request.method == "POST":
         to_save = request.form.to_dict()
         if not current_user.role_edit_shelfs() and to_save.get("is_public") == "on":
-            flash(_(u"Sorry you are not allowed to create a public shelf"), category="error")
+            flash(_("Sorry you are not allowed to create a public shelf"), category="error")
             return redirect(url_for('web.index'))
         is_public = 1 if to_save.get("is_public") == "on" else 0
         if config.config_kobo_sync:
@@ -327,24 +338,24 @@ def create_edit_shelf(shelf, page_title, page, shelf_id=False):
                 shelf.user_id = int(current_user.id)
                 ub.session.add(shelf)
                 shelf_action = "created"
-                flash_text = _(u"Shelf %(title)s created", title=shelf_title)
+                flash_text = _("Shelf %(title)s created", title=shelf_title)
             else:
                 shelf_action = "changed"
-                flash_text = _(u"Shelf %(title)s changed", title=shelf_title)
+                flash_text = _("Shelf %(title)s changed", title=shelf_title)
             try:
                 ub.session.commit()
-                log.info(u"Shelf {} {}".format(shelf_title, shelf_action))
+                log.info("Shelf {} {}".format(shelf_title, shelf_action))
                 flash(flash_text, category="success")
                 return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id))
             except (OperationalError, InvalidRequestError) as ex:
                 ub.session.rollback()
                 log.error_or_exception(ex)
                 log.error_or_exception("Settings Database error: {}".format(ex))
-                flash(_(u"Database error: %(error)s.", error=ex.orig), category="error")
+                flash(_("Oops! Database Error: %(error)s.", error=ex.orig), category="error")
             except Exception as ex:
                 ub.session.rollback()
                 log.error_or_exception(ex)
-                flash(_(u"There was an error"), category="error")
+                flash(_("There was an error"), category="error")
     return render_title_template('shelf_edit.html',
                                  shelf=shelf,
                                  title=page_title,
@@ -366,7 +377,7 @@ def check_shelf_is_unique(title, is_public, shelf_id=False):
 
         if not is_shelf_name_unique:
             log.error("A public shelf with the name '{}' already exists.".format(title))
-            flash(_(u"A public shelf with the name '%(title)s' already exists.", title=title),
+            flash(_("A public shelf with the name '%(title)s' already exists.", title=title),
                   category="error")
     else:
         is_shelf_name_unique = ub.session.query(ub.Shelf) \
@@ -377,7 +388,7 @@ def check_shelf_is_unique(title, is_public, shelf_id=False):
 
         if not is_shelf_name_unique:
             log.error("A private shelf with the name '{}' already exists.".format(title))
-            flash(_(u"A private shelf with the name '%(title)s' already exists.", title=title),
+            flash(_("A private shelf with the name '%(title)s' already exists.", title=title),
                   category="error")
     return is_shelf_name_unique
 
@@ -454,14 +465,14 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param):
             except (OperationalError, InvalidRequestError) as e:
                 ub.session.rollback()
                 log.error_or_exception("Settings Database error: {}".format(e))
-                flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+                flash(_("Oops! Database Error: %(error)s.", error=e.orig), category="error")
 
         return render_title_template(page,
                                      entries=result,
                                      pagination=pagination,
-                                     title=_(u"Shelf: '%(name)s'", name=shelf.name),
+                                     title=_("Shelf: '%(name)s'", name=shelf.name),
                                      shelf=shelf,
                                      page="shelf")
     else:
-        flash(_(u"Error opening shelf. Shelf does not exist or is not accessible"), category="error")
+        flash(_("Error opening shelf. Shelf does not exist or is not accessible"), category="error")
         return redirect(url_for("web.index"))

cps/static/css/caliBlur.css

@@ -3290,10 +3290,13 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropd
     -ms-transform-origin: center top;
     transform-origin: center top;
     border: 0;
-    left: 0 !important;
     overflow-y: auto;
 }
+.dropdown-menu:not(.datepicker-dropdown):not(.profileDropli) {
+    left: 0 !important;
+}
 #add-to-shelves {
+  min-height: 48px;
   max-height: calc(100% - 120px);
   overflow-y: auto;
 }
@@ -4423,38 +4426,6 @@ body.advanced_search > div.container-fluid > div.row-fluid > div.col-sm-10 > div
         left: 49px;
         margin-top: 5px
     }
-
-    body:not(.blur) > .navbar > .container-fluid > .navbar-header:after, body:not(.blur) > .navbar > .container-fluid > .navbar-header:before {
-        color: hsla(0, 0%, 100%, .7);
-        cursor: pointer;
-        display: block;
-        font-family: plex-icons-new, serif;
-        font-size: 20px;
-        font-stretch: 100%;
-        font-style: normal;
-        font-variant-caps: normal;
-        font-variant-east-asian: normal;
-        font-variant-numeric: normal;
-        font-weight: 400;
-        height: 60px;
-        letter-spacing: normal;
-        line-height: 60px;
-        position: absolute
-    }
-
-    body:not(.blur) > .navbar > .container-fluid > .navbar-header:before {
-        content: "\EA30";
-        -webkit-font-variant-ligatures: normal;
-        font-variant-ligatures: normal;
-        left: 20px
-    }
-
-    body:not(.blur) > .navbar > .container-fluid > .navbar-header:after {
-        content: "\EA2F";
-        -webkit-font-variant-ligatures: normal;
-        font-variant-ligatures: normal;
-        left: 60px
-    }
 }
 
 body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > div.row:first-of-type > div.col > h2:before, body.admin > div.container-fluid > div > div.col-sm-10 > div.discover > h2:first-of-type:before, body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > h1:before, body.newuser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.discover > h1:before {
@@ -4842,8 +4813,14 @@ body.advsearch:not(.blur) > div.container-fluid > div.row-fluid > div.col-sm-10
     z-index: 999999999999999999999999999999999999
 }
 
-.search #shelf-actions, body.login .home-btn {
-    display: none
+body.search #shelf-actions button#add-to-shelf {
+  height: 40px;
+}
+@media screen and (max-width: 767px) {
+  body.search .discover, body.advsearch .discover {
+    display: flex;
+    flex-direction: column;
+  }
 }
 
 body.read:not(.blur) a[href*=readbooks] {
@@ -5164,7 +5141,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head
     right: 5px
 }
 
-#shelf-actions > .btn-group.open, .downloadBtn.open, .profileDrop[aria-expanded=true] {
+body:not(.search) #shelf-actions > .btn-group.open, .downloadBtn.open, .profileDrop[aria-expanded=true] {
     pointer-events: none
 }
 
@@ -5181,7 +5158,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head
     color: var(--color-primary)
 }
 
-#shelf-actions, #shelf-actions > .btn-group, #shelf-actions > .btn-group > .empty-ul {
+body:not(.search) #shelf-actions, body:not(.search) #shelf-actions > .btn-group, body:not(.search) #shelf-actions > .btn-group > .empty-ul {
     pointer-events: none
 }
 
@@ -7309,6 +7286,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         float: right
     }
 
+    body.blur #main-nav + #scnd-nav .create-shelf, body.blur #main-nav + .col-sm-2 #scnd-nav .create-shelf {
+      float: none;
+      margin: 5px 0 10px -10px;
+    }
+
     #main-nav + #scnd-nav .nav-head.hidden-xs {
         display: list-item !important;
         width: 225px

cps/static/css/caliBlur_override.css

@@ -22,3 +22,7 @@ body.serieslist.grid-view div.container-fluid > div > div.col-sm-10::before {
   padding: 0 0;
   line-height: 15px;
 }
+
+input.datepicker {color: transparent}
+input.datepicker:focus {color: transparent}
+input.datepicker:focus + input {color: #555}

cps/static/css/epub_themes.css

@@ -0,0 +1,19 @@
+.lightTheme {
+	background: #fff;
+	color: #000;
+}
+
+.darkTheme {
+	background: #202124;
+	color: #fff
+}
+
+.sepiaTheme {
+	background: #ece1ca;
+	color: #000;
+}
+
+.blackTheme {
+	background: #000;
+	color: #fff
+}

cps/static/css/kthoom.css

@@ -149,6 +149,20 @@ body {
   word-wrap: break-word;
 }
 
+#mainContent > canvas {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.long-strip > .mainImage {
+  margin-bottom: 4px;
+}
+
+.long-strip > .mainImage:last-child {
+  margin-bottom: 0px !important;
+}
+
 #titlebar {
   min-height: 25px;
   height: auto;

cps/static/css/libs/images/findbarButton-next-dark.svg

@@ -1,6 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M8 12a1 1 0 0 1-.707-.293l-5-5a1 1 0 0 1 1.414-1.414L8
-9.586l4.293-4.293a1 1 0 0 1 1.414 1.414l-5 5A1 1 0 0 1 8 12z"></path></svg>

cps/static/css/libs/images/findbarButton-previous-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M13 11a1 1 0 0 1-.707-.293L8 6.414l-4.293 4.293a1 1 0 0 1-1.414-1.414l5-5a1 1 0 0 1 1.414 0l5 5A1 1 0 0 1 13 11z"></path></svg>

cps/static/css/libs/images/secondaryToolbarButton-documentProperties-dark.svg

@@ -1,16 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16
-16"
-fill="rgba(255,255,255,1)">
-<path 
-   d="M8 16a8 8 0 1 1 8-8 8.009 8.009 0 0 1-8 8zM8 2a6 6 0 1 0 6 6 6.006 6.006 0 0 0-6-6z">
-</path>
-<path 
-   d="M8 7a1 1 0 0 0-1 1v3a1 1 0 0 0 2 0V8a1 1 0 0 0-1-1z">
-</path>
-<circle
-   cx="8" cy="5" r="1.188">
-</circle>
-</svg>

cps/static/css/libs/images/secondaryToolbarButton-firstPage-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M13 13c-.3 0-.5-.1-.7-.3L8 8.4l-4.3 4.3c-.9.9-2.3-.5-1.4-1.4l5-5c.4-.4 1-.4 1.4 0l5 5c.6.6.2 1.7-.7 1.7zm0-11H3C1.7 2 1.7 4 3 4h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-handTool-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M15 3.7V13c0 1.5-1.53 3-3 3H7.13c-.72 0-1.63-.5-2.13-1l-5-5s.84-1 .87-1c.13-.1.33-.2.53-.2.1 0 .3.1.4.2L4 10.6V2.7c0-.6.4-1 1-1s1 .4 1 1v4.6h1V1c0-.6.4-1 1-1s1 .4 1 1v6.3h1V1.7c0-.6.4-1 1-1s1 .4 1 1v5.7h1V3.7c0-.6.4-1 1-1s1 .4 1 1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-lastPage-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M8 10c-.3 0-.5-.1-.7-.3l-5-5c-.9-.9.5-2.3 1.4-1.4L8 7.6l4.3-4.3c.9-.9 2.3.5 1.4 1.4l-5 5c-.2.2-.4.3-.7.3zm5 2H3c-1.3 0-1.3 2 0 2h10c1.3 0 1.3-2 0-2z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCcw-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M1 1a1 1 0 011 1v2.4A7 7 0 118 15a7 7 0 01-4.9-2 1 1 0 011.4-1.5 5 5 0 10-1-5.5H6a1 1 0 010 2H1a1 1 0 01-1-1V2a1 1 0 011-1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-rotateCw-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M15 1a1 1 0 0 0-1 1v2.418A6.995 6.995 0 1 0 8 15a6.954 6.954 0 0 0 4.95-2.05 1 1 0 0 0-1.414-1.414A5.019 5.019 0 1 1 12.549 6H10a1 1 0 0 0 0 2h5a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1z"></path></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollHorizontal-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M0 4h1.5c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5H0zM9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM16 4h-1.5c-1 0-1.5.5-1.5 1.5v5c0 1 .5 1.5 1.5 1.5H16z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollPage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="M9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollVertical-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M9.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C5 4.5 5.5 4 6.5 4zM11 0v.5c0 1-.5 1.5-1.5 1.5h-3C5.5 2 5 1.5 5 .5V0h6zM11 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-scrollWrapped-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M5.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5C1 4.5 1.5 4 2.5 4zM7 0v.5C7 1.5 6.5 2 5.5 2h-3C1.5 2 1 1.5 1 .5V0h6zM7 16v-.5c0-1-.5-1.5-1.5-1.5h-3c-1 0-1.5.5-1.5 1.5v.5h6zM13.5 4c1 0 1.5.5 1.5 1.5v5c0 1-.5 1.5-1.5 1.5h-3c-1 0-1.5-.5-1.5-1.5v-5c0-1 .5-1.5 1.5-1.5zM15 0v.5c0 1-.5 1.5-1.5 1.5h-3C9.5 2 9 1.5 9 .5V0h6zM15 16v-.507c0-1-.5-1.5-1.5-1.5h-3C9.5 14 9 14.5 9 15.5v.5h6z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-selectTool-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M12.408 8.217l-8.083-6.7A.2.2 0 0 0 4 1.672V12.3a.2.2 0 0 0 .333.146l2.56-2.372 1.857 3.9A1.125 1.125 0 1 0 10.782 13L8.913 9.075l3.4-.51a.2.2 0 0 0 .095-.348z"></path></svg>

cps/static/css/libs/images/secondaryToolbarButton-spreadEven-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M1.5 3.5C.5 3.5 0 4 0 5v6.5c0 1 .5 1.5 1.5 1.5h4c1 0 1.5-.5 1.5-1.5V5c0-1-.5-1.5-1.5-1.5zm2 1.2c.8 0 1.4.2 1.8.6.5.4.7 1 .7 1.7 0 .5-.2 1-.5 1.4-.2.3-.5.7-1 1l-.6.4c-.4.3-.6.4-.75.56-.15.14-.25.24-.35.44H6v1.3H1c0-.6.1-1.1.3-1.5.3-.6.7-1 1.5-1.6.7-.4 1.1-.8 1.28-1 .32-.3.42-.6.42-1 0-.3-.1-.6-.23-.8-.17-.2-.37-.3-.77-.3s-.7.1-.9.5c-.04.2-.1.5-.1.9H1.1c0-.6.1-1.1.3-1.5.4-.7 1.1-1.1 2.1-1.1zM10.54 3.54C9.5 3.54 9 4 9 5v6.5c0 1 .5 1.5 1.54 1.5h4c.96 0 1.46-.5 1.46-1.5V5c0-1-.5-1.46-1.5-1.46zm1.9.95c.7 0 1.3.2 1.7.5.4.4.6.8.6 1.4 0 .4-.1.8-.4 1.1-.2.2-.3.3-.5.4.1 0 .3.1.6.3.4.3.5.8.5 1.4 0 .6-.2 1.2-.6 1.6-.4.5-1.1.7-1.9.7-1 0-1.8-.3-2.2-1-.14-.29-.24-.69-.24-1.29h1.4c0 .3 0 .5.1.7.2.4.5.5 1 .5.3 0 .5-.1.7-.3.2-.2.3-.5.3-.8 0-.5-.2-.8-.6-.95-.2-.05-.5-.15-1-.15v-1c.5 0 .8-.1 1-.14.3-.1.5-.4.5-.9 0-.3-.1-.5-.2-.7-.2-.2-.4-.3-.7-.3-.3 0-.6.1-.75.3-.2.2-.2.5-.2.86h-1.34c0-.4.1-.7.19-1.1 0-.12.2-.32.4-.62.2-.2.4-.3.7-.4.3-.1.6-.1 1-.1z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-spreadNone-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M6 3c-1 0-1.5.5-1.5 1.5v7c0 1 .5 1.5 1.5 1.5h4c1 0 1.5-.5 1.5-1.5v-7c0-1-.5-1.5-1.5-1.5z"/></svg>

cps/static/css/libs/images/secondaryToolbarButton-spreadOdd-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M10.56 3.5C9.56 3.5 9 4 9 5v6.5c0 1 .5 1.5 1.5 1.5h4c1 0 1.5-.5 1.5-1.5V5c0-1-.5-1.5-1.5-1.5zm1.93 1.2c.8 0 1.4.2 1.8.64.5.4.7 1 .7 1.7 0 .5-.2 1-.5 1.44-.2.3-.6.6-1 .93l-.6.4c-.4.3-.6.4-.7.55-.1.1-.2.2-.3.4h3.2v1.27h-5c0-.5.1-1 .3-1.43.2-.49.7-1 1.5-1.54.7-.5 1.1-.8 1.3-1.02.3-.3.4-.7.4-1.05 0-.3-.1-.6-.3-.77-.2-.2-.4-.3-.7-.3-.4 0-.7.2-.9.5-.1.2-.1.5-.2.9h-1.4c0-.6.2-1.1.3-1.5.4-.7 1.1-1.1 2-1.1zM1.54 3.5C.54 3.5 0 4 0 5v6.5c0 1 .5 1.5 1.54 1.5h4c1 0 1.5-.5 1.5-1.5V5c0-1-.5-1.5-1.5-1.5zm1.8 1.125H4.5V12H3V6.9H1.3v-1c.5 0 .8 0 .97-.03.33-.07.53-.17.73-.37.1-.2.2-.3.25-.5.05-.2.05-.3.05-.3z"/></svg>

cps/static/css/libs/images/toolbarButton-bookmark-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M4 16V2s0-1 1-1h6s1 0 1 1v14l-4-5z"/></svg>

cps/static/css/libs/images/toolbarButton-currentOutlineItem.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"><path d="m14 9h-6c-1.3 0-1.3 2 0 2h6c1.3 0 1.3-2 0-2zm-5.2-8h-3.8c-1.3 0-1.3 2 0 2h1.7zm-6.8 0c-1 0-1.3 1-0.7 1.7 0.7 0.6 1.7 0.3 1.7-0.7 0-0.5-0.4-1-1-1zm3 8c-1 0-1.3 1-0.7 1.7 0.6 0.6 1.7 0.2 1.7-0.7 0-0.5-0.4-1-1-1zm0.3-4h-0.3c-1.4 0-1.4 2 0 2h2.3zm-3.3 0c-0.9 0-1.4 1-0.7 1.7 0.7 0.6 1.7 0.2 1.7-0.7 0-0.6-0.5-1-1-1zm12 8h-9c-1.3 0-1.3 2 0 2h9c1.3 0 1.3-2 0-2zm-12 0c-1 0-1.3 1-0.7 1.7 0.7 0.6 1.7 0.2 1.7-0.712 0-0.5-0.4-1-1-1z"/><path d="m7.37 4.838 3.93-3.911v2.138h3.629v3.546h-3.629v2.138l-3.93-3.911"/></svg>

cps/static/css/libs/images/toolbarButton-download-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M14 3h-2v2h2v8H2V5h7V3h-.849L6.584 1.538A2 2 0 0 0 5.219 1H2a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2zM2 3h3.219l1.072 1H2z"></path><path d="M8.146 6.146a.5.5 0 0 0 0 .707l2 2a.5.5 0 0 0 .707 0l2-2a.5.5 0 1 0-.707-.707L11 7.293V.5a.5.5 0 0 0-1 0v6.793L8.854 6.146a.5.5 0 0 0-.708 0z"></path></svg>

cps/static/css/libs/images/toolbarButton-editorFreeText.svg

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- copied from https://www.svgrepo.com/svg/255881/text -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 16 16" style="enable-background:new 0 0 16 16;" xml:space="preserve">
+<g>
+	<g transform="scale(0.03125)">
+		<path d="M405.787,43.574H8.17c-4.513,0-8.17,3.658-8.17,8.17v119.83c0,4.512,3.657,8.17,8.17,8.17h32.681
+			c4.513,0,8.17-3.658,8.17-8.17v-24.511h95.319v119.83c0,4.512,3.657,8.17,8.17,8.17c4.513,0,8.17-3.658,8.17-8.17v-128
+			c0-4.512-3.657-8.17-8.17-8.17H40.851c-4.513,0-8.17,3.658-8.17,8.17v24.511H16.34V59.915h381.277v103.489h-16.34v-24.511
+			c0-4.512-3.657-8.17-8.17-8.17h-111.66c-4.513,0-8.17,3.658-8.17,8.17v288.681c0,4.512,3.657,8.17,8.17,8.17h57.191v16.34H95.319
+			v-16.34h57.191c4.513,0,8.17-3.658,8.17-8.17v-128c0-4.512-3.657-8.17-8.17-8.17c-4.513,0-8.17,3.658-8.17,8.17v119.83H87.149
+			c-4.513,0-8.17,3.658-8.17,8.17v32.681c0,4.512,3.657,8.17,8.17,8.17h239.66c4.513,0,8.17-3.658,8.17-8.17v-32.681
+			c0-4.512-3.657-8.17-8.17-8.17h-57.192v-272.34h95.319v24.511c0,4.512,3.657,8.17,8.17,8.17h32.681c4.513,0,8.17-3.658,8.17-8.17
+			V51.745C413.957,47.233,410.3,43.574,405.787,43.574z"/>
+	</g>
+</g>
+<g>
+	<g transform="scale(0.03125)">
+		<path d="M503.83,452.085h-24.511V59.915h24.511c4.513,0,8.17-3.658,8.17-8.17s-3.657-8.17-8.17-8.17h-65.362
+			c-4.513,0-8.17,3.658-8.17,8.17s3.657,8.17,8.17,8.17h24.511v392.17h-24.511c-4.513,0-8.17,3.658-8.17,8.17s3.657,8.17,8.17,8.17
+			h65.362c4.513,0,8.17-3.658,8.17-8.17S508.343,452.085,503.83,452.085z"/>
+	</g>
+</g>
+</svg>

cps/static/css/libs/images/toolbarButton-editorInk.svg

@@ -0,0 +1,9 @@
+<?xml version='1.0' encoding='utf-8'?>
+<!DOCTYPE svg PUBLIC '-//W3C//DTD SVG 1.1//EN' 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd'>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16" xmlns:xlink="http://www.w3.org/1999/xlink" enable-background="new 0 0 16 16">
+  <g>
+    <g transform="scale(0.03125)">
+      <path d="m455.1,137.9l-32.4,32.4-81-81.1 32.4-32.4c6.6-6.6 18.1-6.6 24.7,0l56.3,56.4c6.8,6.8 6.8,17.9 0,24.7zm-270.7,271l-81-81.1 209.4-209.7 81,81.1-209.4,209.7zm-99.7-42l60.6,60.7-84.4,23.8 23.8-84.5zm399.3-282.6l-56.3-56.4c-11-11-50.7-31.8-82.4,0l-285.3,285.5c-2.5,2.5-4.3,5.5-5.2,8.9l-43,153.1c-2,7.1 0.1,14.7 5.2,20 5.2,5.3 15.6,6.2 20,5.2l153-43.1c3.4-0.9 6.4-2.7 8.9-5.2l285.1-285.5c22.7-22.7 22.7-59.7 0-82.5z"/>
+    </g>
+  </g>
+</svg>

cps/static/css/libs/images/toolbarButton-menuArrow-dark.svg

@@ -1 +0,0 @@
-<svg width="16" height="16" xmlns="http://www.w3.org/2000/svg" fill="rgba(255,255,255,1)"><path d="M8 11a1 1 0 01-.707-.293l-2.99-2.99c-.91-.942.471-2.324 1.414-1.414L8 8.586l2.283-2.283c.943-.91 2.324.472 1.414 1.414l-2.99 2.99A1 1 0 018 11z"/></svg>

cps/static/css/libs/images/toolbarButton-openFile-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M14.859 3.2a1.335 1.335 0 0 1-1.217.8H13v1h1v8H2V5h8V4h-.642a1.365 1.365 0 0 1-1.325-1.11L6.584 1.538A2 2 0 0 0 5.219 1H2a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V5a2 2 0 0 0-1.141-1.8zM2 3h3.219l1.072 1H2zm7.854-.146L11 1.707V8.5a.5.5 0 0 0 1 0V1.707l1.146 1.146a.5.5 0 1 0 .707-.707l-2-2a.5.5 0 0 0-.707 0l-2 2a.5.5 0 0 0 .707.707z"></path></svg>

cps/static/css/libs/images/toolbarButton-pageDown-dark.svg

@@ -1,8 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16
-16"
-fill="rgba(255,255,255,1)"><path transform='rotate(90) translate(0, -16)'
-d="M15.707 7.293l-6-6a1 1 0 0 0-1.414 1.414L12.586 7H1a1 1 0 0 0 0 2h11.586l-4.293
-4.293a1 1 0 1 0 1.414 1.414l6-6a1 1 0 0 0 0-1.414z"></path></svg>

cps/static/css/libs/images/toolbarButton-pageUp-dark.svg

@@ -1,13 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16
-16"
-fill="rgba(255,255,255,1)">
-<path 
-   transform='rotate(90) translate(0, -16)'
-   d="M15 7H3.414l4.293-4.293a1 1 0 0
-0-1.414-1.414l-6 6a1 1 0 0 0 0 1.414l6 6a1 1 0 0 0 1.414-1.414L3.414 9H15a1 1 0 0
-0 0-2z">
-</path>
-</svg>

cps/static/css/libs/images/toolbarButton-presentationMode-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M.5 1H7s0-1 1-1 1 1 1 1h6.5s.5 0 .5.5-.5.5-.5.5H.5S0 2 0 1.5.5 1 .5 1zM1 3h14v7c0 2-1 2-2 2H3c-1 0-2 0-2-2zm5 1v7l6-3.5zM3.72 15.33l.53-2s0-.5.65-.35c.51.13.38.63.38.63l-.53 2s0 .5-.64.35c-.53-.13-.39-.63-.39-.63zM11.24 15.61l-.53-1.99s0-.5.38-.63c.51-.13.64.35.64.35l.53 2s0 .5-.38.63c-.5.13-.64-.35-.65-.35z"/></svg>

cps/static/css/libs/images/toolbarButton-print-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M14 5h-1V1a1 1 0 0 0-1-1H4a1 1 0 0 0-1 1v4H2a2 2 0 0 0-2 2v5h3v3a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1v-3h3V7a2 2 0 0 0-2-2zM2.5 8a.5.5 0 1 1 .5-.5.5.5 0 0 1-.5.5zm9.5 7H4v-5h8zm0-10H4V1h8zm-6.5 7h4a.5.5 0 0 0 0-1h-4a.5.5 0 1 0 0 1zm0 2h5a.5.5 0 0 0 0-1h-5a.5.5 0 1 0 0 1z"></path></svg>

cps/static/css/libs/images/toolbarButton-search-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M15.707 14.293l-4.822-4.822a6.019 6.019 0 1 0-1.414 1.414l4.822 4.822a1 1 0 0 0 1.414-1.414zM6 10a4 4 0 1 1 4-4 4 4 0 0 1-4 4z"></path></svg>

cps/static/css/libs/images/toolbarButton-secondaryToolbarToggle-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M8.707 7.293l-5-5a1 1 0 0 0-1.414 1.414L6.586 8l-4.293 4.293a1 1 0 1 0 1.414 1.414l5-5a1 1 0 0 0 0-1.414zm6 0l-5-5a1 1 0 0 0-1.414 1.414L12.586 8l-4.293 4.293a1 1 0 1 0 1.414 1.414l5-5a1 1 0 0 0 0-1.414z"></path></svg>

cps/static/css/libs/images/toolbarButton-sidebarToggle-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M3 1h10a3.008 3.008 0 0 1 3 3v8a3.009 3.009 0 0 1-3 3H3a3.005 3.005 0 0 1-3-3V4a3.013 3.013 0 0 1 3-3zm11 11V4a1 1 0 0 0-1-1H8v10h5a1 1 0 0 0 1-1zM2 12a1 1 0 0 0 1 1h4V3H3a1 1 0 0 0-1 1v8z"></path><path d="M3.5 5h2a.5.5 0 0 0 0-1h-2a.5.5 0 0 0 0 1zm0 2h2a.5.5 0 0 0 0-1h-2a.5.5 0 0 0 0 1zm1 2h1a.5.5 0 0 0 0-1h-1a.5.5 0 0 0 0 1z"></path></svg>

cps/static/css/libs/images/toolbarButton-viewAttachments-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M6.2 2s.5-.5 1.06 0c.5.5 0 1 0 1l-4.6 4.61s-2.5 2.5 0 5 5 0 5 0L13.8 6.4s1.6-1.6 0-3.2-3.2 0-3.2 0L5.8 8s-.7.7 0 1.4 1.4 0 1.4 0l3.9-3.9s.6-.5 1 0c.5.5 0 1 0 1l-3.8 4s-1.8 1.8-3.5 0C3 8.7 4.8 7 4.8 7l4.7-4.9s2.7-2.6 5.3 0c2.6 2.6 0 5.3 0 5.3l-6.2 6.3s-3.5 3.5-7 0 0-7 0-7z"/></svg>

cps/static/css/libs/images/toolbarButton-viewLayers-dark.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 4.233 4.233" height="16" width="16" fill="rgba(255,255,255,1)"><path d="M.15 2.992c-.198.1-.2.266-.002.365l1.604.802a.93.93 0 00.729-.001l1.602-.801c.198-.1.197-.264 0-.364l-.695-.348c-1.306.595-2.542 0-2.542 0m-.264.53l.658-.329c.6.252 1.238.244 1.754 0l.659.329-1.536.768zM.15 1.935c-.198.1-.198.265 0 .364l1.604.802a.926.926 0 00.727 0l1.603-.802c.198-.099.198-.264 0-.363l-.694-.35c-1.14.56-2.546.001-2.546.001m-.264.53l.664-.332c.52.266 1.261.235 1.75.002l.659.33-1.537.768zM.15.877c-.198.099-.198.264 0 .363l1.604.802a.926.926 0 00.727 0l1.603-.802c.198-.099.198-.264 0-.363L2.481.075a.926.926 0 00-.727 0zm.43.182L2.117.29l1.538.769-1.538.768z"/></svg>

cps/static/css/libs/images/toolbarButton-viewOutline-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" height="16" width="16"
-fill="rgba(255,255,255,1)"><path d="M14 9H8c-1.3 0-1.3 2 0 2h6c1.3 0 1.3-2 0-2zm0-8H5C3.7 1 3.7 3 5 3h9c1.3 0 1.3-2 0-2zM2 1C1 1 .7 2 1.3 2.7 2 3.3 3 3 3 2c0-.5-.4-1-1-1zm3 8c-1 0-1.3 1-.7 1.7.6.6 1.7.2 1.7-.7 0-.5-.4-1-1-1zM14 5H5C3.6 5 3.6 7 5 7h9c1.3 0 1.3-2 0-2zM2 5c-.9 0-1.4 1-.7 1.7C2 7.3 3 6.9 3 6c0-.6-.5-1-1-1zM14 13H5c-1.3 0-1.3 2 0 2h9c1.3 0 1.3-2 0-2zM2 13c-1 0-1.3 1-.7 1.7.7.6 1.7.2 1.7-.712 0-.5-.4-1-1-1z"/></svg>

cps/static/css/libs/images/toolbarButton-viewThumbnail-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><g style="--darkreader-inline-fill:rgba(81, 82, 83, 0.8);" data-darkreader-inline-fill=""><rect x="1" y="1" width="6" height="6" rx="1" ry="1"></rect><rect x="9" y="1" width="6" height="6" rx="1" ry="1"></rect><rect x="1" y="9" width="6" height="6" rx="1" ry="1"></rect><rect x="9" y="9" width="6" height="6" rx="1" ry="1"></rect></g></svg>

cps/static/css/libs/images/toolbarButton-zoomIn-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><path d="M14 7H9V2a1 1 0 0 0-2 0v5H2a1 1 0 0 0 0 2h5v5a1 1 0 0 0 2 0V9h5a1 1 0 0 0 0-2z"></path></svg>

cps/static/css/libs/images/toolbarButton-zoomOut-dark.svg

@@ -1,5 +0,0 @@
-<!-- This Source Code Form is subject to the terms of the Mozilla Public
-   - License, v. 2.0. If a copy of the MPL was not distributed with this
-   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16"
-fill="rgba(255,255,255,1)"><rect x="2" y="7" width="12" height="2" rx="1"></rect></svg>

cps/static/css/libs/images/treeitem-collapsed-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M13 9L6 5v8z"/></svg>

cps/static/css/libs/images/treeitem-expanded-dark.svg

@@ -1,2 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"
-fill="rgba(255,255,255,1)"><path d="M10 13l4-7H6z"/></svg>

cps/static/css/main.css

@@ -567,6 +567,36 @@ input:-moz-placeholder { color: #454545; }
   font-size: 0.8em;
 }
 
+.md-content .themes button {
+  display: inline-block;
+  border: none;
+  text-align: center;
+  text-decoration: none;
+  margin-top: 5%;
+  margin-right: 1%;
+  font-size: 16px;
+}
+
+.md-content .themes button.darkTheme {
+  background-color: #202124;
+  color: white;
+}
+
+.md-content .themes button.whiteTheme {
+  background-color: white;
+  color: black;
+}
+
+.md-content .themes button.sepiaTheme {
+  background-color: #ece1ca;
+  color: black;
+}
+
+.md-content .themes button.blackTheme {
+  background-color: black;
+  color: white;
+}
+
 /* Effect 1: Fade in and scale up */
 .md-effect-1 .md-content {
   -webkit-transform: scale(0.7);

cps/static/css/reader.css

@@ -0,0 +1,29 @@
+.fontSizeWrapper {
+    position: relative;
+}
+.slider {
+    position: absolute;
+    top: 50%;
+    transform: translate(0,-50%);
+    width: 90%;
+    height: 60px;
+    background: transparent;
+    border-radius: 20px;
+    display: flex;
+    align-items: center;
+    box-shadow: 0px 15px 40px #7E6D5766;
+}
+.slider label {
+    font-size: 20px;
+    font-weight: 400;
+    font-family: Open Sans;
+    padding-right: 10px;
+    color: white;
+}
+.slider input[type="range"] {
+    width: 80%;
+    height: 5px;
+    background: black;
+    border: none;
+    outline: none;
+}

cps/static/css/style.css

@@ -140,6 +140,7 @@ table .bg-dark-danger a { color: #fff; }
 
 .container-fluid .book {
   margin-top: 20px;
+  max-width: 180px;
   display: flex;
   flex-direction: column;
 }
@@ -433,3 +434,7 @@ div.log {
 #detailcover:-moz-full-screen { cursor:zoom-out; border: 0; }
 #detailcover:-ms-fullscreen { cursor:zoom-out; border: 0; }
 #detailcover:fullscreen { cursor:zoom-out; border: 0; }
+
+.error-list {
+    margin-top: 5px;
+  }

cps/static/js/caliBlur.js

@@ -16,7 +16,6 @@
  */
 // Move advanced search to side-menu
 $("a[href*='advanced']").parent().insertAfter("#nav_new");
-$("body").addClass("blur");
 $("body.stat").addClass("stats");
 $("body.config").addClass("admin");
 $("body.uiconfig").addClass("admin");
@@ -29,8 +28,8 @@ $("body > div.container-fluid > div > div.col-sm-10 > div.filterheader").attr("s
 // Back button
 curHref = window.location.href.split("/");
 prevHref = document.referrer.split("/");
-$(".navbar-form.navbar-left")
-    .before('<div class="plexBack"><a href="' + encodeURI(document.referrer) + '"></a></div>');
+$(".plexBack a").attr('href', encodeURI(document.referrer));
+
 if (history.length === 1 ||
     curHref[0] +
     curHref[1] +
@@ -44,14 +43,9 @@ if (history.length === 1 ||
 
 //Weird missing a after pressing back from edit.
 setTimeout(function () {
-    if ($(".plexBack a").length < 1) {
-        $(".plexBack").append('<a href="' + encodeURI(document.referrer) + '"></a>');
-    }
+    $(".plexBack a").attr('href', encodeURI(document.referrer));
 }, 10);
 
-// Home button
-$(".plexBack").before('<div class="home-btn"></div>');
-$("a.navbar-brand").clone().appendTo(".home-btn").empty().removeClass("navbar-brand");
 /////////////////////////////////
 // Start of Book Details Work //
 ///////////////////////////////
@@ -201,7 +195,7 @@ if ($("body.book").length > 0) {
 
     // Move dropdown lists higher in dom, replace bootstrap toggle with own toggle.
     $('ul[aria-labelledby="read-in-browser"]').insertBefore(".blur-wrapper").addClass("readinbrowser-drop");
-    $('ul[aria-labelledby="send-to-kindle"]').insertBefore(".blur-wrapper").addClass("sendtokindle-drop");
+    $('ul[aria-labelledby="send-to-kereader"]').insertBefore(".blur-wrapper").addClass("sendtoereader-drop");
     $(".leramslist").insertBefore(".blur-wrapper");
     $('ul[aria-labelledby="btnGroupDrop1"]').insertBefore(".blur-wrapper").addClass("leramslist");
     $("#add-to-shelves").insertBefore(".blur-wrapper");
@@ -215,7 +209,7 @@ if ($("body.book").length > 0) {
     });
 
     $("#sendbtn2").click(function () {
-        $(".sendtokindle-drop").toggle();
+        $(".sendtoereader-drop").toggle();
     });
 
 
@@ -242,12 +236,12 @@ if ($("body.book").length > 0) {
 
         if ($("#sendbtn2").length > 0) {
             position = $("#sendbtn2").offset().left
-            if (position + $(".sendtokindle-drop").width() > $(window).width()) {
-                positionOff = position + $(".sendtokindle-drop").width() - $(window).width();
+            if (position + $(".sendtoereader-drop").width() > $(window).width()) {
+                positionOff = position + $(".sendtoereader-drop").width() - $(window).width();
                 ribPosition = position - positionOff - 5
-                $(".sendtokindle-drop").attr("style", "left: " + ribPosition + "px !important; right: auto; top: " + topPos + "px");
+                $(".sendtoereader-drop").attr("style", "left: " + ribPosition + "px !important; right: auto; top: " + topPos + "px");
             } else {
-                $(".sendtokindle-drop").attr("style", "left: " + position + "px !important; right: auto; top: " + topPos + "px");
+                $(".sendtoereader-drop").attr("style", "left: " + position + "px !important; right: auto; top: " + topPos + "px");
             }
         }
 
@@ -300,7 +294,7 @@ if ($("body.book").length > 0) {
 $(document).mouseup(function (e) {
     var container = new Array();
     container.push($('ul[aria-labelledby="read-in-browser"]'));
-    container.push($(".sendtokindle-drop"));
+    container.push($(".sendtoereader-drop"));
     container.push($(".leramslist"));
     container.push($("#add-to-shelves"));
     container.push($(".navbar-collapse.collapse.in"));
@@ -320,19 +314,11 @@ $(document).mouseup(function (e) {
     });
 });
 
-// Split path name to array and remove blanks
-url = window.location.pathname
-
 // Move create shelf
 $("#nav_createshelf").prependTo(".your-shelves");
 
-// Create drop-down for profile and move elements to it
-$("#main-nav")
-    .prepend('<li class="dropdown"><a href="#" class="dropdown-toggle profileDrop" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><span class="glyphicon glyphicon-user"></span></a><ul class="dropdown-menu profileDropli"></ul></li>');
-$("#top_user").parent().addClass("dropdown").appendTo(".profileDropli");
-$("#nav_about").addClass("dropdown").appendTo(".profileDropli");
-$("#register").parent().addClass("dropdown").appendTo(".profileDropli");
-$("#logout").parent().addClass("dropdown").appendTo(".profileDropli");
+// Move About link it the profile dropdown
+$(".profileDropli #top_user").parent().after($("#nav_about").addClass("dropdown"))
 
 // Remove the modals except from some areas where they are needed
 bodyClass = $("body").attr("class").split(" ");
@@ -371,31 +357,6 @@ $(document).on("click", ".dropdown-toggle", function () {
     });
 });
 
-// Fade out content on page unload
-// delegate all clicks on "a" tag (links)
-/*$(document).on("click", "a:not(.btn-toolbar a, a[href*='shelf/remove'], .identifiers a, .bookinfo , .btn-group > a, #add-to-shelves a, #book-list a, .stat.blur a )", function () {
-
-    // get the href attribute
-    var newUrl = $(this).attr("href");
-
-    // veryfy if the new url exists or is a hash
-    if (!newUrl || newUrl[0] === "#") {
-        // set that hash
-        location.hash = newUrl;
-        return;
-    }
-
-    now, fadeout the html (whole page)
-      $( '.blur-wrapper' ).fadeOut(250);
-    $(".row-fluid .col-sm-10").fadeOut(500,function () {
-        // when the animation is complete, set the new location
-        location = newUrl;
-    });
-
-    // prevent the default browser behavior.
-    return false;
-});*/
-
 // Collapse long text into read-more
 $("div.comments").readmore({
     collapsedHeight: 134,
@@ -408,6 +369,13 @@ $("div.comments").readmore({
 //     End of Global Work     //
 ///////////////////////////////
 
+// Search Results
+if($("body.search").length > 0) {
+  $('div[aria-label="Add to shelves"]').click(function () {
+    $("#add-to-shelves").toggle();
+  });
+}
+
 // Advanced Search Results
 if($("body.advsearch").length > 0) {
   $("#loader + .container-fluid")
@@ -458,6 +426,8 @@ if ($("body.author").length > 0) {
     }
 }
 
+// Split path name to array and remove blanks
+url = window.location.pathname
 // Ereader Page - add class to iframe body on ereader page after it loads.
 backurl = "../../book/" + url[2]
 $("body.epub #title-controls")
@@ -540,6 +510,7 @@ if ($("body.shelf").length > 0) {
 // Rest of Tooltips
 $(".home-btn > a").attr({
     "data-toggle": "tooltip",
+    "href": $(".navbar-brand")[0].href,
     "title": $(document.body).attr("data-text"),    // Home
     "data-placement": "bottom"
 })
@@ -666,7 +637,7 @@ $("#sendbtn").attr({
 
 $("#sendbtn2").attr({
     "data-toggle-two": "tooltip",
-    "title": $("#sendbtn2").text(),                 // "Send to Kindle",
+    "title": $("#sendbtn2").text(),                 // "Send to eReader",
     "data-placement": "bottom",
     "data-viewport": ".btn-toolbar"
 })

cps/static/js/compress/uncompress.js

@@ -125,7 +125,7 @@ function loadArchiveFormats(formats, cb) {
 				_loaded_archive_formats.push(archive_format);
 				break;
 			case 'zip':
-				loadScript(path + 'jszip.js', checkForLoadDone);
+				loadScript(path + 'jszip.min.js', checkForLoadDone);
 				_loaded_archive_formats.push(archive_format);
 				break;
 			case 'tar':

cps/static/js/details.js

@@ -1,5 +1,5 @@
 /* This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
- *    Copyright (C) 2018 jkrehm
+ *    Copyright (C) 2018-2023 jkrehm, OzzieIsaacs
  *
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -17,6 +17,36 @@
 
 /* global _ */
 
+function handleResponse (data) {
+    $(".row-fluid.text-center").remove();
+    $("#flash_danger").remove();
+    $("#flash_success").remove();
+    if (!jQuery.isEmptyObject(data)) {
+        if($("#bookDetailsModal").is(":visible")) {
+            data.forEach(function (item) {
+                $(".modal-header").after('<div id="flash_' + item.type +
+                    '" class="text-center alert alert-' + item.type + '">' + item.message + '</div>');
+            });
+        } else {
+            data.forEach(function (item) {
+                $(".navbar").after('<div class="row-fluid text-center">' +
+                    '<div id="flash_' + item.type + '" class="alert alert-' + item.type + '">' + item.message + '</div>' +
+                    '</div>');
+            });
+        }
+    }
+}
+$(".sendbtn-form").click(function() {
+    $.ajax({
+        method: 'post',
+        url: $(this).data('href'),
+        data: {csrf_token: $("input[name='csrf_token']").val()},
+        success: function (data) {
+            handleResponse(data)
+        }
+    })
+});
+
 $(function() {
     $("#have_read_form").ajaxForm();
 });

cps/static/js/kthoom.js

@@ -62,6 +62,7 @@ var currentImage = 0;
 var imageFiles = [];
 var imageFilenames = [];
 var totalImages = 0;
+var prevScrollPosition = 0;
 
 var settings = {
     hflip: false,
@@ -70,8 +71,9 @@ var settings = {
     fitMode: kthoom.Key.B,
     theme: "light",
     direction: 0, // 0 = Left to Right, 1 = Right to Left
-	nextPage: 0, // 0 = Reset to Top, 1 = Remember Position
-	scrollbar: 1 // 0 = Hide Scrollbar, 1 = Show Scrollbar
+    nextPage: 0, // 0 = Reset to Top, 1 = Remember Position
+	scrollbar: 1, // 0 = Hide Scrollbar, 1 = Show Scrollbar	
+    pageDisplay: 0 // 0 = Single Page, 1 = Long Strip
 };
 
 kthoom.saveSettings = function() {
@@ -130,8 +132,8 @@ var createURLFromArray = function(array, mimeType) {
     }
 
     if ((typeof URL !== "function" && typeof URL !== "object") ||
-      typeof URL.createObjectURL !== "function") {
-        throw "Browser support for Object URLs is missing";
+        typeof URL.createObjectURL !== "function") {
+            throw "Browser support for Object URLs is missing";
     }
 
     return URL.createObjectURL(blob);
@@ -163,23 +165,52 @@ kthoom.ImageFile = function(file) {
             this.mimeType = undefined;
             break;
     }
+
+    // Reset mime type for special files originating from Apple devices
+    // This folder may contain files having image extensions (for example .jpg) but those files are not actual images
+    // Trying to view these files cause corrupted/empty pages in the comic reader and files should be ignored
+    if (this.filename.indexOf("__MACOSX") !== -1) {
+        this.mimeType = undefined;
+    }
+
     if ( this.mimeType !== undefined) {
         this.dataURI = createURLFromArray(file.fileData, this.mimeType);
     }
 };
 
+function updateDirectionButtons(){
+    var left = 1;
+    var right = 1;
+    if (currentImage <= 0 ) {
+        if (settings.direction === 0) {
+            left = 0;
+        } else {
+            right = 0;
+        }
+    }
+    if ((currentImage + 1) >= Math.max(totalImages, imageFiles.length)) {
+        if (settings.direction === 0) {
+            right = 0;
+        } else {
+            left = 0;
+       }
+    }
+    left === 1 ? $("#left").show() : $("#left").hide();
+    right === 1 ? $("#right").show() : $("#right").hide();
+}
 function initProgressClick() {
     $("#progress").click(function(e) {
         var offset = $(this).offset();
         var x = e.pageX - offset.left;
         var rate = settings.direction === 0 ? x / $(this).width() : 1 - x / $(this).width();
         currentImage = Math.max(1, Math.ceil(rate * totalImages)) - 1;
+        updateDirectionButtons();
+        setBookmark();
         updatePage();
     });
 }
 
 function loadFromArrayBuffer(ab) {
-    var lastCompletion = 0;
     const collator = new Intl.Collator('en', { numeric: true, sensitivity: 'base' });
     loadArchiveFormats(['rar', 'zip', 'tar'], function() {
         // Open the file as an archive
@@ -208,9 +239,14 @@ function loadFromArrayBuffer(ab) {
                                         "</a>" +
                                         "</li>"
                                     );
+                                    
+                                    drawCanvas();
+                                    setImage(test.dataURI, null);
+                                    
                                     // display first page if we haven't yet
                                     if (imageFiles.length === currentImage + 1) {
-                                        updatePage(lastCompletion);
+                                        updateDirectionButtons();
+                                        updatePage();
                                     }
                                 } else {
                                     totalImages--;
@@ -225,6 +261,17 @@ function loadFromArrayBuffer(ab) {
 }
 
 function scrollTocToActive() {
+    $(".page").text((currentImage + 1 ) + "/" + totalImages);
+
+    // Mark the current page in the TOC
+    $("#tocView a[data-page]")
+        // Remove the currently active thumbnail
+        .removeClass("active")
+        // Find the new one
+        .filter("[data-page=" + (currentImage + 1) + "]")
+        // Set it to active
+        .addClass("active");
+
     // Scroll to the thumbnail in the TOC on page change
     $("#tocView").stop().animate({
         scrollTop: $("#tocView a.active").position().top
@@ -232,33 +279,33 @@ function scrollTocToActive() {
 }
 
 function updatePage() {
-    $(".page").text((currentImage + 1 ) + "/" + totalImages);
-
-    // Mark the current page in the TOC
-    $("#tocView a[data-page]")
-    // Remove the currently active thumbnail
-        .removeClass("active")
-        // Find the new one
-        .filter("[data-page=" + (currentImage + 1) + "]")
-        // Set it to active
-        .addClass("active");
-
     scrollTocToActive();
+    scrollCurrentImageIntoView();
     updateProgress();
-
-    if (imageFiles[currentImage]) {
-        setImage(imageFiles[currentImage].dataURI);
-    } else {
-        setImage("loading");
-    }
-
-    $("body").toggleClass("dark-theme", settings.theme === "dark");
-	$("#mainContent").toggleClass("disabled-scrollbar", settings.scrollbar === 0);
+    pageDisplayUpdate();
+    setTheme();
 
     kthoom.setSettings();
     kthoom.saveSettings();
 }
 
+function setTheme() {
+    $("body").toggleClass("dark-theme", settings.theme === "dark");
+	$("#mainContent").toggleClass("disabled-scrollbar", settings.scrollbar === 0);
+}
+
+function pageDisplayUpdate() {
+    if(settings.pageDisplay === 0) {
+        $(".mainImage").addClass("hide");
+        $(".mainImage").eq(currentImage).removeClass("hide");
+        $("#mainContent").removeClass("long-strip");
+    } else {
+        $(".mainImage").removeClass("hide");
+        $("#mainContent").addClass("long-strip");
+        scrollCurrentImageIntoView();
+    }
+}
+
 function updateProgress(loadPercentage) {
     if (settings.direction === 0) {
         $("#progress .bar-read")
@@ -290,100 +337,93 @@ function updateProgress(loadPercentage) {
     $("#progress .bar-read").css({ width: totalImages === 0 ? 0 : Math.round((currentImage + 1) / totalImages * 100) + "%"});
 }
 
-function setImage(url) {
-    var canvas = $("#mainImage")[0];
-    var x = $("#mainImage")[0].getContext("2d");
+function setImage(url, _canvas) {
+    var canvas = _canvas || $(".mainImage").slice(-1)[0]; // Select the last item on the array if _canvas is null
+    var x = canvas.getContext("2d");
+
     $("#mainText").hide();
-    if (url === "loading") {
-        updateScale(true);
-        canvas.width = innerWidth - 100;
-        canvas.height = 200;
+    if (url === "error") {
         x.fillStyle = "black";
         x.textAlign = "center";
         x.font = "24px sans-serif";
-        x.strokeStyle = "black";
-        x.fillText("Loading Page #" + (currentImage + 1), innerWidth / 2, 100);
+        x.strokeStyle = (settings.theme === "dark") ? "white" : "black";
+        x.fillText("Unable to decompress image #" + (currentImage + 1), innerWidth / 2, 100);
+
+        $(".mainImage").slice(-1).addClass("error");
     } else {
-        if (url === "error") {
-            updateScale(true);
-            canvas.width = innerWidth - 100;
-            canvas.height = 200;
-            x.fillStyle = "black";
-            x.textAlign = "center";
-            x.font = "24px sans-serif";
-            x.strokeStyle = "black";
-            x.fillText("Unable to decompress image #" + (currentImage + 1), innerWidth / 2, 100);
-        } else {
-            if ($("body").css("scrollHeight") / innerHeight > 1) {
-                $("body").css("overflowY", "scroll");
-            }
-
-            var img = new Image();
-            img.onerror = function() {
-                canvas.width = innerWidth - 100;
-                canvas.height = 300;
-                updateScale(true);
-                x.fillStyle = "black";
-                x.font = "50px sans-serif";
-                x.strokeStyle = "black";
-                x.fillText("Page #" + (currentImage + 1) + " (" +
-                  imageFiles[currentImage].filename + ")", innerWidth / 2, 100);
-                x.fillStyle = "black";
-                x.fillText("Is corrupt or not an image", innerWidth / 2, 200);
-
-                var xhr = new XMLHttpRequest();
-                if (/(html|htm)$/.test(imageFiles[currentImage].filename)) {
-                    xhr.open("GET", url, true);
-                    xhr.onload = function() {
-                        $("#mainText").css("display", "");
-                        $("#mainText").innerHTML("<iframe style=\"width:100%;height:700px;border:0\" src=\"data:text/html," + escape(xhr.responseText) + "\"></iframe>");
-                    };
-                    xhr.send(null);
-                } else if (!/(jpg|jpeg|png|gif|webp)$/.test(imageFiles[currentImage].filename) && imageFiles[currentImage].data.uncompressedSize < 10 * 1024) {
-                    xhr.open("GET", url, true);
-                    xhr.onload = function() {
-                        $("#mainText").css("display", "");
-                        $("#mainText").innerText(xhr.responseText);
-                    };
-                    xhr.send(null);
-                }
-            };
-            img.onload = function() {
-                var h = img.height,
-                    w = img.width,
-                    sw = w,
-                    sh = h;
-                settings.rotateTimes =  (4 + settings.rotateTimes) % 4;
-                x.save();
-                if (settings.rotateTimes % 2 === 1) {
-                    sh = w;
-                    sw = h;
-                }
-                canvas.height = sh;
-                canvas.width = sw;
-                x.translate(sw / 2, sh / 2);
-                x.rotate(Math.PI / 2 * settings.rotateTimes);
-                x.translate(-w / 2, -h / 2);
-                if (settings.vflip) {
-                    x.scale(1, -1);
-                    x.translate(0, -h);
-                }
-                if (settings.hflip) {
-                    x.scale(-1, 1);
-                    x.translate(-w, 0);
-                }
-                canvas.style.display = "none";
-                scrollTo(0, 0);
-                x.drawImage(img, 0, 0);
-
-                updateScale(false);
-
-                canvas.style.display = "";
-                $("body").css("overflowY", "");
-                x.restore();
-            };
-            img.src = url;
+        if ($("body").css("scrollHeight") / innerHeight > 1) {
+            $("body").css("overflowY", "scroll");
         }
+
+        var img = new Image();
+        img.onerror = function() {
+            canvas.width = innerWidth - 100;
+            canvas.height = 300;
+            x.fillStyle = "black";
+            x.font = "50px sans-serif";
+            x.strokeStyle = "black";
+            x.fillText("Page #" + (currentImage + 1) + " (" +
+                imageFiles[currentImage].filename + ")", innerWidth / 2, 100);
+            x.fillStyle = "black";
+            x.fillText("Is corrupt or not an image", innerWidth / 2, 200);
+
+            var xhr = new XMLHttpRequest();
+            if (/(html|htm)$/.test(imageFiles[currentImage].filename)) {
+                xhr.open("GET", url, true);
+                xhr.onload = function() {
+                    $("#mainText").css("display", "");
+                    $("#mainText").innerHTML("<iframe style=\"width:100%;height:700px;border:0\" src=\"data:text/html," + escape(xhr.responseText) + "\"></iframe>");
+                };
+                xhr.send(null);
+            } else if (!/(jpg|jpeg|png|gif|webp)$/.test(imageFiles[currentImage].filename) && imageFiles[currentImage].data.uncompressedSize < 10 * 1024) {
+                xhr.open("GET", url, true);
+                xhr.onload = function() {
+                    $("#mainText").css("display", "");
+                    $("#mainText").innerText(xhr.responseText);
+                };
+                xhr.send(null);
+            }
+        };
+        img.onload = function() {
+            var h = img.height,
+                w = img.width,
+                sw = w,
+                sh = h;
+            settings.rotateTimes =  (4 + settings.rotateTimes) % 4;
+            x.save();
+            if (settings.rotateTimes % 2 === 1) {
+                sh = w;
+                sw = h;
+            }
+            canvas.height = sh;
+            canvas.width = sw;
+            x.translate(sw / 2, sh / 2);
+            x.rotate(Math.PI / 2 * settings.rotateTimes);
+            x.translate(-w / 2, -h / 2);
+            if (settings.vflip) {
+                x.scale(1, -1);
+                x.translate(0, -h);
+            }
+            if (settings.hflip) {
+                x.scale(-1, 1);
+                x.translate(-w, 0);
+            }
+            canvas.style.display = "none";
+            scrollTo(0, 0);
+            x.drawImage(img, 0, 0);
+
+            canvas.style.display = "";
+            $("body").css("overflowY", "");
+            x.restore();
+        };
+        img.src = url;
+    }
+}
+
+// reloadImages is a slow process when multiple images are involved. Only used when rotating/mirroring
+function reloadImages() {
+    for(i=0; i < imageFiles.length; i++) {
+        setImage(imageFiles[i].dataURI, $(".mainImage")[i]);
     }
 }
 
@@ -393,6 +433,7 @@ function showLeftPage() {
     } else {
         showNextPage();
     }
+    setBookmark();
 }
 
 function showRightPage() {
@@ -401,6 +442,7 @@ function showRightPage() {
     } else {
         showPrevPage();
     }
+    setBookmark();
 }
 
 function showPrevPage() {
@@ -410,10 +452,8 @@ function showPrevPage() {
         currentImage++;
     } else {
         updatePage();
-		if (settings.nextPage === 0) {
-			$("#mainContent").scrollTop(0);
-		}
     }
+    updateDirectionButtons();
 }
 
 function showNextPage() {
@@ -423,36 +463,54 @@ function showNextPage() {
         currentImage--;
     } else {
         updatePage();
-		if (settings.nextPage === 0) {
-			$("#mainContent").scrollTop(0);
-		}
+    }
+    updateDirectionButtons();
+}
+
+function scrollCurrentImageIntoView() {
+    if(settings.pageDisplay == 0) {
+        // This will scroll all the way up when Single Page is selected
+		$("#mainContent").scrollTop(0);
+    } else {
+        // This will scroll to the image when Long Strip is selected
+        $("#mainContent").stop().animate({
+            scrollTop: $(".mainImage").eq(currentImage).offset().top + $("#mainContent").scrollTop() - $("#mainContent").offset().top
+        }, 200);
     }
 }
 
-function updateScale(clear) {
-    var mainImageStyle = getElem("mainImage").style;
-    mainImageStyle.width = "";
-    mainImageStyle.height = "";
-    mainImageStyle.maxWidth = "";
-    mainImageStyle.maxHeight = "";
+function updateScale() {
+    var canvasArray = $("#mainContent > canvas");
     var maxheight = innerHeight - 50;
+    
+    canvasArray.css("width", "");
+    canvasArray.css("height", "");
+    canvasArray.css("maxWidth", "");
+    canvasArray.css("maxHeight", "");
 
-    if (!clear) {
-        switch (settings.fitMode) {
-            case kthoom.Key.B:
-                mainImageStyle.maxWidth = "100%";
-                mainImageStyle.maxHeight = maxheight + "px";
-                break;
-            case kthoom.Key.H:
-                mainImageStyle.height = maxheight + "px";
-                break;
-            case kthoom.Key.W:
-                mainImageStyle.width = "100%";
-                break;
-            default:
-                break;
-        }
+    if(settings.pageDisplay === 0) {
+        canvasArray.addClass("hide");
+        pageDisplayUpdate();
     }
+
+    switch (settings.fitMode) {
+        case kthoom.Key.B:
+            canvasArray.css("maxWidth", "100%");
+            canvasArray.css("maxHeight", maxheight + "px");
+            break;
+        case kthoom.Key.H:
+            canvasArray.css("maxHeight", maxheight + "px");
+            break;
+        case kthoom.Key.W:
+            canvasArray.css("width", "100%");
+            break;
+        default:
+            break;
+    }
+
+    $("#mainContent > canvas.error").css("width", innerWidth - 100);
+    $("#mainContent > canvas.error").css("height", 200);
+
     $("#mainContent").css({maxHeight: maxheight + 5});
     kthoom.setSettings();
     kthoom.saveSettings();
@@ -469,6 +527,20 @@ function keyHandler(evt) {
             if (hasModifier) break;
             showRightPage();
             break;
+        case kthoom.Key.S:
+            if (hasModifier) break;
+            settings.pageDisplay = 0;
+            pageDisplayUpdate();
+            kthoom.setSettings();
+            kthoom.saveSettings();
+            break;
+        case kthoom.Key.O:
+            if (hasModifier) break;
+            settings.pageDisplay = 1;
+            pageDisplayUpdate();
+            kthoom.setSettings();
+            kthoom.saveSettings();
+            break;
         case kthoom.Key.L:
             if (hasModifier) break;
             settings.rotateTimes--;
@@ -476,6 +548,7 @@ function keyHandler(evt) {
                 settings.rotateTimes = 3;
             }
             updatePage();
+			reloadImages();
             break;
         case kthoom.Key.R:
             if (hasModifier) break;
@@ -484,6 +557,7 @@ function keyHandler(evt) {
                 settings.rotateTimes = 0;
             }
             updatePage();
+			reloadImages();
             break;
         case kthoom.Key.F:
             if (hasModifier) break;
@@ -499,26 +573,27 @@ function keyHandler(evt) {
                 settings.hflip = true;
             }
             updatePage();
+			reloadImages();
             break;
         case kthoom.Key.W:
             if (hasModifier) break;
             settings.fitMode = kthoom.Key.W;
-            updateScale(false);
+            updateScale();
             break;
         case kthoom.Key.H:
             if (hasModifier) break;
             settings.fitMode = kthoom.Key.H;
-            updateScale(false);
+            updateScale();
             break;
         case kthoom.Key.B:
             if (hasModifier) break;
             settings.fitMode = kthoom.Key.B;
-            updateScale(false);
+            updateScale();
             break;
         case kthoom.Key.N:
             if (hasModifier) break;
             settings.fitMode = kthoom.Key.N;
-            updateScale(false);
+            updateScale();
             break;
         case kthoom.Key.SPACE:
             if (evt.shiftKey) {
@@ -537,37 +612,85 @@ function keyHandler(evt) {
     }
 }
 
+function drawCanvas() {
+    var maxheight = innerHeight - 50;
+    var canvasElement = $("<canvas></canvas>");
+    var x = canvasElement[0].getContext("2d");
+    canvasElement.addClass("mainImage");
+
+    switch (settings.fitMode) {
+        case kthoom.Key.B:
+            canvasElement.css("maxWidth", "100%");
+            canvasElement.css("maxHeight", maxheight + "px");
+            break;
+        case kthoom.Key.H:
+            canvasElement.css("maxHeight", maxheight + "px");
+            break;
+        case kthoom.Key.W:
+            canvasElement.css("width", "100%");
+            break;
+        default:
+            break;
+    }
+
+    if(settings.pageDisplay === 0) {
+        canvasElement.addClass("hide");
+    }
+
+    //Fill with Placeholder text. setImage will override this
+    canvasElement.width = innerWidth - 100;
+    canvasElement.height = 200;
+    x.fillStyle = "black";
+    x.textAlign = "center";
+    x.font = "24px sans-serif";
+    x.strokeStyle = (settings.theme === "dark") ? "white" : "black";
+    x.fillText("Loading Page #" + (currentImage + 1), innerWidth / 2, 100);
+
+    $("#mainContent").append(canvasElement);
+}
+
+function updateArrows() {
+    if ($('input[name="direction"]:checked').val() === "0") {
+        $("#prev_page_key").html("&larr;");
+        $("#next_page_key").html("&rarr;");
+    } else {
+        $("#prev_page_key").html("&rarr;");
+        $("#next_page_key").html("&larr;");
+    }
+};
+
 function init(filename) {
     var request = new XMLHttpRequest();
     request.open("GET", filename);
     request.responseType = "arraybuffer";
-    request.addEventListener("load", function() {
+    request.addEventListener("load", function () {
         if (request.status >= 200 && request.status < 300) {
             loadFromArrayBuffer(request.response);
         } else {
             console.warn(request.statusText, request.responseText);
         }
     });
+    kthoom.loadSettings();
+    setTheme();
+    updateScale();
     request.send();
     initProgressClick();
     document.body.className += /AppleWebKit/.test(navigator.userAgent) ? " webkit" : "";
-    kthoom.loadSettings();
-    updateScale(true);
 
     $(document).keydown(keyHandler);
 
-    $(window).resize(function() {
-        updateScale(false);
+    $(window).resize(function () {
+        updateScale();
     });
 
     // Open TOC menu
-    $("#slider").click(function() {
+    $("#slider").click(function () {
         $("#sidebar").toggleClass("open");
         $("#main").toggleClass("closed");
         $(this).toggleClass("icon-menu icon-right");
 
         // We need this in a timeout because if we call it during the CSS transition, IE11 shakes the page ¯\_(ツ)_/¯
-        setTimeout(function() {
+        setTimeout(function () {
             // Focus on the TOC or the main content area, depending on which is open
             $("#main:not(.closed) #mainContent, #sidebar.open #tocView").focus();
             scrollTocToActive();
@@ -575,12 +698,12 @@ function init(filename) {
     });
 
     // Open Settings modal
-    $("#setting").click(function() {
+    $("#setting").click(function () {
         $("#settings-modal").toggleClass("md-show");
     });
 
     // On Settings input change
-    $("#settings input").on("change", function() {
+    $("#settings input").on("change", function () {
         // Get either the checked boolean or the assigned value
         var value = this.type === "checkbox" ? this.checked : this.value;
 
@@ -588,33 +711,41 @@ function init(filename) {
         value = /^\d+$/.test(value) ? parseInt(value) : value;
 
         settings[this.name] = value;
+
+        if (["hflip", "vflip", "rotateTimes"].includes(this.name)) {
+            reloadImages();
+        } else if (this.name === "direction") {
+            updateDirectionButtons();
+            return updateProgress();
+        }
+
         updatePage();
-        updateScale(false);
+        updateScale();
     });
 
     // Close modal
-    $(".closer, .overlay").click(function() {
+    $(".closer, .overlay").click(function () {
         $(".md-show").removeClass("md-show");
+        $("#mainContent").focus(); // focus back on the main container so you use up/down keys without having to click on it
     });
 
     // TOC thumbnail pagination
-    $("#thumbnails").on("click", "a", function() {
+    $("#thumbnails").on("click", "a", function () {
         currentImage = $(this).data("page") - 1;
         updatePage();
-		if (settings.nextPage === 0) {
-			$("#mainContent").scrollTop(0);
-		}
     });
 
     // Fullscreen mode
     if (typeof screenfull !== "undefined") {
-        $("#fullscreen").click(function() {
+        $("#fullscreen").click(function () {
             screenfull.toggle($("#container")[0]);
+            // Focus on main container so you can use up/down keys immediately after fullscreen
+            $("#mainContent").focus();
         });
 
         if (screenfull.raw) {
             var $button = $("#fullscreen");
-            document.addEventListener(screenfull.raw.fullscreenchange, function() {
+            document.addEventListener(screenfull.raw.fullscreenchange, function () {
                 screenfull.isFullscreen
                     ? $button.addClass("icon-resize-small").removeClass("icon-resize-full")
                     : $button.addClass("icon-resize-full").removeClass("icon-resize-small");
@@ -625,16 +756,16 @@ function init(filename) {
     // Focus the scrollable area so that keyboard scrolling work as expected
     $("#mainContent").focus();
 
-    $("#mainContent").swipe( {
-        swipeRight:function() {
+    $("#mainContent").swipe({
+        swipeRight: function () {
             showLeftPage();
         },
-        swipeLeft:function() {
+        swipeLeft: function () {
             showRightPage();
         },
     });
-    $("#mainImage").click(function(evt) {
-        // Firefox does not support offsetX/Y so we have to manually calculate
+    $(".mainImage").click(function (evt) {
+        // Firefox does not support offsetX/Y, so we have to manually calculate
         // where the user clicked in the image.
         var mainContentWidth = $("#mainContent").width();
         var mainContentHeight = $("#mainContent").height();
@@ -668,5 +799,73 @@ function init(filename) {
             showRightPage();
         }
     });
+
+    // Scrolling up/down will update current image if a new image is into view (for Long Strip Display)
+    $("#mainContent").scroll(function (){
+        var scroll = $("#mainContent").scrollTop();
+        var viewLength = 0;
+        $(".mainImage").each(function(){
+            viewLength += $(this).height();
+        });
+        if (settings.pageDisplay === 0) {
+            // Don't trigger the scroll for Single Page
+        } else if (scroll > prevScrollPosition) {
+            //Scroll Down
+            if (currentImage + 1 < imageFiles.length) {
+                if (currentImageOffset(currentImage + 1) <= 1) {
+                    currentImage = Math.floor((imageFiles.length) / (viewLength-viewLength/(imageFiles.length)) * scroll, 0);
+                    if ( currentImage >= imageFiles.length) {
+                        currentImage = imageFiles.length - 1;
+                    }
+                    console.log(currentImage);
+                    scrollTocToActive();
+                    updateProgress();
+                }
+            }
+        } else {
+            //Scroll Up
+            if (currentImage - 1 > -1) {
+                if (currentImageOffset(currentImage - 1) >= 0) {
+                    currentImage = Math.floor((imageFiles.length) / (viewLength-viewLength/(imageFiles.length)) * scroll, 0);
+                    console.log(currentImage);
+                    scrollTocToActive();
+                    updateProgress();
+                }
+            }
+        }
+        // Update scroll position
+        prevScrollPosition = scroll;
+    });
 }
 
+function currentImageOffset(imageIndex) {
+    return $(".mainImage").eq(imageIndex).offset().top - $("#mainContent").position().top
+}
+
+function setBookmark() {
+  // get csrf_token
+    let csrf_token = $("input[name='csrf_token']").val();
+    //This sends a bookmark update to calibreweb.
+    $.ajax(calibre.bookmarkUrl, {
+        method: "post",
+        data: {
+            csrf_token: csrf_token,
+            bookmark: currentImage
+        }
+    }).fail(function (xhr, status, error) {
+        console.error(error);
+    });
+}
+
+$(function() {
+    $('input[name="direction"]').change(function () {
+        updateArrows();
+    });
+
+    $('#left').click(function () {
+        showLeftPage();
+    });
+    $('#right').click(function () {
+        showRightPage();
+    });
+});