Preparation for release

Thank you.

CONTRIBUTING.md

@@ -16,7 +16,7 @@ We can offer only very limited support regarding configuration of **Reverse-Prox
 
 ### **Translation**
 
-Some of the user languages in Calibre-Web having missing translations. We are happy to add the missing texts if you are translate them. Create a Pull Request, create an issue with the .po file attached, or write an email to "ozzie.fernandez.isaacs@googlemail.com" with attached translation file. To display all book languages in your native language an additional file is used (iso_language_names.py). The content of this file is autogenerated with the corresponding translations of Calibre, please do not edit this file on your own. 
+Some of the user languages in Calibre-Web having missing translations. We are happy to add the missing texts if you translate them. Create a Pull Request, create an issue with the .po file attached, or write an email to "ozzie.fernandez.isaacs@googlemail.com" with attached translation file. To display all book languages in your native language an additional file is used (iso_language_names.py). The content of this file is autogenerated with the corresponding translations of Calibre, please do not edit this file on your own. 
 
 ### **Documentation**
 
@@ -32,7 +32,7 @@ If you're unable to find an **open issue** addressing the problem, open a [new o
 
 ### **Feature Request**
 
-If there is afeature missing in Calibre-Web and you can't find a feature request in the [Issues](https://github.com/janeczku/calibre-web/issues) section, you could create a [feature request](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=feature_request.md&title=). 
+If there is a feature missing in Calibre-Web and you can't find a feature request in the [Issues](https://github.com/janeczku/calibre-web/issues) section, you could create a [feature request](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=feature_request.md&title=). 
 We will not extend Calibre-Web with any more login abilitys or add further files storages, or file syncing ability. Furthermore Calibre-Web is made for home usage for company inhouse usage, so requests regarding any sorts of social interaction capability, payment routines, search engine or web site analytics integration will not be implemeted. 
 
 ### **Contributing code to Calibre-Web**

README.md

@@ -54,9 +54,7 @@ Optionally, to enable on-the-fly conversion from one ebook format to another whe
 
 [Download and install](https://calibre-ebook.com/download) the Calibre desktop program for your platform and enter the folder including program name (normally /opt/calibre/ebook-convert, or C:\Program Files\calibre\ebook-convert.exe) in the field "calibre's converter tool" on the setup page.
 
-\*** DEPRECATED \*** Support will be removed in future releases
-
-[Download](http://www.amazon.com/gp/feature.html?docId=1000765211) Amazon's KindleGen tool for your platform and place the binary named `kindlegen` in the `vendor` folder.
+[Download](https://github.com/geek1011/kepubify/releases/tag/v3.1.2) Kepubify tool for your platform and place the binary starting with `kepubify` in Linux: `\opt\kepubify` Windows: `C:\Program Files\kepubify`.
 
 ## Docker Images
 

cps/__init__.py

@@ -37,6 +37,7 @@ from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
 
+
 mimetypes.init()
 mimetypes.add_type('application/xhtml+xml', '.xhtml')
 mimetypes.add_type('application/epub+zip', '.epub')
@@ -56,10 +57,17 @@ mimetypes.add_type('application/ogg', '.ogg')
 mimetypes.add_type('application/ogg', '.oga')
 
 app = Flask(__name__)
+app.config.update(
+    SESSION_COOKIE_HTTPONLY=True,
+    SESSION_COOKIE_SAMESITE='Lax',
+    REMEMBER_COOKIE_SAMESITE='Lax',  # will be available in flask-login 0.5.1 earliest
+)
+
 
 lm = LoginManager()
 lm.login_view = 'web.login'
 lm.anonymous_user = ub.Anonymous
+lm.session_protection = 'strong'
 
 ub.init_db(cli.settingspath)
 # pylint: disable=no-member
@@ -75,6 +83,8 @@ log = logger.create()
 
 from . import services
 
+calibre_db = db.CalibreDB()
+
 def create_app():
     app.wsgi_app = ReverseProxied(app.wsgi_app)
     # For python2 convert path to unicode
@@ -91,7 +101,8 @@ def create_app():
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-    db.setup_db(config)
+    calibre_db.setup_db(config, cli.settingspath)
+    calibre_db.start()
 
     babel.init_app(app)
     _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

cps/about.py

@@ -30,7 +30,7 @@ import babel, pytz, requests, sqlalchemy
 import werkzeug, flask, flask_login, flask_principal, jinja2
 from flask_babel import gettext as _
 
-from . import db, converter, uploader, server, isoLanguages, constants
+from . import db, calibre_db, converter, uploader, server, isoLanguages, constants
 from .web import render_title_template
 try:
     from flask_login import __version__ as flask_loginVersion
@@ -85,10 +85,12 @@ _VERSIONS.update(uploader.get_versions())
 @about.route("/stats")
 @flask_login.login_required
 def stats():
-    counter = db.session.query(db.Books).count()
-    authors = db.session.query(db.Authors).count()
-    categorys = db.session.query(db.Tags).count()
-    series = db.session.query(db.Series).count()
-    _VERSIONS['ebook converter'] = _(converter.get_version())
+    counter = calibre_db.session.query(db.Books).count()
+    authors = calibre_db.session.query(db.Authors).count()
+    categorys = calibre_db.session.query(db.Tags).count()
+    series = calibre_db.session.query(db.Series).count()
+    _VERSIONS['ebook converter'] = _(converter.get_calibre_version())
+    _VERSIONS['unrar'] = _(converter.get_unrar_version())
+    _VERSIONS['kepubify'] = _(converter.get_kepubify_version())
     return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=_VERSIONS,
                                  categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")

cps/admin.py

@@ -22,6 +22,7 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import re
 import base64
 import json
 import time
@@ -37,8 +38,8 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql.expression import func
 
 from . import constants, logger, helper, services
-from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
-from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash
+from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
+from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano
 
@@ -85,7 +86,7 @@ def shutdown():
     showtext = {}
     if task in (0, 1):  # valid commandos received
         # close all database connections
-        db.dispose()
+        calibre_db.dispose()
         ub.dispose()
 
         if task == 0:
@@ -98,7 +99,7 @@ def shutdown():
 
     if task == 2:
         log.warning("reconnecting to calibre database")
-        db.setup_db(config)
+        calibre_db.setup_db(config, ub.app_DB_path)
         showtext['text'] = _(u'Reconnect successful')
         return json.dumps(showtext)
 
@@ -147,10 +148,10 @@ def configuration():
 @login_required
 @admin_required
 def view_configuration():
-    readColumn = db.session.query(db.Custom_Columns)\
-            .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
-    restrictColumns= db.session.query(db.Custom_Columns)\
-            .filter(and_(db.Custom_Columns.datatype == 'text',db.Custom_Columns.mark_for_delete == 0)).all()
+    readColumn = calibre_db.session.query(db.Custom_Columns)\
+            .filter(and_(db.Custom_Columns.datatype == 'bool', db.Custom_Columns.mark_for_delete == 0)).all()
+    restrictColumns= calibre_db.session.query(db.Custom_Columns)\
+            .filter(and_(db.Custom_Columns.datatype == 'text', db.Custom_Columns.mark_for_delete == 0)).all()
     return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn,
                                  restrictColumns=restrictColumns,
                                  title=_(u"UI Configuration"), page="uiconfig")
@@ -168,7 +169,6 @@ def update_view_configuration():
 
     _config_string("config_calibre_web_title")
     _config_string("config_columns_to_ignore")
-    # _config_string("config_mature_content_tags")
     reboot_required |= _config_string("config_title_regex")
 
     _config_int("config_read_column")
@@ -178,9 +178,6 @@ def update_view_configuration():
     _config_int("config_authors_max")
     _config_int("config_restricted_column")
 
-    if config.config_google_drive_watch_changes_response:
-        config.config_google_drive_watch_changes_response = json.dumps(config.config_google_drive_watch_changes_response)
-
     config.config_default_role = constants.selected_roles(to_save)
     config.config_default_role &= ~constants.ROLE_ANONYMOUS
 
@@ -429,7 +426,6 @@ def delete_restriction(res_type):
     return ""
 
 
-#@admi.route("/ajax/listrestriction/<int:type>/<int:user_id>", defaults={'user_id': '0'})
 @admi.route("/ajax/listrestriction/<int:res_type>")
 @login_required
 @admin_required
@@ -475,6 +471,7 @@ def list_restriction(res_type):
     response.headers["Content-Type"] = "application/json; charset=utf-8"
     return response
 
+
 @admi.route("/config", methods=["GET", "POST"])
 @unconfigured
 def basic_configuration():
@@ -484,19 +481,23 @@ def basic_configuration():
     return _configuration_result()
 
 
-def _configuration_update_helper():
-    reboot_required = False
-    db_change = False
-    to_save = request.form.to_dict()
+def _config_int(to_save, x, func=int):
+    return config.set_from_dictionary(to_save, x, func)
 
-    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-    _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False)
-    _config_checkbox_int = lambda x: config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0)
 
-    db_change |= _config_string("config_calibre_dir")
+def _config_checkbox(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: y == "on", False)
 
-    # Google drive setup
+
+def _config_checkbox_int(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0)
+
+
+def _config_string(to_save, x):
+    return config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
+
+
+def _configuration_gdrive_helper(to_save):
     if not os.path.isfile(gdriveutils.SETTINGS_YAML):
         config.config_use_google_drive = False
 
@@ -515,147 +516,177 @@ def _configuration_update_helper():
 
     # always show google drive settings, but in case of error deny support
     config.config_use_google_drive = (not gdriveError) and ("config_use_google_drive" in to_save)
-    if _config_string("config_google_drive_folder"):
+    if _config_string(to_save, "config_google_drive_folder"):
         gdriveutils.deleteDatabaseOnChange()
+    return gdriveError
 
-    reboot_required |= _config_int("config_port")
+def _configuration_oauth_helper(to_save):
+    active_oauths = 0
+    reboot_required = False
+    for element in oauthblueprints:
+        if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \
+            or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
+            reboot_required = True
+            element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"]
+            element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"]
+        if to_save["config_" + str(element['id']) + "_oauth_client_id"] \
+            and to_save["config_" + str(element['id']) + "_oauth_client_secret"]:
+            active_oauths += 1
+            element["active"] = 1
+        else:
+            element["active"] = 0
+        ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update(
+            {"oauth_client_id": to_save["config_" + str(element['id']) + "_oauth_client_id"],
+             "oauth_client_secret": to_save["config_" + str(element['id']) + "_oauth_client_secret"],
+             "active": element["active"]})
+    return reboot_required
 
-    reboot_required |= _config_string("config_keyfile")
+def _configuration_logfile_helper(to_save, gdriveError):
+    reboot_required = False
+    reboot_required |= _config_int(to_save, "config_log_level")
+    reboot_required |= _config_string(to_save, "config_logfile")
+    if not logger.is_valid_logfile(config.config_logfile):
+        return reboot_required, _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+
+    reboot_required |= _config_checkbox_int(to_save, "config_access_log")
+    reboot_required |= _config_string(to_save, "config_access_logfile")
+    if not logger.is_valid_logfile(config.config_access_logfile):
+        return reboot_required, _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+    return reboot_required, None
+
+def _configuration_ldap_helper(to_save, gdriveError):
+    reboot_required = False
+    reboot_required |= _config_string(to_save, "config_ldap_provider_url")
+    reboot_required |= _config_int(to_save, "config_ldap_port")
+    reboot_required |= _config_int(to_save, "config_ldap_authentication")
+    reboot_required |= _config_string(to_save, "config_ldap_dn")
+    reboot_required |= _config_string(to_save, "config_ldap_serv_username")
+    reboot_required |= _config_string(to_save, "config_ldap_user_object")
+    reboot_required |= _config_string(to_save, "config_ldap_group_object_filter")
+    reboot_required |= _config_string(to_save, "config_ldap_group_members_field")
+    reboot_required |= _config_checkbox(to_save, "config_ldap_openldap")
+    reboot_required |= _config_int(to_save, "config_ldap_encryption")
+    reboot_required |= _config_string(to_save, "config_ldap_cert_path")
+    _config_string(to_save, "config_ldap_group_name")
+    if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "":
+        reboot_required |= 1
+        config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
+    config.save()
+
+    if not config.config_ldap_provider_url \
+        or not config.config_ldap_port \
+        or not config.config_ldap_dn \
+        or not config.config_ldap_user_object:
+        return reboot_required, _configuration_result(_('Please Enter a LDAP Provider, '
+                                       'Port, DN and User Object Identifier'), gdriveError)
+
+    if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
+        if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
+            if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password):
+                return reboot_required, _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError)
+        else:
+            if not config.config_ldap_serv_username:
+                return reboot_required, _configuration_result('Please Enter a LDAP Service Account', gdriveError)
+
+    if config.config_ldap_group_object_filter:
+        if config.config_ldap_group_object_filter.count("%s") != 1:
+            return reboot_required, _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'),
+                                         gdriveError)
+        if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"):
+            return reboot_required, _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'),
+                                         gdriveError)
+
+    if config.config_ldap_user_object.count("%s") != 1:
+        return reboot_required, _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'),
+                                     gdriveError)
+    if config.config_ldap_user_object.count("(") != config.config_ldap_user_object.count(")"):
+        return reboot_required, _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'),
+                                     gdriveError)
+
+    if config.config_ldap_cert_path and not os.path.isdir(config.config_ldap_cert_path):
+        return reboot_required, _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'),
+                                     gdriveError)
+    return reboot_required, None
+
+
+def _configuration_update_helper():
+    reboot_required = False
+    db_change = False
+    to_save = request.form.to_dict()
+
+    to_save['config_calibre_dir'] = re.sub('[\\/]metadata\.db$', '', to_save['config_calibre_dir'], flags=re.IGNORECASE)
+    db_change |= _config_string(to_save, "config_calibre_dir")
+
+    # Google drive setup
+    gdriveError = _configuration_gdrive_helper(to_save)
+
+    reboot_required |= _config_int(to_save, "config_port")
+
+    reboot_required |= _config_string(to_save, "config_keyfile")
     if config.config_keyfile and not os.path.isfile(config.config_keyfile):
         return _configuration_result(_('Keyfile Location is not Valid, Please Enter Correct Path'), gdriveError)
 
-    reboot_required |= _config_string("config_certfile")
+    reboot_required |= _config_string(to_save, "config_certfile")
     if config.config_certfile and not os.path.isfile(config.config_certfile):
         return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path'), gdriveError)
 
-    _config_checkbox_int("config_uploading")
-    _config_checkbox_int("config_anonbrowse")
-    _config_checkbox_int("config_public_reg")
-    reboot_required |= _config_checkbox_int("config_kobo_sync")
-    _config_checkbox_int("config_kobo_proxy")
+    _config_checkbox_int(to_save, "config_uploading")
+    _config_checkbox_int(to_save, "config_anonbrowse")
+    _config_checkbox_int(to_save, "config_public_reg")
+    _config_checkbox_int(to_save, "config_register_email")
+    reboot_required |= _config_checkbox_int(to_save, "config_kobo_sync")
+    _config_checkbox_int(to_save, "config_kobo_proxy")
 
+    _config_string(to_save, "config_upload_formats")
+    constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in config.config_upload_formats.split(',')]
 
-    _config_int("config_ebookconverter")
-    _config_string("config_calibre")
-    _config_string("config_converterpath")
+    _config_string(to_save, "config_calibre")
+    _config_string(to_save, "config_converterpath")
+    _config_string(to_save, "config_kepubifypath")
 
-    reboot_required |= _config_int("config_login_type")
+    reboot_required |= _config_int(to_save, "config_login_type")
 
     #LDAP configurator,
     if config.config_login_type == constants.LOGIN_LDAP:
-        reboot_required |= _config_string("config_ldap_provider_url")
-        reboot_required |= _config_int("config_ldap_port")
-        reboot_required |= _config_int("config_ldap_authentication")
-        reboot_required |= _config_string("config_ldap_dn")
-        reboot_required |= _config_string("config_ldap_serv_username")
-        reboot_required |= _config_string("config_ldap_user_object")
-        reboot_required |= _config_string("config_ldap_group_object_filter")
-        reboot_required |= _config_string("config_ldap_group_members_field")
-        reboot_required |= _config_checkbox("config_ldap_openldap")
-        reboot_required |= _config_int("config_ldap_encryption")
-        reboot_required |= _config_string("config_ldap_cert_path")
-        _config_string("config_ldap_group_name")
-        if "config_ldap_serv_password" in to_save and to_save["config_ldap_serv_password"] != "":
-            reboot_required |= 1
-            config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode, encode='UTF-8')
-        config.save()
-
-        if not config.config_ldap_provider_url \
-            or not config.config_ldap_port \
-            or not config.config_ldap_dn \
-            or not config.config_ldap_user_object:
-                return _configuration_result(_('Please Enter a LDAP Provider, '
-                                             'Port, DN and User Object Identifier'), gdriveError)
-
-        if config.config_ldap_authentication > constants.LDAP_AUTH_ANONYMOUS:
-            if config.config_ldap_authentication > constants.LDAP_AUTH_UNAUTHENTICATE:
-                if not config.config_ldap_serv_username or not bool(config.config_ldap_serv_password):
-                    return _configuration_result('Please Enter a LDAP Service Account and Password', gdriveError)
-            else:
-                if not config.config_ldap_serv_username:
-                    return _configuration_result('Please Enter a LDAP Service Account', gdriveError)
-
-        #_config_checkbox("config_ldap_use_ssl")
-        #_config_checkbox("config_ldap_use_tls")
-        # reboot_required |= _config_checkbox("config_ldap_openldap")
-        # _config_checkbox("config_ldap_require_cert")
-
-        if config.config_ldap_group_object_filter:
-            if config.config_ldap_group_object_filter.count("%s") != 1:
-                return _configuration_result(_('LDAP Group Object Filter Needs to Have One "%s" Format Identifier'),
-                                             gdriveError)
-            if config.config_ldap_group_object_filter.count("(") != config.config_ldap_group_object_filter.count(")"):
-                return _configuration_result(_('LDAP Group Object Filter Has Unmatched Parenthesis'),
-                                             gdriveError)
-
-        if config.config_ldap_user_object.count("%s") != 1:
-            return _configuration_result(_('LDAP User Object Filter needs to Have One "%s" Format Identifier'),
-                                         gdriveError)
-        if config.config_ldap_user_object.count("(") != config.config_ldap_user_object.count(")"):
-            return _configuration_result(_('LDAP User Object Filter Has Unmatched Parenthesis'),
-                                         gdriveError)
-
-        if config.config_ldap_cert_path and not os.path.isdir(config.config_ldap_cert_path):
-            return _configuration_result(_('LDAP Certificate Location is not Valid, Please Enter Correct Path'),
-                                         gdriveError)
+        reboot, message = _configuration_ldap_helper(to_save, gdriveError)
+        if message:
+            return message
+        reboot_required |= reboot
 
     # Remote login configuration
-    _config_checkbox("config_remote_login")
+    _config_checkbox(to_save, "config_remote_login")
     if not config.config_remote_login:
         ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.token_type==0).delete()
 
     # Goodreads configuration
-    _config_checkbox("config_use_goodreads")
-    _config_string("config_goodreads_api_key")
-    _config_string("config_goodreads_api_secret")
+    _config_checkbox(to_save, "config_use_goodreads")
+    _config_string(to_save, "config_goodreads_api_key")
+    _config_string(to_save, "config_goodreads_api_secret")
     if services.goodreads_support:
         services.goodreads_support.connect(config.config_goodreads_api_key,
                                            config.config_goodreads_api_secret,
                                            config.config_use_goodreads)
 
-    _config_int("config_updatechannel")
+    _config_int(to_save, "config_updatechannel")
 
     # Reverse proxy login configuration
-    _config_checkbox("config_allow_reverse_proxy_header_login")
-    _config_string("config_reverse_proxy_login_header_name")
+    _config_checkbox(to_save, "config_allow_reverse_proxy_header_login")
+    _config_string(to_save, "config_reverse_proxy_login_header_name")
 
-    # GitHub OAuth configuration
+    # OAuth configuration
     if config.config_login_type == constants.LOGIN_OAUTH:
-        active_oauths = 0
-
-        for element in oauthblueprints:
-            if to_save["config_" + str(element['id']) + "_oauth_client_id"] != element['oauth_client_id'] \
-                or to_save["config_" + str(element['id']) + "_oauth_client_secret"] != element['oauth_client_secret']:
-                reboot_required = True
-                element['oauth_client_id'] = to_save["config_" + str(element['id']) + "_oauth_client_id"]
-                element['oauth_client_secret'] = to_save["config_" + str(element['id']) + "_oauth_client_secret"]
-            if to_save["config_"+str(element['id'])+"_oauth_client_id"] \
-               and to_save["config_"+str(element['id'])+"_oauth_client_secret"]:
-                active_oauths += 1
-                element["active"] = 1
-            else:
-                element["active"] = 0
-            ub.session.query(ub.OAuthProvider).filter(ub.OAuthProvider.id == element['id']).update(
-                {"oauth_client_id":to_save["config_"+str(element['id'])+"_oauth_client_id"],
-                "oauth_client_secret":to_save["config_"+str(element['id'])+"_oauth_client_secret"],
-                "active":element["active"]})
-
-
-    reboot_required |= _config_int("config_log_level")
-    reboot_required |= _config_string("config_logfile")
-    if not logger.is_valid_logfile(config.config_logfile):
-        return _configuration_result(_('Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
-
-    reboot_required |= _config_checkbox_int("config_access_log")
-    reboot_required |= _config_string("config_access_logfile")
-    if not logger.is_valid_logfile(config.config_access_logfile):
-        return _configuration_result(_('Access Logfile Location is not Valid, Please Enter Correct Path'), gdriveError)
+        reboot_required |= _configuration_oauth_helper(to_save)
 
+    reboot, message = _configuration_logfile_helper(to_save, gdriveError)
+    if message:
+        return message
+    reboot_required |= reboot
     # Rarfile Content configuration
-    _config_string("config_rarfile_location")
-    unrar_status = helper.check_unrar(config.config_rarfile_location)
-    if unrar_status:
-        return _configuration_result(unrar_status, gdriveError)
+    _config_string(to_save, "config_rarfile_location")
+    if "config_rarfile_location" in to_save:
+        unrar_status = helper.check_unrar(config.config_rarfile_location)
+        if unrar_status:
+            return _configuration_result(unrar_status, gdriveError)
 
     try:
         metadata_db = os.path.join(config.config_calibre_dir, "metadata.db")
@@ -666,9 +697,10 @@ def _configuration_update_helper():
         return _configuration_result('%s' % e, gdriveError)
 
     if db_change:
-        # reload(db)
-        if not db.setup_db(config):
+        if not calibre_db.setup_db(config, ub.app_DB_path):
             return _configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdriveError)
+        if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK):
+            flash(_(u"DB is not Writeable"), category="warning")
 
     config.save()
     flash(_(u"Calibre-Web configuration updated"), category="success")
@@ -704,62 +736,155 @@ def _configuration_result(error_flash=None, gdriveError=None):
                                  title=_(u"Basic Configuration"), page="config")
 
 
+def _handle_new_user(to_save, content,languages, translations, kobo_support):
+    content.default_language = to_save["default_language"]
+    # content.mature_content = "Show_mature_content" in to_save
+    content.locale = to_save.get("locale", content.locale)
+
+    content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_'))
+    if "show_detail_random" in to_save:
+        content.sidebar_view |= constants.DETAIL_RANDOM
+
+    content.role = constants.selected_roles(to_save)
+
+    if not to_save["nickname"] or not to_save["email"] or not to_save["password"]:
+        flash(_(u"Please fill out all fields!"), category="error")
+        return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
+                                     registered_oauth=oauth_check, kobo_support=kobo_support,
+                                     title=_(u"Add new user"))
+    content.password = generate_password_hash(to_save["password"])
+    existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower()) \
+        .first()
+    existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
+        .first()
+    if not existing_user and not existing_email:
+        content.nickname = to_save["nickname"]
+        if config.config_public_reg and not check_valid_domain(to_save["email"]):
+            flash(_(u"E-mail is not from valid domain"), category="error")
+            return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
+                                         registered_oauth=oauth_check, kobo_support=kobo_support,
+                                         title=_(u"Add new user"))
+        else:
+            content.email = to_save["email"]
+    else:
+        flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
+        return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
+                                     languages=languages, title=_(u"Add new user"), page="newuser",
+                                     kobo_support=kobo_support, registered_oauth=oauth_check)
+    try:
+        content.allowed_tags = config.config_allowed_tags
+        content.denied_tags = config.config_denied_tags
+        content.allowed_column_value = config.config_allowed_column_value
+        content.denied_column_value = config.config_denied_column_value
+        ub.session.add(content)
+        ub.session.commit()
+        flash(_(u"User '%(user)s' created", user=content.nickname), category="success")
+        return redirect(url_for('admin.admin'))
+    except IntegrityError:
+        ub.session.rollback()
+        flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
+
+
+def _handle_edit_user(to_save, content,languages, translations, kobo_support, downloads):
+    if "delete" in to_save:
+        if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
+                                            ub.User.id != content.id).count():
+            ub.session.query(ub.User).filter(ub.User.id == content.id).delete()
+            ub.session.commit()
+            flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success")
+            return redirect(url_for('admin.admin'))
+        else:
+            flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error")
+            return redirect(url_for('admin.admin'))
+    else:
+        if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
+                                                ub.User.id != content.id).count() and \
+            not 'admin_role' in to_save:
+            flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error")
+            return redirect(url_for('admin.admin'))
+
+        if "password" in to_save and to_save["password"]:
+            content.password = generate_password_hash(to_save["password"])
+        anonymous = content.is_anonymous
+        content.role = constants.selected_roles(to_save)
+        if anonymous:
+            content.role |= constants.ROLE_ANONYMOUS
+        else:
+            content.role &= ~constants.ROLE_ANONYMOUS
+
+        val = [int(k[5:]) for k in to_save if k.startswith('show_')]
+        sidebar = ub.get_sidebar_config()
+        for element in sidebar:
+            value = element['visibility']
+            if value in val and not content.check_visibility(value):
+                content.sidebar_view |= value
+            elif not value in val and content.check_visibility(value):
+                content.sidebar_view &= ~value
+
+        if "Show_detail_random" in to_save:
+            content.sidebar_view |= constants.DETAIL_RANDOM
+        else:
+            content.sidebar_view &= ~constants.DETAIL_RANDOM
+
+        if "default_language" in to_save:
+            content.default_language = to_save["default_language"]
+        if "locale" in to_save and to_save["locale"]:
+            content.locale = to_save["locale"]
+        if to_save["email"] and to_save["email"] != content.email:
+            existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
+                .first()
+            if not existing_email:
+                content.email = to_save["email"]
+            else:
+                flash(_(u"Found an existing account for this e-mail address."), category="error")
+                return render_title_template("user_edit.html",
+                                             translations=translations,
+                                             languages=languages,
+                                             mail_configured=config.get_mail_server_configured(),
+                                             kobo_support=kobo_support,
+                                             new_user=0,
+                                             content=content,
+                                             downloads=downloads,
+                                             registered_oauth=oauth_check,
+                                             title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
+        if "nickname" in to_save and to_save["nickname"] != content.nickname:
+            # Query User nickname, if not existing, change
+            if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar():
+                content.nickname = to_save["nickname"]
+            else:
+                flash(_(u"This username is already taken"), category="error")
+                return render_title_template("user_edit.html",
+                                             translations=translations,
+                                             languages=languages,
+                                             mail_configured=config.get_mail_server_configured(),
+                                             new_user=0, content=content,
+                                             downloads=downloads,
+                                             registered_oauth=oauth_check,
+                                             kobo_support=kobo_support,
+                                             title=_(u"Edit User %(nick)s", nick=content.nickname),
+                                             page="edituser")
+
+        if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail:
+            content.kindle_mail = to_save["kindle_mail"]
+    try:
+        ub.session.commit()
+        flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success")
+    except IntegrityError:
+        ub.session.rollback()
+        flash(_(u"An unknown error occured."), category="error")
+
+
 @admi.route("/admin/user/new", methods=["GET", "POST"])
 @login_required
 @admin_required
 def new_user():
     content = ub.User()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = [LC('en')] + babel.list_translations()
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     if request.method == "POST":
         to_save = request.form.to_dict()
-        content.default_language = to_save["default_language"]
-        # content.mature_content = "Show_mature_content" in to_save
-        content.locale = to_save.get("locale", content.locale)
-
-        content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_'))
-        if "show_detail_random" in to_save:
-            content.sidebar_view |= constants.DETAIL_RANDOM
-
-        content.role = constants.selected_roles(to_save)
-
-        if not to_save["nickname"] or not to_save["email"] or not to_save["password"]:
-            flash(_(u"Please fill out all fields!"), category="error")
-            return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
-                                         registered_oauth=oauth_check, kobo_support=kobo_support,
-                                         title=_(u"Add new user"))
-        content.password = generate_password_hash(to_save["password"])
-        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"].lower())\
-            .first()
-        existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower())\
-            .first()
-        if not existing_user and not existing_email:
-            content.nickname = to_save["nickname"]
-            if config.config_public_reg and not check_valid_domain(to_save["email"]):
-                flash(_(u"E-mail is not from valid domain"), category="error")
-                return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
-                                             registered_oauth=oauth_check, kobo_support=kobo_support,
-                                             title=_(u"Add new user"))
-            else:
-                content.email = to_save["email"]
-        else:
-            flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
-            return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
-                                     languages=languages, title=_(u"Add new user"), page="newuser",
-                                     kobo_support=kobo_support, registered_oauth=oauth_check)
-        try:
-            content.allowed_tags = config.config_allowed_tags
-            content.denied_tags = config.config_denied_tags
-            content.allowed_column_value = config.config_allowed_column_value
-            content.denied_column_value = config.config_denied_column_value
-            ub.session.add(content)
-            ub.session.commit()
-            flash(_(u"User '%(user)s' created", user=content.nickname), category="success")
-            return redirect(url_for('admin.admin'))
-        except IntegrityError:
-            ub.session.rollback()
-            flash(_(u"Found an existing account for this e-mail address or nickname."), category="error")
+        _handle_new_user(to_save, content, languages, translations, kobo_support)
     else:
         content.role = config.config_default_role
         content.sidebar_view = config.config_default_show
@@ -773,8 +898,7 @@ def new_user():
 @admin_required
 def edit_mailsettings():
     content = config.get_mail_settings()
-    # log.debug("edit_mailsettings %r", content)
-    return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"),
+    return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"),
                                  page="mailset")
 
 
@@ -783,17 +907,15 @@ def edit_mailsettings():
 @admin_required
 def update_mailsettings():
     to_save = request.form.to_dict()
-    log.debug("update_mailsettings %r", to_save)
+    # log.debug("update_mailsettings %r", to_save)
 
-    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-
-    _config_string("mail_server")
-    _config_int("mail_port")
-    _config_int("mail_use_ssl")
-    _config_string("mail_login")
-    _config_string("mail_password")
-    _config_string("mail_from")
+    _config_string(to_save, "mail_server")
+    _config_int(to_save, "mail_port")
+    _config_int(to_save, "mail_use_ssl")
+    _config_string(to_save, "mail_login")
+    _config_string(to_save, "mail_password")
+    _config_string(to_save, "mail_from")
+    _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
     config.save()
 
     if to_save.get("test"):
@@ -821,105 +943,18 @@ def edit_user(user_id):
         flash(_(u"User not found"), category="error")
         return redirect(url_for('admin.admin'))
     downloads = list()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     for book in content.downloads:
-        downloadbook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+        downloadbook = calibre_db.get_book(book.book_id)
         if downloadbook:
             downloads.append(downloadbook)
         else:
             ub.delete_download(book.book_id)
-            # ub.session.query(ub.Downloads).filter(book.book_id == ub.Downloads.book_id).delete()
-            # ub.session.commit()
     if request.method == "POST":
         to_save = request.form.to_dict()
-        if "delete" in to_save:
-            if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
-                                                ub.User.id != content.id).count():
-                ub.session.query(ub.User).filter(ub.User.id == content.id).delete()
-                ub.session.commit()
-                flash(_(u"User '%(nick)s' deleted", nick=content.nickname), category="success")
-                return redirect(url_for('admin.admin'))
-            else:
-                flash(_(u"No admin user remaining, can't delete user", nick=content.nickname), category="error")
-                return redirect(url_for('admin.admin'))
-        else:
-            if not ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
-                                                    ub.User.id != content.id).count() and \
-                not 'admin_role' in to_save:
-                flash(_(u"No admin user remaining, can't remove admin role", nick=content.nickname), category="error")
-                return redirect(url_for('admin.admin'))
-
-            if "password" in to_save and to_save["password"]:
-                content.password = generate_password_hash(to_save["password"])
-            anonymous = content.is_anonymous
-            content.role = constants.selected_roles(to_save)
-            if anonymous:
-                content.role |= constants.ROLE_ANONYMOUS
-            else:
-                content.role &= ~constants.ROLE_ANONYMOUS
-
-            val = [int(k[5:]) for k in to_save if k.startswith('show_')]
-            sidebar = ub.get_sidebar_config()
-            for element in sidebar:
-                value = element['visibility']
-                if value in val and not content.check_visibility(value):
-                    content.sidebar_view |= value
-                elif not value in val and content.check_visibility(value):
-                    content.sidebar_view &= ~value
-
-            if "Show_detail_random" in to_save:
-                content.sidebar_view |= constants.DETAIL_RANDOM
-            else:
-                content.sidebar_view &= ~constants.DETAIL_RANDOM
-
-            if "default_language" in to_save:
-                content.default_language = to_save["default_language"]
-            if "locale" in to_save and to_save["locale"]:
-                content.locale = to_save["locale"]
-            if to_save["email"] and to_save["email"] != content.email:
-                existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
-                    .first()
-                if not existing_email:
-                    content.email = to_save["email"]
-                else:
-                    flash(_(u"Found an existing account for this e-mail address."), category="error")
-                    return render_title_template("user_edit.html",
-                                                 translations=translations,
-                                                 languages=languages,
-                                                 mail_configured = config.get_mail_server_configured(),
-                                                 kobo_support=kobo_support,
-                                                 new_user=0,
-                                                 content=content,
-                                                 downloads=downloads,
-                                                 registered_oauth=oauth_check,
-                                                 title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
-            if "nickname" in to_save and to_save["nickname"] != content.nickname:
-                # Query User nickname, if not existing, change
-                if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar():
-                    content.nickname = to_save["nickname"]
-                else:
-                    flash(_(u"This username is already taken"), category="error")
-                    return render_title_template("user_edit.html",
-                                                 translations=translations,
-                                                 languages=languages,
-                                                 mail_configured=config.get_mail_server_configured(),
-                                                 new_user=0, content=content,
-                                                 downloads=downloads,
-                                                 registered_oauth=oauth_check,
-                                                 kobo_support=kobo_support,
-                                                 title=_(u"Edit User %(nick)s", nick=content.nickname),
-                                                 page="edituser")
-
-            if "kindle_mail" in to_save and to_save["kindle_mail"] != content.kindle_mail:
-                content.kindle_mail = to_save["kindle_mail"]
-        try:
-            ub.session.commit()
-            flash(_(u"User '%(nick)s' updated", nick=content.nickname), category="success")
-        except IntegrityError:
-            ub.session.rollback()
-            flash(_(u"An unknown error occured."), category="error")
+        _handle_edit_user(to_save, content, languages, translations, kobo_support, downloads)
     return render_title_template("user_edit.html",
                                  translations=translations,
                                  languages=languages,
@@ -939,7 +974,7 @@ def reset_user_password(user_id):
     if current_user is not None and current_user.is_authenticated:
         ret, message = reset_password(user_id)
         if ret == 1:
-            log.debug(u"Password for user %(user)s reset", user=message)
+            log.debug(u"Password for user %s reset", message)
             flash(_(u"Password for user %(user)s reset", user=message), category="success")
         elif ret == 0:
             log.error(u"An unknown error occurred. Please try again later.")

cps/comic.py

@@ -18,10 +18,17 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import io
 
 from . import logger, isoLanguages
 from .constants import BookMeta
 
+try:
+    from PIL import Image as PILImage
+    use_PIL = True
+except ImportError as e:
+    use_PIL = False
+
 
 log = logger.create()
 
@@ -29,18 +36,43 @@ log = logger.create()
 try:
     from comicapi.comicarchive import ComicArchive, MetaDataStyle
     use_comic_meta = True
+    try:
+        from comicapi import __version__ as comic_version
+    except (ImportError):
+        comic_version = ''
 except ImportError as e:
-    log.debug('cannot import comicapi, extracting comic metadata will not work: %s', e)
+    log.debug('Cannot import comicapi, extracting comic metadata will not work: %s', e)
     import zipfile
     import tarfile
     try:
         import rarfile
         use_rarfile = True
     except ImportError as e:
-        log.debug('cannot import rarfile, extracting cover files from rar files will not work: %s', e)
+        log.debug('Cannot import rarfile, extracting cover files from rar files will not work: %s', e)
         use_rarfile = False
     use_comic_meta = False
 
+def _cover_processing(tmp_file_name, img, extension):
+    if use_PIL:
+        # convert to jpg because calibre only supports jpg
+        if extension in ('.png',  '.webp'):
+            imgc = PILImage.open(io.BytesIO(img))
+            im = imgc.convert('RGB')
+            tmp_bytesio = io.BytesIO()
+            im.save(tmp_bytesio, format='JPEG')
+            img = tmp_bytesio.getvalue()
+
+    prefix = os.path.dirname(tmp_file_name)
+    if img:
+        tmp_cover_name = prefix + '/cover.jpg'
+        image = open(tmp_cover_name, 'wb')
+        image.write(img)
+        image.close()
+    else:
+        tmp_cover_name = None
+    return tmp_cover_name
+
+
 
 def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
     cover_data = extension = None
@@ -50,7 +82,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
             ext = os.path.splitext(name)
             if len(ext) > 1:
                 extension = ext[1].lower()
-                if extension == '.jpg' or extension == '.jpeg':
+                if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                     cover_data = archive.getPage(index)
                     break
     else:
@@ -60,7 +92,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                 ext = os.path.splitext(name)
                 if len(ext) > 1:
                     extension = ext[1].lower()
-                    if extension == '.jpg' or extension == '.jpeg':
+                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                         cover_data = cf.read(name)
                         break
         elif original_file_extension.upper() == '.CBT':
@@ -69,7 +101,7 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                 ext = os.path.splitext(name)
                 if len(ext) > 1:
                     extension = ext[1].lower()
-                    if extension == '.jpg' or extension == '.jpeg':
+                    if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                         cover_data = cf.extractfile(name).read()
                         break
         elif original_file_extension.upper() == '.CBR' and use_rarfile:
@@ -80,21 +112,12 @@ def _extractCover(tmp_file_name, original_file_extension, rarExceutable):
                     ext = os.path.splitext(name)
                     if len(ext) > 1:
                         extension = ext[1].lower()
-                        if extension == '.jpg' or extension == '.jpeg':
+                        if extension in ('.jpg', '.jpeg', '.png', '.webp'):
                             cover_data = cf.read(name)
                             break
             except Exception as e:
                 log.debug('Rarfile failed with error: %s', e)
-
-    prefix = os.path.dirname(tmp_file_name)
-    if cover_data:
-        tmp_cover_name = prefix + '/cover' + extension
-        image = open(tmp_cover_name, 'wb')
-        image.write(cover_data)
-        image.close()
-    else:
-        tmp_cover_name = None
-    return tmp_cover_name
+    return _cover_processing(tmp_file_name, cover_data, extension)
 
 
 def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rarExceutable):

cps/config_sql.py

@@ -53,6 +53,7 @@ class _Settings(_Base):
     mail_login = Column(String, default='mail@example.com')
     mail_password = Column(String, default='mypassword')
     mail_from = Column(String, default='automailer <mail@example.com>')
+    mail_size = Column(Integer, default=25*1024*1024)
 
     config_calibre_dir = Column(String)
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
@@ -96,7 +97,7 @@ class _Settings(_Base):
     config_use_goodreads = Column(Boolean, default=False)
     config_goodreads_api_key = Column(String)
     config_goodreads_api_secret = Column(String)
-
+    config_register_email = Column(Boolean, default=False)
     config_login_type = Column(Integer, default=0)
 
     config_kobo_proxy = Column(Boolean, default=False)
@@ -116,10 +117,11 @@ class _Settings(_Base):
     config_ldap_group_members_field = Column(String, default='memberUid')
     config_ldap_group_name = Column(String, default='calibreweb')
 
-    config_ebookconverter = Column(Integer, default=0)
-    config_converterpath = Column(String)
+    config_kepubifypath = Column(String, default=None)
+    config_converterpath = Column(String, default=None)
     config_calibre = Column(String)
-    config_rarfile_location = Column(String)
+    config_rarfile_location = Column(String, default=None)
+    config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
 
     config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
 
@@ -140,6 +142,22 @@ class _ConfigSQL(object):
         self.config_calibre_dir = None
         self.load()
 
+        change = False
+        if self.config_converterpath == None:
+            change = True
+            self.config_converterpath = autodetect_calibre_binary()
+
+        if self.config_kepubifypath == None:
+            change = True
+            self.config_kepubifypath = autodetect_kepubify_binary()
+
+        if self.config_rarfile_location == None:
+            change = True
+            self.config_rarfile_location = autodetect_unrar_binary()
+        if change:
+            self.save()
+
+
     def _read_from_storage(self):
         if self._settings is None:
             log.debug("_ConfigSQL._read_from_storage")
@@ -264,7 +282,8 @@ class _ConfigSQL(object):
                 setattr(self, k, v)
 
         if self.config_google_drive_watch_changes_response:
-            self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response)
+            self.config_google_drive_watch_changes_response = \
+                json.loads(self.config_google_drive_watch_changes_response)
 
         have_metadata_db = bool(self.config_calibre_dir)
         if have_metadata_db:
@@ -272,13 +291,22 @@ class _ConfigSQL(object):
                 db_file = os.path.join(self.config_calibre_dir, 'metadata.db')
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
-
-        logger.setup(self.config_logfile, self.config_log_level)
+        constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in self.config_upload_formats.split(',')]
+        logfile = logger.setup(self.config_logfile, self.config_log_level)
+        if logfile != self.config_logfile:
+            log.warning("Log path %s not valid, falling back to default", self.config_logfile)
+            self.config_logfile = logfile
+            self._session.merge(s)
+            self._session.commit()
 
     def save(self):
         '''Apply all configuration values to the underlying storage.'''
         s = self._read_from_storage()  # type: _Settings
 
+        if self.config_google_drive_watch_changes_response:
+            self.config_google_drive_watch_changes_response = json.dumps(
+                self.config_google_drive_watch_changes_response)
+
         for k, v in self.__dict__.items():
             if k[0] == '_':
                 continue
@@ -330,17 +358,41 @@ def _migrate_table(session, orm_class):
     if changed:
         session.commit()
 
+
 def autodetect_calibre_binary():
     if sys.platform == "win32":
-        calibre_path = ["C:\\program files\calibre\calibre-convert.exe",
-                        "C:\\program files(x86)\calibre\calibre-convert.exe"]
+        calibre_path = ["C:\\program files\calibre\ebook-convert.exe",
+                        "C:\\program files(x86)\calibre\ebook-convert.exe",
+                        "C:\\program files(x86)\calibre2\ebook-convert.exe",
+                        "C:\\program files\calibre2\ebook-convert.exe"]
     else:
         calibre_path = ["/opt/calibre/ebook-convert"]
     for element in calibre_path:
         if os.path.isfile(element) and os.access(element, os.X_OK):
             return element
-    return None
+    return ""
 
+def autodetect_unrar_binary():
+    if sys.platform == "win32":
+        calibre_path = ["C:\\program files\\WinRar\\unRAR.exe",
+                        "C:\\program files(x86)\\WinRar\\unRAR.exe"]
+    else:
+        calibre_path = ["/usr/bin/unrar"]
+    for element in calibre_path:
+        if os.path.isfile(element) and os.access(element, os.X_OK):
+            return element
+    return ""
+
+def autodetect_kepubify_binary():
+    if sys.platform == "win32":
+        calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe",
+                        "C:\\program files(x86)\\kepubify\\kepubify-windows-64Bit.exe"]
+    else:
+        calibre_path = ["/opt/kepubify/kepubify-linux-64bit", "/opt/kepubify/kepubify-linux-32bit"]
+    for element in calibre_path:
+        if os.path.isfile(element) and os.access(element, os.X_OK):
+            return element
+    return ""
 
 def _migrate_database(session):
     # make sure the table is created, if it does not exist

cps/constants.py

@@ -81,6 +81,7 @@ SIDEBAR_PUBLISHER       = 1 << 12
 SIDEBAR_RATING          = 1 << 13
 SIDEBAR_FORMAT          = 1 << 14
 SIDEBAR_ARCHIVED        = 1 << 15
+# SIDEBAR_LIST            = 1 << 16
 
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS
 ADMIN_USER_SIDEBAR      = (SIDEBAR_ARCHIVED << 1) - 1
@@ -110,12 +111,10 @@ except ValueError:
 del env_CALIBRE_PORT
 
 
-EXTENSIONS_AUDIO    = {'mp3', 'm4a', 'm4b'}
-EXTENSIONS_CONVERT  = {'pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt'}
-EXTENSIONS_UPLOAD   = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx',
-                       'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3',  'm4a', 'm4b'}
-# EXTENSIONS_READER = set(['txt', 'pdf', 'epub', 'zip', 'cbz', 'tar', 'cbt'] +
-#                         (['rar','cbr'] if feature_support['rar'] else []))
+EXTENSIONS_AUDIO    = {'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'}
+EXTENSIONS_CONVERT  = ['pdf', 'epub', 'mobi', 'azw3', 'docx', 'rtf', 'fb2', 'lit', 'lrf', 'txt', 'htmlz', 'rtf', 'odt']
+EXTENSIONS_UPLOAD   = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz', 'cbt', 'djvu', 'prc', 'doc', 'docx',
+                       'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac'}
 
 
 def has_flag(value, bit_flag):
@@ -129,7 +128,7 @@ def selected_roles(dictionary):
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
                                   'series_id, languages')
 
-STABLE_VERSION = {'version': '0.6.7'}
+STABLE_VERSION = {'version': '0.6.8'}
 
 NIGHTLY_VERSION = {}
 NIGHTLY_VERSION[0] = '$Format:%H$'

cps/converter.py

@@ -19,6 +19,7 @@
 from __future__ import division, print_function, unicode_literals
 import os
 import re
+import sys
 from flask_babel import gettext as _
 
 from . import config, logger
@@ -29,8 +30,8 @@ log = logger.create()
 
 # _() necessary to make babel aware of string for translation
 _NOT_CONFIGURED = _('not configured')
-_NOT_INSTALLED = 'not installed'
-_EXECUTION_ERROR = 'Execution permissions missing'
+_NOT_INSTALLED = _('not installed')
+_EXECUTION_ERROR = _('Execution permissions missing')
 
 
 def _get_command_version(path, pattern, argument=None):
@@ -48,10 +49,15 @@ def _get_command_version(path, pattern, argument=None):
     return _NOT_INSTALLED
 
 
-def get_version():
-    version = None
-    if config.config_ebookconverter == 1:
-        version = _get_command_version(config.config_converterpath, r'Amazon kindlegen\(')
-    elif config.config_ebookconverter == 2:
-        version = _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version')
-    return version or _NOT_CONFIGURED
+def get_calibre_version():
+    return _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') \
+           or _NOT_CONFIGURED
+
+
+def get_unrar_version():
+    return _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') or _NOT_CONFIGURED
+
+def get_kepubify_version():
+    return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version') or _NOT_CONFIGURED
+
+

cps/db.py

@@ -22,18 +22,34 @@ import sys
 import os
 import re
 import ast
+import json
+from datetime import datetime
+import threading
 
 from sqlalchemy import create_engine
-from sqlalchemy import Table, Column, ForeignKey
-from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float, DateTime
+from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
+from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.exc import OperationalError
+from flask_login import current_user
+from sqlalchemy.sql.expression import and_, true, false, text, func, or_
+from babel import Locale as LC
+from babel.core import UnknownLocaleError
+from flask_babel import gettext as _
+
+from . import logger, ub, isoLanguages
+from .pagination import Pagination
+
+try:
+    import unidecode
+    use_unidecode = True
+except ImportError:
+    use_unidecode = False
 
 
-session = None
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
-engine = None
 
 Base = declarative_base()
 
@@ -72,9 +88,9 @@ class Identifiers(Base):
     __tablename__ = 'identifiers'
 
     id = Column(Integer, primary_key=True)
-    type = Column(String)
-    val = Column(String)
-    book = Column(Integer, ForeignKey('books.id'))
+    type = Column(String(collation='NOCASE'), nullable=False, default="isbn")
+    val = Column(String(collation='NOCASE'), nullable=False)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
 
     def __init__(self, val, id_type, book):
         self.val = val
@@ -100,14 +116,14 @@ class Identifiers(Base):
             return self.type
 
     def __repr__(self):
-        if self.type == "amazon":
+        if self.type == "amazon" or self.type == "asin":
             return u"https://amzn.com/{0}".format(self.val)
         elif self.type == "isbn":
-            return u"http://www.worldcat.org/isbn/{0}".format(self.val)
+            return u"https://www.worldcat.org/isbn/{0}".format(self.val)
         elif self.type == "doi":
-            return u"http://dx.doi.org/{0}".format(self.val)
+            return u"https://dx.doi.org/{0}".format(self.val)
         elif self.type == "goodreads":
-            return u"http://www.goodreads.com/book/show/{0}".format(self.val)
+            return u"https://www.goodreads.com/book/show/{0}".format(self.val)
         elif self.type == "douban":
             return u"https://book.douban.com/subject/{0}".format(self.val)
         elif self.type == "google":
@@ -115,7 +131,7 @@ class Identifiers(Base):
         elif self.type == "kobo":
             return u"https://www.kobo.com/ebook/{0}".format(self.val)
         elif self.type == "lubimyczytac":
-            return u" http://lubimyczytac.pl/ksiazka/{0}".format(self.val)
+            return u" https://lubimyczytac.pl/ksiazka/{0}".format(self.val)
         elif self.type == "url":
             return u"{0}".format(self.val)
         else:
@@ -126,8 +142,8 @@ class Comments(Base):
     __tablename__ = 'comments'
 
     id = Column(Integer, primary_key=True)
-    text = Column(String)
-    book = Column(Integer, ForeignKey('books.id'))
+    text = Column(String(collation='NOCASE'), nullable=False)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
 
     def __init__(self, text, book):
         self.text = text
@@ -141,7 +157,7 @@ class Tags(Base):
     __tablename__ = 'tags'
 
     id = Column(Integer, primary_key=True, autoincrement=True)
-    name = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
 
     def __init__(self, name):
         self.name = name
@@ -154,9 +170,9 @@ class Authors(Base):
     __tablename__ = 'authors'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
-    link = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
+    sort = Column(String(collation='NOCASE'))
+    link = Column(String, nullable=False, default="")
 
     def __init__(self, name, sort, link):
         self.name = name
@@ -171,8 +187,8 @@ class Series(Base):
     __tablename__ = 'series'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
+    name = Column(String(collation='NOCASE'), unique=True, nullable=False)
+    sort = Column(String(collation='NOCASE'))
 
     def __init__(self, name, sort):
         self.name = name
@@ -186,7 +202,7 @@ class Ratings(Base):
     __tablename__ = 'ratings'
 
     id = Column(Integer, primary_key=True)
-    rating = Column(Integer)
+    rating = Column(Integer, CheckConstraint('rating>-1 AND rating<11'), unique=True)
 
     def __init__(self, rating):
         self.rating = rating
@@ -199,7 +215,7 @@ class Languages(Base):
     __tablename__ = 'languages'
 
     id = Column(Integer, primary_key=True)
-    lang_code = Column(String)
+    lang_code = Column(String(collation='NOCASE'), nullable=False, unique=True)
 
     def __init__(self, lang_code):
         self.lang_code = lang_code
@@ -212,8 +228,8 @@ class Publishers(Base):
     __tablename__ = 'publishers'
 
     id = Column(Integer, primary_key=True)
-    name = Column(String)
-    sort = Column(String)
+    name = Column(String(collation='NOCASE'), nullable=False, unique=True)
+    sort = Column(String(collation='NOCASE'))
 
     def __init__(self, name, sort):
         self.name = name
@@ -225,12 +241,13 @@ class Publishers(Base):
 
 class Data(Base):
     __tablename__ = 'data'
+    __table_args__ = {'schema':'calibre'}
 
     id = Column(Integer, primary_key=True)
-    book = Column(Integer, ForeignKey('books.id'))
-    format = Column(String)
-    uncompressed_size = Column(Integer)
-    name = Column(String)
+    book = Column(Integer, ForeignKey('books.id'), nullable=False)
+    format = Column(String(collation='NOCASE'), nullable=False)
+    uncompressed_size = Column(Integer, nullable=False)
+    name = Column(String, nullable=False)
 
     def __init__(self, book, book_format, uncompressed_size, name):
         self.book = book
@@ -247,17 +264,20 @@ class Books(Base):
 
     DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00"
 
-    id = Column(Integer, primary_key=True)
-    title = Column(String)
-    sort = Column(String)
-    author_sort = Column(String)
-    timestamp = Column(TIMESTAMP)
-    pubdate = Column(String)
-    series_index = Column(String)
-    last_modified = Column(TIMESTAMP)
-    path = Column(String)
-    has_cover = Column(Integer)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    title = Column(String(collation='NOCASE'), nullable=False, default='Unknown')
+    sort = Column(String(collation='NOCASE'))
+    author_sort = Column(String(collation='NOCASE'))
+    timestamp = Column(TIMESTAMP, default=datetime.utcnow)
+    pubdate = Column(String) # , default=datetime.utcnow)
+    series_index = Column(String, nullable=False, default="1.0")
+    last_modified = Column(TIMESTAMP, default=datetime.utcnow)
+    path = Column(String, default="", nullable=False)
+    has_cover = Column(Integer, default=0)
     uuid = Column(String)
+    isbn = Column(String(collation='NOCASE'), default="")
+    # Iccn = Column(String(collation='NOCASE'), default="")
+    flags = Column(Integer, nullable=False, default=1)
 
     authors = relationship('Authors', secondary=books_authors_link, backref='books')
     tags = relationship('Tags', secondary=books_tags_link, backref='books',order_by="Tags.name")
@@ -310,131 +330,323 @@ class Custom_Columns(Base):
         return display_dict
 
 
-def update_title_sort(config, conn=None):
-    # user defined sort function for calibre databases (Series, etc.)
-    def _title_sort(title):
-        # calibre sort stuff
-        title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
-        match = title_pat.search(title)
-        if match:
-            prep = match.group(1)
-            title = title.replace(prep, '') + ', ' + prep
-        return title.strip()
+class CalibreDB(threading.Thread):
 
-    conn = conn or session.connection().connection.connection
-    conn.create_function("title_sort", 1, _title_sort)
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.engine = None
+        self.session = None
+        self.queue = None
+        self.log = None
+        self.config = None
+
+    def add_queue(self,queue):
+        self.queue = queue
+        self.log = logger.create()
+
+    def run(self):
+        while True:
+            i = self.queue.get()
+            if i == 'dummy':
+                self.queue.task_done()
+                break
+            if i['task'] == 'add_format':
+                cur_book = self.session.query(Books).filter(Books.id == i['id']).first()
+                cur_book.data.append(i['format'])
+                try:
+                    # db.session.merge(cur_book)
+                    self.session.commit()
+                except OperationalError as e:
+                    self.session.rollback()
+                    self.log.error("Database error: %s", e)
+                    # self._handleError(_(u"Database error: %(error)s.", error=e))
+                    # return
+            self.queue.task_done()
 
 
-def setup_db(config):
-    dispose()
-    global engine
+    def stop(self):
+        self.queue.put('dummy')
 
-    if not config.config_calibre_dir:
-        config.invalidate()
-        return False
+    def setup_db(self, config, app_db_path):
+        self.config = config
+        self.dispose()
+        # global engine
 
-    dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
-    if not os.path.exists(dbpath):
-        config.invalidate()
-        return False
+        if not config.config_calibre_dir:
+            config.invalidate()
+            return False
 
-    try:
-        engine = create_engine('sqlite:///{0}'.format(dbpath),
-                               echo=False,
-                               isolation_level="SERIALIZABLE",
-                               connect_args={'check_same_thread': False})
-        conn = engine.connect()
-        # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
-    except Exception as e:
-        config.invalidate(e)
-        return False
+        dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+        if not os.path.exists(dbpath):
+            config.invalidate()
+            return False
 
-    config.db_configured = True
-    update_title_sort(config, conn.connection)
+        try:
+            #engine = create_engine('sqlite:///{0}'.format(dbpath),
+            self.engine = create_engine('sqlite://',
+                                   echo=False,
+                                   isolation_level="SERIALIZABLE",
+                                   connect_args={'check_same_thread': False})
+            self.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
 
-    if not cc_classes:
-        cc = conn.execute("SELECT id, datatype FROM custom_columns")
+            conn = self.engine.connect()
+            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
+        except Exception as e:
+            config.invalidate(e)
+            return False
 
-        cc_ids = []
-        books_custom_column_links = {}
-        for row in cc:
-            if row.datatype not in cc_exceptions:
-                books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
-                                                          Column('book', Integer, ForeignKey('books.id'),
-                                                                 primary_key=True),
-                                                          Column('value', Integer,
-                                                                 ForeignKey('custom_column_' + str(row.id) + '.id'),
-                                                                 primary_key=True)
-                                                          )
-                cc_ids.append([row.id, row.datatype])
-                if row.datatype == 'bool':
-                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                              'id': Column(Integer, primary_key=True),
-                              'book': Column(Integer, ForeignKey('books.id')),
-                              'value': Column(Boolean)}
-                elif row.datatype == 'int':
-                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                              'id': Column(Integer, primary_key=True),
-                              'book': Column(Integer, ForeignKey('books.id')),
-                              'value': Column(Integer)}
-                elif row.datatype == 'float':
-                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                              'id': Column(Integer, primary_key=True),
-                              'book': Column(Integer, ForeignKey('books.id')),
-                              'value': Column(Float)}
+        config.db_configured = True
+        self.update_title_sort(config, conn.connection)
+
+        if not cc_classes:
+            cc = conn.execute("SELECT id, datatype FROM custom_columns")
+
+            cc_ids = []
+            books_custom_column_links = {}
+            for row in cc:
+                if row.datatype not in cc_exceptions:
+                    books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
+                                                              Column('book', Integer, ForeignKey('books.id'),
+                                                                     primary_key=True),
+                                                              Column('value', Integer,
+                                                                     ForeignKey('custom_column_' + str(row.id) + '.id'),
+                                                                     primary_key=True)
+                                                              )
+                    cc_ids.append([row.id, row.datatype])
+                    if row.datatype == 'bool':
+                        ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                                  'id': Column(Integer, primary_key=True),
+                                  'book': Column(Integer, ForeignKey('books.id')),
+                                  'value': Column(Boolean)}
+                    elif row.datatype == 'int':
+                        ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                                  'id': Column(Integer, primary_key=True),
+                                  'book': Column(Integer, ForeignKey('books.id')),
+                                  'value': Column(Integer)}
+                    elif row.datatype == 'float':
+                        ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                                  'id': Column(Integer, primary_key=True),
+                                  'book': Column(Integer, ForeignKey('books.id')),
+                                  'value': Column(Float)}
+                    else:
+                        ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                                  'id': Column(Integer, primary_key=True),
+                                  'value': Column(String)}
+                    cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict)
+
+            for cc_id in cc_ids:
+                if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'):
+                    setattr(Books,
+                            'custom_column_' + str(cc_id[0]),
+                            relationship(cc_classes[cc_id[0]],
+                                         primaryjoin=(
+                                         Books.id == cc_classes[cc_id[0]].book),
+                                         backref='books'))
                 else:
-                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                              'id': Column(Integer, primary_key=True),
-                              'value': Column(String)}
-                cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict)
+                    setattr(Books,
+                            'custom_column_' + str(cc_id[0]),
+                            relationship(cc_classes[cc_id[0]],
+                                         secondary=books_custom_column_links[cc_id[0]],
+                                         backref='books'))
 
-        for cc_id in cc_ids:
-            if (cc_id[1] == 'bool') or (cc_id[1] == 'int') or (cc_id[1] == 'float'):
-                setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]],
-                                                                           primaryjoin=(
-                                                                           Books.id == cc_classes[cc_id[0]].book),
-                                                                           backref='books'))
-            else:
-                setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]],
-                                                                           secondary=books_custom_column_links[cc_id[0]],
-                                                                           backref='books'))
+        Session = scoped_session(sessionmaker(autocommit=False,
+                                              autoflush=False,
+                                              bind=self.engine))
+        self.session = Session()
+        return True
 
+    def get_book(self, book_id):
+        return self.session.query(Books).filter(Books.id == book_id).first()
 
-    global session
-    Session = scoped_session(sessionmaker(autocommit=False,
-                                             autoflush=False,
-                                             bind=engine))
-    session = Session()
-    return True
+    def get_filtered_book(self, book_id, allow_show_archived=False):
+        return self.session.query(Books).filter(Books.id == book_id).\
+            filter(self.common_filters(allow_show_archived)).first()
 
+    def get_book_by_uuid(self, book_uuid):
+        return self.session.query(Books).filter(Books.uuid == book_uuid).first()
 
-def dispose():
-    global session
+    def get_book_format(self, book_id, format):
+        return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first()
 
-    old_session = session
-    session = None
-    if old_session:
-        try: old_session.close()
-        except: pass
-        if old_session.bind:
-            try: old_session.bind.dispose()
-            except Exception: pass
+    # Language and content filters for displaying in the UI
+    def common_filters(self, allow_show_archived=False):
+        if not allow_show_archived:
+            archived_books = (
+                ub.session.query(ub.ArchivedBook)
+                    .filter(ub.ArchivedBook.user_id == int(current_user.id))
+                    .filter(ub.ArchivedBook.is_archived == True)
+                    .all()
+            )
+            archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+            archived_filter = Books.id.notin_(archived_book_ids)
+        else:
+            archived_filter = true()
 
-    for attr in list(Books.__dict__.keys()):
-        if attr.startswith("custom_column_"):
-            setattr(Books, attr, None)
+        if current_user.filter_language() != "all":
+            lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language())
+        else:
+            lang_filter = true()
+        negtags_list = current_user.list_denied_tags()
+        postags_list = current_user.list_allowed_tags()
+        neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list))
+        pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list))
+        if self.config.config_restricted_column:
+            pos_cc_list = current_user.allowed_column_value.split(',')
+            pos_content_cc_filter = true() if pos_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
+            neg_cc_list = current_user.denied_column_value.split(',')
+            neg_content_cc_filter = false() if neg_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
+        else:
+            pos_content_cc_filter = true()
+            neg_content_cc_filter = false()
+        return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
+                    pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
 
-    for db_class in cc_classes.values():
-        Base.metadata.remove(db_class.__table__)
-    cc_classes.clear()
+    # Fill indexpage with all requested data from database
+    def fill_indexpage(self, page, database, db_filter, order, *join):
+        return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
 
-    for table in reversed(Base.metadata.sorted_tables):
-        name = table.key
-        if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
-            if table is not None:
-                Base.metadata.remove(table)
+    def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join):
+        if current_user.show_detail_random():
+            randm = self.session.query(Books) \
+                .filter(self.common_filters(allow_show_archived)) \
+                .order_by(func.random()) \
+                .limit(self.config.config_random_books)
+        else:
+            randm = false()
+        off = int(int(self.config.config_books_per_page) * (page - 1))
+        query = self.session.query(database) \
+            .join(*join, isouter=True) \
+            .filter(db_filter) \
+            .filter(self.common_filters(allow_show_archived))
+        pagination = Pagination(page, self.config.config_books_per_page,
+                                len(query.all()))
+        entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all()
+        for book in entries:
+            book = self.order_authors(book)
+        return entries, randm, pagination
 
-def reconnect_db(config):
-    session.close()
-    engine.dispose()
-    setup_db(config)
+    # Orders all Authors in the list according to authors sort
+    def order_authors(self, entry):
+        sort_authors = entry.author_sort.split('&')
+        authors_ordered = list()
+        error = False
+        for auth in sort_authors:
+            # ToDo: How to handle not found authorname
+            result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first()
+            if not result:
+                error = True
+                break
+            authors_ordered.append(result)
+        if not error:
+            entry.authors = authors_ordered
+        return entry
+
+    def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()):
+        query = query or ''
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        entries = self.session.query(database).filter(tag_filter). \
+            filter(func.lower(database.name).ilike("%" + query + "%")).all()
+        json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
+        return json_dumps
+
+    def check_exists_book(self, authr, title):
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        q = list()
+        authorterms = re.split(r'\s*&\s*', authr)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+
+        return self.session.query(Books)\
+            .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
+
+    # read search results from calibre-database and return it (function is used for feed and simple search
+    def get_search_results(self, term):
+        term.strip().lower()
+        self.session.connection().connection.connection.create_function("lower", 1, lcase)
+        q = list()
+        authorterms = re.split("[, ]+", term)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+        return self.session.query(Books).filter(self.common_filters(True)).filter(
+            or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
+                Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
+                Books.authors.any(and_(*q)),
+                Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
+                func.lower(Books.title).ilike("%" + term + "%")
+                )).order_by(Books.sort).all()
+
+    # Creates for all stored languages a translated speaking name in the array for the UI
+    def speaking_language(self, languages=None):
+        from . import get_locale
+
+        if not languages:
+            languages = self.session.query(Languages) \
+                .join(books_languages_link) \
+                .join(Books) \
+                .filter(self.common_filters()) \
+                .group_by(text('books_languages_link.lang_code')).all()
+        for lang in languages:
+            try:
+                cur_l = LC.parse(lang.lang_code)
+                lang.name = cur_l.get_language_name(get_locale())
+            except UnknownLocaleError:
+                lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
+        return languages
+
+    def update_title_sort(self, config, conn=None):
+        # user defined sort function for calibre databases (Series, etc.)
+        def _title_sort(title):
+            # calibre sort stuff
+            title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
+            match = title_pat.search(title)
+            if match:
+                prep = match.group(1)
+                title = title[len(prep):] + ', ' + prep
+            return title.strip()
+
+        conn = conn or self.session.connection().connection.connection
+        conn.create_function("title_sort", 1, _title_sort)
+
+    def dispose(self):
+        # global session
+
+        old_session = self.session
+        self.session = None
+        if old_session:
+            try: old_session.close()
+            except: pass
+            if old_session.bind:
+                try: old_session.bind.dispose()
+                except Exception: pass
+
+        for attr in list(Books.__dict__.keys()):
+            if attr.startswith("custom_column_"):
+                setattr(Books, attr, None)
+
+        for db_class in cc_classes.values():
+            Base.metadata.remove(db_class.__table__)
+        cc_classes.clear()
+
+        for table in reversed(Base.metadata.sorted_tables):
+            name = table.key
+            if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
+                if table is not None:
+                    Base.metadata.remove(table)
+
+    def reconnect_db(self, config, app_db_path):
+        self.session.close()
+        self.engine.dispose()
+        self.setup_db(config, app_db_path)
+
+def lcase(s):
+    try:
+        return unidecode.unidecode(s.lower())
+    except Exception as e:
+        log = logger.create()
+        log.exception(e)
+        return s.lower()

cps/editbooks.py

@@ -24,16 +24,17 @@ from __future__ import division, print_function, unicode_literals
 import os
 from datetime import datetime
 import json
-from shutil import move, copyfile
+from shutil import copyfile
 from uuid import uuid4
 
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
 from flask_login import current_user, login_required
+from sqlalchemy.exc import OperationalError
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
-from . import config, get_locale, db, ub, worker
-from .helper import order_authors, common_filters
+from . import config, get_locale, ub, worker, db
+from . import calibre_db
 from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
 
 
@@ -174,13 +175,15 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
 @login_required
 def delete_book(book_id, book_format):
     if current_user.role_delete_books():
-        book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+        book = calibre_db.get_book(book_id)
         if book:
             try:
                 result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
                 if not result:
                     flash(error, category="error")
                     return redirect(url_for('editbook.edit_book', book_id=book_id))
+                if error:
+                    flash(error, category="warning")
                 if not book_format:
                     # delete book from Shelfs, Downloads, Read list
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
@@ -190,13 +193,13 @@ def delete_book(book_id, book_format):
 
                     # check if only this book links to:
                     # author, language, series, tags, custom columns
-                    modify_database_object([u''], book.authors, db.Authors, db.session, 'author')
-                    modify_database_object([u''], book.tags, db.Tags, db.session, 'tags')
-                    modify_database_object([u''], book.series, db.Series, db.session, 'series')
-                    modify_database_object([u''], book.languages, db.Languages, db.session, 'languages')
-                    modify_database_object([u''], book.publishers, db.Publishers, db.session, 'publishers')
+                    modify_database_object([u''], book.authors, db.Authors, calibre_db.session, 'author')
+                    modify_database_object([u''], book.tags, db.Tags, calibre_db.session, 'tags')
+                    modify_database_object([u''], book.series, db.Series, calibre_db.session, 'series')
+                    modify_database_object([u''], book.languages, db.Languages, calibre_db.session, 'languages')
+                    modify_database_object([u''], book.publishers, db.Publishers, calibre_db.session, 'publishers')
 
-                    cc = db.session.query(db.Custom_Columns).\
+                    cc = calibre_db.session.query(db.Custom_Columns).\
                         filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
                     for c in cc:
                         cc_string = "custom_column_" + str(c.id)
@@ -206,32 +209,32 @@ def delete_book(book_id, book_format):
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     log.debug('remove ' + str(c.id))
-                                    db.session.delete(del_cc)
-                                    db.session.commit()
+                                    calibre_db.session.delete(del_cc)
+                                    calibre_db.session.commit()
                                 elif c.datatype == 'rating':
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     if len(del_cc.books) == 0:
                                         log.debug('remove ' + str(c.id))
-                                        db.session.delete(del_cc)
-                                        db.session.commit()
+                                        calibre_db.session.delete(del_cc)
+                                        calibre_db.session.commit()
                                 else:
                                     del_cc = getattr(book, cc_string)[0]
                                     getattr(book, cc_string).remove(del_cc)
                                     log.debug('remove ' + str(c.id))
-                                    db.session.delete(del_cc)
-                                    db.session.commit()
+                                    calibre_db.session.delete(del_cc)
+                                    calibre_db.session.commit()
                         else:
                             modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id],
-                                                   db.session, 'custom')
-                    db.session.query(db.Books).filter(db.Books.id == book_id).delete()
+                                                   calibre_db.session, 'custom')
+                    calibre_db.session.query(db.Books).filter(db.Books.id == book_id).delete()
                 else:
-                    db.session.query(db.Data).filter(db.Data.book == book.id).\
+                    calibre_db.session.query(db.Data).filter(db.Data.book == book.id).\
                         filter(db.Data.format == book_format).delete()
-                db.session.commit()
+                calibre_db.session.commit()
             except Exception as e:
                 log.debug(e)
-                db.session.rollback()
+                calibre_db.session.rollback()
         else:
             # book not found
             log.error('Book with id "%s" could not be deleted: not found', book_id)
@@ -244,11 +247,9 @@ def delete_book(book_id, book_format):
 
 
 def render_edit_book(book_id):
-    db.update_title_sort(config)
-    cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    book = db.session.query(db.Books)\
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
-
+    calibre_db.update_title_sort(config)
+    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    book = calibre_db.get_filtered_book(book_id)
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
@@ -256,7 +257,7 @@ def render_edit_book(book_id):
     for lang in book.languages:
         lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
 
-    book = order_authors(book)
+    book = calibre_db.order_authors(book)
 
     author_names = []
     for authr in book.authors:
@@ -264,19 +265,25 @@ def render_edit_book(book_id):
 
     # Option for showing convertbook button
     valid_source_formats=list()
-    if config.config_ebookconverter == 2:
+    allowed_conversion_formats = list()
+    kepub_possible=None
+    if config.config_converterpath:
         for file in book.data:
             if file.format.lower() in constants.EXTENSIONS_CONVERT:
                 valid_source_formats.append(file.format.lower())
+    if config.config_kepubifypath and 'epub' in [file.format.lower() for file in book.data]:
+        kepub_possible = True
+        if not config.config_converterpath:
+            valid_source_formats.append('epub')
 
     # Determine what formats don't already exist
-    allowed_conversion_formats = constants.EXTENSIONS_CONVERT.copy()
-    for file in book.data:
-        try:
-            allowed_conversion_formats.remove(file.format.lower())
-        except Exception:
-            log.warning('%s already removed from list.', file.format.lower())
-
+    if config.config_converterpath:
+        allowed_conversion_formats = constants.EXTENSIONS_CONVERT[:]
+        for file in book.data:
+            if file.format.lower() in allowed_conversion_formats:
+                allowed_conversion_formats.remove(file.format.lower())
+    if kepub_possible:
+        allowed_conversion_formats.append('kepub')
     return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
                                  title=_(u"edit metadata"), page="editbook",
                                  conversion_formats=allowed_conversion_formats,
@@ -293,7 +300,7 @@ def edit_book_ratings(to_save, book):
         ratingx2 = int(float(to_save["rating"]) * 2)
         if ratingx2 != old_rating:
             changed = True
-            is_rating = db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first()
+            is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == ratingx2).first()
             if is_rating:
                 book.ratings.append(is_rating)
             else:
@@ -307,15 +314,59 @@ def edit_book_ratings(to_save, book):
             changed = True
     return changed
 
+def edit_book_tags(tags, book):
+    input_tags = tags.split(',')
+    input_tags = list(map(lambda it: it.strip(), input_tags))
+    # if input_tags[0] !="": ??
+    return modify_database_object(input_tags, book.tags, db.Tags, calibre_db.session, 'tags')
 
-def edit_book_languages(to_save, book):
-    input_languages = to_save["languages"].split(',')
+
+def edit_book_series(series, book):
+    input_series = [series.strip()]
+    input_series = [x for x in input_series if x != '']
+    return modify_database_object(input_series, book.series, db.Series, calibre_db.session, 'series')
+
+
+def edit_book_series_index(series_index, book):
+    # Add default series_index to book
+    modif_date = False
+    series_index = series_index or '1'
+    #if series_index == '':
+    #    series_index = '1'
+    if book.series_index != series_index:
+        book.series_index = series_index
+        modif_date = True
+    return modif_date
+
+# Handle book comments/description
+def edit_book_comments(comments, book):
+    modif_date = False
+    if len(book.comments):
+        if book.comments[0].text != comments:
+            book.comments[0].text = comments
+            modif_date = True
+    else:
+        if comments:
+            book.comments.append(db.Comments(text=comments, book=book.id))
+            modif_date = True
+    return modif_date
+
+
+def edit_book_languages(languages, book, upload=False):
+    input_languages = languages.split(',')
     unknown_languages = []
     input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages)
     for l in unknown_languages:
         log.error('%s is not a valid language', l)
-        flash(_(u"%(langname)s is not a valid language", langname=l), category="error")
-    return modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages')
+        flash(_(u"%(langname)s is not a valid language", langname=l), category="warning")
+    # ToDo: Not working correct
+    if upload and len(input_l) == 1:
+        # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
+        # the book it's language is set to the filter language
+        if input_l[0] != current_user.filter_language() and current_user.filter_language() != "all":
+            input_l[0] = calibre_db.session.query(db.Languages). \
+                filter(db.Languages.lang_code == current_user.filter_language()).first()
+    return modify_database_object(input_l, book.languages, db.Languages, calibre_db.session, 'languages')
 
 
 def edit_book_publisher(to_save, book):
@@ -323,15 +374,15 @@ def edit_book_publisher(to_save, book):
     if to_save["publisher"]:
         publisher = to_save["publisher"].rstrip().strip()
         if len(book.publishers) == 0 or (len(book.publishers) > 0 and publisher != book.publishers[0].name):
-            changed |= modify_database_object([publisher], book.publishers, db.Publishers, db.session, 'publisher')
+            changed |= modify_database_object([publisher], book.publishers, db.Publishers, calibre_db.session, 'publisher')
     elif len(book.publishers):
-        changed |= modify_database_object([], book.publishers, db.Publishers, db.session, 'publisher')
+        changed |= modify_database_object([], book.publishers, db.Publishers, calibre_db.session, 'publisher')
     return changed
 
 
 def edit_cc_data(book_id, book, to_save):
     changed = False
-    cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     for c in cc:
         cc_string = "custom_column_" + str(c.id)
         if not c.is_multiple:
@@ -354,12 +405,12 @@ def edit_cc_data(book_id, book, to_save):
                             else:
                                 del_cc = getattr(book, cc_string)[0]
                                 getattr(book, cc_string).remove(del_cc)
-                                db.session.delete(del_cc)
+                                calibre_db.session.delete(del_cc)
                                 changed = True
                         else:
                             cc_class = db.cc_classes[c.id]
                             new_cc = cc_class(value=to_save[cc_string], book=book_id)
-                            db.session.add(new_cc)
+                            calibre_db.session.add(new_cc)
                             changed = True
 
                 else:
@@ -371,18 +422,18 @@ def edit_cc_data(book_id, book, to_save):
                             del_cc = getattr(book, cc_string)[0]
                             getattr(book, cc_string).remove(del_cc)
                             if len(del_cc.books) == 0:
-                                db.session.delete(del_cc)
+                                calibre_db.session.delete(del_cc)
                                 changed = True
                         cc_class = db.cc_classes[c.id]
-                        new_cc = db.session.query(cc_class).filter(
+                        new_cc = calibre_db.session.query(cc_class).filter(
                             cc_class.value == to_save[cc_string].strip()).first()
                         # if no cc val is found add it
                         if new_cc is None:
                             new_cc = cc_class(value=to_save[cc_string].strip())
-                            db.session.add(new_cc)
+                            calibre_db.session.add(new_cc)
                             changed = True
-                            db.session.flush()
-                            new_cc = db.session.query(cc_class).filter(
+                            calibre_db.session.flush()
+                            new_cc = calibre_db.session.query(cc_class).filter(
                                 cc_class.value == to_save[cc_string].strip()).first()
                         # add cc value to book
                         getattr(book, cc_string).append(new_cc)
@@ -392,13 +443,16 @@ def edit_cc_data(book_id, book, to_save):
                     del_cc = getattr(book, cc_string)[0]
                     getattr(book, cc_string).remove(del_cc)
                     if not del_cc.books or len(del_cc.books) == 0:
-                        db.session.delete(del_cc)
+                        calibre_db.session.delete(del_cc)
                         changed = True
         else:
             input_tags = to_save[cc_string].split(',')
             input_tags = list(map(lambda it: it.strip(), input_tags))
-            changed |= modify_database_object(input_tags, getattr(book, cc_string), db.cc_classes[c.id], db.session,
-                                   'custom')
+            changed |= modify_database_object(input_tags,
+                                              getattr(book, cc_string),
+                                              db.cc_classes[c.id],
+                                              calibre_db.session,
+                                              'custom')
     return changed
 
 def upload_single_file(request, book, book_id):
@@ -435,17 +489,22 @@ def upload_single_file(request, book, book_id):
                 return redirect(url_for('web.show_book', book_id=book.id))
 
             file_size = os.path.getsize(saved_filename)
-            is_format = db.session.query(db.Data).filter(db.Data.book == book_id).\
-                filter(db.Data.format == file_ext.upper()).first()
+            is_format = calibre_db.get_book_format(book_id, file_ext.upper())
 
             # Format entry already exists, no need to update the database
             if is_format:
                 log.warning('Book format %s already existing', file_ext.upper())
             else:
-                db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
-                db.session.add(db_format)
-                db.session.commit()
-                db.update_title_sort(config)
+                try:
+                    db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
+                    calibre_db.session.add(db_format)
+                    calibre_db.session.commit()
+                    calibre_db.update_title_sort(config)
+                except OperationalError as e:
+                    calibre_db.session.rollback()
+                    log.error('Database error: %s', e)
+                    flash(_(u"Database error: %(error)s.", error=e), category="error")
+                    return redirect(url_for('web.show_book', book_id=book.id))
 
             # Queue uploader info
             uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
@@ -453,7 +512,8 @@ def upload_single_file(request, book, book_id):
                 "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
 
             return uploader.process(
-                saved_filename, *os.path.splitext(requested_file.filename), config.config_rarfile_location)
+                saved_filename, *os.path.splitext(requested_file.filename),
+                rarExecutable=config.config_rarfile_location)
 
 
 def upload_cover(request, book):
@@ -480,9 +540,8 @@ def edit_book(book_id):
         return render_edit_book(book_id)
 
     # create the function for sorting...
-    db.update_title_sort(config)
-    book = db.session.query(db.Books)\
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
+    calibre_db.update_title_sort(config)
+    book = calibre_db.get_filtered_book(book_id)
 
     # Book not found
     if not book:
@@ -513,13 +572,13 @@ def edit_book(book_id):
         if input_authors == ['']:
             input_authors = [_(u'Unknown')]  # prevent empty Author
 
-        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author')
+        modif_date |= modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
 
         # Search for each author if author is in database, if not, authorname and sorted authorname is generated new
         # everything then is assembled for sorted author field in database
         sort_authors_list = list()
         for inp in input_authors:
-            stored_author = db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
             if not stored_author:
                 stored_author = helper.get_sorted_author(inp)
             else:
@@ -547,33 +606,21 @@ def edit_book(book_id):
                 else:
                     flash(error, category="error")
 
-            if book.series_index != to_save["series_index"]:
-                book.series_index = to_save["series_index"]
-                modif_date = True
+            # Add default series_index to book
+            modif_date |= edit_book_series_index(to_save["series_index"], book)
 
             # Handle book comments/description
-            if len(book.comments):
-                if book.comments[0].text != to_save["description"]:
-                    book.comments[0].text = to_save["description"]
-                    modif_date = True
-            else:
-                if to_save["description"]:
-                    book.comments.append(db.Comments(text=to_save["description"], book=book.id))
-                    modif_date = True
+            modif_date |= edit_book_comments(to_save["description"], book)
 
                     # Handle identifiers
             input_identifiers = identifier_list(to_save, book)
-            modif_date |= modify_identifiers(input_identifiers, book.identifiers, db.session)
+            modif_date |= modify_identifiers(input_identifiers, book.identifiers, calibre_db.session)
 
             # Handle book tags
-            input_tags = to_save["tags"].split(',')
-            input_tags = list(map(lambda it: it.strip(), input_tags))
-            modif_date |= modify_database_object(input_tags, book.tags, db.Tags, db.session, 'tags')
+            modif_date |= edit_book_tags(to_save['tags'], book)
 
             # Handle book series
-            input_series = [to_save["series"].strip()]
-            input_series = [x for x in input_series if x != '']
-            modif_date |= modify_database_object(input_series, book.series, db.Series, db.session, 'series')
+            modif_date |= edit_book_series(to_save["series"], book)
 
             if to_save["pubdate"]:
                 try:
@@ -587,7 +634,7 @@ def edit_book(book_id):
             modif_date |= edit_book_publisher(to_save, book)
 
             # handle book languages
-            modif_date |= edit_book_languages(to_save, book)
+            modif_date |= edit_book_languages(to_save['languages'], book)
 
             # handle book ratings
             modif_date |= edit_book_ratings(to_save, book)
@@ -597,7 +644,7 @@ def edit_book(book_id):
 
             if modif_date:
                 book.last_modified = datetime.utcnow()
-            db.session.commit()
+            calibre_db.session.commit()
             if config.config_use_google_drive:
                 gdriveutils.updateGdriveCalibreFromLocal()
             if "detail_view" in to_save:
@@ -606,12 +653,12 @@ def edit_book(book_id):
                 flash(_("Metadata successfully updated"), category="success")
                 return render_edit_book(book_id)
         else:
-            db.session.rollback()
+            calibre_db.session.rollback()
             flash(error, category="error")
             return render_edit_book(book_id)
     except Exception as e:
         log.exception(e)
-        db.session.rollback()
+        calibre_db.session.rollback()
         flash(_("Error editing book, please check logfile for details"), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
 
@@ -651,179 +698,172 @@ def upload():
         abort(404)
     if request.method == 'POST' and 'btn-upload' in request.files:
         for requested_file in request.files.getlist("btn-upload"):
-            # create the function for sorting...
-            db.update_title_sort(config)
-            db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
-
-            # check if file extension is correct
-            if '.' in requested_file.filename:
-                file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
-                if file_ext not in constants.EXTENSIONS_UPLOAD:
-                    flash(
-                        _("File extension '%(ext)s' is not allowed to be uploaded to this server",
-                          ext=file_ext), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-            else:
-                flash(_('File to be uploaded must have an extension'), category="error")
-                return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-
-            # extract metadata from file
             try:
-                meta = uploader.upload(requested_file, config.config_rarfile_location)
-            except (IOError, OSError):
-                log.error("File %s could not saved to temp dir", requested_file.filename)
-                flash(_(u"File %(filename)s could not saved to temp dir",
-                        filename= requested_file.filename), category="error")
-                return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-            title = meta.title
-            authr = meta.author
-            tags = meta.tags
-            series = meta.series
-            series_index = meta.series_id
-            title_dir = helper.get_valid_filename(title)
-            author_dir = helper.get_valid_filename(authr)
-            filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir)
-            saved_filename = os.path.join(filepath, title_dir + meta.extension.lower())
+                modif_date = False
+                # create the function for sorting...
+                calibre_db.update_title_sort(config)
+                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
 
-            if title != _(u'Unknown') and authr != _(u'Unknown'):
-                entry = helper.check_exists_book(authr, title)
-                if entry:
-                    log.info("Uploaded book probably exists in library")
-                    flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
-                        + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
-
-            # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
-            if not os.path.exists(filepath):
-                try:
-                    os.makedirs(filepath)
-                except OSError:
-                    log.error("Failed to create path %s (Permission denied)", filepath)
-                    flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
+                # check if file extension is correct
+                if '.' in requested_file.filename:
+                    file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
+                    if file_ext not in constants.EXTENSIONS_UPLOAD:
+                        flash(
+                            _("File extension '%(ext)s' is not allowed to be uploaded to this server",
+                              ext=file_ext), category="error")
+                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+                else:
+                    flash(_('File to be uploaded must have an extension'), category="error")
                     return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-            try:
-                copyfile(meta.file_path, saved_filename)
-                os.unlink(meta.file_path)
-            except OSError as e:
-                log.error("Failed to move file %s: %s", saved_filename, e)
-                flash(_(u"Failed to Move File %(file)s: %(error)s", file=saved_filename, error=e), category="error")
-                return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
-            if meta.cover is None:
-                has_cover = 0
-                copyfile(os.path.join(constants.STATIC_DIR, 'generic_cover.jpg'),
-                         os.path.join(filepath, "cover.jpg"))
-            else:
-                has_cover = 1
+                # extract metadata from file
                 try:
-                    copyfile(meta.cover, os.path.join(filepath, "cover.jpg"))
-                    os.unlink(meta.cover)
+                    meta = uploader.upload(requested_file, config.config_rarfile_location)
+                except (IOError, OSError):
+                    log.error("File %s could not saved to temp dir", requested_file.filename)
+                    flash(_(u"File %(filename)s could not saved to temp dir",
+                            filename= requested_file.filename), category="error")
+                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+                title = meta.title
+                authr = meta.author
+
+                if title != _(u'Unknown') and authr != _(u'Unknown'):
+                    entry = calibre_db.check_exists_book(authr, title)
+                    if entry:
+                        log.info("Uploaded book probably exists in library")
+                        flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
+                            + Markup(render_title_template('book_exists_flash.html', entry=entry)), category="warning")
+
+                # handle authors
+                input_authors = authr.split('&')
+                # handle_authors(input_authors)
+                input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
+                # we have all author names now
+                if input_authors == ['']:
+                    input_authors = [_(u'Unknown')]  # prevent empty Author
+
+                sort_authors_list=list()
+                db_author = None
+                for inp in input_authors:
+                    stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+                    if not stored_author:
+                        if not db_author:
+                            db_author = db.Authors(inp, helper.get_sorted_author(inp), "")
+                            calibre_db.session.add(db_author)
+                            calibre_db.session.commit()
+                        sort_author = helper.get_sorted_author(inp)
+                    else:
+                        if not db_author:
+                            db_author = stored_author
+                        sort_author = stored_author.sort
+                    sort_authors_list.append(sort_author) # helper.get_sorted_author(sort_author))
+                sort_authors = ' & '.join(sort_authors_list)
+
+                title_dir = helper.get_valid_filename(title)
+                author_dir = helper.get_valid_filename(db_author.name)
+                filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir)
+                saved_filename = os.path.join(filepath, title_dir + meta.extension.lower())
+
+                # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
+                if not os.path.exists(filepath):
+                    try:
+                        os.makedirs(filepath)
+                    except OSError:
+                        log.error("Failed to create path %s (Permission denied)", filepath)
+                        flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
+                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+                try:
+                    copyfile(meta.file_path, saved_filename)
+                    os.unlink(meta.file_path)
                 except OSError as e:
-                    log.error("Failed to move cover file %s: %s", os.path.join(filepath, "cover.jpg"), e)
-                    flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=os.path.join(filepath, "cover.jpg"),
-                            error=e),
-                          category="error")
+                    log.error("Failed to move file %s: %s", saved_filename, e)
+                    flash(_(u"Failed to Move File %(file)s: %(error)s", file=saved_filename, error=e), category="error")
                     return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
-            # handle authors
-            is_author = db.session.query(db.Authors).filter(db.Authors.name == authr).first()
-            if is_author:
-                db_author = is_author
-            else:
-                db_author = db.Authors(authr, helper.get_sorted_author(authr), "")
-                db.session.add(db_author)
-
-            # handle series
-            db_series = None
-            is_series = db.session.query(db.Series).filter(db.Series.name == series).first()
-            if is_series:
-                db_series = is_series
-            elif series != '':
-                db_series = db.Series(series, "")
-                db.session.add(db_series)
-
-            # add language actually one value in list
-            input_language = meta.languages
-            db_language = None
-            if input_language != "":
-                input_language = isoLanguages.get(name=input_language).part3
-                hasLanguage = db.session.query(db.Languages).filter(db.Languages.lang_code == input_language).first()
-                if hasLanguage:
-                    db_language = hasLanguage
+                if meta.cover is None:
+                    has_cover = 0
+                    copyfile(os.path.join(constants.STATIC_DIR, 'generic_cover.jpg'),
+                             os.path.join(filepath, "cover.jpg"))
                 else:
-                    db_language = db.Languages(input_language)
-                    db.session.add(db_language)
+                    has_cover = 1
 
-            # If the language of the file is excluded from the users view, it's not imported, to allow the user to view
-            # the book it's language is set to the filter language
-            if db_language != current_user.filter_language() and current_user.filter_language() != "all":
-                db_language = db.session.query(db.Languages).\
-                    filter(db.Languages.lang_code == current_user.filter_language()).first()
+                # combine path and normalize path from windows systems
+                path = os.path.join(author_dir, title_dir).replace('\\', '/')
+                # Calibre adds books with utc as timezone
+                db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
+                                   '1', datetime.utcnow(), path, has_cover, db_author, [], "")
 
-            # combine path and normalize path from windows systems
-            path = os.path.join(author_dir, title_dir).replace('\\', '/')
-            # Calibre adds books with utc as timezone
-            db_book = db.Books(title, "", db_author.sort, datetime.utcnow(), datetime(101, 1, 1),
-                               series_index, datetime.utcnow(), path, has_cover, db_author, [], db_language)
-            db_book.authors.append(db_author)
-            if db_series:
-                db_book.series.append(db_series)
-            if db_language is not None:
-                db_book.languages.append(db_language)
-            file_size = os.path.getsize(saved_filename)
-            db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
+                modif_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
+                                                     'author')
 
-            # handle tags
-            input_tags = tags.split(',')
-            input_tags = list(map(lambda it: it.strip(), input_tags))
-            if input_tags[0] !="":
-                modify_database_object(input_tags, db_book.tags, db.Tags, db.session, 'tags')
+                # Add series_index to book
+                modif_date |= edit_book_series_index(meta.series_id, db_book)
 
-            # flush content, get db_book.id available
-            db_book.data.append(db_data)
-            db.session.add(db_book)
-            db.session.flush()
+                # add languages
+                modif_date |= edit_book_languages(meta.languages, db_book, upload=True)
 
-            # add comment
-            book_id = db_book.id
-            upload_comment = Markup(meta.description).unescape()
-            if upload_comment != "":
-                db.session.add(db.Comments(upload_comment, book_id))
+                # handle tags
+                modif_date |= edit_book_tags(meta.tags, db_book)
 
-            # save data to database, reread data
-            db.session.commit()
-            db.update_title_sort(config)
-            # Reread book. It's important not to filter the result, as it could have language which hide it from
-            # current users view (tags are not stored/extracted from metadata and could also be limited)
-            book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-            # upload book to gdrive if nesseccary and add "(bookid)" to folder name
-            if config.config_use_google_drive:
-                gdriveutils.updateGdriveCalibreFromLocal()
-            error = helper.update_dir_stucture(book.id, config.config_calibre_dir)
-            db.session.commit()
-            if config.config_use_google_drive:
-                gdriveutils.updateGdriveCalibreFromLocal()
-            if error:
-                flash(error, category="error")
-            uploadText=_(u"File %(file)s uploaded", file=book.title)
-            worker.add_upload(current_user.nickname,
-                "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
+                # handle series
+                modif_date |= edit_book_series(meta.series, db_book)
 
-            # create data for displaying display Full language name instead of iso639.part3language
-            if db_language is not None:
-                book.languages[0].language_name = _(meta.languages)
-            author_names = []
-            for author in db_book.authors:
-                author_names.append(author.name)
-            if len(request.files.getlist("btn-upload")) < 2:
-                if current_user.role_edit() or current_user.role_admin():
-                    resp = {"location": url_for('editbook.edit_book', book_id=db_book.id)}
-                    return Response(json.dumps(resp), mimetype='application/json')
-                else:
-                    resp = {"location": url_for('web.show_book', book_id=db_book.id)}
-                    return Response(json.dumps(resp), mimetype='application/json')
+                # Add file to book
+                file_size = os.path.getsize(saved_filename)
+                db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
+                db_book.data.append(db_data)
+                calibre_db.session.add(db_book)
+
+                # flush content, get db_book.id available
+                calibre_db.session.flush()
+
+                # Comments needs book id therfore only possiblw after flush
+                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
+
+                book_id = db_book.id
+                title = db_book.title
+
+                error = helper.update_dir_stucture(book_id, config.config_calibre_dir, input_authors[0])
+
+                # move cover to final directory, including book id
+                if has_cover:
+                    try:
+                        new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
+                        copyfile(meta.cover, new_coverpath)
+                        os.unlink(meta.cover)
+                    except OSError as e:
+                        log.error("Failed to move cover file %s: %s", new_coverpath, e)
+                        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
+                                error=e),
+                              category="error")
+
+                # save data to database, reread data
+                calibre_db.session.commit()
+                #calibre_db.setup_db(config, ub.app_DB_path)
+                # Reread book. It's important not to filter the result, as it could have language which hide it from
+                # current users view (tags are not stored/extracted from metadata and could also be limited)
+                #book = calibre_db.get_book(book_id)
+                if config.config_use_google_drive:
+                    gdriveutils.updateGdriveCalibreFromLocal()
+                if error:
+                    flash(error, category="error")
+                uploadText=_(u"File %(file)s uploaded", file=title)
+                worker.add_upload(current_user.nickname,
+                    "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>")
+
+                if len(request.files.getlist("btn-upload")) < 2:
+                    if current_user.role_edit() or current_user.role_admin():
+                        resp = {"location": url_for('editbook.edit_book', book_id=book_id)}
+                        return Response(json.dumps(resp), mimetype='application/json')
+                    else:
+                        resp = {"location": url_for('web.show_book', book_id=book_id)}
+                        return Response(json.dumps(resp), mimetype='application/json')
+            except OperationalError as e:
+                calibre_db.session.rollback()
+                log.error("Database error: %s", e)
+                flash(_(u"Database error: %(error)s.", error=e), category="error")
         return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
-
 @editbook.route("/admin/book/convert/<int:book_id>", methods=['POST'])
 @login_required_if_no_ano
 @edit_required

cps/gdrive.py

@@ -39,7 +39,7 @@ try:
 except ImportError:
     pass
 
-from . import logger, gdriveutils, config, db
+from . import logger, gdriveutils, config, ub, calibre_db
 from .web import admin_required
 
 
@@ -145,7 +145,8 @@ def on_received_watch_confirmation():
                         dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
                     else:
                         dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode()
-                    if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != hashlib.md5(dbpath):
+                    if not response['deleted'] and response['file']['title'] == 'metadata.db' \
+                       and response['file']['md5Checksum'] != hashlib.md5(dbpath):
                         tmpDir = tempfile.gettempdir()
                         log.info('Database file updated')
                         copyfile(dbpath, os.path.join(tmpDir, "metadata.db_" + str(current_milli_time())))
@@ -154,7 +155,7 @@ def on_received_watch_confirmation():
                         log.info('Setting up new DB')
                         # prevent error on windows, as os.rename does on exisiting files
                         move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath)
-                        db.setup_db(config)
+                        calibre_db.setup_db(config, ub.app_DB_path)
             except Exception as e:
                 log.exception(e)
         updateMetaData()

cps/helper.py

@@ -23,7 +23,6 @@ import os
 import io
 import json
 import mimetypes
-import random
 import re
 import shutil
 import time
@@ -32,8 +31,6 @@ from datetime import datetime, timedelta
 from tempfile import gettempdir
 
 import requests
-from babel import Locale as LC
-from babel.core import UnknownLocaleError
 from babel.dates import format_datetime
 from babel.units import format_unit
 from flask import send_from_directory, make_response, redirect, abort
@@ -42,6 +39,7 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, or_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
+from . import calibre_db
 
 try:
     from urllib.parse import quote
@@ -56,6 +54,7 @@ except ImportError:
 
 try:
     from PIL import Image as PILImage
+    from PIL import UnidentifiedImageError
     use_PIL = True
 except ImportError:
     use_PIL = False
@@ -74,8 +73,8 @@ log = logger.create()
 
 # Convert existing book entry to new format
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first()
+    book = calibre_db.get_book(book_id)
+    data = calibre_db.get_book_format(book.id, old_book_format)
     if not data:
         error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
@@ -142,25 +141,27 @@ def check_send_to_kindle(entry):
     """
     if len(entry.data):
         bookformats = list()
-        if config.config_ebookconverter == 0:
+        if not config.config_converterpath:
             # no converter - only for mobi and pdf formats
             for ele in iter(entry.data):
-                if 'MOBI' in ele.format:
-                    bookformats.append({'format': 'Mobi',
-                                        'convert': 0,
-                                        'text': _('Send %(format)s to Kindle', format='Mobi')})
-                if 'PDF' in ele.format:
-                    bookformats.append({'format': 'Pdf',
-                                        'convert': 0,
-                                        'text': _('Send %(format)s to Kindle', format='Pdf')})
-                if 'AZW' in ele.format:
-                    bookformats.append({'format': 'Azw',
-                                        'convert': 0,
-                                        'text': _('Send %(format)s to Kindle', format='Azw')})
+                if ele.uncompressed_size < config.mail_size:
+                    if 'MOBI' in ele.format:
+                        bookformats.append({'format': 'Mobi',
+                                            'convert': 0,
+                                            'text': _('Send %(format)s to Kindle', format='Mobi')})
+                    if 'PDF' in ele.format:
+                        bookformats.append({'format': 'Pdf',
+                                            'convert': 0,
+                                            'text': _('Send %(format)s to Kindle', format='Pdf')})
+                    if 'AZW' in ele.format:
+                        bookformats.append({'format': 'Azw',
+                                            'convert': 0,
+                                            'text': _('Send %(format)s to Kindle', format='Azw')})
         else:
             formats = list()
             for ele in iter(entry.data):
-                formats.append(ele.format)
+                if ele.uncompressed_size < config.mail_size:
+                    formats.append(ele.format)
             if 'MOBI' in formats:
                 bookformats.append({'format': 'Mobi',
                                     'convert': 0,
@@ -173,14 +174,13 @@ def check_send_to_kindle(entry):
                 bookformats.append({'format': 'Pdf',
                                     'convert': 0,
                                     'text': _('Send %(format)s to Kindle', format='Pdf')})
-            if config.config_ebookconverter >= 1:
+            if config.config_converterpath:
                 if 'EPUB' in formats and not 'MOBI' in formats:
                     bookformats.append({'format': 'Mobi',
                                         'convert':1,
                                         'text': _('Convert %(orig)s to %(format)s and send to Kindle',
                                                   orig='Epub',
                                                   format='Mobi')})
-            if config.config_ebookconverter == 2:
                 if 'AZW3' in formats and not 'MOBI' in formats:
                     bookformats.append({'format': 'Mobi',
                                         'convert': 2,
@@ -211,7 +211,7 @@ def check_read_formats(entry):
 # 3: If Pdf file is existing, it's directly send to kindle email
 def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
     """Send email with attachments"""
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
 
     if convert == 1:
         # returns None if success, otherwise errormessage
@@ -317,13 +317,13 @@ def delete_book_file(book, calibrepath, book_format=None):
                 return True, None
             else:
                 log.error("Deleting book %s failed, book path not valid: %s", book.id, book.path)
-                return False, _("Deleting book %(id)s failed, book path not valid: %(path)s",
+                return True, _("Deleting book %(id)s, book path not valid: %(path)s",
                                      id=book.id,
                                      path=book.path)
 
 
 def update_dir_structure_file(book_id, calibrepath, first_author):
-    localbook = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    localbook = calibre_db.get_book(book_id)
     path = os.path.join(calibrepath, localbook.path)
 
     authordir = localbook.path.split('/')[0]
@@ -382,7 +382,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author):
 
 def update_dir_structure_gdrive(book_id, first_author):
     error = False
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     path = book.path
 
     authordir = book.path.split('/')[0]
@@ -450,11 +450,11 @@ def reset_password(user_id):
     existing_user = ub.session.query(ub.User).filter(ub.User.id == user_id).first()
     if not existing_user:
         return 0, None
-    password = generate_random_password()
-    existing_user.password = generate_password_hash(password)
     if not config.get_mail_server_configured():
         return 2, None
     try:
+        password = generate_random_password()
+        existing_user.password = generate_password_hash(password)
         ub.session.commit()
         send_registration_mail(existing_user.email, existing_user.nickname, password, True)
         return 1, existing_user.nickname
@@ -466,7 +466,7 @@ def reset_password(user_id):
 def generate_random_password():
     s = "abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%&*()?"
     passlen = 8
-    return "".join(random.sample(s, passlen))
+    return "".join(s[c % len(s)] for c in os.urandom(passlen))
 
 ################################## External interface
 
@@ -493,18 +493,17 @@ def get_cover_on_failure(use_generic_cover):
 
 
 def get_book_cover(book_id):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     return get_book_cover_internal(book, use_generic_cover_on_failure=True)
 
 
 def get_book_cover_with_uuid(book_uuid,
                              use_generic_cover_on_failure=True):
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     return get_book_cover_internal(book, use_generic_cover_on_failure)
 
 
-def get_book_cover_internal(book,
-                            use_generic_cover_on_failure):
+def get_book_cover_internal(book, use_generic_cover_on_failure):
     if book and book.has_cover:
         if config.config_use_google_drive:
             try:
@@ -532,8 +531,19 @@ def get_book_cover_internal(book,
 
 # saves book cover from url
 def save_cover_from_url(url, book_path):
-    img = requests.get(url, timeout=10)      # ToDo: Error Handling
-    return save_cover(img, book_path)
+    try:
+        img = requests.get(url, timeout=(10, 200))      # ToDo: Error Handling
+        img.raise_for_status()
+        return save_cover(img, book_path)
+    except (requests.exceptions.HTTPError,
+            requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout) as ex:
+        log.info(u'Cover Download Error %s', ex)
+        return False, _("Error Downloading Cover")
+    except UnidentifiedImageError as ex:
+        log.info(u'File Format Error %s', ex)
+        return False, _("Cover Format Error")
+
 
 
 def save_cover_from_filestorage(filepath, saved_filename, img):
@@ -594,7 +604,8 @@ def save_cover(img, book_path):
         return save_cover_from_filestorage(os.path.join(config.config_calibre_dir, book_path), "cover.jpg", img)
 
 
-def do_download_file(book, book_format, data, headers):
+
+def do_download_file(book, book_format, client, data, headers):
     if config.config_use_google_drive:
         startTime = time.time()
         df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
@@ -608,8 +619,14 @@ def do_download_file(book, book_format, data, headers):
         if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
             # ToDo: improve error handling
             log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
+
+        if client == "kobo" and book_format == "kepub":
+            headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
+
         response = make_response(send_from_directory(filename, data.name + "." + book_format))
-        response.headers = headers
+        # ToDo Check headers parameter
+        for element in headers:
+            response.headers[element[0]] = element[1]
         return response
 
 ##################################
@@ -627,11 +644,12 @@ def check_unrar(unrarLocation):
             unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
         unrarLocation = [unrarLocation]
         for lines in process_wait(unrarLocation):
-            value = re.search('UNRAR (.*) freeware', lines)
+            value = re.search('UNRAR (.*) freeware', lines, re.IGNORECASE)
             if value:
                 version = value.group(1)
                 log.debug("unrar version %s", version)
-    except OSError as err:
+                break
+    except (OSError, UnicodeDecodeError) as err:
         log.exception(err)
         return _('Error excecuting UnRar')
 
@@ -717,66 +735,12 @@ def render_task_status(tasklist):
     return renderedtasklist
 
 
-# Language and content filters for displaying in the UI
-def common_filters(allow_show_archived=False):
-    if not allow_show_archived:
-        archived_books = (
-            ub.session.query(ub.ArchivedBook)
-            .filter(ub.ArchivedBook.user_id == int(current_user.id))
-            .filter(ub.ArchivedBook.is_archived == True)
-            .all()
-        )
-        archived_book_ids = [archived_book.book_id for archived_book in archived_books]
-        archived_filter = db.Books.id.notin_(archived_book_ids)
-    else:
-        archived_filter = true()
-
-    if current_user.filter_language() != "all":
-        lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())
-    else:
-        lang_filter = true()
-    negtags_list = current_user.list_denied_tags()
-    postags_list = current_user.list_allowed_tags()
-    neg_content_tags_filter = false() if negtags_list == [''] else db.Books.tags.any(db.Tags.name.in_(negtags_list))
-    pos_content_tags_filter = true() if postags_list == [''] else db.Books.tags.any(db.Tags.name.in_(postags_list))
-    if config.config_restricted_column:
-        pos_cc_list = current_user.allowed_column_value.split(',')
-        pos_content_cc_filter = true() if pos_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(pos_cc_list))
-        neg_cc_list = current_user.denied_column_value.split(',')
-        neg_content_cc_filter = false() if neg_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(neg_cc_list))
-    else:
-        pos_content_cc_filter = true()
-        neg_content_cc_filter = false()
-    return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
-                pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
-
-
 def tags_filters():
     negtags_list = current_user.list_denied_tags()
     postags_list = current_user.list_allowed_tags()
     neg_content_tags_filter = false() if negtags_list == [''] else db.Tags.name.in_(negtags_list)
     pos_content_tags_filter = true() if postags_list == [''] else db.Tags.name.in_(postags_list)
     return and_(pos_content_tags_filter, ~neg_content_tags_filter)
-    # return ~(false()) if postags_list == [''] else db.Tags.in_(postags_list)
-
-
-# Creates for all stored languages a translated speaking name in the array for the UI
-def speaking_language(languages=None):
-    if not languages:
-        languages = db.session.query(db.Languages).join(db.books_languages_link).join(db.Books)\
-            .filter(common_filters())\
-            .group_by(text('books_languages_link.lang_code')).all()
-    for lang in languages:
-        try:
-            cur_l = LC.parse(lang.lang_code)
-            lang.name = cur_l.get_language_name(get_locale())
-        except UnknownLocaleError:
-            lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
-    return languages
 
 
 # checks if domain is in database (including wildcards)
@@ -793,93 +757,28 @@ def check_valid_domain(domain_text):
     return not len(result)
 
 
-# Orders all Authors in the list according to authors sort
-def order_authors(entry):
-    sort_authors = entry.author_sort.split('&')
-    authors_ordered = list()
-    error = False
-    for auth in sort_authors:
-        # ToDo: How to handle not found authorname
-        result = db.session.query(db.Authors).filter(db.Authors.sort == auth.lstrip().strip()).first()
-        if not result:
-            error = True
-            break
-        authors_ordered.append(result)
-    if not error:
-        entry.authors = authors_ordered
-    return entry
-
-
-# Fill indexpage with all requested data from database
-def fill_indexpage(page, database, db_filter, order, *join):
-    return fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
-
-
-def fill_indexpage_with_archived_books(page, database, db_filter, order, allow_show_archived, *join):
-    if current_user.show_detail_random():
-        randm = db.session.query(db.Books).filter(common_filters(allow_show_archived))\
-            .order_by(func.random()).limit(config.config_random_books)
-    else:
-        randm = false()
-    off = int(int(config.config_books_per_page) * (page - 1))
-    pagination = Pagination(page, config.config_books_per_page,
-                            len(db.session.query(database).filter(db_filter)
-                                .filter(common_filters(allow_show_archived)).all()))
-    entries = db.session.query(database).join(*join, isouter=True).filter(db_filter)\
-        .filter(common_filters(allow_show_archived))\
-        .order_by(*order).offset(off).limit(config.config_books_per_page).all()
-    for book in entries:
-        book = order_authors(book)
-    return entries, randm, pagination
-
-
-def get_typeahead(database, query, replace=('', ''), tag_filter=true()):
-    query = query or ''
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    entries = db.session.query(database).filter(tag_filter).\
-        filter(func.lower(database.name).ilike("%" + query + "%")).all()
-    json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
-    return json_dumps
-
-
-# read search results from calibre-database and return it (function is used for feed and simple search
-def get_search_results(term):
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split("[, ]+", term)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + term + "%"))
-
-    return db.session.query(db.Books).filter(common_filters()).filter(
-        or_(db.Books.tags.any(func.lower(db.Tags.name).ilike("%" + term + "%")),
-            db.Books.series.any(func.lower(db.Series.name).ilike("%" + term + "%")),
-            db.Books.authors.any(and_(*q)),
-            db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + term + "%")),
-            func.lower(db.Books.title).ilike("%" + term + "%")
-            )).order_by(db.Books.sort).all()
-
-
-def get_cc_columns():
-    tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+def get_cc_columns(filter_config_custom_read=False):
+    tmpcc = calibre_db.session.query(db.Custom_Columns)\
+        .filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    cc = []
+    r = None
     if config.config_columns_to_ignore:
-        cc = []
-        for col in tmpcc:
-            r = re.compile(config.config_columns_to_ignore)
-            if not r.match(col.name):
-                cc.append(col)
-    else:
-        cc = tmpcc
+        r = re.compile(config.config_columns_to_ignore)
+
+    for col in tmpcc:
+        if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id:
+            continue
+        if r and r.match(col.name):
+            continue
+        cc.append(col)
+
     return cc
 
-
-def get_download_link(book_id, book_format):
+def get_download_link(book_id, book_format, client):
     book_format = book_format.split(".")[0]
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if book:
-        data1 = db.session.query(db.Data).filter(db.Data.book == book.id)\
-            .filter(db.Data.format == book_format.upper()).first()
+        data1 = calibre_db.get_book_format(book.id, book_format.upper())
     else:
         abort(404)
     if data1:
@@ -894,28 +793,6 @@ def get_download_link(book_id, book_format):
         headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
         headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % (
             quote(file_name.encode('utf-8')), book_format, quote(file_name.encode('utf-8')), book_format)
-        return do_download_file(book, book_format, data1, headers)
+        return do_download_file(book, book_format, client, data1, headers)
     else:
         abort(404)
-
-
-def check_exists_book(authr, title):
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split(r'\s*&\s*', authr)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    return db.session.query(db.Books).filter(
-        and_(db.Books.authors.any(and_(*q)),
-             func.lower(db.Books.title).ilike("%" + title + "%")
-             )).first()
-
-############### Database Helper functions
-
-
-def lcase(s):
-    try:
-        return unidecode.unidecode(s.lower())
-    except Exception as e:
-        log.exception(e)

cps/isoLanguages.py

@@ -57,12 +57,12 @@ def get_language_name(locale, lang_code):
 
 def get_language_codes(locale, language_names, remainder=None):
     language_names = set(x.strip().lower() for x in language_names if x)
-
+    languages = list()
     for k, v in get_language_names(locale).items():
         v = v.lower()
         if v in language_names:
+            languages.append(k)
             language_names.remove(v)
-            yield k
-
     if remainder is not None:
         remainder.extend(language_names)
+    return languages

cps/jinjia.py

@@ -111,10 +111,3 @@ def timestamptodate(date, fmt=None):
 @jinjia.app_template_filter('yesno')
 def yesno(value, yes, no):
     return yes if value else no
-
-
-'''@jinjia.app_template_filter('canread')
-def canread(ext):
-    if isinstance(ext, db.Data):
-        ext = ext.format
-    return ext.lower() in EXTENSIONS_READER'''

cps/kobo.py

@@ -48,7 +48,7 @@ from sqlalchemy.sql.expression import and_, or_
 from sqlalchemy.exc import StatementError
 import requests
 
-from . import config, logger, kobo_auth, db, helper, shelf as shelf_lib, ub
+from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub
 from .services import SyncToken as SyncToken
 from .web import download_required
 from .kobo_auth import requires_kobo_auth
@@ -143,7 +143,7 @@ def HandleSyncRequest():
 
     # We reload the book database so that the user get's a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
-    db.reconnect_db(config)
+    calibre_db.reconnect_db(config, ub.app_DB_path)
 
     archived_books = (
         ub.session.query(ub.ArchivedBook)
@@ -170,7 +170,7 @@ def HandleSyncRequest():
     # It looks like it's treating the db.Books.last_modified field as a string and may fail
     # the comparison because of the +00:00 suffix.
     changed_entries = (
-        db.session.query(db.Books)
+        calibre_db.session.query(db.Books)
         .join(db.Data)
         .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified,
                     db.Books.id.in_(recently_restored_or_archived_books)))
@@ -207,7 +207,7 @@ def HandleSyncRequest():
                      ub.KoboReadingState.user_id == current_user.id,
                      ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
     for kobo_reading_state in changed_reading_states.all():
-        book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none()
+        book = calibre_db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none()
         if book:
             sync_results.append({
                 "ChangedReadingState": {
@@ -256,7 +256,7 @@ def HandleMetadataRequest(book_uuid):
     if not current_app.wsgi_app.is_proxied:
         log.debug('Kobo: Received unproxied request, changed request port to server port')
     log.info("Kobo library metadata request received for book %s" % book_uuid)
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -332,6 +332,9 @@ def get_series(book):
         return None
     return book.series[0].name
 
+def get_seriesindex(book):
+    return book.series_index or 1
+
 
 def get_metadata(book):
     download_urls = []
@@ -386,8 +389,8 @@ def get_metadata(book):
             name = get_series(book)
         metadata["Series"] = {
             "Name": get_series(book),
-            "Number": book.series_index,        # ToDo Check int() ?
-            "NumberFloat": float(book.series_index),
+            "Number": get_seriesindex(book),        # ToDo Check int() ?
+            "NumberFloat": float(get_seriesindex(book)),
             # Get a deterministic id based on the series name.
             "Id": uuid.uuid3(uuid.NAMESPACE_DNS, name),
         }
@@ -471,7 +474,7 @@ def add_items_to_shelf(items, shelf):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -542,7 +545,7 @@ def HandleTagRemoveItem(tag_id):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -610,7 +613,7 @@ def create_kobo_tag(shelf):
         "Type": "UserTag"
     }
     for book_shelf in shelf.books:
-        book = db.session.query(db.Books).filter(db.Books.id == book_shelf.book_id).one_or_none()
+        book = calibre_db.get_book(book_shelf.book_id)
         if not book:
             log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database",  book_shelf.book_id, shelf.id)
             continue
@@ -626,7 +629,7 @@ def create_kobo_tag(shelf):
 @kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
 @login_required
 def HandleStateRequest(book_uuid):
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -801,7 +804,7 @@ def TopLevelEndpoint():
 @login_required
 def HandleBookDeletionRequest(book_uuid):
     log.info("Kobo book deletion request received for book %s" % book_uuid)
-    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -979,7 +982,7 @@ def NATIVE_KOBO_RESOURCES():
         "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"},
         "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}",
         "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}",
-        "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}",
+        "book_detail_page_rakuten": "https://books.rakuten.co.jp/rk/{crossrevisionid}",
         "book_landing_page": "https://store.kobobooks.com/ebooks",
         "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions",
         "categories": "https://storeapi.kobo.com/v1/categories",
@@ -1017,7 +1020,7 @@ def NATIVE_KOBO_RESOURCES():
         "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests",
         "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader",
         "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem",
-        "help_page": "http://www.kobo.com/help",
+        "help_page": "https://www.kobo.com/help",
         "kobo_audiobooks_enabled": "False",
         "kobo_audiobooks_orange_deal_enabled": "False",
         "kobo_audiobooks_subscriptions_enabled": "False",

cps/kobo_auth.py

@@ -121,7 +121,11 @@ kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth")
 @kobo_auth.route("/generate_auth_token/<int:user_id>")
 @login_required
 def generate_auth_token(user_id):
-    host = ':'.join(request.host.rsplit(':')[0:-1])
+    host_list = request.host.rsplit(':')
+    if len(host_list) == 1:
+        host = ':'.join(host_list)
+    else:
+        host = ':'.join(host_list[0:-1])
     if host.startswith('127.') or host.lower() == 'localhost' or host.startswith('[::ffff:7f'):
         warning = _('PLease access calibre-web from non localhost to get valid api_endpoint for kobo device')
         return render_title_template(

cps/logger.py

@@ -82,7 +82,6 @@ def _absolute_log_file(log_file, default_log_file):
         if not os.path.dirname(log_file):
             log_file = os.path.join(_CONFIG_DIR, log_file)
         return os.path.abspath(log_file)
-
     return default_log_file
 
 
@@ -115,7 +114,7 @@ def setup(log_file, log_level=None):
     if previous_handler:
         # if the log_file has not changed, don't create a new handler
         if getattr(previous_handler, 'baseFilename', None) == log_file:
-            return
+            return "" if log_file == DEFAULT_LOG_FILE else log_file
         logging.debug("logging to %s level %s", log_file, r.level)
 
     if log_file == LOG_TO_STDERR or log_file == LOG_TO_STDOUT:
@@ -132,12 +131,14 @@ def setup(log_file, log_level=None):
             if log_file == DEFAULT_LOG_FILE:
                 raise
             file_handler = RotatingFileHandler(DEFAULT_LOG_FILE, maxBytes=50000, backupCount=2)
+            log_file = ""
     file_handler.setFormatter(FORMATTER)
 
     for h in r.handlers:
         r.removeHandler(h)
         h.close()
     r.addHandler(file_handler)
+    return "" if log_file == DEFAULT_LOG_FILE else log_file
 
 
 def create_access_log(log_file, log_name, formatter):
@@ -150,11 +151,18 @@ def create_access_log(log_file, log_name, formatter):
     access_log = logging.getLogger(log_name)
     access_log.propagate = False
     access_log.setLevel(logging.INFO)
+    try:
+        file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2)
+    except IOError:
+        if log_file == DEFAULT_ACCESS_LOG:
+            raise
+        file_handler = RotatingFileHandler(DEFAULT_ACCESS_LOG, maxBytes=50000, backupCount=2)
+        log_file = ""
 
-    file_handler = RotatingFileHandler(log_file, maxBytes=50000, backupCount=2)
     file_handler.setFormatter(formatter)
     access_log.addHandler(file_handler)
-    return access_log
+    return access_log, \
+           "" if _absolute_log_file(log_file, DEFAULT_ACCESS_LOG) == DEFAULT_ACCESS_LOG else log_file
 
 
 # Enable logging of smtp lib debug output

cps/oauth.py

@@ -40,7 +40,7 @@ try:
         Stores and retrieves OAuth tokens using a relational database through
         the `SQLAlchemy`_ ORM.
 
-        .. _SQLAlchemy: http://www.sqlalchemy.org/
+        .. _SQLAlchemy: https://www.sqlalchemy.org/
         """
         def __init__(self, model, session, provider_id,
                      user=None, user_id=None, user_required=None, anon_user=None,

cps/opds.py

@@ -30,10 +30,10 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import func, text, or_, and_
 from werkzeug.security import check_password_hash
 
-from . import constants, logger, config, db, ub, services, get_locale, isoLanguages
-from .helper import fill_indexpage, get_download_link, get_book_cover, speaking_language
+from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages
+from .helper import get_download_link, get_book_cover
 from .pagination import Pagination
-from .web import common_filters, get_search_results, render_read_books, download_required
+from .web import render_read_books, download_required
 from flask_babel import gettext as _
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
@@ -100,15 +100,15 @@ def feed_normal_search():
 @requires_basic_auth_if_no_ano
 def feed_new():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books, True, [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books, True, [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
 @opds.route("/opds/discover")
 @requires_basic_auth_if_no_ano
 def feed_discover():
-    entries = db.session.query(db.Books).filter(common_filters()).order_by(func.random())\
+    entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\
         .limit(config.config_books_per_page)
     pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page))
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -118,9 +118,9 @@ def feed_discover():
 @requires_basic_auth_if_no_ano
 def feed_best_rated():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -133,16 +133,13 @@ def feed_hot():
     hot_books = all_books.offset(off).limit(config.config_books_per_page)
     entries = list()
     for book in hot_books:
-        downloadBook = db.session.query(db.Books).filter(db.Books.id == book.Downloads.book_id).first()
+        downloadBook = calibre_db.get_book(book.Downloads.book_id)
         if downloadBook:
             entries.append(
-                db.session.query(db.Books).filter(common_filters())
-                .filter(db.Books.id == book.Downloads.book_id).first()
+                calibre_db.get_filtered_book(book.Downloads.book_id)
             )
         else:
             ub.delete_download(book.Downloads.book_id)
-            # ub.session.query(ub.Downloads).filter(book.Downloads.book_id == ub.Downloads.book_id).delete()
-            # ub.session.commit()
     numBooks = entries.__len__()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1),
                             config.config_books_per_page, numBooks)
@@ -153,11 +150,13 @@ def feed_hot():
 @requires_basic_auth_if_no_ano
 def feed_authorindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Authors).join(db.books_authors_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).limit(config.config_books_per_page)\
+    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_authors_link.author'))\
+        .order_by(db.Authors.sort).limit(config.config_books_per_page)\
         .offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Authors).all()))
+                            len(calibre_db.session.query(db.Authors).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_author', pagination=pagination)
 
 
@@ -165,10 +164,10 @@ def feed_authorindex():
 @requires_basic_auth_if_no_ano
 def feed_author(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.authors.any(db.Authors.id == book_id),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.authors.any(db.Authors.id == book_id),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -176,11 +175,14 @@ def feed_author(book_id):
 @requires_basic_auth_if_no_ano
 def feed_publisherindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Publishers).join(db.books_publishers_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.sort)\
+    entries = calibre_db.session.query(db.Publishers)\
+        .join(db.books_publishers_link)\
+        .join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(text('books_publishers_link.publisher'))\
+        .order_by(db.Publishers.sort)\
         .limit(config.config_books_per_page).offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Publishers).all()))
+                            len(calibre_db.session.query(db.Publishers).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_publisher', pagination=pagination)
 
 
@@ -188,10 +190,10 @@ def feed_publisherindex():
 @requires_basic_auth_if_no_ano
 def feed_publisher(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.publishers.any(db.Publishers.id == book_id),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.publishers.any(db.Publishers.id == book_id),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -199,10 +201,16 @@ def feed_publisher(book_id):
 @requires_basic_auth_if_no_ano
 def feed_categoryindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).offset(off).limit(config.config_books_per_page)
+    entries = calibre_db.session.query(db.Tags)\
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name)\
+        .offset(off)\
+        .limit(config.config_books_per_page)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Tags).all()))
+                            len(calibre_db.session.query(db.Tags).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination)
 
 
@@ -210,10 +218,10 @@ def feed_categoryindex():
 @requires_basic_auth_if_no_ano
 def feed_category(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.tags.any(db.Tags.id == book_id),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.tags.any(db.Tags.id == book_id),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -221,10 +229,15 @@ def feed_category(book_id):
 @requires_basic_auth_if_no_ano
 def feed_seriesindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters())\
-        .group_by(text('books_series_link.series')).order_by(db.Series.sort).offset(off).all()
+    entries = calibre_db.session.query(db.Series)\
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.sort)\
+        .offset(off).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
-                            len(db.session.query(db.Series).all()))
+                            len(calibre_db.session.query(db.Series).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination)
 
 
@@ -232,10 +245,10 @@ def feed_seriesindex():
 @requires_basic_auth_if_no_ano
 def feed_series(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.series.any(db.Series.id == book_id),
-                                             [db.Books.series_index])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.series.any(db.Series.id == book_id),
+                                                        [db.Books.series_index])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -243,10 +256,13 @@ def feed_series(book_id):
 @requires_basic_auth_if_no_ano
 def feed_ratingindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
+    entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                (db.Ratings.rating / 2).label('name')) \
-        .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
+        .join(db.books_ratings_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_ratings_link.rating'))\
+        .order_by(db.Ratings.rating).all()
 
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
@@ -260,10 +276,10 @@ def feed_ratingindex():
 @requires_basic_auth_if_no_ano
 def feed_ratings(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.ratings.any(db.Ratings.id == book_id),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.ratings.any(db.Ratings.id == book_id),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -271,8 +287,10 @@ def feed_ratings(book_id):
 @requires_basic_auth_if_no_ano
 def feed_formatindex():
     off = request.args.get("offset") or 0
-    entries = db.session.query(db.Data).join(db.Books).filter(common_filters()) \
-        .group_by(db.Data.format).order_by(db.Data.format).all()
+    entries = calibre_db.session.query(db.Data).join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
 
@@ -286,10 +304,10 @@ def feed_formatindex():
 @requires_basic_auth_if_no_ano
 def feed_format(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.data.any(db.Data.format == book_id.upper()),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.data.any(db.Data.format == book_id.upper()),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -299,13 +317,13 @@ def feed_format(book_id):
 def feed_languagesindex():
     off = request.args.get("offset") or 0
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         try:
             cur_l = LC.parse(current_user.filter_language())
         except UnknownLocaleError:
             cur_l = None
-        languages = db.session.query(db.Languages).filter(
+        languages = calibre_db.session.query(db.Languages).filter(
             db.Languages.lang_code == current_user.filter_language()).all()
         if cur_l:
             languages[0].name = cur_l.get_language_name(get_locale())
@@ -320,10 +338,10 @@ def feed_languagesindex():
 @requires_basic_auth_if_no_ano
 def feed_languages(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
-                                             db.Books,
-                                             db.Books.languages.any(db.Languages.id == book_id),
-                                             [db.Books.timestamp.desc()])
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+                                                        db.Books,
+                                                        db.Books.languages.any(db.Languages.id == book_id),
+                                                        [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
 
@@ -356,7 +374,7 @@ def feed_shelf(book_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by(
             ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+            cur_book = calibre_db.get_book(book.book_id)
             result.append(cur_book)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(result))
@@ -367,14 +385,18 @@ def feed_shelf(book_id):
 @requires_basic_auth_if_no_ano
 @download_required
 def opds_download_link(book_id, book_format):
-    return get_download_link(book_id, book_format.lower())
+    if "Kobo" in request.headers.get('User-Agent'):
+        client = "kobo"
+    else:
+        client = ""
+    return get_download_link(book_id, book_format.lower(), client)
 
 
 @opds.route("/ajax/book/<string:uuid>/<library>")
 @opds.route("/ajax/book/<string:uuid>", defaults={'library': ""})
 @requires_basic_auth_if_no_ano
 def get_metadata_calibre_companion(uuid, library):
-    entry = db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first()
+    entry = calibre_db.session.query(db.Books).filter(db.Books.uuid.like("%" + uuid + "%")).first()
     if entry is not None:
         js = render_template('json.txt', entry=entry)
         response = make_response(js)
@@ -386,8 +408,7 @@ def get_metadata_calibre_companion(uuid, library):
 
 def feed_search(term):
     if term:
-        term = term.strip().lower()
-        entries = get_search_results(term)
+        entries = calibre_db.get_search_results(term)
         entriescount = len(entries) if len(entries) > 0 else 1
         pagination = Pagination(1, entriescount, entriescount)
         return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)

cps/server.py

@@ -72,7 +72,11 @@ class WebServer(object):
         if config.config_access_log:
             log_name = "gevent.access" if _GEVENT else "tornado.access"
             formatter = logger.ACCESS_FORMATTER_GEVENT if _GEVENT else logger.ACCESS_FORMATTER_TORNADO
-            self.access_logger = logger.create_access_log(config.config_access_logfile, log_name, formatter)
+            self.access_logger, logfile = logger.create_access_log(config.config_access_logfile, log_name, formatter)
+            if logfile != config.config_access_logfile:
+                log.warning("Accesslog path %s not valid, falling back to default", config.config_access_logfile)
+                config.config_access_logfile = logfile
+                config.save()
         else:
             if not _GEVENT:
                 logger.get('tornado.access').disabled = True
@@ -196,6 +200,9 @@ class WebServer(object):
     def stop(self, restart=False):
         from . import updater_thread
         updater_thread.stop()
+        from . import calibre_db
+        calibre_db.stop()
+
 
         log.info("webserver stop (restart=%s)", restart)
         self.restart = restart

cps/services/goodreads_support.py

@@ -20,7 +20,10 @@ from __future__ import division, print_function, unicode_literals
 import time
 from functools import reduce
 
-from goodreads.client import GoodreadsClient
+try:
+    from goodreads.client import GoodreadsClient
+except ImportError:
+    from betterreads.client import GoodreadsClient
 
 try: import Levenshtein
 except ImportError: Levenshtein = False
@@ -95,8 +98,12 @@ def get_other_books(author_info, library_books=None):
     for book in author_info.books:
         if book.isbn in identifiers:
             continue
-        if book.gid["#text"] in identifiers:
-            continue
+        if isinstance(book.gid, int):
+            if book.gid in identifiers:
+                continue
+        else:
+            if book.gid["#text"] in identifiers:
+                continue
 
         if Levenshtein and library_titles:
             goodreads_title = book._book_dict['title_without_series']

cps/shelf.py

@@ -28,9 +28,8 @@ from flask_babel import gettext as _
 from flask_login import login_required, current_user
 from sqlalchemy.sql.expression import func
 
-from . import logger, ub, searched_ids, db
+from . import logger, ub, searched_ids, db, calibre_db
 from .web import render_title_template
-from .helper import common_filters
 
 
 shelf = Blueprint('shelf', __name__)
@@ -320,11 +319,11 @@ def show_shelf(shelf_type, shelf_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append(cur_book)
             else:
-                cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 if not cur_book:
                     log.info('Not existing book %s in %s deleted', book.book_id, shelf)
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete()
@@ -356,7 +355,7 @@ def order_shelf(shelf_id):
         books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf2:
-            cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append({'title': cur_book.title,
                                'id': cur_book.id,
@@ -364,7 +363,7 @@ def order_shelf(shelf_id):
                                'series': cur_book.series,
                                'series_index': cur_book.series_index})
             else:
-                cur_book = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 result.append({'title': _('Hidden Book'),
                                'id': cur_book.id,
                                'author': [],

cps/static/css/caliBlur_override.css

@@ -0,0 +1,17 @@
+body.serieslist.grid-view div.container-fluid>div>div.col-sm-10:before{
+    display: none;
+}
+
+.cover .badge{
+    position: absolute;
+    top: 0;
+    right: 0;
+    background-color: #cc7b19;
+    border-radius: 0;
+    padding: 0 8px;
+    box-shadow: 0 0 4px rgba(0,0,0,.6);
+    line-height: 24px;
+}
+.cover{
+    box-shadow: 0 0 4px rgba(0,0,0,.6);
+}

cps/static/css/style.css

@@ -109,7 +109,7 @@ a { color: #45b29d; }
 
 .container-fluid .book .cover img {
   border: 1px solid #fff;
-  box-sizeing: border-box;
+  box-sizing: border-box;
   height: 100%;
   bottom: 0;
   position: absolute;
@@ -165,7 +165,7 @@ span.glyphicon.glyphicon-tags {
 
 .container-fluid .single .cover img {
   border: 1px solid #fff;
-  box-sizeing: border-box;
+  box-sizing: border-box;
   -webkit-box-shadow: 0 5px 8px -6px #777;
   -moz-box-shadow: 0 5px 8px -6px #777;
   box-shadow: 0 5px 8px -6px #777;
@@ -174,6 +174,12 @@ span.glyphicon.glyphicon-tags {
 .navbar-default .navbar-toggle .icon-bar {background-color: #000; }
 .navbar-default .navbar-toggle {border-color: #000; }
 .cover { margin-bottom: 10px; }
+.cover .badge{
+   position: absolute;
+   top: 2px;
+   left: 2px;
+   background-color: #777;
+}
 .cover-height { max-height: 100px;}
 
 .col-sm-2 a .cover-small {
@@ -207,7 +213,7 @@ span.glyphicon.glyphicon-tags {
 .panel-body {background-color: #f5f5f5; }
 .spinner {margin: 0 41%; }
 .spinner2 {margin: 0 41%; }
-
+.intend-form { margin-left:20px; }
 table .bg-dark-danger {background-color: #d9534f; color: #fff; }
 table .bg-dark-danger a {color: #fff; }
 table .bg-dark-danger:hover {background-color: #c9302c; }
@@ -296,3 +302,4 @@ div.log {
   white-space: nowrap;
   padding: 0.5em;
 }
+

cps/static/js/filter_grid.js

@@ -0,0 +1,54 @@
+/* This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+ *    Copyright (C) 2018 OzzieIsaacs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+var $list = $("#list").isotope({
+    itemSelector: ".book",
+    layoutMode: "fitRows",
+    getSortData: {
+        title: ".title",
+    }
+});
+
+$("#desc").click(function() {
+    $list.isotope({
+        sortBy: "name",
+        sortAscending: true
+    });
+    return;
+});
+
+$("#asc").click(function() {
+    $list.isotope({
+        sortBy: "name",
+        sortAscending: false
+    });
+    return;
+});
+
+$("#all").click(function() {
+    // go through all elements and make them visible
+    $list.isotope({ filter: function() {
+        return true;
+      } })
+});
+
+$(".char").click(function() {
+    var character = this.innerText;
+    $list.isotope({ filter: function() {
+        return this.attributes["data-id"].value.charAt(0).toUpperCase() == character;
+      } })
+});

cps/static/js/main.js

@@ -323,4 +323,17 @@ $(function() {
         $(".discover .row").isotope("layout");
     });
 
+    $(".update-view").click(function(e) {
+        var target = $(this).data("target");
+        var view = $(this).data("view");
+
+        e.preventDefault();
+        e.stopPropagation();
+        var data = {};
+        data[target] = view;
+        console.debug("Updating view data: ", data);
+        $.post( "/ajax/view", data).done(function( ) {
+            location.reload();
+        });
+    });
 });

cps/static/js/table.js

@@ -124,7 +124,7 @@ $(function() {
                 if (field === 3) {
                     $.ajax ({
                         type: "Post",
-                        data: "id=" + row.id + "&type=" + row.type + "&Element=" + row.Element,
+                        data: "id=" + row.id + "&type=" + row.type + "&Element=" + encodeURIComponent(row.Element),
                         url: path + "/../../ajax/deleterestriction/" + type,
                         async: true,
                         timeout: 900,

cps/subproc_wrapper.py

@@ -22,7 +22,7 @@ import os
 import subprocess
 
 
-def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE):
+def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE, newlines=True):
     # Linux py2.7 encode as list without quotes no empty element for parameters
     # linux py3.x no encode and as list without quotes no empty element for parameters
     # windows py2.7 encode as string with quotes empty element for parameters is okay
@@ -41,12 +41,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro
         else:
             exc_command = [x for x in command]
 
-    return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env)
+    return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=newlines, env=env)
 
 
 def process_wait(command, serr=subprocess.PIPE):
     # Run command, wait for process to terminate, and return an iterator over lines of its output.
-    p = process_open(command, serr=serr)
+    newlines = os.name != 'nt'
+    p = process_open(command, serr=serr, newlines=newlines)
     p.wait()
     for line in p.stdout.readlines():
         if isinstance(line, bytes):

cps/templates/admin.html

@@ -13,11 +13,14 @@
             <th>{{_('E-mail Address')}}</th>
             <th>{{_('Send to Kindle E-mail Address')}}</th>
             <th>{{_('Downloads')}}</th>
-            <th class="hidden-xs">{{_('Admin')}}</th>
-            <th class="hidden-xs">{{_('Download')}}</th>
-            <th class="hidden-xs">{{_('View Books')}}</th>
-            <th class="hidden-xs">{{_('Upload')}}</th>
-            <th class="hidden-xs">{{_('Edit')}}</th>
+            <th class="hidden-xs ">{{_('Admin')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Password')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Upload')}}</th>
+            <th class="hidden-xs hidden-sm">{{_('Download')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('View Books')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Edit')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Delete')}}</th>
+            <th class="hidden-xs hidden-sm hidden-md">{{_('Public Shelf')}}</th>
         </tr>
         {% for user in allUser %}
           {% if not user.role_anonymous() or config.config_anonbrowse %}
@@ -27,10 +30,13 @@
             <td>{{user.kindle_mail}}</td>
             <td>{{user.downloads.count()}}</td>
             <td class="hidden-xs">{{ display_bool_setting(user.role_admin()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_download()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_viewer()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_upload()) }}</td>
-            <td class="hidden-xs">{{ display_bool_setting(user.role_edit()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_passwd()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_upload()) }}</td>
+            <td class="hidden-xs hidden-sm">{{ display_bool_setting(user.role_download()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_viewer()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_edit()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_delete_books()) }}</td>
+            <td class="hidden-xs hidden-sm hidden-md">{{ display_bool_setting(user.role_edit_shelfs()) }}</td>
           </tr>
           {% endif %}
         {% endfor %}

cps/templates/book_edit.html

@@ -86,7 +86,7 @@
     </div>
     <div class="form-group">
       <label for="series_index">{{_('Series ID')}}</label>
-      <input type="number" step="0.01" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
+      <input type="number" step="0.01" min="0" placeholder="1" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
     </div>
     <div class="form-group">
       <label for="rating">{{_('Rating')}}</label>
@@ -305,8 +305,8 @@
     // create a random identifier type to have a valid name in form. This will not be used when dealing with the form
     var rand_id = Math.floor(Math.random() * 1000000).toString();
     var line = '<tr>';
-    line += '<td><input type="text" class="form-control" name="identifier-type-'+ rand_id +'" required="required" placeholder="{{_('Identifier Type')}}"></td>';
-    line += '<td><input type="text" class="form-control" name="identifier-val-'+ rand_id +'" required="required" placeholder="{{_('Identifier Value')}}"></td>';
+    line += '<td><input type="text" class="form-control" name="identifier-type-'+ rand_id +'" required="required" placeholder={{_('Identifier Type')|safe|tojson}}></td>';
+    line += '<td><input type="text" class="form-control" name="identifier-val-'+ rand_id +'" required="required" placeholder={{_('Identifier Value')|safe|tojson}}></td>';
     line += '<td><a class="btn btn-default" onclick="removeIdentifierLine(this)">{{_('Remove')}}</a></td>';
     line += '</tr>';
     $("#identifier-table").append(line);

cps/templates/book_table.html

@@ -0,0 +1,59 @@
+{% extends "layout.html" %}
+{% block body %}
+<h1 class="{{page}}">{{_(title)}}</h1>
+
+    <div class="filterheader hidden-xs hidden-sm">
+      {% if entries.__len__() %}
+       {% if data == 'author' %}
+        <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
+       {% endif %}
+      {% endif %}
+      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      {% if charlist|length %}
+      <button id="all" class="btn btn-primary">{{_('All')}}</button>
+      {% endif %}
+      <div class="btn-group character" role="group">
+        {% for char in charlist%}
+        <button class="btn btn-primary char">{{char.char}}</button>
+        {% endfor %}
+      </div>
+
+      {% if title == "Series" %}
+      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
+      {% endif %}
+    </div>
+  <div class="container">
+    <div id="list" class="col-xs-12 col-sm-6">
+    {% for entry in entries %}
+      {% if loop.index0 == (loop.length/2+loop.length%2)|int and loop.length > 20 %}
+        </div>
+        <div id="second" class="col-xs-12 col-sm-6">
+      {% endif %}
+      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
+        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
+        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort='new', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].id )}}{% endif %}">
+          {% if entry.name %}
+          <div class="rating">
+          {% for number in range(entry.name) %}
+            <span class="glyphicon glyphicon-star good"></span>
+            {% if loop.last and loop.index < 5 %}
+              {% for numer in range(5 - loop.index) %}
+                <span class="glyphicon glyphicon-star"></span>
+              {% endfor %}
+            {% endif %}
+          {% endfor %}
+          </div>
+          {% else %}
+          {% if entry.format %}
+            {{entry.format}}
+          {% else %}
+            {{entry[0].name}}{% endif %}{% endif %}</a></div>
+      </div>
+    {% endfor %}
+    </div>
+  </div>
+{% endblock %}
+{% block js %}
+<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
+{% endblock %}

cps/templates/config_edit.html

@@ -3,7 +3,7 @@
 <div class="discover">
   <h2>{{title}}</h2>
   <form role="form" method="POST" autocomplete="off">
-<div class="panel-group">
+<div class="panel-group col-md-10 col-lg-6">
   <div class="panel panel-default">
     <div class="panel-heading">
       <h4 class="panel-title">
@@ -15,10 +15,13 @@
     </div>
     <div id="collapseOne" class="panel-collapse collapse in">
       <div class="panel-body">
-    <div class="form-group required">
-      <label for="config_calibre_dir">{{_('Location of Calibre Database')}}</label>
-      <input type="text" class="form-control" name="config_calibre_dir" id="config_calibre_dir" value="{% if config.config_calibre_dir != None %}{{ config.config_calibre_dir }}{% endif %}" autocomplete="off">
-    </div>
+       <div class="form-group required input-group">
+        <label for="config_calibre_dir" class="sr-only">{{_('Location of Calibre Database')}}</label>
+        <input type="text" class="form-control" id="config_calibre_dir" name="config_calibre_dir" value="{% if config.config_calibre_dir != None %}{{ config.config_calibre_dir }}{% endif %}" autocomplete="off">
+        <span class="input-group-btn">
+          <button type="button" id="library_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+        </span>
+      </div>
     {% if feature_support['gdrive']  %}
     <div class="form-group required">
       <input type="checkbox" id="config_use_google_drive" name="config_use_google_drive" data-control="gdrive_settings" {% if config.config_use_google_drive %}checked{% endif %} >
@@ -87,21 +90,25 @@
           <label for="config_port">{{_('Server Port')}}</label>
           <input type="number" min="1" max="65535" class="form-control" name="config_port" id="config_port" value="{% if config.config_port != None %}{{ config.config_port }}{% endif %}" autocomplete="off" required>
         </div>
-        <div class="form-group">
-          <label for="config_certfile">{{_('SSL certfile location (leave it empty for non-SSL Servers)')}}</label>
-          <input type="text" class="form-control" name="config_certfile" id="config_certfile" value="{% if config.config_certfile != None %}{{ config.config_certfile }}{% endif %}" autocomplete="off">
+        <label for="config_certfile">{{_('SSL certfile location (leave it empty for non-SSL Servers)')}}</label>
+         <div class="form-group input-group">
+          <input type="text" class="form-control" id="config_certfile" name="config_certfile" value="{% if config.config_certfile != None %}{{ config.config_certfile }}{% endif %}" autocomplete="off">
+          <span class="input-group-btn">
+            <button type="button" id="certfile_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+          </span>
         </div>
-        <div class="form-group">
-          <label for="config_keyfile">{{_('SSL Keyfile location (leave it empty for non-SSL Servers)')}}</label>
-          <input type="text" class="form-control" name="config_keyfile" id="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
+          <label for="config_calibre_dir" >{{_('SSL Keyfile location (leave it empty for non-SSL Servers)')}}</label>
+         <div class="form-group input-group">
+          <input type="text" class="form-control" id="config_keyfile" name="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
+          <span class="input-group-btn">
+            <button type="button" id="keyfile_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+          </span>
         </div>
         <div class="form-group">
           <label for="config_updatechannel">{{_('Update Channel')}}</label>
             <select name="config_updatechannel" id="config_updatechannel" class="form-control">
-                    <option value="0" {% if config.config_updatechannel == 0 %}selected{% endif %}>{{_('Stable')}}</option>
-                    <!--option value="1" {% if config.config_updatechannel == 1 %}selected{% endif %}>{{_('Stable (Automatic)')}}</option-->
-                    <option value="2" {% if config.config_updatechannel == 2 %}selected{% endif %}>{{_('Nightly')}}</option>
-                    <!--option-- value="3" {% if config.config_updatechannel == 3 %}selected{% endif %}>{{_('Nightly (Automatic)')}}</option-->
+              <option value="0" {% if config.config_updatechannel == 0 %}selected{% endif %}>{{_('Stable')}}</option>
+              <option value="2" {% if config.config_updatechannel == 2 %}selected{% endif %}>{{_('Nightly')}}</option>
             </select>
         </div>
       </div>
@@ -154,17 +161,29 @@
     <div id="collapsefive" class="panel-collapse collapse">
       <div class="panel-body">
     <div class="form-group">
-        <input type="checkbox" id="config_uploading" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
+        <input type="checkbox" id="config_uploading" data-control="upload_settings" name="config_uploading" {% if config.config_uploading %}checked{% endif %}>
         <label for="config_uploading">{{_('Enable Uploads')}}</label>
     </div>
+    <div data-related="upload_settings">
+      <div class="form-group">
+        <label for="config_upload_formats">{{_('Allowed Upload Fileformats')}}</label>
+        <input type="text" class="form-control" name="config_upload_formats" id="config_upload_formats" value="{% if config.config_upload_formats != None %}{{ config.config_upload_formats }}{% endif %}" autocomplete="off">
+      </div>
+    </div>
     <div class="form-group">
         <input type="checkbox" id="config_anonbrowse" name="config_anonbrowse" {% if config.config_anonbrowse %}checked{% endif %}>
         <label for="config_anonbrowse">{{_('Enable Anonymous Browsing')}}</label>
     </div>
     <div class="form-group">
-        <input type="checkbox" id="config_public_reg" name="config_public_reg" {% if config.config_public_reg %}checked{% endif %}>
+        <input type="checkbox" id="config_public_reg" data-control="register_settings" name="config_public_reg" {% if config.config_public_reg %}checked{% endif %}>
         <label for="config_public_reg">{{_('Enable Public Registration')}}</label>
     </div>
+    <div data-related="register_settings">
+      <div class="form-group intend-form">
+        <input type="checkbox" id="config_register_email" name="config_register_email" {% if config.config_register_email %}checked{% endif %}>
+        <label for="config_register_email">{{_('Use E-Mail as Username')}}</label>
+      </div>
+    </div>
     <div class="form-group">
       <input type="checkbox" id="config_remote_login" name="config_remote_login" {% if config.config_remote_login %}checked{% endif %}>
       <label for="config_remote_login">{{_('Enable Magic Link Remote Login')}}</label>
@@ -326,30 +345,33 @@
     </div>
     <div id="collapseeight" class="panel-collapse collapse">
       <div class="panel-body">
-          <div class="form-group">
-              <div><input type="radio" name="config_ebookconverter" id="converter0" value="0" {% if config.config_ebookconverter == 0 %}checked{% endif %}>
-              <label for="converter0">{{_('No Converter')}}</label></div>
-              <div><input type="radio" name="config_ebookconverter" id="converter1" value="1" {% if config.config_ebookconverter == 1 %}checked{% endif %}>
-              <label for="converter1">{{_('Use Kindlegen')}}</label></div>
-              <div><input type="radio" name="config_ebookconverter" id="converter2" value="2" {% if config.config_ebookconverter == 2 %}checked{% endif %}>
-              <label for="converter2">{{_('Use calibre\'s ebook converter')}}</label></div>
+           <label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label>
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="converter_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
+           </div>
+           <div class="form-group">
+              <label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label>
+              <input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if config.config_calibre != None %}{{ config.config_calibre }}{% endif %}" autocomplete="off">
+           </div>
+        <label for="config_kepubifypath">{{_('Path to Kepubify E-Book Converter')}}</label>
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_kepubifypath" name="config_kepubifypath" value="{% if config.config_kepubifypath != None %}{{ config.config_kepubifypath }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="kepubify_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
           </div>
-          <div data-related="calibre">
-            <div class="form-group">
-                <label for="config_calibre">{{_('E-Book converter settings')}}</label>
-                <input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if config.config_calibre != None %}{{ config.config_calibre }}{% endif %}" autocomplete="off">
-            </div>
-            <div class="form-group">
-                <label for="config_calibre">{{_('Path to convertertool')}}</label>
-                <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
-            </div>
+        {% if feature_support['rar'] %}
+            <label for="config_rarfile_location">{{_('Location of Unrar binary')}}</label>
+           <div class="form-group input-group">
+            <input type="text" class="form-control" id="config_rarfile_location" name="config_rarfile_location" value="{% if config.config_rarfile_location != None %}{{ config.config_rarfile_location }}{% endif %}" autocomplete="off">
+            <span class="input-group-btn">
+              <button type="button" id="unrar_path" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+            </span>
           </div>
-          {% if feature_support['rar'] %}
-            <div class="form-group">
-                <label for="config_rarfile_location">{{_('Location of Unrar binary')}}</label>
-                <input type="text" class="form-control" name="config_rarfile_location" id="config_rarfile_location" value="{% if config.config_rarfile_location != None %}{{ config.config_rarfile_location }}{% endif %}" autocomplete="off">
-            </div>
-          {% endif %}
+        {% endif %}
       </div>
     </div>
   </div>

cps/templates/config_view_edit.html

@@ -6,7 +6,7 @@
 {% block body %}
 <div class="discover">
   <h2>{{title}}</h2>
-  <form role="form" method="POST" autocomplete="off">
+  <form role="form" method="POST" autocomplete="off" class="col-md-10 col-lg-6">
 <div class="panel-group">
   <div class="panel panel-default">
     <div class="panel-heading">
@@ -113,7 +113,7 @@
         </div>
         <div class="form-group">
           <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if conf.role_edit_shelfs() %}checked{% endif %}>
-          <label for="edit_shelf_role">{{_('Allow Editing Public Shelfs')}}</label>
+          <label for="edit_shelf_role">{{_('Allow Editing Public Shelves')}}</label>
         </div>
       </div>
     </div>

cps/templates/detail.html

@@ -202,6 +202,7 @@
             </label>
           </form>
           </p>
+          {% if g.user.check_visibility(32768) %}
           <p>
             <form id="archived_form" action="{{ url_for('web.toggle_archived', book_id=entry.id)}}" method="POST">
               <label class="block-label">
@@ -210,6 +211,7 @@
               </label>
             </form>
           </p>
+          {% endif %}
         </div>
       {% endif %}
 
@@ -240,7 +242,7 @@
                      data-remove-href="{{ url_for('shelf.remove_from_shelf', book_id=entry.id, shelf_id=shelf.id) }}"
                      data-shelf-action="add"
                   >
-                    {{shelf.name}}
+                    {{shelf.name}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %}
                   </a>
                 </li>
               {% endif %}
@@ -257,7 +259,7 @@
                 >
                   <span {% if not shelf.is_public or g.user.role_edit_shelfs() %}
                         class="glyphicon glyphicon-remove"
-                        {% endif %}></span> {{shelf.name}}
+                        {% endif %}></span> {{shelf.name}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %}
                 </a>
               {% endif %}
             {%endfor%}

cps/templates/email_edit.html

@@ -6,14 +6,14 @@
 {% block body %}
 <div class="discover">
   <h1>{{title}}</h1>
-  <form role="form" method="POST">
+  <form role="form" class="col-md-10 col-lg-6" method="POST">
     <div class="form-group">
       <label for="mail_server">{{_('SMTP Hostname')}}</label>
       <input type="text" class="form-control" name="mail_server" id="mail_server" value="{{content.mail_server}}">
     </div>
     <div class="form-group">
       <label for="mail_port">{{_('SMTP Port')}}</label>
-      <input type="text" class="form-control" name="mail_port" id="mail_port" value="{{content.mail_port}}">
+      <input type="number" min="1" max="65535" step="1" class="form-control" name="mail_port" id="mail_port" value="{% if content.mail_port != None %}{{ content.mail_port }}{% endif %}" autocomplete="off">
     </div>
     <div class="form-group">
       <label for="mail_use_ssl">{{_('Encryption')}}</label>
@@ -35,11 +35,19 @@
       <label for="mail_from">{{_('From E-mail')}}</label>
       <input type="text" class="form-control" name="mail_from" id="mail_from" value="{{content.mail_from}}">
     </div>
+    <label for="mail_size">{{_('Attachment Size Limit')}}</label>
+     <div class="form-group input-group">
+      <input type="number" min="1" max="600" step="1" class="form-control" name="mail_size" id="mail_size" value="{% if content.mail_size != None %}{{ (content.mail_size / 1024 / 1024)|int }}{% endif %}">
+      <span class="input-group-btn">
+        <button type="button" id="attachement_size" class="btn btn-default" disabled>MB</button>
+      </span>
+    </div>
     <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
     <button type="submit" name="test" value="test" class="btn btn-default">{{_('Save and Send Test E-mail')}}</button>
     <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Cancel')}}</a>
   </form>
     {% if g.allow_registration %}
+  <div class="col-md-10 col-lg-6">
     <h2>{{_('Allowed Domains (Whitelist)')}}</h2>
     <form id="domain_add_allow" action="{{ url_for('admin.add_domain',allow=1)}}" method="POST">
     <div class="form-group required">
@@ -74,6 +82,7 @@
     </div>
     <button id="domain_deny_submit" class="btn btn-default">{{_('Add')}}</button>
     </form>
+    </div>
 
     {% endif %}
 </div>

cps/templates/feed.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<feed xmlns="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/terms/">
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/terms/" xmlns:dcterms="http://purl.org/dc/terms/">
   <id>urn:uuid:2853dacf-ed79-42f5-8e8a-a7bb3d1ae6a2</id>
   <updated>{{ current_time }}</updated>
   <link rel="self"

cps/templates/grid.html

@@ -0,0 +1,49 @@
+{% extends "layout.html" %}
+{% block body %}
+<h1 class="{{page}}">{{_(title)}}</h1>
+
+    <div class="filterheader hidden-xs hidden-sm">
+      {% if entries.__len__() %}
+       {% if entries[0][0].sort %}
+        <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
+       {% endif %}
+      {% endif %}
+      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      {% if charlist|length %}
+      <button id="all" class="btn btn-primary">{{_('All')}}</button>
+      {% endif %}
+      <div class="btn-group character" role="group">
+        {% for char in charlist%}
+        <button class="btn btn-primary char">{{char.char}}</button>
+        {% endfor %}
+      </div>
+
+        <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="list">List</button>
+    </div>
+
+    {% if entries[0] %}
+        <div id="list" class="row">
+          {% for entry in entries %}
+              <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
+                  <div class="cover">
+                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                          <img src="{{ url_for('web.get_cover', book_id=entry[0].id) }}" alt="{{ entry[0].name }}"/>
+                          <span class="badge">{{entry.count}}</span>
+                      </a>
+                  </div>
+                  <div class="meta">
+                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                          <p class="title">{{entry[0].series[0].name|shortentitle}}</p>
+                      </a>
+                  </div>
+              </div>
+        {% endfor %}
+        </div>
+    {% endif %}
+
+
+{% endblock %}
+{% block js %}
+<script src="{{ url_for('static', filename='js/filter_grid.js') }}"></script>
+{% endblock %}

cps/templates/layout.html

@@ -17,6 +17,7 @@
     <link href="{{ url_for('static', filename='css/upload.css') }}" rel="stylesheet" media="screen">
     {% if g.current_theme == 1 %}
        <link href="{{ url_for('static', filename='css/caliBlur.min.css') }}" rel="stylesheet" media="screen">
+       <link href="{{ url_for('static', filename='css/caliBlur_override.css') }}" rel="stylesheet" media="screen">
     {% endif %}
     <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
@@ -25,7 +26,7 @@
       <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
     <![endif]-->
   </head>
-  <body class="{{ page }}" data-text="{{_('Home')}}" data-textback="{{_('Back')}}">
+  <body class="{{ page }} {{ bodyClass }}" data-text="{{_('Home')}}" data-textback="{{_('Back')}}">
     <!-- Static navbar -->
     <div class="navbar navbar-default navbar-static-top" role="navigation">
       <div class="container-fluid">
@@ -227,7 +228,9 @@
             $(document).ready(function() {
               var inp = $('#query').first()
               var val = inp.val()
-              inp.val('').blur().focus().val(val)
+              if (val.length) {
+                  inp.val('').blur().focus().val(val)
+              }
             });
         });
     </script>

cps/templates/list.html

@@ -18,6 +18,10 @@
         <button class="btn btn-primary char">{{char.char}}</button>
         {% endfor %}
       </div>
+
+      {% if data == "series" %}
+      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
+      {% endif %}
     </div>
   <div class="container">
     <div id="list" class="col-xs-12 col-sm-6">

cps/templates/register.html

@@ -3,10 +3,12 @@
 <div class="well col-sm-6 col-sm-offset-2">
   <h2 style="margin-top: 0">{{_('Register New Account')}}</h2>
   <form method="POST" role="form">
+    {% if not config.config_register_email %}
     <div class="form-group required">
       <label for="nickname">{{_('Username')}}</label>
       <input type="text" class="form-control" id="nickname" name="nickname" placeholder="{{_('Choose a username')}}" required>
     </div>
+    {% endif %}
     <div class="form-group required">
       <label for="email">{{_('E-mail Address')}}</label>
       <input type="email" class="form-control" id="email" name="email" placeholder="{{_('Your email address')}}" required>

cps/templates/search.html

@@ -15,14 +15,11 @@
                 <span class="caret"></span>
               </button>
               <ul id="add-to-shelves" class="dropdown-menu" aria-labelledby="add-to-shelf">
-                {% for shelf in g.user.shelf %}
-                  {% if shelf.is_public != 1 %}
-                    <li><a href="{{ url_for('shelf.search_to_shelf', shelf_id=shelf.id) }}"> {{shelf.name}}</a></li>
-                  {% endif %}
-                {% endfor %}
                 {% for shelf in g.shelves_access %}
-                  <li><a href="{{ url_for('shelf.search_to_shelf', shelf_id=shelf.id) }}">{{shelf.name}}</a></li>
-                {% endfor %}
+                  {% if not shelf.id in books_shelfs and ( not shelf.is_public or g.user.role_edit_shelfs() ) %}
+                    <li><a href="{{ url_for('shelf.search_to_shelf', shelf_id=shelf.id) }}"> {{shelf.name}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %}</a></li>
+                  {% endif %}
+                {%endfor%}
               </ul>
             </div>
           </div>

cps/templates/search_form.html

@@ -1,6 +1,6 @@
 {% extends "layout.html" %}
 {% block body %}
-<div class="col-sm-8">
+<div class="col-md-10 col-lg-6">
   <form role="form" id="search" action="{{ url_for('web.advanced_search') }}" method="GET">
     <div class="form-group">
       <label for="book_title">{{_('Book Title')}}</label>
@@ -143,6 +143,10 @@
           <input type="number" step="1" class="form-control" name="{{ 'custom_column_' ~ c.id }}" id="{{ 'custom_column_' ~ c.id }}" value="">
         {% endif %}
 
+        {% if c.datatype == 'float' %}
+          <input type="number" step="0.01" class="form-control" name="{{ 'custom_column_' ~ c.id }}" id="{{ 'custom_column_' ~ c.id }}" value="">
+        {% endif %}
+
         {% if c.datatype in ['text', 'series'] and not c.is_multiple %}
           <input type="text" class="form-control" name="{{ 'custom_column_' ~ c.id }}" id="{{ 'custom_column_' ~ c.id }}" value="">
         {% endif %}

cps/templates/user_edit.html

@@ -3,6 +3,7 @@
 <div class="discover">
   <h1>{{title}}</h1>
   <form role="form" method="POST" autocomplete="off">
+    <div class="col-md-10 col-lg-8">
     {% if new_user or ( g.user and content.nickname != "Guest" and g.user.role_admin() ) %}
     <div class="form-group required">
       <label for="nickname">{{_('Username')}}</label>
@@ -65,6 +66,7 @@
       <div class="btn btn-danger" id="config_delete_kobo_token" data-toggle="modal" data-target="#modalDeleteToken" data-remote="false" {% if not content.remote_auth_token.first() %} style="display: none;" {% endif %}>{{_('Delete')}}</div>
     </div>
     {% endif %}
+    </div>
     <div class="col-sm-6">
         {% for element in sidebar %}
           {% if element['config_show'] %}
@@ -119,7 +121,7 @@
         </div>
         <div class="form-group">
           <input type="checkbox" name="edit_shelf_role" id="edit_shelf_role" {% if content.role_edit_shelfs() %}checked{% endif %}>
-          <label for="edit_shelf_role">{{_('Allow Editing Public Shelfs')}}</label>
+          <label for="edit_shelf_role">{{_('Allow Editing Public Shelves')}}</label>
         </div>
       {% endif %}
     {% endif %}

cps/ub.py

@@ -49,6 +49,7 @@ from . import constants
 
 
 session = None
+app_DB_path = None
 Base = declarative_base()
 
 
@@ -107,6 +108,11 @@ def get_sidebar_config(kwargs=None):
         {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
          "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
          "show_text": _('Show archived books'), "config_show": content})
+    '''sidebar.append(
+        {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_list', "id": "list",
+         "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list",
+         "show_text": _('Show Books List'), "config_show": content})'''
+
     return sidebar
 
 
@@ -211,6 +217,7 @@ class User(UserBase, Base):
     denied_column_value = Column(String, default="")
     allowed_column_value = Column(String, default="")
     remote_auth_token = relationship('RemoteAuthToken', backref='user', lazy='dynamic')
+    series_view = Column(String(10), default="list")
 
 
 if oauth_support:
@@ -251,6 +258,7 @@ class Anonymous(AnonymousUserMixin, UserBase):
         self.allowed_tags = data.allowed_tags
         self.denied_column_value = data.denied_column_value
         self.allowed_column_value = data.allowed_column_value
+        self.series_view = data.series_view
 
     def role_admin(self):
         return False
@@ -447,7 +455,7 @@ class RemoteAuthToken(Base):
 
 
 # Migrate database to current version, has to be updated after every database change. Currently migration from
-# everywhere to curent should work. Migration is done by checking if relevant coloums are existing, and than adding
+# everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding
 # rows with SQL commands
 def migrate_Database(session):
     engine = session.bind
@@ -494,6 +502,10 @@ def migrate_Database(session):
         conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME")
         conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0")
         session.commit()
+    test = session.query(ReadBook).filter(ReadBook.last_modified == None).all()
+    for book in test:
+        book.last_modified = datetime.datetime.utcnow()
+    session.commit()
     try:
         session.query(exists().where(Shelf.uuid)).scalar()
     except exc.OperationalError:
@@ -552,6 +564,13 @@ def migrate_Database(session):
         conn.execute("ALTER TABLE user ADD column `allowed_tags` String DEFAULT ''")
         conn.execute("ALTER TABLE user ADD column `denied_column_value` DEFAULT ''")
         conn.execute("ALTER TABLE user ADD column `allowed_column_value` DEFAULT ''")
+        session.commit()
+    try:
+        session.query(exists().where(User.series_view)).scalar()
+    except exc.OperationalError:
+        conn = engine.connect()
+        conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'")
+
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() \
         is None:
         create_anonymous_user(session)
@@ -563,7 +582,7 @@ def migrate_Database(session):
         # Create new table user_id and copy contents of table user into it
         conn = engine.connect()
         conn.execute("CREATE TABLE user_id (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,"
-                     " nickname VARCHAR(64),"
+                     "nickname VARCHAR(64),"
                      "email VARCHAR(120),"
                      "role SMALLINT,"
                      "password VARCHAR,"
@@ -571,10 +590,11 @@ def migrate_Database(session):
                      "locale VARCHAR(2),"
                      "sidebar_view INTEGER,"
                      "default_language VARCHAR(3),"
+                     "series_view VARCHAR(10),"
                      "UNIQUE (nickname),"
                      "UNIQUE (email))")
         conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale,"
-                     "sidebar_view, default_language) "
+                     "sidebar_view, default_language, series_view) "
                      "SELECT id, nickname, email, role, password, kindle_mail, locale,"
                      "sidebar_view, default_language FROM user")
         # delete old user table and rename new user_id table to user:
@@ -611,8 +631,7 @@ def delete_download(book_id):
     session.query(Downloads).filter(book_id == Downloads.book_id).delete()
     session.commit()
 
-
-# Generate user Guest (translated text), as anoymous user, no rights
+# Generate user Guest (translated text), as anonymous user, no rights
 def create_anonymous_user(session):
     user = User()
     user.nickname = "Guest"
@@ -646,7 +665,9 @@ def create_admin_user(session):
 def init_db(app_db_path):
     # Open session for database connection
     global session
+    global app_DB_path
 
+    app_DB_path = app_db_path
     engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False)
 
     Session = sessionmaker()

cps/uploader.py

@@ -40,7 +40,7 @@ try:
     from wand.exceptions import PolicyError
     use_generic_pdf_cover = False
 except (ImportError, RuntimeError) as e:
-    log.debug('cannot import Image, generating pdf covers for pdf uploads will not work: %s', e)
+    log.debug('Cannot import Image, generating pdf covers for pdf uploads will not work: %s', e)
     use_generic_pdf_cover = True
 
 try:
@@ -48,21 +48,21 @@ try:
     from PyPDF2 import __version__ as PyPdfVersion
     use_pdf_meta = True
 except ImportError as e:
-    log.debug('cannot import PyPDF2, extracting pdf metadata will not work: %s', e)
+    log.debug('Cannot import PyPDF2, extracting pdf metadata will not work: %s', e)
     use_pdf_meta = False
 
 try:
     from . import epub
     use_epub_meta = True
 except ImportError as e:
-    log.debug('cannot import epub, extracting epub metadata will not work: %s', e)
+    log.debug('Cannot import epub, extracting epub metadata will not work: %s', e)
     use_epub_meta = False
 
 try:
     from . import fb2
     use_fb2_meta = True
 except ImportError as e:
-    log.debug('cannot import fb2, extracting fb2 metadata will not work: %s', e)
+    log.debug('Cannot import fb2, extracting fb2 metadata will not work: %s', e)
     use_fb2_meta = False
 
 try:
@@ -70,27 +70,25 @@ try:
     from PIL import __version__ as PILversion
     use_PIL = True
 except ImportError as e:
-    log.debug('cannot import Pillow, using png and webp images as cover will not work: %s', e)
+    log.debug('Cannot import Pillow, using png and webp images as cover will not work: %s', e)
     use_PIL = False
 
-__author__ = 'lemmsh'
 
-
-def process(tmp_file_path, original_file_name, original_file_extension, rarExcecutable):
+def process(tmp_file_path, original_file_name, original_file_extension, rarExecutable):
     meta = None
+    extension_upper = original_file_extension.upper()
     try:
-        if ".PDF" == original_file_extension.upper():
+        if ".PDF" == extension_upper:
             meta = pdf_meta(tmp_file_path, original_file_name, original_file_extension)
-        if ".EPUB" == original_file_extension.upper() and use_epub_meta is True:
+        elif extension_upper in [".KEPUB", ".EPUB"] and use_epub_meta is True:
             meta = epub.get_epub_info(tmp_file_path, original_file_name, original_file_extension)
-        if ".FB2" == original_file_extension.upper() and use_fb2_meta is True:
+        elif ".FB2" == extension_upper and use_fb2_meta is True:
             meta = fb2.get_fb2_info(tmp_file_path, original_file_extension)
-        if original_file_extension.upper() in ['.CBZ', '.CBT', '.CBR']:
+        elif extension_upper in ['.CBZ', '.CBT', '.CBR']:
             meta = comic.get_comic_info(tmp_file_path,
                                         original_file_name,
                                         original_file_extension,
-                                        rarExcecutable)
-
+                                        rarExecutable)
     except Exception as ex:
         log.warning('cannot parse metadata, using default: %s', ex)
 
@@ -98,8 +96,7 @@ def process(tmp_file_path, original_file_name, original_file_extension, rarExcec
         if meta.author.lower() == 'unknown':
             meta = meta._replace(author=_(u'Unknown'))
         return meta
-    else:
-        return default_meta(tmp_file_path, original_file_name, original_file_extension)
+    return default_meta(tmp_file_path, original_file_name, original_file_extension)
 
 
 def default_meta(tmp_file_path, original_file_name, original_file_extension):
@@ -117,14 +114,11 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
 
 
 def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
-
+    doc_info = None
     if use_pdf_meta:
-        pdf = PdfFileReader(open(tmp_file_path, 'rb'))
-        doc_info = pdf.getDocumentInfo()
-    else:
-        doc_info = None
+        doc_info = PdfFileReader(open(tmp_file_path, 'rb')).getDocumentInfo()
 
-    if doc_info is not None:
+    if doc_info:
         author = doc_info.author if doc_info.author else u'Unknown'
         title = doc_info.title if doc_info.title else original_file_name
         subject = doc_info.subject
@@ -149,21 +143,20 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
 def pdf_preview(tmp_file_path, tmp_dir):
     if use_generic_pdf_cover:
         return None
-    else:
-        try:
-            cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jpg"
-            with Image() as img:
-                img.options["pdf:use-cropbox"] = "true"
-                img.read(filename=tmp_file_path + '[0]', resolution = 150)
-                img.compression_quality = 88
-                img.save(filename=os.path.join(tmp_dir, cover_file_name))
-            return cover_file_name
-        except PolicyError as ex:
-            log.warning('Pdf extraction forbidden by Imagemagick policy: %s', ex)
-            return None
-        except Exception as ex:
-            log.warning('Cannot extract cover image, using default: %s', ex)
-            return None
+    try:
+        cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jpg"
+        with Image() as img:
+            img.options["pdf:use-cropbox"] = "true"
+            img.read(filename=tmp_file_path + '[0]', resolution=150)
+            img.compression_quality = 88
+            img.save(filename=os.path.join(tmp_dir, cover_file_name))
+        return cover_file_name
+    except PolicyError as ex:
+        log.warning('Pdf extraction forbidden by Imagemagick policy: %s', ex)
+        return None
+    except Exception as ex:
+        log.warning('Cannot extract cover image, using default: %s', ex)
+        return None
 
 
 def get_versions():
@@ -186,7 +179,7 @@ def get_versions():
     else:
         PILVersion = u'not installed'
     if comic.use_comic_meta:
-        ComicVersion = u'installed'
+        ComicVersion = comic.comic_version or u'installed'
     else:
         ComicVersion = u'not installed'
     return {'Image Magick': IVersion,
@@ -202,12 +195,11 @@ def upload(uploadfile, rarExcecutable):
 
     if not os.path.isdir(tmp_dir):
         os.mkdir(tmp_dir)
+
     filename = uploadfile.filename
     filename_root, file_extension = os.path.splitext(filename)
-    md5 = hashlib.md5()
-    md5.update(filename.encode('utf-8'))
-    tmp_file_path = os.path.join(tmp_dir, md5.hexdigest())
+    md5 = hashlib.md5(filename.encode('utf-8')).hexdigest()
+    tmp_file_path = os.path.join(tmp_dir, md5)
     log.debug("Temporary file: %s", tmp_file_path)
     uploadfile.save(tmp_file_path)
-    meta = process(tmp_file_path, filename_root, file_extension, rarExcecutable)
-    return meta
+    return process(tmp_file_path, filename_root, file_extension, rarExcecutable)

cps/web.py

@@ -37,9 +37,10 @@ from flask import Blueprint
 from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
 from flask_babel import gettext as _
 from flask_login import login_user, logout_user, login_required, current_user
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
 from sqlalchemy.sql.expression import text, func, true, false, not_, and_, or_
-from werkzeug.exceptions import default_exceptions
+from werkzeug.exceptions import default_exceptions, InternalServerError
+from sqlalchemy.sql.functions import coalesce
 try:
     from werkzeug.exceptions import FailedDependency
 except ImportError:
@@ -47,13 +48,13 @@ except ImportError:
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash, check_password_hash
 
-from . import constants, logger, isoLanguages, services, worker
+from . import constants, logger, isoLanguages, services, worker, cli
 from . import searched_ids, lm, babel, db, ub, config, get_locale, app
+from . import calibre_db
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import common_filters, get_search_results, fill_indexpage, fill_indexpage_with_archived_books, \
-    speaking_language, check_valid_domain, order_authors, get_typeahead, render_task_status, json_serial, \
+from .helper import check_valid_domain, render_task_status, json_serial, \
     get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
-    send_registration_mail, check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password
+    send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
 
@@ -123,6 +124,21 @@ if feature_support['ldap']:
         log.debug('LDAP server not accessible while trying to login to opds feed')
         return error_http(FailedDependency())
 
+# @app.errorhandler(InvalidRequestError)
+#@app.errorhandler(OperationalError)
+#def handle_db_exception(e):
+#    db.session.rollback()
+#    log.error('Database request error: %s',e)
+#    return internal_error(InternalServerError(e))
+
+@app.after_request
+def add_security_headers(resp):
+    # resp.headers['Content-Security-Policy']= "script-src 'self' https://www.googleapis.com https://api.douban.com https://comicvine.gamespot.com;"
+    resp.headers['X-Content-Type-Options'] = 'nosniff'
+    resp.headers['X-Frame-Options'] = 'SAMEORIGIN'
+    resp.headers['X-XSS-Protection'] = '1; mode=block'
+    # resp.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
+    return resp
 
 web = Blueprint('web', __name__)
 log = logger.create()
@@ -316,7 +332,7 @@ def import_ldap_users():
             match = re.search("([a-zA-Z0-9-]+)=%s", config.config_ldap_user_object, re.IGNORECASE | re.UNICODE)
             if match:
                 match_filter = match.group(1)
-                match = re.search(match_filter + "=([[\d\w-]+)", user, re.IGNORECASE | re.UNICODE)
+                match = re.search(match_filter + "=([\d\s\w-]+)", user, re.IGNORECASE | re.UNICODE)
                 if match:
                     user = match.group(1)
                 else:
@@ -422,21 +438,24 @@ def toggle_read(book_id):
         ub.session.commit()
     else:
         try:
-            db.update_title_sort(config)
-            book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+            calibre_db.update_title_sort(config)
+            book = calibre_db.get_filtered_book(book_id)
             read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
             if len(read_status):
                 read_status[0].value = not read_status[0].value
-                db.session.commit()
+                calibre_db.session.commit()
             else:
                 cc_class = db.cc_classes[config.config_read_column]
                 new_cc = cc_class(value=1, book=book_id)
-                db.session.add(new_cc)
-                db.session.commit()
+                calibre_db.session.add(new_cc)
+                calibre_db.session.commit()
         except KeyError:
             log.error(u"Custom Column No.%d is not exisiting in calibre database", config.config_read_column)
-    return ""
+        except OperationalError as e:
+            calibre_db.session.rollback()
+            log.error(u"Read status could not set: %e", e)
 
+    return ""
 
 @web.route("/ajax/togglearchived/<int:book_id>", methods=['POST'])
 @login_required
@@ -454,11 +473,30 @@ def toggle_archived(book_id):
     return ""
 
 
+@web.route("/ajax/view", methods=["POST"])
+@login_required
+def update_view():
+    to_save = request.form.to_dict()
+    allowed_view = ['grid', 'list']
+    if "series_view" in to_save and to_save["series_view"] in allowed_view:
+        current_user.series_view = to_save["series_view"]
+    else:
+        log.error("Invalid request received: %r %r", request, to_save)
+        return "Invalid request", 400
+
+    try:
+        ub.session.commit()
+    except InvalidRequestError:
+        log.error("Invalid request received: %r ", request, )
+        return "Invalid request", 400
+    return "", 200
+
+
 '''
 @web.route("/ajax/getcomic/<int:book_id>/<book_format>/<int:page>")
 @login_required
 def get_comic_book(book_id, book_format, page):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     if not book:
         return "", 204
     else:
@@ -512,25 +550,25 @@ def get_comic_book(book_id, book_format, page):
 @web.route("/get_authors_json", methods=['GET'])
 @login_required_if_no_ano
 def get_authors_json():
-    return get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_publishers_json", methods=['GET'])
 @login_required_if_no_ano
 def get_publishers_json():
-    return get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_tags_json", methods=['GET'])
 @login_required_if_no_ano
 def get_tags_json():
-    return get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
+    return calibre_db.get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
 
 
 @web.route("/get_series_json", methods=['GET'])
 @login_required_if_no_ano
 def get_series_json():
-    return get_typeahead(db.Series, request.args.get('q'))
+    return calibre_db.get_typeahead(db.Series, request.args.get('q'))
 
 
 @web.route("/get_languages_json", methods=['GET'])
@@ -551,8 +589,8 @@ def get_languages_json():
 @login_required_if_no_ano
 def get_matching_tags():
     tag_dict = {'tags': []}
-    q = db.session.query(db.Books)
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True))
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
     author_input = request.args.get('author_name') or ''
     title_input = request.args.get('book_title') or ''
     include_tag_inputs = request.args.getlist('include_tag') or ''
@@ -582,7 +620,7 @@ def get_matching_tags():
 @web.route('/page/<int:page>')
 @login_required_if_no_ano
 def index(page):
-    entries, random, pagination = fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
+    entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                  title=_(u"Recently Added Books"), page="root")
 
@@ -609,15 +647,17 @@ def books_list(data, sort, book_id, page):
 
     if data == "rated":
         if current_user.check_visibility(constants.SIDEBAR_BEST_RATED):
-            entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
-                                                         order)
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
+                                                                    db.Books.ratings.any(db.Ratings.rating > 9),
+                                                                    order)
             return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                          id=book_id, title=_(u"Top Rated Books"), page="rated")
         else:
             abort(404)
     elif data == "discover":
         if current_user.check_visibility(constants.SIDEBAR_RANDOM):
-            entries, __, pagination = fill_indexpage(page, db.Books, True, [func.randomblob(2)])
+            entries, __, pagination = calibre_db.fill_indexpage(page, db.Books, True, [func.randomblob(2)])
             pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page)
             return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id,
                                          title=_(u"Discover (Random Books)"), page="discover")
@@ -646,7 +686,7 @@ def books_list(data, sort, book_id, page):
     elif data == "archived":
         return render_archived_books(page, order)
     else:
-        entries, random, pagination = fill_indexpage(page, db.Books, True, order)
+        entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, order)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                      title=_(u"Books"), page="newest")
 
@@ -654,7 +694,7 @@ def books_list(data, sort, book_id, page):
 def render_hot_books(page):
     if current_user.check_visibility(constants.SIDEBAR_HOT):
         if current_user.show_detail_random():
-            random = db.session.query(db.Books).filter(common_filters()) \
+            random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \
                 .order_by(func.random()).limit(config.config_random_books)
         else:
             random = false()
@@ -664,7 +704,7 @@ def render_hot_books(page):
         hot_books = all_books.offset(off).limit(config.config_books_per_page)
         entries = list()
         for book in hot_books:
-            downloadBook = db.session.query(db.Books).filter(common_filters()).filter(
+            downloadBook = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter(
                 db.Books.id == book.Downloads.book_id).first()
             if downloadBook:
                 entries.append(downloadBook)
@@ -681,15 +721,18 @@ def render_hot_books(page):
 
 
 def render_author_books(page, author_id, order):
-    entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id),
-                                             [order[0], db.Series.name, db.Books.series_index],
-                                             db.books_series_link, db.Series)
+    entries, __, pagination = calibre_db.fill_indexpage(page,
+                                                        db.Books,
+                                                        db.Books.authors.any(db.Authors.id == author_id),
+                                                        [order[0], db.Series.name, db.Books.series_index],
+                                                        db.books_series_link,
+                                                        db.Series)
     if entries is None or not len(entries):
         flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"),
               category="error")
         return redirect(url_for("web.index"))
 
-    author = db.session.query(db.Authors).get(author_id)
+    author = calibre_db.session.query(db.Authors).get(author_id)
     author_name = author.name.replace('|', ',')
 
     author_info = None
@@ -704,12 +747,14 @@ def render_author_books(page, author_id, order):
 
 
 def render_publisher_books(page, book_id, order):
-    publisher = db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
+    publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
     if publisher:
-        entries, random, pagination = fill_indexpage(page, db.Books,
-                                                     db.Books.publishers.any(db.Publishers.id == book_id),
-                                                     [db.Series.name, order[0], db.Books.series_index],
-                                                     db.books_series_link, db.Series)
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.publishers.any(db.Publishers.id == book_id),
+                                                                [db.Series.name, order[0], db.Books.series_index],
+                                                                db.books_series_link,
+                                                                db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
                                      title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher")
     else:
@@ -717,10 +762,12 @@ def render_publisher_books(page, book_id, order):
 
 
 def render_series_books(page, book_id, order):
-    name = db.session.query(db.Series).filter(db.Series.id == book_id).first()
+    name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.series.any(db.Series.id == book_id),
-                                                     [db.Books.series_index, order[0]])
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.series.any(db.Series.id == book_id),
+                                                                [db.Books.series_index, order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"Series: %(serie)s", serie=name.name), page="series")
     else:
@@ -728,9 +775,11 @@ def render_series_books(page, book_id, order):
 
 
 def render_ratings_books(page, book_id, order):
-    name = db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.id == book_id),
-                                                 [db.Books.timestamp.desc(), order[0]])
+    name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.ratings.any(db.Ratings.id == book_id),
+                                                            [db.Books.timestamp.desc(), order[0]])
     if name and name.rating <= 10:
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings")
@@ -739,11 +788,12 @@ def render_ratings_books(page, book_id, order):
 
 
 def render_formats_books(page, book_id, order):
-    name = db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
+    name = calibre_db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books,
-                                                     db.Books.data.any(db.Data.format == book_id.upper()),
-                                                     [db.Books.timestamp.desc(), order[0]])
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.data.any(db.Data.format == book_id.upper()),
+                                                                [db.Books.timestamp.desc(), order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"File format: %(format)s", format=name.format), page="formats")
     else:
@@ -751,11 +801,13 @@ def render_formats_books(page, book_id, order):
 
 
 def render_category_books(page, book_id, order):
-    name = db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
+    name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.tags.any(db.Tags.id == book_id),
-                                                     [order[0], db.Series.name, db.Books.series_index],
-                                                     db.books_series_link, db.Series)
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.tags.any(db.Tags.id == book_id),
+                                                                [order[0], db.Series.name, db.Books.series_index],
+                                                                db.books_series_link, db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
                                      title=_(u"Category: %(name)s", name=name.name), page="category")
     else:
@@ -771,21 +823,29 @@ def render_language_books(page, name, order):
             lang_name = _(isoLanguages.get(part3=name).name)
         except KeyError:
             abort(404)
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.languages.any(db.Languages.lang_code == name),
-                                                 [db.Books.timestamp.desc(), order[0]])
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.languages.any(db.Languages.lang_code == name),
+                                                            [db.Books.timestamp.desc(), order[0]])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language")
 
 
+'''@web.route("/table")
+@login_required_if_no_ano
+def books_table():
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
+                                 title=_(u"Language: %(name)s", name=lang_name), page="language")'''
+
 @web.route("/author")
 @login_required_if_no_ano
 def author_list():
     if current_user.check_visibility(constants.SIDEBAR_AUTHOR):
-        entries = db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).all()
-        charlist = db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
         for entry in entries:
             entry.Authors.name = entry.Authors.name.replace('|', ',')
@@ -799,11 +859,11 @@ def author_list():
 @login_required_if_no_ano
 def publisher_list():
     if current_user.check_visibility(constants.SIDEBAR_PUBLISHER):
-        entries = db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
-            .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.sort).all()
-        charlist = db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
+            .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.name).all()
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher")
@@ -815,14 +875,25 @@ def publisher_list():
 @login_required_if_no_ano
 def series_list():
     if current_user.check_visibility(constants.SIDEBAR_SERIES):
-        entries = db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
-            .join(db.books_series_link).join(db.Books).filter(common_filters()) \
-            .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
-        charlist = db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
-            .join(db.books_series_link).join(db.Books).filter(common_filters()) \
-            .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
-        return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
-                                     title=_(u"Series"), page="serieslist", data="series")
+        if current_user.series_view == 'list':
+            entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
+                .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
+            charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
+                .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
+            return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
+                                         title=_(u"Series"), page="serieslist", data="series")
+        else:
+            entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count')) \
+                .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \
+                .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
+            charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
+                .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
+
+            return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist,
+                                         title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view")
     else:
         abort(404)
 
@@ -831,9 +902,9 @@ def series_list():
 @login_required_if_no_ano
 def ratings_list():
     if current_user.check_visibility(constants.SIDEBAR_RATING):
-        entries = db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
+        entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                    (db.Ratings.rating / 2).label('name')) \
-            .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings")
@@ -845,8 +916,10 @@ def ratings_list():
 @login_required_if_no_ano
 def formats_list():
     if current_user.check_visibility(constants.SIDEBAR_FORMAT):
-        entries = db.session.query(db.Data, func.count('data.book').label('count'), db.Data.format.label('format')) \
-            .join(db.Books).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Data,
+                                           func.count('data.book').label('count'),
+                                           db.Data.format.label('format')) \
+            .join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(db.Data.format).order_by(db.Data.format).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats")
@@ -860,20 +933,20 @@ def language_overview():
     if current_user.check_visibility(constants.SIDEBAR_LANGUAGE):
         charlist = list()
         if current_user.filter_language() == u"all":
-            languages = speaking_language()
+            languages = calibre_db.speaking_language()
             # ToDo: generate first character list for languages
         else:
             try:
                 cur_l = LC.parse(current_user.filter_language())
             except UnknownLocaleError:
                 cur_l = None
-            languages = db.session.query(db.Languages).filter(
+            languages = calibre_db.session.query(db.Languages).filter(
                 db.Languages.lang_code == current_user.filter_language()).all()
             if cur_l:
                 languages[0].name = cur_l.get_language_name(get_locale())
             else:
                 languages[0].name = _(isoLanguages.get(part3=languages[0].lang_code).name)
-        lang_counter = db.session.query(db.books_languages_link,
+        lang_counter = calibre_db.session.query(db.books_languages_link,
                                         func.count('books_languages_link.book').label('bookcount')).group_by(
             text('books_languages_link.lang_code')).all()
         return render_title_template('languages.html', languages=languages, lang_counter=lang_counter,
@@ -887,11 +960,11 @@ def language_overview():
 @login_required_if_no_ano
 def category_list():
     if current_user.check_visibility(constants.SIDEBAR_CATEGORY):
-        entries = db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
-            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(common_filters()) \
+        entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
+            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
-        charlist = db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
-            .join(db.books_tags_link).join(db.Books).filter(common_filters()) \
+        charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
+            .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Categories"), page="catlist", data="category")
@@ -911,21 +984,21 @@ def get_tasks_status():
     return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks")
 
 
-# ################################### Search functions ################################################################
-
 @app.route("/reconnect")
 def reconnect():
-    db.reconnect_db(config)
+    db.reconnect_db(config, ub.app_DB_path)
     return json.dumps({})
 
 
+# ################################### Search functions ################################################################
+
+
 @web.route("/search", methods=["GET"])
 @login_required_if_no_ano
 def search():
     term = request.args.get("query")
     if term:
-        term.strip().lower()
-        entries = get_search_results(term)
+        entries = calibre_db.get_search_results(term)
         ids = list()
         for element in entries:
             ids.append(element.id)
@@ -947,9 +1020,9 @@ def search():
 @login_required_if_no_ano
 def advanced_search():
     # Build custom columns names
-    cc = get_cc_columns()
-    db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = db.session.query(db.Books).filter(common_filters()).order_by(db.Books.sort)
+    cc = get_cc_columns(filter_config_custom_read=True)
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True)).order_by(db.Books.sort)
 
     include_tag_inputs = request.args.getlist('include_tag')
     exclude_tag_inputs = request.args.getlist('exclude_tag')
@@ -1002,13 +1075,13 @@ def advanced_search():
                                                format='medium', locale=get_locale())])
             except ValueError:
                 pub_start = u""
-        tag_names = db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all()
+        tag_names = calibre_db.session.query(db.Tags).filter(db.Tags.id.in_(include_tag_inputs)).all()
         searchterm.extend(tag.name for tag in tag_names)
-        serie_names = db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all()
+        serie_names = calibre_db.session.query(db.Series).filter(db.Series.id.in_(include_series_inputs)).all()
         searchterm.extend(serie.name for serie in serie_names)
-        language_names = db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all()
+        language_names = calibre_db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all()
         if language_names:
-            language_names = speaking_language(language_names)
+            language_names = calibre_db.speaking_language(language_names)
         searchterm.extend(language.name for language in language_names)
         if rating_high:
             searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)])
@@ -1063,13 +1136,16 @@ def advanced_search():
         # search custom culumns
         for c in cc:
             custom_query = request.args.get('custom_column_' + str(c.id))
-            if custom_query:
+            if custom_query != '' and custom_query is not None:
                 if c.datatype == 'bool':
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         db.cc_classes[c.id].value == (custom_query == "True")))
-                elif c.datatype == 'int':
+                elif c.datatype == 'int' or c.datatype == 'float':
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         db.cc_classes[c.id].value == custom_query))
+                elif c.datatype == 'rating':
+                    q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
+                        db.cc_classes[c.id].value == int(custom_query) * 2))
                 else:
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
@@ -1081,15 +1157,26 @@ def advanced_search():
         return render_title_template('search.html', adv_searchterm=searchterm,
                                      entries=q, title=_(u"search"), page="search")
     # prepare data for search-form
-    tags = db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).all()
-    series = db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters()) \
-        .group_by(text('books_series_link.series')).order_by(db.Series.name).filter(common_filters()).all()
-    extensions = db.session.query(db.Data).join(db.Books).filter(common_filters()) \
-        .group_by(db.Data.format).order_by(db.Data.format).all()
-
+    tags = calibre_db.session.query(db.Tags)\
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name).all()
+    series = calibre_db.session.query(db.Series)\
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.name)\
+        .filter(calibre_db.common_filters()).all()
+    extensions = calibre_db.session.query(db.Data)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         languages = None
     return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
@@ -1099,34 +1186,43 @@ def advanced_search():
 def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs):
     order = order or []
     if not config.config_read_column:
-        readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
-            .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
-        readBookIds = [x.book_id for x in readBooks]
+        if are_read:
+            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
+                             ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
+        else:
+            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db_filter,
+                                                                order,
+                                                                ub.ReadBook, db.Books.id==ub.ReadBook.book_id)
     else:
         try:
-            readBooks = db.session.query(db.cc_classes[config.config_read_column]) \
-                .filter(db.cc_classes[config.config_read_column].value == True).all()
-            readBookIds = [x.book for x in readBooks]
+            if are_read:
+                db_filter = db.cc_classes[config.config_read_column].value == True
+            else:
+                db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
+                                                                    db_filter,
+                                                                    order,
+                                                                    db.cc_classes[config.config_read_column])
         except KeyError:
             log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
-            readBookIds = []
-
-    if are_read:
-        db_filter = db.Books.id.in_(readBookIds)
-    else:
-        db_filter = ~db.Books.id.in_(readBookIds)
-
-    entries, random, pagination = fill_indexpage(page, db.Books, db_filter, order)
-
+            if not as_xml:
+                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                        column=config.config_read_column),
+                      category="error")
+                return redirect(url_for("web.index"))
+            # ToDo: Handle error Case for opds
     if as_xml:
         return entries, pagination
     else:
         if are_read:
-            name = _(u'Read Books') + ' (' + str(len(readBookIds)) + ')'
+            name = _(u'Read Books') + ' (' + str(pagination.total_count) + ')'
             pagename = "read"
         else:
-            total_books = db.session.query(func.count(db.Books.id)).filter(common_filters()).scalar()
-            name = _(u'Unread Books') + ' (' + str(total_books - len(readBookIds)) + ')'
+            name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')'
             pagename = "unread"
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                      title=name, page=pagename)
@@ -1144,8 +1240,11 @@ def render_archived_books(page, order):
 
     archived_filter = db.Books.id.in_(archived_book_ids)
 
-    entries, random, pagination = fill_indexpage_with_archived_books(page, db.Books, archived_filter, order,
-                                                                     allow_show_archived=True)
+    entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page,
+                                                                                db.Books,
+                                                                                archived_filter,
+                                                                                order,
+                                                                                allow_show_archived=True)
 
     name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
     pagename = "archived"
@@ -1167,9 +1266,8 @@ def get_cover(book_id):
 @viewer_required
 def serve_book(book_id, book_format, anyname):
     book_format = book_format.split(".")[0]
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == book_format.upper()) \
-        .first()
+    book = calibre_db.get_book(book_id)
+    data = calibre_db.get_book_format(book.id, book_format.upper())
     log.info('Serving book: %s', data.name)
     if config.config_use_google_drive:
         headers = Headers()
@@ -1185,7 +1283,12 @@ def serve_book(book_id, book_format, anyname):
 @login_required_if_no_ano
 @download_required
 def download_link(book_id, book_format, anyname):
-    return get_download_link(book_id, book_format.lower())
+    if "Kobo" in request.headers.get('User-Agent'):
+        client = "kobo"
+    else:
+        client=""
+
+    return get_download_link(book_id, book_format, client)
 
 
 @web.route('/send/<int:book_id>/<book_format>/<int:convert>')
@@ -1226,30 +1329,33 @@ def register():
 
     if request.method == "POST":
         to_save = request.form.to_dict()
-        if not to_save["nickname"] or not to_save["email"]:
+        if config.config_register_email:
+            nickname = to_save["email"]
+        else:
+            nickname = to_save["nickname"]
+        if not nickname or not to_save["email"]:
             flash(_(u"Please fill out all fields!"), category="error")
             return render_title_template('register.html', title=_(u"register"), page="register")
 
-        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == to_save["nickname"]
+
+        existing_user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == nickname
                                                          .lower()).first()
         existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()).first()
         if not existing_user and not existing_email:
             content = ub.User()
-            # content.password = generate_password_hash(to_save["password"])
             if check_valid_domain(to_save["email"]):
-                content.nickname = to_save["nickname"]
+                content.nickname = nickname
                 content.email = to_save["email"]
                 password = generate_random_password()
                 content.password = generate_password_hash(password)
                 content.role = config.config_default_role
                 content.sidebar_view = config.config_default_show
-                # content.mature_content = bool(config.config_default_show & constants.MATURE_CONTENT)
                 try:
                     ub.session.add(content)
                     ub.session.commit()
                     if feature_support['oauth']:
                         register_user_with_oauth(content)
-                    send_registration_mail(to_save["email"], to_save["nickname"], password)
+                    send_registration_mail(to_save["email"], nickname, password)
                 except Exception:
                     ub.session.rollback()
                     flash(_(u"An unknown error occurred. Please try again later."), category="error")
@@ -1443,7 +1549,7 @@ def token_verified():
 @login_required
 def profile():
     downloads = list()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     if feature_support['oauth']:
@@ -1452,9 +1558,9 @@ def profile():
         oauth_status = None
 
     for book in current_user.downloads:
-        downloadBook = db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+        downloadBook = calibre_db.get_book(book.book_id)
         if downloadBook:
-            downloads.append(db.session.query(db.Books).filter(db.Books.id == book.book_id).first())
+            downloads.append(downloadBook)
         else:
             ub.delete_download(book.book_id)
     if request.method == "POST":
@@ -1467,7 +1573,7 @@ def profile():
             current_user.kindle_mail = to_save["kindle_mail"]
         if "allowed_tags" in to_save and to_save["allowed_tags"] != current_user.allowed_tags:
             current_user.allowed_tags = to_save["allowed_tags"].strip()
-        if to_save["email"] and to_save["email"] != current_user.email:
+        if "email" in to_save and to_save["email"] != current_user.email:
             if config.config_public_reg and not check_valid_domain(to_save["email"]):
                 flash(_(u"E-mail is not from valid domain"), category="error")
                 return render_title_template("user_edit.html", content=current_user, downloads=downloads,
@@ -1533,10 +1639,10 @@ def profile():
 @login_required_if_no_ano
 @viewer_required
 def read_book(book_id, book_format):
-    book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if not book:
-        flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")
-        log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
+        flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
+        log.debug(u"Error opening eBook. File does not exist or file is not accessible")
         return redirect(url_for("web.index"))
 
     # check if book has bookmark
@@ -1555,9 +1661,9 @@ def read_book(book_id, book_format):
         log.debug(u"Start txt reader for %d", book_id)
         return render_title_template('readtxt.html', txtfile=book_id, title=_(u"Read a Book"))
     else:
-        for fileExt in ["mp3", "m4b", "m4a"]:
+        for fileExt in constants.EXTENSIONS_AUDIO:
             if book_format.lower() == fileExt:
-                entries = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+                entries = calibre_db.get_filtered_book(book_id)
                 log.debug(u"Start mp3 listening for %d", book_id)
                 return render_title_template('listenmp3.html', mp3file=book_id, audioformat=book_format.lower(),
                                              title=_(u"Read a Book"), entry=entries, bookmark=bookmark)
@@ -1575,25 +1681,24 @@ def read_book(book_id, book_format):
         #     if book_format.lower() == fileext:
         #         return render_title_template('readcbr.html', comicfile=book_id,
         #         extension=fileext, title=_(u"Read a Book"), book=book)
-        log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
-        flash(_(u"Error opening eBook. File does not exist or file is not accessible."), category="error")
+        log.debug(u"Error opening eBook. File does not exist or file is not accessible")
+        flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
 
 
 @web.route("/book/<int:book_id>")
 @login_required_if_no_ano
 def show_book(book_id):
-    entries = db.session.query(db.Books).filter(and_(db.Books.id == book_id,
-                                                     common_filters(allow_show_archived=True))).first()
+    entries = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     if entries:
         for index in range(0, len(entries.languages)):
             try:
-                entries.languages[index].language_name = LC.parse(entries.languages[index].lang_code).get_language_name(
-                    get_locale())
+                entries.languages[index].language_name = LC.parse(entries.languages[index].lang_code)\
+                    .get_language_name(get_locale())
             except UnknownLocaleError:
                 entries.languages[index].language_name = _(
                     isoLanguages.get(part3=entries.languages[index].lang_code).name)
-        cc = get_cc_columns()
+        cc = get_cc_columns(filter_config_custom_read=True)
         book_in_shelfs = []
         shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all()
         for entry in shelfs:
@@ -1624,7 +1729,7 @@ def show_book(book_id):
 
         entries.tags = sort(entries.tags, key=lambda tag: tag.name)
 
-        entries = order_authors(entries)
+        entries = calibre_db.order_authors(entries)
 
         kindle_list = check_send_to_kindle(entries)
         reader_list = check_read_formats(entries)
@@ -1638,6 +1743,6 @@ def show_book(book_id):
                                      is_xhr=request.headers.get('X-Requested-With')=='XMLHttpRequest', title=entries.title, books_shelfs=book_in_shelfs,
                                      have_read=have_read, is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, page="book")
     else:
-        log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
-        flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")
+        log.debug(u"Error opening eBook. File does not exist or file is not accessible")
+        flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))

cps/worker.py

@@ -24,6 +24,12 @@ import smtplib
 import socket
 import time
 import threading
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+from glob import glob
+from shutil import copyfile
 from datetime import datetime
 
 try:
@@ -43,9 +49,10 @@ from email.utils import make_msgid
 from email.generator import Generator
 from flask_babel import gettext as _
 
-from . import logger, config, db, gdriveutils
+from . import calibre_db, db
+from . import logger, config
 from .subproc_wrapper import process_open
-
+from . import gdriveutils
 
 log = logger.create()
 
@@ -96,7 +103,7 @@ def get_attachment(bookpath, filename):
             data = file_.read()
             file_.close()
         except IOError as e:
-            log.exception(e) # traceback.print_exc()
+            log.exception(e)
             log.error(u'The requested file could not be read. Maybe wrong permissions?')
             return None
 
@@ -187,6 +194,8 @@ class WorkerThread(threading.Thread):
         self.UIqueue = list()
         self.asyncSMTP = None
         self.id = 0
+        self.db_queue = queue.Queue()
+        calibre_db.add_queue(self.db_queue)
         self.doLock = threading.Lock()
 
     # Main thread loop starting the different tasks
@@ -273,7 +282,7 @@ class WorkerThread(threading.Thread):
         index = self.current
         self.doLock.release()
         file_path = self.queue[index]['file_path']
-        bookid = self.queue[index]['bookid']
+        book_id = self.queue[index]['bookid']
         format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower()
         format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower()
 
@@ -281,95 +290,51 @@ class WorkerThread(threading.Thread):
         # if it does - mark the conversion task as complete and return a success
         # this will allow send to kindle workflow to continue to work
         if os.path.isfile(file_path + format_new_ext):
-            log.info("Book id %d already converted to %s", bookid, format_new_ext)
-            cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
+            log.info("Book id %d already converted to %s", book_id, format_new_ext)
+            cur_book = calibre_db.get_book(book_id)
             self.queue[index]['path'] = file_path
             self.queue[index]['title'] = cur_book.title
             self._handleSuccess()
             return file_path + format_new_ext
         else:
-            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext)
+            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
+                     book_id,
+                     format_new_ext)
 
-        # check if converter-executable is existing
-        if not os.path.exists(config.config_converterpath):
-            # ToDo Text is not translated
-            self._handleError(u"Convertertool %s not found" % config.config_converterpath)
-            return
-
-        try:
-            # check which converter to use kindlegen is "1"
-            if format_old_ext == '.epub' and format_new_ext == '.mobi':
-                if config.config_ebookconverter == 1:
-                    command = [config.config_converterpath, file_path + u'.epub']
-                    quotes = [1]
-            if config.config_ebookconverter == 2:
-                # Linux py2.7 encode as list without quotes no empty element for parameters
-                # linux py3.x no encode and as list without quotes no empty element for parameters
-                # windows py2.7 encode as string with quotes empty element for parameters is okay
-                # windows py 3.x no encode and as string with quotes empty element for parameters is okay
-                # separate handling for windows and linux
-                quotes = [1,2]
-                command = [config.config_converterpath, (file_path + format_old_ext),
-                    (file_path + format_new_ext)]
-                quotes_index = 3
-                if config.config_calibre:
-                    parameters = config.config_calibre.split(" ")
-                    for param in parameters:
-                        command.append(param)
-                        quotes.append(quotes_index)
-                        quotes_index += 1
-            p = process_open(command, quotes)
-            # p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
-        except OSError as e:
-            self._handleError(_(u"Ebook-converter failed: %(error)s", error=e))
-            return
-
-        if config.config_ebookconverter == 1:
-            nextline = p.communicate()[0]
-            # Format of error message (kindlegen translates its output texts):
-            # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
-            conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
-            # If error occoures, store error message for logfile
-            if conv_error:
-                error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
-                                  error=conv_error.group(1), message=conv_error.group(2).strip())
-            log.debug("convert_kindlegen: %s", nextline)
+        if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
+            check, error_message = self._convert_kepubify(file_path,
+                                                          format_old_ext,
+                                                          format_new_ext,
+                                                          index)
         else:
-            while p.poll() is None:
-                nextline = p.stdout.readline()
-                if os.name == 'nt' and sys.version_info < (3, 0):
-                    nextline = nextline.decode('windows-1252')
-                elif os.name == 'posix' and sys.version_info < (3, 0):
-                    nextline = nextline.decode('utf-8')
-                log.debug(nextline.strip('\r\n'))
-                # parse progress string from calibre-converter
-                progress = re.search(r"(\d+)%\s.*", nextline)
-                if progress:
-                    self.UIqueue[index]['progress'] = progress.group(1) + ' %'
+            # check if calibre converter-executable is existing
+            if not os.path.exists(config.config_converterpath):
+                # ToDo Text is not translated
+                self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
+                return
+            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index)
 
-        # process returncode
-        check = p.returncode
-        calibre_traceback = p.stderr.readlines()
-        for ele in calibre_traceback:
-            if sys.version_info < (3, 0):
-                ele = ele.decode('utf-8')
-            log.debug(ele.strip('\n'))
-            if not ele.startswith('Traceback') and not ele.startswith('  File'):
-                error_message = "Calibre failed with error: %s" % ele.strip('\n')
-
-        # kindlegen returncodes
-        # 0 = Info(prcgen):I1036: Mobi file built successfully
-        # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
-        # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
-        if (check < 2 and config.config_ebookconverter == 1) or \
-            (check == 0 and config.config_ebookconverter == 2):
-            cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
+        if check == 0:
+            cur_book = calibre_db.get_book(book_id)
             if os.path.isfile(file_path + format_new_ext):
+                # self.db_queue.join()
                 new_format = db.Data(name=cur_book.data[0].name,
                                          book_format=self.queue[index]['settings']['new_book_format'].upper(),
-                                         book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
-                cur_book.data.append(new_format)
-                db.session.commit()
+                                         book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
+                task = {'task':'add_format','id': book_id, 'format': new_format}
+                self.db_queue.put(task)
+                # To Do how to handle error?
+
+                '''cur_book.data.append(new_format)
+                try:
+                    # db.session.merge(cur_book)
+                    calibre_db.session.commit()
+                except OperationalError as e:
+                    calibre_db.session.rollback()
+                    log.error("Database error: %s", e)
+                    self._handleError(_(u"Database error: %(error)s.", error=e))
+                    return'''
+
                 self.queue[index]['path'] = cur_book.path
                 self.queue[index]['title'] = cur_book.title
                 if config.config_use_google_drive:
@@ -385,6 +350,87 @@ class WorkerThread(threading.Thread):
         return
 
 
+    def _convert_calibre(self, file_path, format_old_ext, format_new_ext, index):
+        try:
+            # Linux py2.7 encode as list without quotes no empty element for parameters
+            # linux py3.x no encode and as list without quotes no empty element for parameters
+            # windows py2.7 encode as string with quotes empty element for parameters is okay
+            # windows py 3.x no encode and as string with quotes empty element for parameters is okay
+            # separate handling for windows and linux
+            quotes = [1, 2]
+            command = [config.config_converterpath, (file_path + format_old_ext),
+                       (file_path + format_new_ext)]
+            quotes_index = 3
+            if config.config_calibre:
+                parameters = config.config_calibre.split(" ")
+                for param in parameters:
+                    command.append(param)
+                    quotes.append(quotes_index)
+                    quotes_index += 1
+
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Ebook-converter failed: %(error)s", error=e)
+
+        while p.poll() is None:
+            nextline = p.stdout.readline()
+            if os.name == 'nt' and sys.version_info < (3, 0):
+                nextline = nextline.decode('windows-1252')
+            elif os.name == 'posix' and sys.version_info < (3, 0):
+                nextline = nextline.decode('utf-8')
+            log.debug(nextline.strip('\r\n'))
+            # parse progress string from calibre-converter
+            progress = re.search(r"(\d+)%\s.*", nextline)
+            if progress:
+                self.UIqueue[index]['progress'] = progress.group(1) + ' %'
+
+        # process returncode
+        check = p.returncode
+        calibre_traceback = p.stderr.readlines()
+        error_message = ""
+        for ele in calibre_traceback:
+            if sys.version_info < (3, 0):
+                ele = ele.decode('utf-8')
+            log.debug(ele.strip('\n'))
+            if not ele.startswith('Traceback') and not ele.startswith('  File'):
+                error_message = "Calibre failed with error: %s" % ele.strip('\n')
+        return check, error_message
+
+
+    def _convert_kepubify(self, file_path, format_old_ext, format_new_ext, index):
+        quotes = [1, 3]
+        command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
+        try:
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Kepubify-converter failed: %(error)s", error=e)
+        self.UIqueue[index]['progress'] = '1 %'
+        while True:
+            nextline = p.stdout.readlines()
+            nextline = [x.strip('\n') for x in nextline if x != '\n']
+            if sys.version_info < (3, 0):
+                nextline = [x.decode('utf-8') for x in nextline]
+            for line in nextline:
+                log.debug(line)
+            if p.poll() is not None:
+                break
+
+        # ToD Handle
+        # process returncode
+        check = p.returncode
+
+        # move file
+        if check == 0:
+            converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
+            if len(converted_file) == 1:
+                copyfile(converted_file[0], (file_path + format_new_ext))
+                os.unlink(converted_file[0])
+            else:
+                return 1, _(u"Converted file not found or more than one file in folder %(folder)s",
+                            folder=os.path.dirname(file_path))
+        return check, None
+
+
     def add_convert(self, file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
         self.doLock.acquire()
         if self.last >= 20:
@@ -426,10 +472,6 @@ class WorkerThread(threading.Thread):
         if self.last >= 20:
             self._delete_completed_tasks()
         # progress=100%, runtime=0, and status finished
-        log.debug("Last " + str(self.last))
-        log.debug("Current " + str(self.current))
-        log.debug("id" + str(self.id))
-
         self.id += 1
         starttime = datetime.now()
         self.queue.append({'starttime': starttime, 'taskType': TASK_UPLOAD})
@@ -501,7 +543,8 @@ class WorkerThread(threading.Thread):
                 smtplib.stderr = org_smtpstderr
 
         except (MemoryError) as e:
-            self._handleError(u'Error sending email: ' + e.message)
+            log.exception(e)
+            self._handleError(u'MemoryError sending email: ' + str(e))
             return None
         except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
             if hasattr(e, "smtp_error"):
@@ -509,11 +552,12 @@ class WorkerThread(threading.Thread):
             elif hasattr(e, "message"):
                 text = e.message
             else:
+                log.exception(e)
                 text = ''
-            self._handleError(u'Error sending email: ' + text)
+            self._handleError(u'Smtplib Error sending email: ' + text)
             return None
         except (socket.error) as e:
-            self._handleError(u'Error sending email: ' + e.strerror)
+            self._handleError(u'Socket Error sending email: ' + e.strerror)
             return None
 
     def _handleError(self, error_message):
@@ -535,10 +579,6 @@ class WorkerThread(threading.Thread):
         self.UIqueue[index]['formRuntime'] = datetime.now() - self.queue[index]['starttime']
 
 
-_worker = WorkerThread()
-_worker.start()
-
-
 def get_taskstatus():
     return _worker.get_taskstatus()
 
@@ -553,3 +593,7 @@ def add_upload(user_name, taskMessage):
 
 def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
     return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)
+
+
+_worker = WorkerThread()
+_worker.start()

optional-requirements.txt

@@ -1,9 +1,9 @@
 # GDrive Integration
 google-api-python-client==1.7.11,<1.8.0
-gevent>=1.2.1,<1.6.0
+#gevent>=1.2.1,<20.6.0
 greenlet>=0.4.12,<0.5.0
 httplib2>=0.9.2,<0.18.0
-oauth2client>=4.0.0,<4.14.0
+oauth2client>=4.0.0,<4.1.4
 uritemplate>=3.0.0,<3.1.0
 pyasn1-modules>=0.0.8,<0.3.0
 pyasn1>=0.1.9,<0.5.0
@@ -31,7 +31,7 @@ rarfile>=2.7
 
 # other
 natsort>=2.2.0,<7.1.0
-git+https://github.com/OzzieIsaacs/comicapi.git@15dff9ce4e1ffed29ba4a2feadfcdb6bed00bcad#egg=comicapi
+git+https://github.com/OzzieIsaacs/comicapi.git@3e15b950b72724b1b8ca619c36580b5fbaba9784#egg=comicapi
 
 #Kobo integration
 jsonschema>=3.2.0,<3.3.0