Update setup.cfg

Fix sorting in comics (#950)
Fix log error on Calibre converter error (#953)
Fix long running tasks (#954)

MANIFEST.in

@@ -0,0 +1,3 @@
+include cps/static/*
+include cps/templates/*
+include cps/translations/*

cps/book_formats.py

@@ -35,6 +35,7 @@ logger = logging.getLogger("book_formats")
 try:
     from wand.image import Image
     from wand import version as ImageVersion
+    from wand.exceptions import PolicyError
     use_generic_pdf_cover = False
 except (ImportError, RuntimeError) as e:
     logger.warning('cannot import Image, generating pdf covers for pdf uploads will not work: %s', e)
@@ -61,6 +62,13 @@ except ImportError as e:
     logger.warning('cannot import fb2, extracting fb2 metadata will not work: %s', e)
     use_fb2_meta = False
 
+try:
+    from PIL import Image
+    from PIL import __version__ as PILversion
+    use_PIL = True
+except ImportError:
+    use_PIL = False
+
 
 def process(tmp_file_path, original_file_name, original_file_extension):
     meta = None
@@ -100,7 +108,7 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
 def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
 
     if use_pdf_meta:
-        pdf = PdfFileReader(open(tmp_file_path, 'rb'))
+        pdf = PdfFileReader(open(tmp_file_path, 'rb'), strict=False)
         doc_info = pdf.getDocumentInfo()
     else:
         doc_info = None
@@ -130,12 +138,58 @@ def pdf_preview(tmp_file_path, tmp_dir):
     if use_generic_pdf_cover:
         return None
     else:
-        cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jpg"
-        with Image(filename=tmp_file_path + "[0]", resolution=150) as img:
-            img.compression_quality = 88
-            img.save(filename=os.path.join(tmp_dir, cover_file_name))
-        return cover_file_name
+        if use_PIL:
+            try:
+                input1 = PdfFileReader(open(tmp_file_path, 'rb'), strict=False)
+                page0 = input1.getPage(0)
+                xObject = page0['/Resources']['/XObject'].getObject()
 
+                for obj in xObject:
+                    if xObject[obj]['/Subtype'] == '/Image':
+                        size = (xObject[obj]['/Width'], xObject[obj]['/Height'])
+                        data = xObject[obj]._data # xObject[obj].getData()
+                        if xObject[obj]['/ColorSpace'] == '/DeviceRGB':
+                            mode = "RGB"
+                        else:
+                            mode = "P"
+                        if '/Filter' in xObject[obj]:
+                            if xObject[obj]['/Filter'] == '/FlateDecode':
+                                img = Image.frombytes(mode, size, data)
+                                cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.png"
+                                img.save(filename=os.path.join(tmp_dir, cover_file_name))
+                                return cover_file_name
+                                # img.save(obj[1:] + ".png")
+                            elif xObject[obj]['/Filter'] == '/DCTDecode':
+                                cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jpg"
+                                img = open(cover_file_name, "wb")
+                                img.write(data)
+                                img.close()
+                                return cover_file_name
+                            elif xObject[obj]['/Filter'] == '/JPXDecode':
+                                cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jp2"
+                                img = open(cover_file_name, "wb")
+                                img.write(data)
+                                img.close()
+                                return cover_file_name
+                        else:
+                            img = Image.frombytes(mode, size, data)
+                            cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.png"
+                            img.save(filename=os.path.join(tmp_dir, cover_file_name))
+                            return cover_file_name
+            except Exception as ex:
+                print(ex)
+        try:
+            cover_file_name = os.path.splitext(tmp_file_path)[0] + ".cover.jpg"
+            with Image(filename=tmp_file_path + "[0]", resolution=150) as img:
+                img.compression_quality = 88
+                img.save(filename=os.path.join(tmp_dir, cover_file_name))
+            return cover_file_name
+        except PolicyError as ex:
+            logger.warning('Pdf extraction forbidden by Imagemagick policy: %s', ex)
+            return None
+        except Exception as ex:
+            logger.warning('Cannot extract cover image, using default: %s', ex)
+            return None
 
 def get_versions():
     if not use_generic_pdf_cover:
@@ -152,4 +206,12 @@ def get_versions():
         XVersion = 'v'+'.'.join(map(str, lxmlversion))
     else:
         XVersion = _(u'not installed')
-    return {'Image Magick': IVersion, 'PyPdf': PVersion, 'lxml':XVersion, 'Wand Version': WVersion}
+    if use_PIL:
+        PILVersion = 'v' + PILversion
+    else:
+        PILVersion = _(u'not installed')
+    return {'Image Magick': IVersion,
+            'PyPdf': PVersion,
+            'lxml':XVersion,
+            'Wand': WVersion,
+            'Pillow': PILVersion}

cps/comic.py

@@ -17,41 +17,104 @@
 #   You should have received a copy of the GNU General Public License
 #   along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-import zipfile
-import tarfile
 import os
 import uploader
+import logging
+from iso639 import languages as isoLanguages
+
+
+logger = logging.getLogger("book_formats")
+
+try:
+    from comicapi.comicarchive import ComicArchive, MetaDataStyle
+    use_comic_meta = True
+except ImportError as e:
+    logger.warning('cannot import comicapi, extracting comic metadata will not work: %s', e)
+    import zipfile
+    import tarfile
+    use_comic_meta = False
 
 
 def extractCover(tmp_file_name, original_file_extension):
-    if original_file_extension.upper() == '.CBZ':
-        cf = zipfile.ZipFile(tmp_file_name)
-        compressed_name = cf.namelist()[0]
-        cover_data = cf.read(compressed_name)
-    elif original_file_extension.upper() == '.CBT':
-        cf = tarfile.TarFile(tmp_file_name)
-        compressed_name = cf.getnames()[0]
-        cover_data = cf.extractfile(compressed_name).read()
-
+    if use_comic_meta:
+        archive = ComicArchive(tmp_file_name)
+        cover_data = None
+        ext = os.path.splitext(archive.getPageName(0))
+        if len(ext) > 1:
+            extension = ext[1].lower()
+            if extension == '.jpg' or extension == '.jpeg':
+                cover_data = archive.getPage(0)
+    else:
+        if original_file_extension.upper() == '.CBZ':
+            cf = zipfile.ZipFile(tmp_file_name)
+            for name in cf.namelist():
+                ext = os.path.splitext(name)
+                if len(ext) > 1:
+                    extension = ext[1].lower()
+                    if extension == '.jpg':
+                        cover_data = cf.read(name)
+                        break
+        elif original_file_extension.upper() == '.CBT':
+            cf = tarfile.TarFile(tmp_file_name)
+            for name in cf.getnames():
+                ext = os.path.splitext(name)
+                if len(ext) > 1:
+                    extension = ext[1].lower()
+                    if extension == '.jpg':
+                        cover_data = cf.extractfile(name).read()
+                        break
     prefix = os.path.dirname(tmp_file_name)
-
-    tmp_cover_name = prefix + '/cover' + os.path.splitext(compressed_name)[1]
-    image = open(tmp_cover_name, 'wb')
-    image.write(cover_data)
-    image.close()
+    if cover_data:
+        tmp_cover_name = prefix + '/cover' + extension
+        image = open(tmp_cover_name, 'wb')
+        image.write(cover_data)
+        image.close()
+    else:
+        tmp_cover_name = None
     return tmp_cover_name
 
 
 def get_comic_info(tmp_file_path, original_file_name, original_file_extension):
+    if use_comic_meta:
+        archive = ComicArchive(tmp_file_path)
+        if archive.seemsToBeAComicArchive():
+            if archive.hasMetadata(MetaDataStyle.CIX):
+                style = MetaDataStyle.CIX
+            elif archive.hasMetadata(MetaDataStyle.CBI):
+                style = MetaDataStyle.CBI
+            else:
+                style = None
 
-    coverfile = extractCover(tmp_file_path, original_file_extension)
+            if style is not None:
+                loadedMetadata = archive.readMetadata(style)
 
-    return uploader.BookMeta(
+        lang = loadedMetadata.language
+        if len(lang) == 2:
+             loadedMetadata.language = isoLanguages.get(part1=lang).name
+        elif len(lang) == 3:
+             loadedMetadata.language = isoLanguages.get(part3=lang).name
+        else:
+             loadedMetadata.language = ""
+
+        return uploader.BookMeta(
+                file_path=tmp_file_path,
+                extension=original_file_extension,
+                title=loadedMetadata.title or original_file_name,
+                author=" & ".join([credit["person"] for credit in loadedMetadata.credits if credit["role"] == "Writer"]) or u"Unknown",
+                cover=extractCover(tmp_file_path, original_file_extension),
+                description=loadedMetadata.comments or "",
+                tags="",
+                series=loadedMetadata.series or "",
+                series_id=loadedMetadata.issue or "",
+                languages=loadedMetadata.language)
+    else:
+
+        return uploader.BookMeta(
             file_path=tmp_file_path,
             extension=original_file_extension,
             title=original_file_name,
             author=u"Unknown",
-            cover=coverfile,
+            cover=extractCover(tmp_file_path, original_file_extension),
             description="",
             tags="",
             series="",

cps/db.py

@@ -27,6 +27,7 @@ import ast
 from ub import config
 import ub
 import sys
+import unidecode
 
 session = None
 cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series']
@@ -46,7 +47,7 @@ def title_sort(title):
 
 
 def lcase(s):
-    return s.lower()
+    return unidecode.unidecode(s.lower())
 
 
 def ucase(s):
@@ -112,6 +113,8 @@ class Identifiers(Base):
             return u"Google Books"
         elif self.type == "kobo":
             return u"Kobo"
+        if self.type == "lubimyczytac":
+            return u"Lubimyczytac"
         else:
             return self.type
 
@@ -130,6 +133,8 @@ class Identifiers(Base):
             return u"https://books.google.com/books?id={0}".format(self.val)
         elif self.type == "kobo":
             return u"https://www.kobo.com/ebook/{0}".format(self.val)
+        elif self.type == "lubimyczytac":
+            return u" http://lubimyczytac.pl/ksiazka/{0}".format(self.val)
         elif self.type == "url":
             return u"{0}".format(self.val)
         else:
@@ -355,8 +360,8 @@ def setup_db():
     ub.session.commit()
     config.loadSettings()
     conn.connection.create_function('title_sort', 1, title_sort)
-    conn.connection.create_function('lower', 1, lcase)
-    conn.connection.create_function('upper', 1, ucase)
+    # conn.connection.create_function('lower', 1, lcase)
+    # conn.connection.create_function('upper', 1, ucase)
 
     if not cc_classes:
         cc = conn.execute("SELECT id, datatype FROM custom_columns")

cps/epub.py

@@ -21,7 +21,7 @@ import zipfile
 from lxml import etree
 import os
 import uploader
-from iso639 import languages as isoLanguages
+import isoLanguages
 
 
 def extractCover(zipFile, coverFile, coverpath, tmp_file_name):

cps/gdriveutils.py

@@ -20,7 +20,7 @@
 try:
     from pydrive.auth import GoogleAuth
     from pydrive.drive import GoogleDrive
-    from pydrive.auth import RefreshError
+    from pydrive.auth import RefreshError, InvalidConfigError
     from apiclient import errors
     gdrive_support = True
 except ImportError:
@@ -31,12 +31,9 @@ from ub import config
 import cli
 import shutil
 from flask import Response, stream_with_context
-
 from sqlalchemy import *
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import *
-
-
 import web
 
 class Singleton:
@@ -166,7 +163,10 @@ def getDrive(drive=None, gauth=None):
         # Save the current credentials to a file
         return GoogleDrive(gauth)
     if drive.auth.access_token_expired:
-        drive.auth.Refresh()
+        try:
+            drive.auth.Refresh()
+        except RefreshError as e:
+            web.app.logger.error("Google Drive error: " + e.message)
     return drive
 
 def listRootFolders():
@@ -184,7 +184,7 @@ def getFolderInFolder(parentId, folderName, drive):
     # drive = getDrive(drive)
     query=""
     if folderName:
-        query = "title = '%s' and " % folderName.replace("'", "\\'")
+        query = "title = '%s' and " % folderName.replace("'", r"\'")
     folder = query + "'%s' in parents and mimeType = 'application/vnd.google-apps.folder'" \
                      " and trashed = false" % parentId
     fileList = drive.ListFile({'q': folder}).GetList()
@@ -211,7 +211,7 @@ def getEbooksFolderId(drive=None):
 
 
 def getFile(pathId, fileName, drive):
-    metaDataFile = "'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'"))
+    metaDataFile = "'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", r"\'"))
     fileList = drive.ListFile({'q': metaDataFile}).GetList()
     if fileList.__len__() == 0:
         return None
@@ -246,7 +246,7 @@ def getFolderId(path, drive):
                         dbChange = True
                         currentFolderId = currentFolder['id']
                     else:
-                        currentFolderId= None
+                        currentFolderId = None
                         break
         if dbChange:
             session.commit()
@@ -268,16 +268,9 @@ def getFileFromEbooksFolder(path, fileName):
         return None
 
 
-'''def copyDriveFileRemote(drive, origin_file_id, copy_title):
-    drive = getDrive(drive)
-    copied_file = {'title': copy_title}
-    try:
-        file_data = drive.auth.service.files().copy(
-        fileId = origin_file_id, body=copied_file).execute()
-        return drive.CreateFile({'id': file_data['id']})
-    except errors.HttpError as error:
-        print ('An error occurred: %s' % error)
-    return None'''
+def moveGdriveFileRemote(origin_file_id, new_title):
+    origin_file_id['title']= new_title
+    origin_file_id.Upload()
 
 
 # Download metadata.db from gdrive
@@ -289,9 +282,10 @@ def downloadFile(path, filename, output):
 def moveGdriveFolderRemote(origin_file, target_folder):
     drive = getDrive(Gdrive.Instance().drive)
     previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')])
+    children = drive.auth.service.children().list(folderId=previous_parents).execute()
     gFileTargetDir = getFileFromEbooksFolder(None, target_folder)
     if not gFileTargetDir:
-        # Folder is not exisiting, create, and move folder
+        # Folder is not existing, create, and move folder
         gFileTargetDir = drive.CreateFile(
             {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}],
              "mimeType": "application/vnd.google-apps.folder"})
@@ -301,13 +295,10 @@ def moveGdriveFolderRemote(origin_file, target_folder):
                                       addParents=gFileTargetDir['id'],
                                       removeParents=previous_parents,
                                       fields='id, parents').execute()
-    # if previous_parents has no childs anymore, delete originfileparent
-    # is not working correctly, because of slow update on gdrive -> could cause trouble in gdrive.db
-    # (nonexisting folder has id)
-    # children = drive.auth.service.children().list(folderId=previous_parents).execute()
-    # if not len(children['items']):
-    #    drive.auth.service.files().delete(fileId=previous_parents).execute()
-
+    # if previous_parents has no childs anymore, delete original fileparent
+    if len(children['items']) == 1:
+        deleteDatabaseEntry(previous_parents)
+        drive.auth.service.files().delete(fileId=previous_parents).execute()
 
 
 def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
@@ -320,7 +311,7 @@ def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
         parent = getEbooksFolder(drive)
     if os.path.isdir(os.path.join(prevDir,uploadFile)):
         existingFolder = drive.ListFile({'q': "title = '%s' and '%s' in parents and trashed = false" %
-                                              (os.path.basename(uploadFile), parent['id'])}).GetList()
+                                              (os.path.basename(uploadFile).replace("'", r"\'"), parent['id'])}).GetList()
         if len(existingFolder) == 0 and (not isInitial or createRoot):
             parent = drive.CreateFile({'title': os.path.basename(uploadFile),
                                        'parents': [{"kind": "drive#fileLink", 'id': parent['id']}],
@@ -335,11 +326,11 @@ def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
     else:
         if os.path.basename(uploadFile) not in ignoreFiles:
             existingFiles = drive.ListFile({'q': "title = '%s' and '%s' in parents and trashed = false" %
-                                                 (os.path.basename(uploadFile), parent['id'])}).GetList()
+                                                 (os.path.basename(uploadFile).replace("'", r"\'"), parent['id'])}).GetList()
             if len(existingFiles) > 0:
                 driveFile = existingFiles[0]
             else:
-                driveFile = drive.CreateFile({'title': os.path.basename(uploadFile),
+                driveFile = drive.CreateFile({'title': os.path.basename(uploadFile).replace("'", r"\'"),
                                               'parents': [{"kind":"drive#fileLink", 'id': parent['id']}], })
             driveFile.SetContentFile(os.path.join(prevDir, uploadFile))
             driveFile.Upload()
@@ -352,7 +343,7 @@ def uploadFileToEbooksFolder(destFile, f):
     for i, x in enumerate(splitDir):
         if i == len(splitDir)-1:
             existingFiles = drive.ListFile({'q': "title = '%s' and '%s' in parents and trashed = false" %
-                                                 (x, parent['id'])}).GetList()
+                                                 (x.replace("'", r"\'"), parent['id'])}).GetList()
             if len(existingFiles) > 0:
                 driveFile = existingFiles[0]
             else:
@@ -361,7 +352,7 @@ def uploadFileToEbooksFolder(destFile, f):
             driveFile.Upload()
         else:
             existingFolder = drive.ListFile({'q': "title = '%s' and '%s' in parents and trashed = false" %
-                                                  (x, parent['id'])}).GetList()
+                                                  (x.replace("'", r"\'"), parent['id'])}).GetList()
             if len(existingFolder) == 0:
                 parent = drive.CreateFile({'title': x, 'parents': [{"kind": "drive#fileLink", 'id': parent['id']}],
                     "mimeType": "application/vnd.google-apps.folder"})
@@ -454,6 +445,10 @@ def getChangeById (drive, change_id):
     except (errors.HttpError) as error:
         web.app.logger.info(error.message)
         return None
+    except Exception as e:
+        web.app.logger.info(e)
+        return None
+
 
 # Deletes the local hashes database to force search for new folder names
 def deleteDatabaseOnChange():
@@ -468,9 +463,10 @@ def updateGdriveCalibreFromLocal():
 
 # update gdrive.db on edit of books title
 def updateDatabaseOnEdit(ID,newPath):
+    sqlCheckPath = newPath if newPath[-1] == '/' else newPath + u'/'
     storedPathName = session.query(GdriveId).filter(GdriveId.gdrive_id == ID).first()
     if storedPathName:
-        storedPathName.path = newPath
+        storedPathName.path = sqlCheckPath
         session.commit()
 
 

cps/helper.py

@@ -22,30 +22,27 @@
 import db
 import ub
 from flask import current_app as app
-# import logging
 from tempfile import gettempdir
 import sys
+import io
 import os
 import re
 import unicodedata
-# from io import BytesIO
 import worker
 import time
 from flask import send_from_directory, make_response, redirect, abort
 from flask_babel import gettext as _
 from flask_login import current_user
 from babel.dates import format_datetime
-from datetime import datetime
-# import threading
+from babel.units import format_unit
+from datetime import datetime, timedelta
 import shutil
 import requests
-# import zipfile
 try:
     import gdriveutils as gd
 except ImportError:
     pass
 import web
-# import server
 import random
 import subprocess
 
@@ -55,6 +52,12 @@ try:
 except ImportError:
     use_unidecode = False
 
+try:
+    from PIL import Image
+    use_PIL = True
+except ImportError:
+    use_PIL = False
+
 # Global variables
 # updater_thread = None
 global_WorkerThread = worker.WorkerThread()
@@ -128,7 +131,7 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
     text += "Sincerely\r\n\r\n"
     text += "Your Calibre-Web team"
     global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, ub.get_mail_settings(),
-                                  e_mail, user_name, _(u"Registration e-mail for user: %(name)s", name=user_name), text)
+                                  e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text)
     return
 
 def check_send_to_kindle(entry):
@@ -146,8 +149,8 @@ def check_send_to_kindle(entry):
                     bookformats.append({'format':'Pdf','convert':0,'text':_('Send %(format)s to Kindle',format='Pdf')})
                 if 'AZW' in ele.format:
                     bookformats.append({'format':'Azw','convert':0,'text':_('Send %(format)s to Kindle',format='Azw')})
-                if 'AZW3' in ele.format:
-                    bookformats.append({'format':'Azw3','convert':0,'text':_('Send %(format)s to Kindle',format='Azw3')})
+                '''if 'AZW3' in ele.format:
+                    bookformats.append({'format':'Azw3','convert':0,'text':_('Send %(format)s to Kindle',format='Azw3')})'''
         else:
             formats = list()
             for ele in iter(entry.data):
@@ -156,18 +159,16 @@ def check_send_to_kindle(entry):
                 bookformats.append({'format': 'Mobi','convert':0,'text':_('Send %(format)s to Kindle',format='Mobi')})
             if 'AZW' in formats:
                 bookformats.append({'format': 'Azw','convert':0,'text':_('Send %(format)s to Kindle',format='Azw')})
-            if 'AZW3' in formats:
-                bookformats.append({'format': 'Azw3','convert':0,'text':_('Send %(format)s to Kindle',format='Azw3')})
             if 'PDF' in formats:
                 bookformats.append({'format': 'Pdf','convert':0,'text':_('Send %(format)s to Kindle',format='Pdf')})
             if ub.config.config_ebookconverter >= 1:
                 if 'EPUB' in formats and not 'MOBI' in formats:
                     bookformats.append({'format': 'Mobi','convert':1,
                             'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Mobi')})
-            if ub.config.config_ebookconverter == 2:
+            '''if ub.config.config_ebookconverter == 2:
                 if 'EPUB' in formats and not 'AZW3' in formats:
                     bookformats.append({'format': 'Azw3','convert':1,
-                            'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Azw3')})
+                            'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Azw3')})'''
         return bookformats
     else:
         app.logger.error(u'Cannot find book entry %d', entry.id)
@@ -177,7 +178,7 @@ def check_send_to_kindle(entry):
 # Check if a reader is existing for any of the book formats, if not, return empty list, otherwise return
 # list with supported formats
 def check_read_formats(entry):
-    EXTENSIONS_READER = {'TXT', 'PDF', 'EPUB', 'ZIP', 'CBZ', 'TAR', 'CBT', 'RAR', 'CBR'}
+    EXTENSIONS_READER = {'TXT', 'PDF', 'EPUB', 'CBZ', 'CBT', 'CBR'}
     bookformats = list()
     if len(entry.data):
         for ele in iter(entry.data):
@@ -235,7 +236,10 @@ def get_valid_filename(value, replace_whitespace=True):
     value = value[:128]
     if not value:
         raise ValueError("Filename cannot be empty")
-    return value
+    if sys.version_info.major == 3:
+        return value
+    else:
+        return value.decode('utf-8')
 
 
 def get_sorted_author(value):
@@ -324,11 +328,11 @@ def update_dir_structure_file(book_id, calibrepath, first_author):
     # Rename all files from old names to new names
     if authordir != new_authordir or titledir != new_titledir:
         try:
+            new_name = get_valid_filename(localbook.title) + ' - ' + get_valid_filename(new_authordir)
+            path_name = os.path.join(calibrepath, new_authordir, os.path.basename(path))
             for file_format in localbook.data:
-                path_name = os.path.join(calibrepath, new_authordir, os.path.basename(path))
-                new_name = get_valid_filename(localbook.title) + ' - ' + get_valid_filename(new_authordir)
                 os.renames(os.path.join(path_name, file_format.name + '.' + file_format.format.lower()),
-                           os.path.join(path_name,new_name + '.' + file_format.format.lower()))
+                           os.path.join(path_name, new_name + '.' + file_format.format.lower()))
                 file_format.name = new_name
         except OSError as ex:
             web.app.logger.error("Rename file in path " + path + " to " + new_name + ": " + str(ex))
@@ -341,6 +345,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author):
 def update_dir_structure_gdrive(book_id, first_author):
     error = False
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    path = book.path
 
     authordir = book.path.split('/')[0]
     if first_author:
@@ -348,40 +353,39 @@ def update_dir_structure_gdrive(book_id, first_author):
     else:
         new_authordir = get_valid_filename(book.authors[0].name)
     titledir = book.path.split('/')[1]
-    new_titledir = get_valid_filename(book.title) + " (" + str(book_id) + ")"
+    new_titledir = get_valid_filename(book.title) + u" (" + str(book_id) + u")"
 
     if titledir != new_titledir:
         gFile = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir)
         if gFile:
             gFile['title'] = new_titledir
-
             gFile.Upload()
-            book.path = book.path.split('/')[0] + '/' + new_titledir
+            book.path = book.path.split('/')[0] + u'/' + new_titledir
+            path = book.path
             gd.updateDatabaseOnEdit(gFile['id'], book.path)     # only child folder affected
         else:
             error = _(u'File %(file)s not found on Google Drive', file=book.path) # file not found
 
     if authordir != new_authordir:
-        gFile = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir)
+        gFile = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir)
         if gFile:
-            gd.moveGdriveFolderRemote(gFile,new_authordir)
-            book.path = new_authordir + '/' + book.path.split('/')[1]
+            gd.moveGdriveFolderRemote(gFile, new_authordir)
+            book.path = new_authordir + u'/' + book.path.split('/')[1]
+            path = book.path
             gd.updateDatabaseOnEdit(gFile['id'], book.path)
         else:
             error = _(u'File %(file)s not found on Google Drive', file=authordir) # file not found
     # Rename all files from old names to new names
-    # ToDo: Rename also all bookfiles with new author name and new title name
-    '''
+
     if authordir != new_authordir or titledir != new_titledir:
-        for format in book.data:
-            # path_name = os.path.join(calibrepath, new_authordir, os.path.basename(path))
-            new_name = get_valid_filename(book.title) + ' - ' + get_valid_filename(book)
-            format.name = new_name
-            if gFile:
-                pass
-            else:
-                error = _(u'File %(file)s not found on Google Drive', file=format.name)  # file not found
-                break'''
+        new_name = get_valid_filename(book.title) + u' - ' + get_valid_filename(new_authordir)
+        for file_format in book.data:
+            gFile = gd.getFileFromEbooksFolder(path, file_format.name + u'.' + file_format.format.lower())
+            if not gFile:
+                error = _(u'File %(file)s not found on Google Drive', file=file_format.name)  # file not found
+                break
+            gd.moveGdriveFileRemote(gFile, new_name + u'.' + file_format.format.lower())
+            file_format.name = new_name
     return error
 
 
@@ -427,6 +431,8 @@ def delete_book(book, calibrepath, book_format):
 def get_book_cover(cover_path):
     if ub.config.config_use_google_drive:
         try:
+            if not web.is_gdrive_ready():
+                return send_from_directory(os.path.join(os.path.dirname(__file__), "static"), "generic_cover.jpg")
             path=gd.get_cover_via_gdrive(cover_path)
             if path:
                 return redirect(path)
@@ -434,7 +440,7 @@ def get_book_cover(cover_path):
                 web.app.logger.error(cover_path + '/cover.jpg not found on Google Drive')
                 return send_from_directory(os.path.join(os.path.dirname(__file__), "static"), "generic_cover.jpg")
         except Exception as e:
-            web.app.logger.error("Error Message: "+e.message)
+            web.app.logger.error("Error Message: " + e.message)
             web.app.logger.exception(e)
             # traceback.print_exc()
             return send_from_directory(os.path.join(os.path.dirname(__file__), "static"),"generic_cover.jpg")
@@ -442,27 +448,71 @@ def get_book_cover(cover_path):
         return send_from_directory(os.path.join(ub.config.config_calibre_dir, cover_path), "cover.jpg")
 
 
-# saves book cover to gdrive or locally
-def save_cover(url, book_path):
+# saves book cover from url
+def save_cover_from_url(url, book_path):
     img = requests.get(url)
-    if img.headers.get('content-type') != 'image/jpeg':
-        web.app.logger.error("Cover is no jpg file, can't save")
-        return False
+    return save_cover(img, book_path)
+
+
+def save_cover_from_filestorage(filepath, saved_filename, img):
+    if hasattr(img,'_content'):
+        f = open(os.path.join(filepath, saved_filename), "wb")
+        f.write(img._content)
+        f.close()
+    else:
+        # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
+        if not os.path.exists(filepath):
+            try:
+                os.makedirs(filepath)
+            except OSError:
+                web.app.logger.error(u"Failed to create path for cover")
+                return False
+        try:
+            img.save(os.path.join(filepath, saved_filename))
+        except OSError:
+            web.app.logger.error(u"Failed to store cover-file")
+            return False
+        except IOError:
+            web.app.logger.error(u"Cover-file is not a valid image file")
+            return False
+    return True
+
+
+# saves book cover to gdrive or locally
+def save_cover(img, book_path):
+    content_type = img.headers.get('content-type')
+
+    if use_PIL:
+        if content_type not in ('image/jpeg', 'image/png', 'image/webp'):
+            web.app.logger.error("Only jpg/jpeg/png/webp files are supported as coverfile")
+            return False
+        # convert to jpg because calibre only supports jpg
+        if content_type in ('image/png', 'image/webp'):
+            if hasattr(img,'stream'):
+                imgc = Image.open(img.stream)
+            else:
+                imgc = Image.open(io.BytesIO(img.content))
+            im = imgc.convert('RGB')
+            tmp_bytesio = io.BytesIO()
+            im.save(tmp_bytesio, format='JPEG')
+            img._content = tmp_bytesio.getvalue()
+    else:
+        if content_type not in ('image/jpeg'):
+            web.app.logger.error("Only jpg/jpeg files are supported as coverfile")
+            return False
 
     if ub.config.config_use_google_drive:
         tmpDir = gettempdir()
-        f = open(os.path.join(tmpDir, "uploaded_cover.jpg"), "wb")
-        f.write(img.content)
-        f.close()
-        gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'), os.path.join(tmpDir, f.name))
-        web.app.logger.info("Cover is saved on Google Drive")
-        return True
+        if save_cover_from_filestorage(tmpDir, "uploaded_cover.jpg", img) is True:
+            gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'),
+                                        os.path.join(tmpDir, "uploaded_cover.jpg"))
+            web.app.logger.info("Cover is saved on Google Drive")
+            return True
+        else:
+            return False
+    else:
+        return save_cover_from_filestorage(os.path.join(ub.config.config_calibre_dir, book_path), "cover.jpg", img)
 
-    f = open(os.path.join(ub.config.config_calibre_dir, book_path, "cover.jpg"), "wb")
-    f.write(img.content)
-    f.close()
-    web.app.logger.info("Cover is saved")
-    return True
 
 
 def do_download_file(book, book_format, data, headers):
@@ -487,7 +537,6 @@ def do_download_file(book, book_format, data, headers):
 
 
 
-
 def check_unrar(unrarLocation):
     error = False
     if os.path.exists(unrarLocation):
@@ -518,23 +567,49 @@ def json_serial(obj):
 
     if isinstance(obj, (datetime)):
         return obj.isoformat()
+    if isinstance(obj, (timedelta)):
+        return {
+            '__type__': 'timedelta',
+            'days': obj.days,
+            'seconds': obj.seconds,
+            'microseconds': obj.microseconds,
+        }
     raise TypeError ("Type %s not serializable" % type(obj))
 
 
+# helper function for displaying the runtime of tasks
+def format_runtime(runtime):
+    retVal = ""
+    if runtime.days:
+        retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', '
+    mins, seconds = divmod(runtime.seconds, 60)
+    hours, minutes = divmod(mins, 60)
+    # ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ?
+    if hours:
+        retVal += '{:d}:{:02d}:{:02d}s'.format(hours, minutes, seconds)
+    elif minutes:
+        retVal += '{:2d}:{:02d}s'.format(minutes, seconds)
+    else:
+        retVal += '{:2d}s'.format(seconds)
+    return retVal
+
+
+# helper function to apply localize status information in tasklist entries
 def render_task_status(tasklist):
-    #helper function to apply localize status information in tasklist entries
     renderedtasklist=list()
-    # task2 = task
     for task in tasklist:
         if task['user'] == current_user.nickname or current_user.role_admin():
-            # task2 = copy.deepcopy(task) # = task
             if task['formStarttime']:
                 task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=web.get_locale())
-            # task2['formStarttime'] = ""
             else:
                 if 'starttime' not in task:
                     task['starttime'] = ""
 
+            if 'formRuntime' not in task:
+                task['runtime'] = ""
+            else:
+                task['runtime'] = format_runtime(task['formRuntime'])
+
             # localize the task status
             if isinstance( task['stat'], int ):
                 if task['stat'] == worker.STAT_WAITING:

cps/isoLanguages.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2019 pwr
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+try:
+    from iso639 import languages, __version__
+    get = languages.get
+except ImportError:
+    from pycountry import languages as pyc_languages
+    try:
+        import pkg_resources
+        __version__ = pkg_resources.get_distribution('pycountry').version + ' (PyCountry)'
+        del pkg_resources
+    except (ImportError, Exception):
+        __version__ = "? (PyCountry)"
+
+    def _copy_fields(l):
+        l.part1 = l.alpha_2
+        l.part3 = l.alpha_3
+        return l
+
+    def get(name=None, part1=None, part3=None):
+        if (part3 is not None):
+            return _copy_fields(pyc_languages.get(alpha_3=part3))
+        if (part1 is not None):
+            return _copy_fields(pyc_languages.get(alpha_2=part1))
+        if (name is not None):
+            return _copy_fields(pyc_languages.get(name=name))

cps/reverseproxy.py

@@ -1,21 +1,41 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
-#    Copyright (C) 2018 cervinko, janeczku, OzzieIsaacs
+#  Flask License
 #
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
+#  Copyright © 2010 by the Pallets team, cervinko, janeczku, OzzieIsaacs
 #
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
+#  Some rights reserved.
 #
-#  You should have received a copy of the GNU General Public License
-#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+#  Redistribution and use in source and binary forms of the software as
+#  well as documentation, with or without modification, are permitted
+#  provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice,
+#  this list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+#  CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+#  BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+#  FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+#  COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+#  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+#  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#  USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+#  ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+#  THIS SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF
+#  SUCH DAMAGE.
+#
+# Inspired by http://flask.pocoo.org/snippets/35/
 
 class ReverseProxied(object):
     """Wrap the application in this middleware and configure the
@@ -49,7 +69,7 @@ class ReverseProxied(object):
         scheme = environ.get('HTTP_X_SCHEME', '')
         if scheme:
             environ['wsgi.url_scheme'] = scheme
-        servr = environ.get('HTTP_X_FORWARDED_SERVER', '')
+        servr = environ.get('HTTP_X_FORWARDED_HOST', '')
         if servr:
             environ['HTTP_HOST'] = servr
         return self.app(environ, start_response)

cps/server.py

@@ -103,6 +103,7 @@ class server:
                             ssl_options=ssl)
                 http_server.listen(web.ub.config.config_port)
                 self.wsgiserver=IOLoop.instance()
+                web.py3_gevent_link = self.wsgiserver
                 self.wsgiserver.start()
                 # wait for stop signal
                 self.wsgiserver.close(True)
@@ -142,10 +143,10 @@ class server:
         # ToDo: Somehow caused by circular import under python3 refactor
         if sys.version_info > (3, 0):
             if not self.wsgiserver:
-                if gevent_present:
-                    self.wsgiserver = web.py3_gevent_link
-                else:
-                    self.wsgiserver = IOLoop.instance()
+                # if gevent_present:
+                self.wsgiserver = web.py3_gevent_link
+                #else:
+                #    self.wsgiserver = IOLoop.instance()
         if self.wsgiserver:
             if gevent_present:
                 self.wsgiserver.close()

cps/static/css/kthoom.css

@@ -152,11 +152,11 @@ body {
   max-width: 70%;
 }
 
-#prev {
+#left {
   left: 40px;
 }
 
-#next {
+#right {
   right: 40px;
 }