Merge branch 'master' into Develop
# Conflicts: # cps/admin.py # cps/config_sql.py # cps/search.py # cps/templates/admin.html # cps/web.py # setup.cfg # test/Calibre-Web TestSummary_Linux.html
This commit is contained in:
Ozzie Isaacs
+11
-11
@@ -64,13 +64,13 @@ class TaskConvert(CalibreTask):
|
||||
if df:
|
||||
datafile = os.path.join(config.config_calibre_dir,
|
||||
cur_book.path,
|
||||
data.name + u"." + self.settings['old_book_format'].lower())
|
||||
data.name + "." + self.settings['old_book_format'].lower())
|
||||
if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
|
||||
os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
|
||||
df.GetContentFile(datafile)
|
||||
worker_db.session.close()
|
||||
else:
|
||||
error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
|
||||
error_message = _("%(format)s not found on Google Drive: %(fn)s",
|
||||
format=self.settings['old_book_format'],
|
||||
fn=data.name + "." + self.settings['old_book_format'].lower())
|
||||
worker_db.session.close()
|
||||
@@ -78,7 +78,7 @@ class TaskConvert(CalibreTask):
|
||||
|
||||
filename = self._convert_ebook_format()
|
||||
if config.config_use_google_drive:
|
||||
os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower())
|
||||
os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
|
||||
|
||||
if filename:
|
||||
if config.config_use_google_drive:
|
||||
@@ -89,7 +89,7 @@ class TaskConvert(CalibreTask):
|
||||
# if we're sending to E-Reader after converting, create a one-off task and run it immediately
|
||||
# todo: figure out how to incorporate this into the progress
|
||||
try:
|
||||
EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title))
|
||||
EmailText = N_("%(book)s send to E-Reader", book=escape(self.title))
|
||||
worker_thread.add(self.user, TaskEmail(self.settings['subject'],
|
||||
self.results["path"],
|
||||
filename,
|
||||
@@ -107,8 +107,8 @@ class TaskConvert(CalibreTask):
|
||||
local_db = db.CalibreDB(expire_on_commit=False, init=True)
|
||||
file_path = self.file_path
|
||||
book_id = self.book_id
|
||||
format_old_ext = u'.' + self.settings['old_book_format'].lower()
|
||||
format_new_ext = u'.' + self.settings['new_book_format'].lower()
|
||||
format_old_ext = '.' + self.settings['old_book_format'].lower()
|
||||
format_new_ext = '.' + self.settings['new_book_format'].lower()
|
||||
|
||||
# check to see if destination format already exists - or if book is in database
|
||||
# if it does - mark the conversion task as complete and return a success
|
||||
@@ -133,7 +133,7 @@ class TaskConvert(CalibreTask):
|
||||
local_db.session.rollback()
|
||||
log.error("Database error: %s", e)
|
||||
local_db.session.close()
|
||||
self._handleError(N_("Database error: %(error)s.", error=e))
|
||||
self._handleError(N_("Oops! Database Error: %(error)s.", error=e))
|
||||
return
|
||||
self._handleSuccess()
|
||||
local_db.session.close()
|
||||
@@ -150,7 +150,7 @@ class TaskConvert(CalibreTask):
|
||||
else:
|
||||
# check if calibre converter-executable is existing
|
||||
if not os.path.exists(config.config_converterpath):
|
||||
self._handleError(N_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
|
||||
self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
|
||||
return
|
||||
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext)
|
||||
|
||||
@@ -199,7 +199,7 @@ class TaskConvert(CalibreTask):
|
||||
try:
|
||||
p = process_open(command, quotes)
|
||||
except OSError as e:
|
||||
return 1, N_(u"Kepubify-converter failed: %(error)s", error=e)
|
||||
return 1, N_("Kepubify-converter failed: %(error)s", error=e)
|
||||
self.progress = 0.01
|
||||
while True:
|
||||
nextline = p.stdout.readlines()
|
||||
@@ -220,7 +220,7 @@ class TaskConvert(CalibreTask):
|
||||
copyfile(converted_file[0], (file_path + format_new_ext))
|
||||
os.unlink(converted_file[0])
|
||||
else:
|
||||
return 1, N_(u"Converted file not found or more than one file in folder %(folder)s",
|
||||
return 1, N_("Converted file not found or more than one file in folder %(folder)s",
|
||||
folder=os.path.dirname(file_path))
|
||||
return check, None
|
||||
|
||||
@@ -244,7 +244,7 @@ class TaskConvert(CalibreTask):
|
||||
|
||||
p = process_open(command, quotes, newlines=False)
|
||||
except OSError as e:
|
||||
return 1, N_(u"Ebook-converter failed: %(error)s", error=e)
|
||||
return 1, N_("Ebook-converter failed: %(error)s", error=e)
|
||||
|
||||
while p.poll() is None:
|
||||
nextline = p.stdout.readline()
|
||||
|
||||
+8
-7
@@ -152,7 +152,7 @@ class TaskEmail(CalibreTask):
|
||||
main_type, sub_type = content_type.split('/', 1)
|
||||
message.add_attachment(data, maintype=main_type, subtype=sub_type, filename=self.attachment)
|
||||
else:
|
||||
self._handleError(u"Attachment not found")
|
||||
self._handleError("Attachment not found")
|
||||
return
|
||||
return message
|
||||
|
||||
@@ -166,7 +166,7 @@ class TaskEmail(CalibreTask):
|
||||
self.send_gmail_email(msg)
|
||||
except MemoryError as e:
|
||||
log.error_or_exception(e, stacklevel=3)
|
||||
self._handleError(u'MemoryError sending e-mail: {}'.format(str(e)))
|
||||
self._handleError('MemoryError sending e-mail: {}'.format(str(e)))
|
||||
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
|
||||
log.error_or_exception(e, stacklevel=3)
|
||||
if hasattr(e, "smtp_error"):
|
||||
@@ -177,13 +177,13 @@ class TaskEmail(CalibreTask):
|
||||
text = '\n'.join(e.args)
|
||||
else:
|
||||
text = ''
|
||||
self._handleError(u'Smtplib Error sending e-mail: {}'.format(text))
|
||||
self._handleError('Smtplib Error sending e-mail: {}'.format(text))
|
||||
except (socket.error) as e:
|
||||
log.error_or_exception(e, stacklevel=3)
|
||||
self._handleError(u'Socket Error sending e-mail: {}'.format(e.strerror))
|
||||
self._handleError('Socket Error sending e-mail: {}'.format(e.strerror))
|
||||
except Exception as ex:
|
||||
log.error_or_exception(ex, stacklevel=3)
|
||||
self._handleError(u'Error sending e-mail: {}'.format(ex))
|
||||
self._handleError('Error sending e-mail: {}'.format(ex))
|
||||
|
||||
def send_standard_email(self, msg):
|
||||
use_ssl = int(self.settings.get('mail_use_ssl', 0))
|
||||
@@ -216,7 +216,8 @@ class TaskEmail(CalibreTask):
|
||||
log.debug("E-mail send successfully")
|
||||
|
||||
def send_gmail_email(self, message):
|
||||
return gmail.send_messsage(self.settings.get('mail_gmail_token', None), message)
|
||||
gmail.send_messsage(self.settings.get('mail_gmail_token', None), message)
|
||||
self._handleSuccess()
|
||||
|
||||
@property
|
||||
def progress(self):
|
||||
@@ -256,7 +257,7 @@ class TaskEmail(CalibreTask):
|
||||
file_.close()
|
||||
except IOError as e:
|
||||
log.error_or_exception(e, stacklevel=3)
|
||||
log.error(u'The requested file could not be read. Maybe wrong permissions?')
|
||||
log.error('The requested file could not be read. Maybe wrong permissions?')
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
|
||||
# Copyright (C) 2020 monkey
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
from lxml import objectify
|
||||
from urllib.request import urlopen
|
||||
from lxml import etree
|
||||
from html import escape
|
||||
|
||||
from cps import config, db, fs, gdriveutils, logger, ub
|
||||
from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED
|
||||
from flask_babel import lazy_gettext as N_
|
||||
|
||||
OPF_NAMESPACE = "http://www.idpf.org/2007/opf"
|
||||
PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/"
|
||||
|
||||
OPF = "{%s}" % OPF_NAMESPACE
|
||||
PURL = "{%s}" % PURL_NAMESPACE
|
||||
|
||||
etree.register_namespace("opf", OPF_NAMESPACE)
|
||||
etree.register_namespace("dc", PURL_NAMESPACE)
|
||||
|
||||
OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix)
|
||||
NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
|
||||
|
||||
|
||||
class TaskBackupMetadata(CalibreTask):
|
||||
|
||||
def __init__(self, export_language="en",
|
||||
translated_title="Cover",
|
||||
set_dirty=False,
|
||||
task_message=N_('Backing up Metadata')):
|
||||
super(TaskBackupMetadata, self).__init__(task_message)
|
||||
self.log = logger.create()
|
||||
self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
|
||||
self.export_language = export_language
|
||||
self.translated_title = translated_title
|
||||
self.set_dirty = set_dirty
|
||||
|
||||
def run(self, worker_thread):
|
||||
if self.set_dirty:
|
||||
self.set_all_books_dirty()
|
||||
else:
|
||||
self.backup_metadata()
|
||||
|
||||
def set_all_books_dirty(self):
|
||||
try:
|
||||
books = self.calibre_db.session.query(db.Books).all()
|
||||
for book in books:
|
||||
self.calibre_db.set_metadata_dirty(book.id)
|
||||
self.calibre_db.session.commit()
|
||||
self._handleSuccess()
|
||||
except Exception as ex:
|
||||
self.log.debug('Error adding book for backup: ' + str(ex))
|
||||
self._handleError('Error adding book for backup: ' + str(ex))
|
||||
self.calibre_db.session.rollback()
|
||||
self.calibre_db.session.close()
|
||||
|
||||
def backup_metadata(self):
|
||||
try:
|
||||
metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all()
|
||||
custom_columns = self.calibre_db.session.query(db.CustomColumns).order_by(db.CustomColumns.label).all()
|
||||
count = len(metadata_backup)
|
||||
i = 0
|
||||
for backup in metadata_backup:
|
||||
book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
|
||||
self.calibre_db.session.query(db.Metadata_Dirtied).filter(
|
||||
db.Metadata_Dirtied.book == backup.book).delete()
|
||||
self.calibre_db.session.commit()
|
||||
if book:
|
||||
self.open_metadata(book, custom_columns)
|
||||
else:
|
||||
self.log.error("Book {} not found in database".format(backup.book))
|
||||
# self._handleError("Book {} not found in database".format(backup.book))
|
||||
i += 1
|
||||
self.progress = (1.0 / count) * i
|
||||
self._handleSuccess()
|
||||
self.calibre_db.session.close()
|
||||
|
||||
except Exception as ex:
|
||||
b = "NaN" if not hasattr(book, 'id') else book.id
|
||||
self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex))
|
||||
self._handleError('Error creating metadata backup: ' + str(ex))
|
||||
self.calibre_db.session.rollback()
|
||||
self.calibre_db.session.close()
|
||||
|
||||
def open_metadata(self, book, custom_columns):
|
||||
if config.config_use_google_drive:
|
||||
if not gdriveutils.is_gdrive_ready():
|
||||
raise Exception('Google Drive is configured but not ready')
|
||||
|
||||
web_content_link = gdriveutils.get_metadata_backup_via_gdrive(book.path)
|
||||
if not web_content_link:
|
||||
raise Exception('Google Drive cover url not found')
|
||||
|
||||
stream = None
|
||||
try:
|
||||
stream = urlopen(web_content_link)
|
||||
except Exception as ex:
|
||||
# Bubble exception to calling function
|
||||
self.log.debug('Error reading metadata.opf: ' + str(ex)) # ToDo Check whats going on
|
||||
raise ex
|
||||
finally:
|
||||
if stream is not None:
|
||||
stream.close()
|
||||
else:
|
||||
# ToDo: Handle book folder not found or not readable
|
||||
book_metadata_filepath = os.path.join(config.config_calibre_dir, book.path, 'metadata.opf')
|
||||
#if not os.path.isfile(book_metadata_filepath):
|
||||
self.create_new_metadata_backup(book, custom_columns, book_metadata_filepath)
|
||||
# else:
|
||||
'''namespaces = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE}
|
||||
test = etree.parse(book_metadata_filepath)
|
||||
root = test.getroot()
|
||||
for i in root.iter():
|
||||
self.log.info(i)
|
||||
title = root.find("dc:metadata", namespaces)
|
||||
pass
|
||||
with open(book_metadata_filepath, "rb") as f:
|
||||
xml = f.read()
|
||||
|
||||
root = objectify.fromstring(xml)
|
||||
# root.metadata['{http://purl.org/dc/elements/1.1/}title']
|
||||
# root.metadata[PURL + 'title']
|
||||
# getattr(root.metadata, PURL +'title')
|
||||
# test = objectify.parse()
|
||||
pass
|
||||
# backup not found has to be created
|
||||
#raise Exception('Book cover file not found')'''
|
||||
|
||||
def create_new_metadata_backup(self, book, custom_columns, book_metadata_filepath):
|
||||
# generate root package element
|
||||
package = etree.Element(OPF + "package", nsmap=OPF_NS)
|
||||
package.set("unique-identifier", "uuid_id")
|
||||
package.set("version", "2.0")
|
||||
|
||||
# generate metadata element and all subelements of it
|
||||
metadata = etree.SubElement(package, "metadata", nsmap=NSMAP)
|
||||
identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP)
|
||||
identifier.set(OPF + "scheme", "calibre")
|
||||
identifier.text = str(book.id)
|
||||
identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP)
|
||||
identifier2.set(OPF + "scheme", "uuid")
|
||||
identifier2.text = book.uuid
|
||||
title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP)
|
||||
title.text = book.title
|
||||
for author in book.authors:
|
||||
creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP)
|
||||
creator.text = str(author.name)
|
||||
creator.set(OPF + "file-as", book.author_sort) # ToDo Check
|
||||
creator.set(OPF + "role", "aut")
|
||||
contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP)
|
||||
contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]"
|
||||
contributor.set(OPF + "file-as", "calibre") # ToDo Check
|
||||
contributor.set(OPF + "role", "bkp")
|
||||
|
||||
date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP)
|
||||
date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate)
|
||||
if book.comments:
|
||||
for b in book.comments:
|
||||
description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP)
|
||||
description.text = b.text
|
||||
if not book.languages:
|
||||
language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
|
||||
language.text = self.export_language
|
||||
else:
|
||||
for b in book.languages:
|
||||
language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP)
|
||||
language.text = str(b.lang_code)
|
||||
for b in book.tags:
|
||||
subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP)
|
||||
subject.text = str(b.name)
|
||||
etree.SubElement(metadata, "meta", name="calibre:author_link_map",
|
||||
content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}",
|
||||
nsmap=NSMAP)
|
||||
for b in book.series:
|
||||
etree.SubElement(metadata, "meta", name="calibre:series",
|
||||
content=str(str(b.name)),
|
||||
nsmap=NSMAP)
|
||||
if book.series:
|
||||
etree.SubElement(metadata, "meta", name="calibre:series_index",
|
||||
content=str(book.series_index),
|
||||
nsmap=NSMAP)
|
||||
etree.SubElement(metadata, "meta", name="calibre:timestamp",
|
||||
content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(
|
||||
d=book.timestamp),
|
||||
nsmap=NSMAP)
|
||||
etree.SubElement(metadata, "meta", name="calibre:title_sort",
|
||||
content=book.sort,
|
||||
nsmap=NSMAP)
|
||||
sequence = 0
|
||||
for cc in custom_columns:
|
||||
value = None
|
||||
extra = None
|
||||
cc_entry = getattr(book, "custom_column_" + str(cc.id))
|
||||
if cc_entry.__len__():
|
||||
value = cc_entry[0].get("value")
|
||||
extra = cc_entry[0].get("extra")
|
||||
etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label),
|
||||
content=cc.to_json(value, extra, sequence),
|
||||
nsmap=NSMAP)
|
||||
sequence += 1
|
||||
|
||||
# generate guide element and all sub elements of it
|
||||
# Title is translated from default export language
|
||||
guide = etree.SubElement(package, "guide")
|
||||
etree.SubElement(guide, "reference", type="cover", title=self.translated_title, href="cover.jpg")
|
||||
|
||||
# prepare finalize everything and output
|
||||
doc = etree.ElementTree(package)
|
||||
# doc = etree.tostring(package, xml_declaration=True, encoding='utf-8', pretty_print=True) # .replace(b"&quot;", b""")
|
||||
try:
|
||||
with open(book_metadata_filepath, 'wb') as f:
|
||||
# f.write(doc)
|
||||
doc.write(f, xml_declaration=True, encoding='utf-8', pretty_print=True)
|
||||
except Exception:
|
||||
# ToDo: Folder not writeable error
|
||||
pass
|
||||
@property
|
||||
def name(self):
|
||||
return "Metadata backup"
|
||||
|
||||
# needed for logging
|
||||
def __str__(self):
|
||||
if self.set_dirty:
|
||||
return "Queue all books for metadata backup"
|
||||
else:
|
||||
return "Perform metadata backup"
|
||||
|
||||
@property
|
||||
def is_cancellable(self):
|
||||
return True
|
||||
@@ -92,7 +92,7 @@ class TaskGenerateCoverThumbnails(CalibreTask):
|
||||
|
||||
if generated > 0:
|
||||
total_generated += generated
|
||||
self.message = N_(u'Generated %(count)s cover thumbnails', count=total_generated)
|
||||
self.message = N_('Generated %(count)s cover thumbnails', count=total_generated)
|
||||
|
||||
# Check if job has been cancelled or ended
|
||||
if self.stat == STAT_CANCELLED:
|
||||
|
||||
Reference in New Issue
Block a user