From 3cda0a0f32a6f5b5037ac02b90b23dac0648e2f4 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 11 Jul 2024 19:39:33 +0200 Subject: [PATCH 1/9] Testupdate --- test/Calibre-Web TestSummary_Linux.html | 2002 ++--------------------- 1 file changed, 161 insertions(+), 1841 deletions(-) diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 042bdb8d7..c5f120c03 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-07-09 19:39:05

+

Start Time: 2024-07-10 20:16:18

-

Stop Time: 2024-07-10 13:56:20

+

Stop Time: 2024-07-11 03:24:32

-

Duration: 17h 7 min

+

Duration: 5h 56 min

@@ -234,11 +234,11 @@ - + TestBackupMetadata 21 - 20 - 1 + 21 + 0 0 0 @@ -374,31 +374,11 @@ - +
TestBackupMetadata - test_backup_change_custom_categories
- -
- FAIL -
- - - - + PASS @@ -722,286 +702,57 @@ AssertionError: ['Kulo', 'Smudo'] != None - + TestEbookConvertCalibre - 18 - 9 - 1 - 8 + 15 + 15 + 0 + 0 0 - Detail + Detail - +
TestEbookConvertCalibre - test_calibre_log
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibre - test_calibre_log
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibre - test_convert_deactivate
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibre - test_convert_deactivate
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibre - test_convert_email
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibre - test_convert_email
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibre - test_convert_failed_and_email
- -
- ERROR -
- - - - + PASS - +
TestEbookConvertCalibre - test_convert_only
@@ -1010,37 +761,16 @@ ValueError: not enough values to unpack (expected 2, got 1) - +
TestEbookConvertCalibre - test_convert_options
- -
- FAIL -
- - - - + PASS - +
TestEbookConvertCalibre - test_convert_parameter
@@ -1049,7 +779,7 @@ AssertionError: Items in the first set but not the second: - +
TestEbookConvertCalibre - test_convert_wrong_excecutable
@@ -1058,7 +788,7 @@ AssertionError: Items in the first set but not the second: - +
TestEbookConvertCalibre - test_convert_xss
@@ -1067,7 +797,7 @@ AssertionError: Items in the first set but not the second: - +
TestEbookConvertCalibre - test_email_failed
@@ -1076,36 +806,16 @@ AssertionError: Items in the first set but not the second: - +
TestEbookConvertCalibre - test_email_only
- -
- ERROR -
- - - - + PASS - +
TestEbookConvertCalibre - test_kindle_send_not_configured
@@ -1114,7 +824,7 @@ ValueError: too many values to unpack (expected 2) - +
TestEbookConvertCalibre - test_ssl_smtp_setup_error
@@ -1123,7 +833,7 @@ ValueError: too many values to unpack (expected 2) - +
TestEbookConvertCalibre - test_starttls_smtp_setup_error
@@ -1132,7 +842,7 @@ ValueError: too many values to unpack (expected 2) - +
TestEbookConvertCalibre - test_user_convert_xss
@@ -1142,1276 +852,79 @@ ValueError: too many values to unpack (expected 2) - + TestEbookConvertCalibreGDrive - 14 + 7 + 7 + 0 0 - 1 - 13 0 - Detail + Detail - +
TestEbookConvertCalibreGDrive - test_convert_email
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_convert_email
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_convert_only
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_convert_only
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_convert_parameter
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_convert_parameter
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_email_failed
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_email_failed
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_email_only
- -
- ERROR -
- - - - + PASS - - -
TestEbookConvertCalibreGDrive - test_email_only
- - -
- ERROR -
- - - - - - - - - +
TestEbookConvertCalibreGDrive - test_thumbnail_cache
- -
- FAIL -
- - - - - - - - - - -
TestEbookConvertCalibreGDrive - test_thumbnail_cache
- - -
- ERROR -
- - - - + PASS @@ -3709,13 +2222,13 @@ IndexError: list index out of range - + TestSTARTTLS 3 - 0 - 0 3 0 + 0 + 0 Detail @@ -3723,141 +2236,40 @@ IndexError: list index out of range - +
TestSTARTTLS - test_STARTTLS
- -
- ERROR -
- - - - + PASS - +
TestSTARTTLS - test_STARTTLS_SSL_setup_error
- -
- ERROR -
- - - - + PASS - +
TestSTARTTLS - test_STARTTLS_resend_password
- -
- ERROR -
- - - - + PASS - + TestSSL 7 - 2 - 1 - 4 + 7 + 0 + 0 0 Detail @@ -3884,186 +2296,47 @@ dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:136:16 - +
TestSSL - test_SSL_logging_email
- -
- FAIL -
- - - - + PASS - +
TestSSL - test_SSL_non_admin_user
- -
- ERROR -
- - - - + PASS - +
TestSSL - test_SSL_only
- -
- ERROR -
- - - - + PASS - +
TestSSL - test_email_limit
- -
- ERROR -
- - - - + PASS - +
TestSSL - test_filepicker_two_file
- -
- ERROR -
- - - - + PASS @@ -5572,43 +3845,50 @@ AssertionError: False is not true - - TestPipInstall - 3 - 3 + + _FailedTest + 1 0 0 + 1 0 - Detail + Detail - + -
TestPipInstall - test_command_start
+
_FailedTest - test_pip_install
- PASS - - - - - - -
TestPipInstall - test_foldername_database_location
+ +
+ ERROR +
+ + + - PASS - - - - - - -
TestPipInstall - test_module_start
- - PASS @@ -6276,11 +4556,11 @@ AssertionError: False is not true - + TestUpdater 9 - 8 - 0 + 6 + 2 0 1 @@ -6335,11 +4615,31 @@ AssertionError: False is not true - +
TestUpdater - test_perform_update_stable_errors
- PASS + +
+ FAIL +
+ + + + @@ -6379,11 +4679,31 @@ AssertionError: False is not true - +
TestUpdater - test_update_write_protect
- PASS + +
+ FAIL +
+ + + + @@ -7353,10 +5673,10 @@ AssertionError: False is not true Total - 511 - 467 - 5 - 29 + 499 + 484 + 3 + 2 10   @@ -7889,7 +6209,7 @@ AssertionError: False is not true From 840f4d2010521a3752f0c477a2b984476f704c12 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 11 Jul 2024 20:10:15 +0200 Subject: [PATCH 2/9] Update to make upload of m4a and m4b files work (#3102) --- cps/__init__.py | 4 ++-- cps/file_helper.py | 7 +++---- optional-requirements.txt | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index fead0d2f4..642f6a6e1 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -70,8 +70,8 @@ mimetypes.add_type('image/vnd.djv', '.djv') mimetypes.add_type('image/vnd.djv', '.djvu') mimetypes.add_type('application/mpeg', '.mpeg') mimetypes.add_type('audio/mpeg', '.mp3') -mimetypes.add_type('application/mp4', '.m4a') -mimetypes.add_type('application/mp4', '.m4b') +mimetypes.add_type('audio/x-m4a', '.m4a') +mimetypes.add_type('audio/x-m4a', '.m4b') mimetypes.add_type('audio/ogg', '.ogg') mimetypes.add_type('application/ogg', '.oga') mimetypes.add_type('text/css', '.css') diff --git a/cps/file_helper.py b/cps/file_helper.py index 095b5ee09..d61823110 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -47,11 +47,11 @@ def del_temp_dir(): def validate_mime_type(file_buffer, allowed_extensions): mime = magic.Magic(mime=True) - allowed_mimetypes =list() + allowed_mimetypes = list() for x in allowed_extensions: try: allowed_mimetypes.append(mimetypes.types_map["." + x]) - except KeyError as e: + except KeyError: log.error("Unkown mimetype for Extension: {}".format(x)) tmp_mime_type = mime.from_buffer(file_buffer.read()) file_buffer.seek(0) @@ -66,6 +66,5 @@ def validate_mime_type(file_buffer, allowed_extensions): return True except: file_buffer.seek(0) - pass - + log.error("Mimetype '{}' not found in allowed types".format(tmp_mime_type)) return False diff --git a/optional-requirements.txt b/optional-requirements.txt index 77f41e1e2..3f6b6e37d 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -42,7 +42,7 @@ natsort>=2.2.0,<8.5.0 comicapi>=2.2.0,<3.3.0 # Kobo integration -jsonschema>=3.2.0,<4.23.0 +jsonschema>=3.2.0,<4.24.0 # Hide console Window on Windows pywin32>=220,<310 ; sys_platform == 'win32' From 6996e813b5a3079e071ca8703e78a097d2c46478 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 11 Jul 2024 20:12:26 +0200 Subject: [PATCH 3/9] Fixed error during debug logging on upload --- cps/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/helper.py b/cps/helper.py index b7d4452fa..b90d0fbc0 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -562,7 +562,7 @@ def move_files_on_change(calibre_path, new_author_dir, new_titledir, localbook, if not os.path.isdir(new_path): os.makedirs(new_path) shutil.move(original_filepath, os.path.join(new_path, db_filename)) - log.debug("Moving title: %s to %s/%s", original_filepath, new_path) + log.debug("Moving title: %s to %s", original_filepath, new_path) else: # Check new path is not valid path if not os.path.exists(new_path): From 02d2c2bbf8c7a0528505b8f69673e98192e4eedd Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 11 Jul 2024 20:30:35 +0200 Subject: [PATCH 4/9] Better error message on failed convert because of running calibre instance (#3100) --- cps/tasks/convert.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 5354dce20..e6af356f3 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -269,10 +269,18 @@ class TaskConvert(CalibreTask): '--with-library', library_path] p = process_open(opf_command, quotes, my_env) p.wait() - path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") - with open(path_tmp_opf, 'w') as fd: - copyfileobj(p.stdout, fd) - + check = p.returncode + calibre_traceback = p.stderr.readlines() + if check == 0: + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + else: + error_message = "" + for ele in calibre_traceback: + if not ele.startswith('Traceback') and not ele.startswith(' File'): + error_message = N_("Calibre failed with error: %(error)s", error=ele) + return check, error_message quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext)] From 7367e96d2bc807a89f706b0256f828d57238d201 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 18 Jul 2024 20:32:47 +0200 Subject: [PATCH 5/9] Fix hiding console window on windows --- cps.py | 20 ++++++++------------ optional-requirements.txt | 3 --- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cps.py b/cps.py index 8af6cef1f..039c2a5e1 100755 --- a/cps.py +++ b/cps.py @@ -30,19 +30,15 @@ from cps.main import main def hide_console_windows(): import ctypes - import os - hwnd = ctypes.windll.kernel32.GetConsoleWindow() - if hwnd != 0: - try: - import win32process - except ImportError: - print("To hide console window install 'pywin32' using 'pip install pywin32'") - return - ctypes.windll.user32.ShowWindow(hwnd, 0) - ctypes.windll.kernel32.CloseHandle(hwnd) - _, pid = win32process.GetWindowThreadProcessId(hwnd) - os.system('taskkill /PID ' + str(pid) + ' /f') + kernel32 = ctypes.WinDLL('kernel32') + user32 = ctypes.WinDLL('user32') + + SW_HIDE = 0 + + hWnd = kernel32.GetConsoleWindow() + if hWnd: + user32.ShowWindow(hWnd, SW_HIDE) if __name__ == '__main__': diff --git a/optional-requirements.txt b/optional-requirements.txt index 3f6b6e37d..d63bc41c5 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -43,6 +43,3 @@ comicapi>=2.2.0,<3.3.0 # Kobo integration jsonschema>=3.2.0,<4.24.0 - -# Hide console Window on Windows -pywin32>=220,<310 ; sys_platform == 'win32' From e33a7a3f082e8ee0babd705477ffbafa1760b130 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Fri, 12 Jul 2024 21:09:05 +0200 Subject: [PATCH 6/9] Testupdate --- test/Calibre-Web TestSummary_Linux.html | 114 ++++++++++++------------ 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index c5f120c03..5b79eaf61 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-07-10 20:16:18

+

Start Time: 2024-07-11 20:34:31

-

Stop Time: 2024-07-11 03:24:32

+

Stop Time: 2024-07-12 03:34:37

-

Duration: 5h 56 min

+

Duration: 5h 47 min

@@ -2021,11 +2021,11 @@ IndexError: list index out of range - + TestEditBooksOnGdrive 18 - 18 - 0 + 16 + 2 0 0 @@ -2152,11 +2152,31 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_edit_rating
- PASS + +
+ FAIL +
+ + + + @@ -2188,11 +2208,31 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_watch_metadata
- PASS + +
+ FAIL +
+ + + + @@ -4556,11 +4596,11 @@ ModuleNotFoundError: No module named 'build_release' - + TestUpdater 9 - 6 - 2 + 8 + 0 0 1 @@ -4615,31 +4655,11 @@ ModuleNotFoundError: No module named 'build_release' - +
TestUpdater - test_perform_update_stable_errors
- -
- FAIL -
- - - - + PASS @@ -4679,31 +4699,11 @@ AssertionError: False is not true - +
TestUpdater - test_update_write_protect
- -
- FAIL -
- - - - + PASS From a59d20bd0f32de6f04400d939a6f310e822c7a93 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Jul 2024 11:31:31 +0200 Subject: [PATCH 7/9] Output error message in case import of libmagic fails (#3104) --- cps/file_helper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cps/file_helper.py b/cps/file_helper.py index d61823110..9d5406d3c 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -29,8 +29,9 @@ log = logger.create() try: import magic + error = None except ImportError as e: - log.error("Cannot import python-magic, checking uploaded file metadata will not work: %s", e) + error = "Cannot import python-magic, checking uploaded file metadata will not work: {}".format(e) def get_temp_dir(): @@ -46,6 +47,9 @@ def del_temp_dir(): def validate_mime_type(file_buffer, allowed_extensions): + if error: + log.error(error) + return False mime = magic.Magic(mime=True) allowed_mimetypes = list() for x in allowed_extensions: From 8df45d150d71c286384d84a63e1c0b82935e6675 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Jul 2024 13:51:45 +0200 Subject: [PATCH 8/9] Added hints for libmagic on Windows (fix #3104) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d6efaf5c..a400b3b20 100755 --- a/README.md +++ b/README.md @@ -89,8 +89,9 @@ Refer to the Wiki for additional installation examples: [manual installation](ht ## Requirements -- Python 3.5+ +- Python 3.7+ - [Imagemagick](https://imagemagick.org/script/download.php) for cover extraction from EPUBs (Windows users may need to install [Ghostscript](https://ghostscript.com/releases/gsdnld.html) for PDF cover extraction) +- Windows users need to install [libmagic for 32bit python](https://gnuwin32.sourceforge.net/downlinks/file.php) or [libmagic for 64bit python](https://github.com/nscaife/file-windows/releases/tag/20170108), depending on the python version; The files need to be installed in path (e.g. script folder of your Calibre-Web venv, or in the root folder of Calibre-Web - Optional: [Calibre desktop program](https://calibre-ebook.com/download) for on-the-fly conversion and metadata editing (set "calibre's converter tool" path on the setup page) - Optional: [Kepubify tool](https://github.com/pgaskin/kepubify/releases/latest) for Kobo device support (place the binary in `/opt/kepubify` on Linux or `C:\Program Files\kepubify` on Windows) From 1c5d1e24f769314102e579891fe3b39462839728 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Jul 2024 13:58:53 +0200 Subject: [PATCH 9/9] Update test results --- test/Calibre-Web TestSummary_Linux.html | 124 ++++++------------------ 1 file changed, 32 insertions(+), 92 deletions(-) diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 5b79eaf61..288c74437 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-07-11 20:34:31

+

Start Time: 2024-07-18 20:53:44

-

Stop Time: 2024-07-12 03:34:37

+

Stop Time: 2024-07-19 03:48:09

-

Duration: 5h 47 min

+

Duration: 5h 43 min

@@ -2009,7 +2009,7 @@ 
Traceback (most recent call last):
   File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_metadata.py", line 90, in test_load_metadata
-    elif 'https://amazon.com/' == results[20]['source']:
+    elif len(results)>19 and 'https://amazon.com/' == results[20]['source']:
 IndexError: list index out of range
@@ -2021,11 +2021,11 @@ IndexError: list index out of range - + TestEditBooksOnGdrive 18 - 16 - 2 + 18 + 0 0 0 @@ -2152,31 +2152,11 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_edit_rating
- -
- FAIL -
- - - - + PASS @@ -2208,31 +2188,11 @@ AssertionError: 1 != 0 - +
TestEditBooksOnGdrive - test_watch_metadata
- -
- FAIL -
- - - - + PASS @@ -3340,11 +3300,11 @@ AssertionError: False is not true - + TestLogin 19 - 18 - 1 + 19 + 0 0 0 @@ -3498,31 +3458,11 @@ AssertionError: False is not true - +
TestLogin - test_proxy_login_multi_user
- -
- FAIL -
- - - - + PASS @@ -5674,8 +5614,8 @@ ModuleNotFoundError: No module named 'build_release' Total 499 - 484 - 3 + 487 + 0 2 10   @@ -5705,7 +5645,7 @@ ModuleNotFoundError: No module named 'build_release' Platform - Linux 6.5.0-41-generic #41~22.04.2-Ubuntu SMP PREEMPT_DYNAMIC Mon Jun 3 11:32:55 UTC 2 x86_64 x86_64 + Linux 6.5.0-44-generic #44~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Tue Jun 18 14:36:16 UTC 2 x86_64 x86_64 Basic @@ -5758,14 +5698,14 @@ ModuleNotFoundError: No module named 'build_release' - Flask-Limiter - 3.5.1 + Flask-HTTPAuth + 4.8.0 Basic - Flask-Login - 0.6.3 + Flask-Limiter + 3.5.1 Basic @@ -5891,7 +5831,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestBackupMetadataGdrive @@ -5921,7 +5861,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestCliGdrivedb @@ -5951,7 +5891,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEbookConvertCalibreGDrive @@ -5981,7 +5921,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEbookConvertGDriveKepubify @@ -6029,7 +5969,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEditAuthorsGdrive @@ -6065,7 +6005,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEditBooksOnGdrive @@ -6107,7 +6047,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEmbedMetadataGdrive @@ -6137,7 +6077,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestSetupGdrive @@ -6209,7 +6149,7 @@ ModuleNotFoundError: No module named 'build_release'