From 9bdc72865bda0fe77b454d6904338077a725f34d Mon Sep 17 00:00:00 2001 From: Joel Goguen Date: Wed, 26 Jun 2024 17:59:34 -0400 Subject: [PATCH] Allow using calibredb for upload processing Some things calibre is able to do via plugins that would be helpful to handle in calibre-web, but there's not much point in re-implementing those in calibre-web directly. Things like importing ACSM files and getting the resulting ebook directly, or importing KFX files that require extra processing. To accommodate this, and allowing everything to happen in one interface instead of switching between calibre and calibre-web, this PR adds: - A new setting, linked to the main upload permission, to use calibredb for uploading files - The location the linuxserver Docker image installs calibre to the list of paths searched for calibre binaries - The chunk of code to `upload()` in `editbooks.py` to shell out to `calibredb` - The original file extension to the temporary file path, since calibredb needs that for format detection --- cps/admin.py | 1 + cps/config_sql.py | 3 +- cps/editbooks.py | 132 ++++++++++++++++++++++++--------- cps/templates/config_edit.html | 4 + cps/uploader.py | 2 +- 5 files changed, 105 insertions(+), 37 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index a39b43426e..f6b1d0ccae 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -1762,6 +1762,7 @@ def _configuration_update_helper(): return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path')) _config_checkbox_int(to_save, "config_uploading") + _config_checkbox_int(to_save, "config_upload_with_calibredb") _config_checkbox_int(to_save, "config_unicode_filename") _config_checkbox_int(to_save, "config_embed_metadata") # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case diff --git a/cps/config_sql.py b/cps/config_sql.py index 33dba62303..23a28cebe1 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -148,6 +148,7 @@ class _Settings(_Base): config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_unicode_filename = Column(Boolean, default=False) config_embed_metadata = Column(Boolean, default=True) + config_upload_with_calibredb = Column(Boolean, default=False) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) @@ -503,7 +504,7 @@ def autodetect_calibre_binaries(): "C:\\program files(x86)\\calibre2\\", "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/"] + calibre_path = ["/opt/calibre/", "/app/calibre"] for element in calibre_path: supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] diff --git a/cps/editbooks.py b/cps/editbooks.py index 802c3b09a0..d2a43e53c2 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -23,7 +23,9 @@ import os from datetime import datetime import json +import re from shutil import copyfile +import subprocess from uuid import uuid4 from markupsafe import escape, Markup # dependency of flask from functools import wraps @@ -33,6 +35,8 @@ from flask_babel import lazy_gettext as N_ from flask_babel import get_locale from flask_login import current_user, login_required +from requests.sessions import Request +from sqlalchemy import Column, Integer from sqlalchemy.exc import OperationalError, IntegrityError, InterfaceError from sqlalchemy.orm.exc import StaleDataError from sqlalchemy.sql.expression import func @@ -240,51 +244,109 @@ def upload(): if not config.config_uploading: abort(404) if request.method == 'POST' and 'btn-upload' in request.files: + calibredb_binarypath = os.path.join(config.config_binariesdir, constants.SUPPORTED_CALIBRE_BINARIES["calibredb"]) + log.debug(f"Looking for calibredb binary at {calibredb_binarypath}") + for requested_file in request.files.getlist("btn-upload"): try: modify_date = False - # create the function for sorting... - calibre_db.update_title_sort(config) - calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) meta, error = file_handling_on_upload(requested_file) if error: return error - db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta) - - # Comments need book id therefore only possible after flush - modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) - - book_id = db_book.id - title = db_book.title - if config.config_use_google_drive: - helper.upload_new_file_gdrive(book_id, - input_authors[0], - renamed_authors, - title, - title_dir, - meta.file_path, - meta.extension.lower()) + if config.config_upload_with_calibredb and os.path.exists(calibredb_binarypath): + if not os.path.exists(meta.file_path): + flash( + _("Uploaded book not found!"), + category="error", + ) + log.error(f"Expected to find temp file at {meta.file_path} but no file exists") + return Response( + json.dumps({"location": url_for("web.index")}), + mimetype="application/json", + ) + + log.debug(f"Running calibredb to add {meta.file_path}") + proc = subprocess.run( + [ + calibredb_binarypath, + "add", + f"--library-path={config.config_calibre_dir}", + meta.file_path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + if proc.returncode != 0: + flash(_("calibredb failed importing {requested_file}"), category="error") + log.error(f"calibredb failed to import {requested_file}: {proc.stderr}") + return Response( + json.dumps({"location": url_for("web.index")}), + mimetype="application/json", + ) + + # The output contains a line with the new book's ID + title = meta.title + book_id = -1 + for line in proc.stdout.split("\n"): + line = line.strip() + matches = re.match(r"^Added book ids: (\d+)$", line) + if matches is None: + continue + book_id = int(matches.group(1)) + break + log.debug(f"New calibre book ID {book_id}") + + if book_id == -1: + msg = "No ID found in calibredb output" + flash(_(msg), category="error") + log.error(f"{msg}: {proc.stdout}") + return Response( + json.dumps({"location": url_for("web.index")}), + mimetype="application/json", + ) else: - error = helper.update_dir_structure(book_id, - config.get_book_path(), - input_authors[0], - meta.file_path, - title_dir + meta.extension.lower(), - renamed_author=renamed_authors) - - move_coverfile(meta, db_book) - - if modify_date: - calibre_db.set_metadata_dirty(book_id) - # save data to database, reread data - calibre_db.session.commit() + # create the function for sorting... + calibre_db.update_title_sort(config) + calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) + + db_book, input_authors, title_dir, renamed_authors = create_book_on_upload(modify_date, meta) + + # Comments need book id therefore only possible after flush + modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book) + + book_id = db_book.id + title = db_book.title + if config.config_use_google_drive: + helper.upload_new_file_gdrive(book_id, + input_authors[0], + renamed_authors, + title, + title_dir, + meta.file_path, + meta.extension.lower()) + else: + error = helper.update_dir_structure(book_id, + config.get_book_path(), + input_authors[0], + meta.file_path, + title_dir + meta.extension.lower(), + renamed_author=renamed_authors) + + move_coverfile(meta, db_book) + + if modify_date: + calibre_db.set_metadata_dirty(book_id) + # save data to database, reread data + calibre_db.session.commit() + + if config.config_use_google_drive: + gdriveutils.updateGdriveCalibreFromLocal() + if error: + flash(error, category="error") - if config.config_use_google_drive: - gdriveutils.updateGdriveCalibreFromLocal() - if error: - flash(error, category="error") link = '{}'.format(url_for('web.show_book', book_id=book_id), escape(title)) upload_text = N_("File %(file)s uploaded", file=link) WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title))) diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 0d0a695f66..a0632366ef 100755 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -112,6 +112,10 @@

+
+ + +
diff --git a/cps/uploader.py b/cps/uploader.py index 8f20762f29..57ef7b01d9 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -254,7 +254,7 @@ def upload(uploadfile, rar_excecutable): filename = uploadfile.filename filename_root, file_extension = os.path.splitext(filename) md5 = hashlib.md5(filename.encode('utf-8')).hexdigest() # nosec - tmp_file_path = os.path.join(tmp_dir, md5) + tmp_file_path = os.path.join(tmp_dir, md5) + file_extension log.debug("Temporary file: %s", tmp_file_path) uploadfile.save(tmp_file_path) return process(tmp_file_path, filename_root, file_extension, rar_excecutable)