Allow using calibredb for upload processing

Some things calibre is able to do via plugins that would be helpful to handle in calibre-web, but there's not much point in re-implementing those in calibre-web directly. Things like importing ACSM files and getting the resulting ebook directly, or importing KFX files that require extra processing. To accommodate this, and allowing everything to happen in one interface instead of switching between calibre and calibre-web, this PR adds:

- A new setting, linked to the main upload permission, to use calibredb for uploading files
- The location the linuxserver Docker image installs calibre to the list of paths searched for calibre binaries
- The chunk of code to `upload()` in `editbooks.py` to shell out to `calibredb`
- The original file extension to the temporary file path, since calibredb needs that for format detection

cps/admin.py

@@ -1762,6 +1762,7 @@ def _configuration_update_helper():
             return _configuration_result(_('Certfile Location is not Valid, Please Enter Correct Path'))
 
         _config_checkbox_int(to_save, "config_uploading")
+        _config_checkbox_int(to_save, "config_upload_with_calibredb")
         _config_checkbox_int(to_save, "config_unicode_filename")
         _config_checkbox_int(to_save, "config_embed_metadata")
         # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case

cps/config_sql.py

@@ -148,6 +148,7 @@ class _Settings(_Base):
     config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
     config_unicode_filename = Column(Boolean, default=False)
     config_embed_metadata = Column(Boolean, default=True)
+    config_upload_with_calibredb = Column(Boolean, default=False)
 
     config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
 
@@ -504,7 +505,7 @@ def autodetect_calibre_binaries():
                         "C:\\program files(x86)\\calibre2\\",
                         "C:\\program files\\calibre2\\"]
     else:
-        calibre_path = ["/opt/calibre/"]
+        calibre_path = ["/opt/calibre/", "/app/calibre"]
     for element in calibre_path:
         supported_binary_paths = [os.path.join(element, binary)
                                   for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()]

cps/editbooks.py

@@ -23,7 +23,9 @@
 import os
 from datetime import datetime
 import json
-from shutil import copyfile, move
+import re
+from shutil import copyfile
+import subprocess
 from uuid import uuid4
 from markupsafe import escape, Markup  # dependency of flask
 from functools import wraps
@@ -239,52 +241,111 @@ def upload():
     if not config.config_uploading:
         abort(404)
     if request.method == 'POST' and 'btn-upload' in request.files:
+        calibredb_binarypath = os.path.join(config.config_binariesdir, constants.SUPPORTED_CALIBRE_BINARIES["calibredb"])
+        log.debug(f"Looking for calibredb binary at {calibredb_binarypath}")
+
         for requested_file in request.files.getlist("btn-upload"):
             try:
                 modify_date = False
-                # create the function for sorting...
-                calibre_db.update_title_sort(config)
-                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
 
                 meta, error = file_handling_on_upload(requested_file)
                 if error:
                     return error
 
-                db_book, input_authors, title_dir = create_book_on_upload(modify_date, meta)
-
-                # Comments need book id therefore only possible after flush
-                modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
-
-                book_id = db_book.id
-                title = db_book.title
-                if config.config_use_google_drive:
-                    helper.upload_new_file_gdrive(book_id,
-                                                  input_authors[0],
-                                                  title,
-                                                  title_dir,
-                                                  meta.file_path,
-                                                  meta.extension.lower())
-                    for file_format in db_book.data:
-                        file_format.name = (helper.get_valid_filename(title, chars=42) + ' - '
-                                            + helper.get_valid_filename(input_authors[0], chars=42))
+                if config.config_upload_with_calibredb and os.path.exists(calibredb_binarypath):
+                    if not os.path.exists(meta.file_path):
+                        flash(
+                            _("Uploaded book not found!"),
+                            category="error",
+                        )
+                        log.error(f"Expected to find temp file at {meta.file_path} but no file exists")
+                        return Response(
+                            json.dumps({"location": url_for("web.index")}),
+                            mimetype="application/json",
+                        )
+
+                    log.debug(f"Running calibredb to add {meta.file_path}")
+                    proc = subprocess.run(
+                        [
+                            calibredb_binarypath,
+                            "add",
+                            f"--library-path={config.config_calibre_dir}",
+                            meta.file_path,
+                        ],
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        universal_newlines=True,
+                    )
+                    if proc.returncode != 0:
+                        flash(_("calibredb failed importing {requested_file}"), category="error")
+                        log.error(f"calibredb failed to import {requested_file}: {proc.stderr}")
+                        return Response(
+                            json.dumps({"location": url_for("web.index")}),
+                            mimetype="application/json",
+                        )
+
+                    # The output contains a line with the new book's ID
+                    title = meta.title
+                    book_id = -1
+                    for line in proc.stdout.split("\n"):
+                        line = line.strip()
+                        matches = re.match(r"^Added book ids: (\d+)$", line)
+                        if matches is None:
+                            continue
+                        book_id = int(matches.group(1))
+                        break
+                    log.debug(f"New calibre book ID {book_id}")
+
+                    if book_id == -1:
+                        msg = "No ID found in calibredb output"
+                        flash(_(msg), category="error")
+                        log.error(f"{msg}: {proc.stdout}")
+                        return Response(
+                            json.dumps({"location": url_for("web.index")}),
+                            mimetype="application/json",
+                        )
                 else:
-                    error = helper.update_dir_structure(book_id,
-                                                        config.get_book_path(),
-                                                        input_authors[0],
-                                                        meta.file_path,
-                                                        title_dir + meta.extension.lower())
+                    # create the function for sorting...
+                    calibre_db.update_title_sort(config)
+                    calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
+
+                    db_book, input_authors, title_dir = create_book_on_upload(modify_date, meta)
+
+                    # Comments need book id therefore only possible after flush
+                    modify_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
+
+                    book_id = db_book.id
+                    title = db_book.title
+                    if config.config_use_google_drive:
+                        helper.upload_new_file_gdrive(book_id,
+                                                    input_authors[0],
+                                                    renamed_authors,
+                                                    title,
+                                                    title_dir,
+                                                    meta.file_path,
+                                                    meta.extension.lower())
+                        for file_format in db_book.data:
+                            file_format.name = (helper.get_valid_filename(title, chars=42) + ' - '
+                                                + helper.get_valid_filename(input_authors[0], chars=42))
+                    else:
+                        error = helper.update_dir_structure(book_id,
+                                                            config.get_book_path(),
+                                                            input_authors[0],
+                                                            meta.file_path,
+                                                            title_dir + meta.extension.lower())
 
-                move_coverfile(meta, db_book)
+                    move_coverfile(meta, db_book)
 
-                if modify_date:
-                    calibre_db.set_metadata_dirty(book_id)
-                # save data to database, reread data
-                calibre_db.session.commit()
+                    if modify_date:
+                        calibre_db.set_metadata_dirty(book_id)
+                    # save data to database, reread data
+                    calibre_db.session.commit()
+
+                    if config.config_use_google_drive:
+                        gdriveutils.updateGdriveCalibreFromLocal()
+                    if error:
+                        flash(error, category="error")
 
-                if config.config_use_google_drive:
-                    gdriveutils.updateGdriveCalibreFromLocal()
-                if error:
-                    flash(error, category="error")
                 link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book_id), escape(title))
                 upload_text = N_("File %(file)s uploaded", file=link)
                 WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title)))

cps/templates/config_edit.html

@@ -112,6 +112,10 @@ <h4 class="panel-title">
         <label for="config_uploading">{{_('Enable Uploads')}} {{_('(Please ensure that users also have upload permissions)')}}</label>
     </div>
     <div data-related="upload_settings">
+      <div class = "form-group">
+          <input type="checkbox" id="config_upload_with_calibredb" name="config_upload_with_calibredb" {% if config.config_upload_with_calibredb %}checked{% endif %}>
+          <label for="config_upload_with_calibredb">{{_('Upload with calibredb')}}</label>
+      </div>
       <div class="form-group">
         <label for="config_upload_formats">{{_('Allowed Upload Fileformats')}}</label>
         <input type="text" class="form-control" name="config_upload_formats" id="config_upload_formats" value="{% if config.config_upload_formats != None %}{{ config.config_upload_formats }}{% endif %}" autocomplete="off">

cps/uploader.py

@@ -254,7 +254,7 @@ def upload(uploadfile, rar_excecutable):
     filename = uploadfile.filename
     filename_root, file_extension = os.path.splitext(filename)
     md5 = hashlib.md5(filename.encode('utf-8')).hexdigest()  # nosec
-    tmp_file_path = os.path.join(tmp_dir, md5)
+    tmp_file_path = os.path.join(tmp_dir, md5) + file_extension
     log.debug("Temporary file: %s", tmp_file_path)
     uploadfile.save(tmp_file_path)
     return process(tmp_file_path, filename_root, file_extension, rar_excecutable)