diff --git a/src/common/checksumcalculator.cpp b/src/common/checksumcalculator.cpp new file mode 100644 index 000000000000..588ff5e1c833 --- /dev/null +++ b/src/common/checksumcalculator.cpp @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#include "checksumcalculator.h" + +#include + +#include +#include +#include + +namespace +{ +constexpr qint64 bufSize = 500 * 1024; +} + +namespace OCC { + +Q_LOGGING_CATEGORY(lcChecksumCalculator, "nextcloud.common.checksumcalculator", QtInfoMsg) + +static QCryptographicHash::Algorithm algorithmTypeToQCryptoHashAlgorithm(ChecksumCalculator::AlgorithmType algorithmType) +{ + switch (algorithmType) { + case ChecksumCalculator::AlgorithmType::Undefined: + case ChecksumCalculator::AlgorithmType::Adler32: + qCWarning(lcChecksumCalculator) << "Invalid algorithm type" << static_cast(algorithmType); + return static_cast(-1); + case ChecksumCalculator::AlgorithmType::MD5: + return QCryptographicHash::Algorithm::Md5; + case ChecksumCalculator::AlgorithmType::SHA1: + return QCryptographicHash::Algorithm::Sha1; + case ChecksumCalculator::AlgorithmType::SHA256: + return QCryptographicHash::Algorithm::Sha256; + case ChecksumCalculator::AlgorithmType::SHA3_256: + return QCryptographicHash::Algorithm::Sha3_256; + } + return static_cast(-1); +} + +ChecksumCalculator::ChecksumCalculator(QSharedPointer sharedDevice, const QByteArray &checksumTypeName) + : _device(sharedDevice) +{ + if (checksumTypeName == checkSumMD5C) { + _algorithmType = AlgorithmType::MD5; + } else if (checksumTypeName == checkSumSHA1C) { + _algorithmType = AlgorithmType::SHA1; + } else if (checksumTypeName == checkSumSHA2C) { + _algorithmType = AlgorithmType::SHA256; + } else if (checksumTypeName == checkSumSHA3C) { + _algorithmType = AlgorithmType::SHA3_256; + } else if (checksumTypeName == checkSumAdlerC) { + _algorithmType = AlgorithmType::Adler32; + } + + initChecksumAlgorithm(); +} + +ChecksumCalculator::~ChecksumCalculator() +{ + QMutexLocker locker(&_deviceMutex); + if (_device && _device->isOpen()) { + _device->close(); + } +} + +QByteArray ChecksumCalculator::calculate() +{ + QByteArray result; + + if (!_isInitialized) { + return result; + } + + Q_ASSERT(!_device->isOpen()); + if (_device->isOpen()) { + qCWarning(lcChecksumCalculator) << "Device already open. Ignoring."; + } + + if (!_device->isOpen() && !_device->open(QIODevice::ReadOnly)) { + if (auto file = qobject_cast(_device.data())) { + qCWarning(lcChecksumCalculator) << "Could not open file" << file->fileName() << "for reading to compute a checksum" << file->errorString(); + } else { + qCWarning(lcChecksumCalculator) << "Could not open device" << _device.data() << "for reading to compute a checksum" << _device->errorString(); + } + return result; + } + + bool isAnyChunkAdded = false; + + for (;;) { + QMutexLocker locker(&_deviceMutex); + if (!_device->isOpen() || _device->atEnd()) { + break; + } + const auto toRead = qMin(_device->bytesAvailable(), bufSize); + if (toRead <= 0) { + break; + } + QByteArray buf(toRead, Qt::Uninitialized); + const auto sizeRead = _device->read(buf.data(), toRead); + if (sizeRead <= 0) { + break; + } + if (!addChunk(buf, sizeRead)) { + break; + } + isAnyChunkAdded = true; + } + + { + QMutexLocker locker(&_deviceMutex); + if (!_device->isOpen()) { + return result; + } + } + + if (isAnyChunkAdded) { + if (_algorithmType == AlgorithmType::Adler32) { + result = QByteArray::number(_adlerHash, 16); + } else { + Q_ASSERT(_cryptographicHash); + if (_cryptographicHash) { + result = _cryptographicHash->result().toHex(); + } + } + } + + { + QMutexLocker locker(&_deviceMutex); + if (_device->isOpen()) { + _device->close(); + } + } + + return result; +} + +void ChecksumCalculator::initChecksumAlgorithm() +{ + if (_algorithmType == AlgorithmType::Undefined) { + qCWarning(lcChecksumCalculator) << "_algorithmType is Undefined, impossible to init Checksum Algorithm"; + return; + } + + { + QMutexLocker locker(&_deviceMutex); + if (_device->size() == 0) { + return; + } + } + + if (_algorithmType == AlgorithmType::Adler32) { + _adlerHash = adler32(0L, Z_NULL, 0); + } else { + _cryptographicHash.reset(new QCryptographicHash(algorithmTypeToQCryptoHashAlgorithm(_algorithmType))); + } + + _isInitialized = true; +} + +bool ChecksumCalculator::addChunk(const QByteArray &chunk, const qint64 size) +{ + Q_ASSERT(_algorithmType != AlgorithmType::Undefined); + if (_algorithmType == AlgorithmType::Undefined) { + qCWarning(lcChecksumCalculator) << "_algorithmType is Undefined, impossible to add a chunk!"; + return false; + } + + if (_algorithmType == AlgorithmType::Adler32) { + _adlerHash = adler32(_adlerHash, (const Bytef *)chunk.data(), size); + return true; + } else { + Q_ASSERT(_cryptographicHash); + if (_cryptographicHash) { + _cryptographicHash->addData(chunk.data(), size); + return true; + } + } + return false; +} + +} diff --git a/src/common/checksumcalculator.h b/src/common/checksumcalculator.h new file mode 100644 index 000000000000..c9c90bbf3d21 --- /dev/null +++ b/src/common/checksumcalculator.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#pragma once + +#include "ocsynclib.h" +#include "config.h" +#include "checksumconsts.h" + +#include +#include +#include +#include + +#include + +class QCryptographicHash; + +namespace OCC { +class OCSYNC_EXPORT ChecksumCalculator +{ + Q_DISABLE_COPY(ChecksumCalculator) + +public: + enum class AlgorithmType { + Undefined = -1, + MD5, + SHA1, + SHA256, + SHA3_256, + Adler32, + }; + + ChecksumCalculator(QSharedPointer sharedDevice, const QByteArray &checksumTypeName); + ~ChecksumCalculator(); + [[nodiscard]] QByteArray calculate(); + +private: + void initChecksumAlgorithm(); + bool addChunk(const QByteArray &chunk, const qint64 size); + QSharedPointer _device; + QScopedPointer _cryptographicHash; + unsigned int _adlerHash = 0; + bool _isInitialized = false; + AlgorithmType _algorithmType = AlgorithmType::Undefined; + QMutex _deviceMutex; +}; +} diff --git a/src/common/checksumconsts.h b/src/common/checksumconsts.h new file mode 100644 index 000000000000..6770fddae2dd --- /dev/null +++ b/src/common/checksumconsts.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 by Oleksandr Zolotov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#pragma once + +namespace OCC +{ +/** + * Tags for checksum headers values. + * They are here for being shared between Upload- and Download Job + */ +static constexpr auto checkSumMD5C = "MD5"; +static constexpr auto checkSumSHA1C = "SHA1"; +static constexpr auto checkSumSHA2C = "SHA256"; +static constexpr auto checkSumSHA3C = "SHA3-256"; +static constexpr auto checkSumAdlerC = "Adler32"; +} diff --git a/src/common/checksums.cpp b/src/common/checksums.cpp index a298e5d2fc75..9bc70b41b60a 100644 --- a/src/common/checksums.cpp +++ b/src/common/checksums.cpp @@ -18,6 +18,7 @@ #include "config.h" #include "filesystembase.h" #include "common/checksums.h" +#include "checksumcalculator.h" #include "asserts.h" #include @@ -90,48 +91,6 @@ Q_LOGGING_CATEGORY(lcChecksums, "nextcloud.sync.checksums", QtInfoMsg) #define BUFSIZE qint64(500 * 1024) // 500 KiB -static QByteArray calcCryptoHash(QIODevice *device, QCryptographicHash::Algorithm algo) -{ - QByteArray arr; - QCryptographicHash crypto( algo ); - - if (crypto.addData(device)) { - arr = crypto.result().toHex(); - } - return arr; - } - -QByteArray calcMd5(QIODevice *device) -{ - return calcCryptoHash(device, QCryptographicHash::Md5); -} - -QByteArray calcSha1(QIODevice *device) -{ - return calcCryptoHash(device, QCryptographicHash::Sha1); -} - -#ifdef ZLIB_FOUND -QByteArray calcAdler32(QIODevice *device) -{ - if (device->size() == 0) - { - return QByteArray(); - } - QByteArray buf(BUFSIZE, Qt::Uninitialized); - - unsigned int adler = adler32(0L, Z_NULL, 0); - qint64 size = 0; - while (!device->atEnd()) { - size = device->read(buf.data(), BUFSIZE); - if (size > 0) - adler = adler32(adler, (const Bytef *)buf.data(), size); - } - - return QByteArray::number(adler, 16); -} -#endif - QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum) { if (checksumType.isEmpty() || checksum.isEmpty()) @@ -228,87 +187,44 @@ QByteArray ComputeChecksum::checksumType() const void ComputeChecksum::start(const QString &filePath) { qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of" << filePath << "in a thread"; - startImpl(std::make_unique(filePath)); + startImpl(QSharedPointer::create(filePath)); } -void ComputeChecksum::start(std::unique_ptr device) +void ComputeChecksum::start(QSharedPointer device) { ENFORCE(device); qCInfo(lcChecksums) << "Computing" << checksumType() << "checksum of device" << device.get() << "in a thread"; ASSERT(!device->parent()); - startImpl(std::move(device)); + startImpl(device); } -void ComputeChecksum::startImpl(std::unique_ptr device) +void ComputeChecksum::startImpl(QSharedPointer device) { connect(&_watcher, &QFutureWatcherBase::finished, this, &ComputeChecksum::slotCalculationDone, Qt::UniqueConnection); - // We'd prefer to move the unique_ptr into the lambda, but that's - // awkward with the C++ standard we're on - auto sharedDevice = QSharedPointer(device.release()); - - // Bug: The thread will keep running even if ComputeChecksum is deleted. - auto type = checksumType(); - _watcher.setFuture(QtConcurrent::run([sharedDevice, type]() { - if (!sharedDevice->open(QIODevice::ReadOnly)) { - if (auto file = qobject_cast(sharedDevice.data())) { - qCWarning(lcChecksums) << "Could not open file" << file->fileName() - << "for reading to compute a checksum" << file->errorString(); - } else { - qCWarning(lcChecksums) << "Could not open device" << sharedDevice.data() - << "for reading to compute a checksum" << sharedDevice->errorString(); - } - return QByteArray(); - } - auto result = ComputeChecksum::computeNow(sharedDevice.data(), type); - sharedDevice->close(); - return result; + _checksumCalculator.reset(new ChecksumCalculator(device, _checksumType)); + _watcher.setFuture(QtConcurrent::run([this]() { + return _checksumCalculator->calculate(); })); } QByteArray ComputeChecksum::computeNowOnFile(const QString &filePath, const QByteArray &checksumType) { - QFile file(filePath); - if (!file.open(QIODevice::ReadOnly)) { - qCWarning(lcChecksums) << "Could not open file" << filePath << "for reading and computing checksum" << file.errorString(); - return QByteArray(); - } - - return computeNow(&file, checksumType); + return computeNow(QSharedPointer::create(filePath), checksumType); } -QByteArray ComputeChecksum::computeNow(QIODevice *device, const QByteArray &checksumType) +QByteArray ComputeChecksum::computeNow(QSharedPointer device, const QByteArray &checksumType) { if (!checksumComputationEnabled()) { qCWarning(lcChecksums) << "Checksum computation disabled by environment variable"; return QByteArray(); } - if (checksumType == checkSumMD5C) { - return calcMd5(device); - } else if (checksumType == checkSumSHA1C) { - return calcSha1(device); - } else if (checksumType == checkSumSHA2C) { - return calcCryptoHash(device, QCryptographicHash::Sha256); - } -#if QT_VERSION >= QT_VERSION_CHECK(5, 9, 0) - else if (checksumType == checkSumSHA3C) { - return calcCryptoHash(device, QCryptographicHash::Sha3_256); - } -#endif -#ifdef ZLIB_FOUND - else if (checksumType == checkSumAdlerC) { - return calcAdler32(device); - } -#endif - // for an unknown checksum or no checksum, we're done right now - if (!checksumType.isEmpty()) { - qCWarning(lcChecksums) << "Unknown checksum type:" << checksumType; - } - return QByteArray(); + ChecksumCalculator checksumCalculator(device, checksumType); + return checksumCalculator.calculate(); } void ComputeChecksum::slotCalculationDone() @@ -354,10 +270,11 @@ void ValidateChecksumHeader::start(const QString &filePath, const QByteArray &ch calculator->start(filePath); } -void ValidateChecksumHeader::start(std::unique_ptr device, const QByteArray &checksumHeader) +void ValidateChecksumHeader::start(QSharedPointer device, const QByteArray &checksumHeader) { - if (auto calculator = prepareStart(checksumHeader)) - calculator->start(std::move(device)); + if (auto calculator = prepareStart(checksumHeader)) { + calculator->start(device); + } } QByteArray ValidateChecksumHeader::calculatedChecksumType() const diff --git a/src/common/checksums.h b/src/common/checksums.h index 3c0e5528f240..9496c29edc16 100644 --- a/src/common/checksums.h +++ b/src/common/checksums.h @@ -31,16 +31,7 @@ class QFile; namespace OCC { -/** - * Tags for checksum headers values. - * They are here for being shared between Upload- and Download Job - */ -static const char checkSumMD5C[] = "MD5"; -static const char checkSumSHA1C[] = "SHA1"; -static const char checkSumSHA2C[] = "SHA256"; -static const char checkSumSHA3C[] = "SHA3-256"; -static const char checkSumAdlerC[] = "Adler32"; - +class ChecksumCalculator; class SyncJournalDb; /** @@ -65,13 +56,6 @@ OCSYNC_EXPORT QByteArray parseChecksumHeaderType(const QByteArray &header); /// Checks OWNCLOUD_DISABLE_CHECKSUM_UPLOAD OCSYNC_EXPORT bool uploadChecksumEnabled(); -// Exported functions for the tests. -QByteArray OCSYNC_EXPORT calcMd5(QIODevice *device); -QByteArray OCSYNC_EXPORT calcSha1(QIODevice *device); -#ifdef ZLIB_FOUND -QByteArray OCSYNC_EXPORT calcAdler32(QIODevice *device); -#endif - /** * Computes the checksum of a file. * \ingroup libsync @@ -105,12 +89,12 @@ class OCSYNC_EXPORT ComputeChecksum : public QObject * The device ownership transfers into the thread that * will compute the checksum. It must not have a parent. */ - void start(std::unique_ptr device); + void start(QSharedPointer device); /** * Computes the checksum synchronously. */ - static QByteArray computeNow(QIODevice *device, const QByteArray &checksumType); + static QByteArray computeNow(QSharedPointer device, const QByteArray &checksumType); /** * Computes the checksum synchronously on file. Convenience wrapper for computeNow(). @@ -124,12 +108,14 @@ private slots: void slotCalculationDone(); private: - void startImpl(std::unique_ptr device); + void startImpl(QSharedPointer device); QByteArray _checksumType; // watcher for the checksum calculation thread QFutureWatcher _watcher; + + QScopedPointer _checksumCalculator; }; /** @@ -167,7 +153,7 @@ class OCSYNC_EXPORT ValidateChecksumHeader : public QObject * The device ownership transfers into the thread that * will compute the checksum. It must not have a parent. */ - void start(std::unique_ptr device, const QByteArray &checksumHeader); + void start(QSharedPointer device, const QByteArray &checksumHeader); [[nodiscard]] QByteArray calculatedChecksumType() const; [[nodiscard]] QByteArray calculatedChecksum() const; diff --git a/src/common/common.cmake b/src/common/common.cmake index ebe69f565281..671973579f0b 100644 --- a/src/common/common.cmake +++ b/src/common/common.cmake @@ -3,6 +3,7 @@ # help keep track of the different code licenses. set(common_SOURCES ${CMAKE_CURRENT_LIST_DIR}/checksums.cpp + ${CMAKE_CURRENT_LIST_DIR}/checksumcalculator.cpp ${CMAKE_CURRENT_LIST_DIR}/filesystembase.cpp ${CMAKE_CURRENT_LIST_DIR}/ownsql.cpp ${CMAKE_CURRENT_LIST_DIR}/preparedsqlquerymanager.cpp diff --git a/test/testchecksumvalidator.cpp b/test/testchecksumvalidator.cpp index c72e0e7bd752..e0684c8d7423 100644 --- a/test/testchecksumvalidator.cpp +++ b/test/testchecksumvalidator.cpp @@ -11,6 +11,8 @@ #include "common/checksums.h" #include "networkjobs.h" +#include "common/checksumcalculator.h" +#include "common/checksumconsts.h" #include "common/utility.h" #include "filesystem.h" #include "propagatorjobs.h" @@ -84,10 +86,10 @@ using namespace OCC::Utility; QFileInfo fi(file); QVERIFY(fi.exists()); - QFile fileDevice(file); - fileDevice.open(QIODevice::ReadOnly); - QByteArray sum = calcMd5(&fileDevice); - fileDevice.close(); + auto sharedFile(QSharedPointer::create(file)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumMD5C); + + const auto sum = checksumCalculator.calculate(); QByteArray sSum = shellSum("md5sum", file); if (sSum.isEmpty()) @@ -104,10 +106,10 @@ using namespace OCC::Utility; QFileInfo fi(file); QVERIFY(fi.exists()); - QFile fileDevice(file); - fileDevice.open(QIODevice::ReadOnly); - QByteArray sum = calcSha1(&fileDevice); - fileDevice.close(); + auto sharedFile(QSharedPointer::create(file)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumSHA1C); + + const auto sum = checksumCalculator.calculate(); QByteArray sSum = shellSum("sha1sum", file); if (sSum.isEmpty()) @@ -127,9 +129,11 @@ using namespace OCC::Utility; connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated); - auto file = new QFile(_testfile, vali); - file->open(QIODevice::ReadOnly); - _expected = calcAdler32(file); + auto sharedFile(QSharedPointer::create(_testfile)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumAdlerC); + + _expected = checksumCalculator.calculate(); + qDebug() << "XX Expected Checksum: " << _expected; vali->start(_testfile); @@ -148,9 +152,10 @@ using namespace OCC::Utility; vali->setChecksumType(_expectedType); connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated); - auto file = new QFile(_testfile, vali); - file->open(QIODevice::ReadOnly); - _expected = calcMd5(file); + auto sharedFile(QSharedPointer::create(_testfile)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumMD5C); + + _expected = checksumCalculator.calculate(); vali->start(_testfile); QEventLoop loop; @@ -167,9 +172,9 @@ using namespace OCC::Utility; vali->setChecksumType(_expectedType); connect(vali, &ComputeChecksum::done, this, &TestChecksumValidator::slotUpValidated); - auto file = new QFile(_testfile, vali); - file->open(QIODevice::ReadOnly); - _expected = calcSha1(file); + auto sharedFile(QSharedPointer::create(_testfile)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumSHA1C); + _expected = checksumCalculator.calculate(); vali->start(_testfile); @@ -188,15 +193,16 @@ using namespace OCC::Utility; connect(vali, &ValidateChecksumHeader::validated, this, &TestChecksumValidator::slotDownValidated); connect(vali, &ValidateChecksumHeader::validationFailed, this, &TestChecksumValidator::slotDownError); - auto file = new QFile(_testfile, vali); - file->open(QIODevice::ReadOnly); - _expected = calcAdler32(file); + auto sharedFile(QSharedPointer::create(_testfile)); + ChecksumCalculator checksumCalculator(sharedFile, OCC::checkSumAdlerC); + _expected = checksumCalculator.calculate(); QByteArray adler = checkSumAdlerC; adler.append(":"); adler.append(_expected); - file->seek(0); + sharedFile->open(QIODevice::ReadOnly); + sharedFile->seek(0); _successDown = false; vali->start(_testfile, adler); @@ -205,14 +211,14 @@ using namespace OCC::Utility; _expectedError = QStringLiteral("The downloaded file does not match the checksum, it will be resumed. \"543345\" != \"%1\"").arg(QString::fromUtf8(_expected)); _expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumMismatch; _errorSeen = false; - file->seek(0); + sharedFile->seek(0); vali->start(_testfile, "Adler32:543345"); QTRY_VERIFY(_errorSeen); _expectedError = QLatin1String("The checksum header contained an unknown checksum type \"Klaas32\""); _expectedFailureReason = ValidateChecksumHeader::FailureReason::ChecksumTypeUnknown; _errorSeen = false; - file->seek(0); + sharedFile->seek(0); vali->start(_testfile, "Klaas32:543345"); QTRY_VERIFY(_errorSeen);