Upload: Persistent initial & failsafe chunk size

Signed-off-by: Juergen Kellerer <[email protected]>

src/gui/folder.cpp

@@ -27,6 +27,7 @@
 #include "syncresult.h"
 #include "clientproxy.h"
 #include "syncengine.h"
+#include "owncloudpropagator.h"
 #include "syncrunfilelog.h"
 #include "socketapi/socketapi.h"
 #include "theme.h"
@@ -916,10 +917,10 @@ SyncOptions Folder::initializeSyncOptions() const
     opt._initialChunkSize = cfgFile.chunkSize();
     opt._minChunkSize = cfgFile.minChunkSize();
     opt._maxChunkSize = cfgFile.maxChunkSize();
-    opt._failsafeMaxChunkSize = cfgFile.failsafeMaxChunkSize();
     opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration();
 
     opt.fillFromEnvironmentVariables();
+    opt.fillFromJournal(_journal);
     opt.verifyChunkSizes();
 
     return opt;
@@ -1004,6 +1005,11 @@ void Folder::slotSyncFinished(bool success)
     }
 
     if (_syncResult.status() == SyncResult::Success && success) {
+        // Remember initial chunk size to speed-up auto-sensing in next sync
+        if (const auto propagator = _engine->getPropagator()) {
+            SyncOptions::persistInitialChunkSize(_journal, propagator->_chunkSize);
+        }
+
         // Clear the white list as all the folders that should be on that list are sync-ed
         journalDb()->setSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList, QStringList());
     }

src/libsync/configfile.cpp

@@ -81,7 +81,6 @@ static const char timeoutC[] = "timeout";
 static const char chunkSizeC[] = "chunkSize";
 static const char minChunkSizeC[] = "minChunkSize";
 static const char maxChunkSizeC[] = "maxChunkSize";
-static const char failsafeMaxChunkSizeC[] = "failsafeMaxChunkSize";
 static const char targetChunkUploadDurationC[] = "targetChunkUploadDuration";
 static const char automaticLogDirC[] = "logToTemporaryLogDir";
 static const char logDirC[] = "logDir";
@@ -247,12 +246,6 @@ qint64 ConfigFile::maxChunkSize() const
     return settings.value(QLatin1String(maxChunkSizeC), DefaultMaxChunkSize).toLongLong(); // default to 1000 MB
 }
 
-qint64 ConfigFile::failsafeMaxChunkSize() const
-{
-    QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(failsafeMaxChunkSizeC), DefaultFailsafeMaxChunkSize).toLongLong(); // default to 100 MB
-}
-
 qint64 ConfigFile::minChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);

src/libsync/configfile.h

@@ -161,7 +161,6 @@ class OWNCLOUDSYNC_EXPORT ConfigFile
     int timeout() const;
     qint64 chunkSize() const;
     qint64 maxChunkSize() const;
-    qint64 failsafeMaxChunkSize() const;
     qint64 minChunkSize() const;
     std::chrono::milliseconds targetChunkUploadDuration() const;
 

src/libsync/propagateupload.cpp

@@ -699,8 +699,13 @@ void PropagateUploadFileCommon::commonErrorHandling(AbstractNetworkJob *job)
                 ? opts._failsafeMaxChunkSize
                 : qMax(opts._minChunkSize, requestSize / 2);
 
+            // Set new upload limit
             propagator()->account()->setMaxRequestSizeIfLower(maxRequestSize);
 
+            // Remember next _failsafeMaxChunkSize 
+            const auto nextFailsafeSize = qMax(LowestPossibleFailsafeMaxChunkSize, propagator()->account()->getMaxRequestSize());
+            SyncOptions::persistFailsafeMaxChunkSize(*propagator()->_journal, nextFailsafeSize);
+
             // Apply to this propagator (limit is applied in setSyncOptions())
             propagator()->setSyncOptions(opts);
             const auto adjustedOpts = propagator()->syncOptions();

src/libsync/syncoptions.cpp

@@ -40,10 +40,6 @@ void SyncOptions::fillFromEnvironmentVariables()
     if (!maxChunkSizeEnv.isEmpty())
         _maxChunkSize = maxChunkSizeEnv.toUInt();
 
-    QByteArray failsafeMaxChunkSizeEnv = qgetenv("OWNCLOUD_FAILSAFE_MAX_CHUNK_SIZE");
-    if (!failsafeMaxChunkSizeEnv.isEmpty())
-        _failsafeMaxChunkSize = failsafeMaxChunkSizeEnv.toUInt();
-
     QByteArray targetChunkUploadDurationEnv = qgetenv("OWNCLOUD_TARGET_CHUNK_UPLOAD_DURATION");
     if (!targetChunkUploadDurationEnv.isEmpty())
         _targetChunkUploadDuration = std::chrono::milliseconds(targetChunkUploadDurationEnv.toUInt());
@@ -53,6 +49,28 @@ void SyncOptions::fillFromEnvironmentVariables()
         _parallelNetworkJobs = maxParallel;
 }
 
+void SyncOptions::fillFromJournal(SyncJournalDb &journal) 
+{
+    _initialChunkSize = journal.keyValueStoreGetInt(KeyInitialChunkSize, _initialChunkSize);
+    _failsafeMaxChunkSize = journal.keyValueStoreGetInt(KeyFailsafeMaxChunkSize, _failsafeMaxChunkSize);
+}
+
+void SyncOptions::resetJournal(SyncJournalDb &journal) 
+{
+    journal.keyValueStoreDelete(KeyInitialChunkSize);
+    journal.keyValueStoreDelete(KeyFailsafeMaxChunkSize);
+}
+
+void SyncOptions::persistInitialChunkSize(SyncJournalDb &journal, qint64 value)
+{
+    journal.keyValueStoreSet(KeyInitialChunkSize, value);
+}
+
+void SyncOptions::persistFailsafeMaxChunkSize(SyncJournalDb &journal, qint64 value)
+{
+    journal.keyValueStoreSet(KeyFailsafeMaxChunkSize, value);
+}
+
 void SyncOptions::verifyChunkSizes(qint64 maxUpperLimit)
 {
     if (maxUpperLimit > 0) {
@@ -64,7 +82,7 @@ void SyncOptions::verifyChunkSizes(qint64 maxUpperLimit)
     _initialChunkSize = qMax(LowestPossibleChunkSize, _initialChunkSize);
     _minChunkSize = qMax(LowestPossibleChunkSize, _minChunkSize);
     _maxChunkSize = qMax(LowestPossibleChunkSize, _maxChunkSize);
-    _failsafeMaxChunkSize = qMax(LowestPossibleChunkSize, _failsafeMaxChunkSize);
+    _failsafeMaxChunkSize = qMax(LowestPossibleFailsafeMaxChunkSize, _failsafeMaxChunkSize);
 
     _minChunkSize = qMin(_minChunkSize, _initialChunkSize);
     _maxChunkSize = qMax(_maxChunkSize, _initialChunkSize);

src/libsync/syncoptions.h

@@ -16,6 +16,7 @@
 
 #include "owncloudlib.h"
 #include "common/vfs.h"
+#include "common/syncjournaldb.h"
 
 #include <QRegularExpression>
 #include <QSharedPointer>
@@ -38,9 +39,19 @@ const qint64 DefaultMaxChunkSize = 1000 * 1000 * 1000;
 /** Default value for SyncOptions::_failsafeMaxChunkSize ( 100MB) */
 const qint64 DefaultFailsafeMaxChunkSize = 100 * 1000 * 1000;
 
+/** _failsafeMaxChunkSize cannot be set lower than this  (  25MB) */
+const qint64 LowestPossibleFailsafeMaxChunkSize = 25 * 1000 * 1000;
+
 /** No chunk size can be set to a value lower than this  ( 512KB) */
 const qint64 LowestPossibleChunkSize = 512 * 1000;
 
+
+/** Journal DB storage key for _initialChunkSize (for persisting the auto-sensing results) */
+const auto KeyInitialChunkSize = QStringLiteral("initial_chunk_size");
+
+/** Journal DB storage key for _failsafeMaxChunkSize (for persisting the auto-sensing results) */
+const auto KeyFailsafeMaxChunkSize = QStringLiteral("failsafe_max_chunk_size");
+
 /**
  * Value class containing the options given to the sync engine
  */
@@ -99,6 +110,21 @@ class OWNCLOUDSYNC_EXPORT SyncOptions
      */
     void fillFromEnvironmentVariables();
 
+    /** Reads persistable settings from the journal DB where available.
+     * 
+     * Currently reads _initialChunkSize and _failsafeMaxChunkSize
+     */
+    void fillFromJournal(SyncJournalDb &journal);
+
+    /** Removes all persisted values from journal DB. */
+    static void resetJournal(SyncJournalDb &journal);
+
+    /** Stores InitialChunkSize in JournalDB. */
+    static void persistInitialChunkSize(SyncJournalDb &journal, qint64 value);
+
+    /** Stores FailsafeMaxChunkSize in JournalDB. */
+    static void persistFailsafeMaxChunkSize(SyncJournalDb &journal, qint64 value);
+
     /** Ensure min <= initial <= max
      *
      * Previously min/max chunk size values didn't exist, so users might

test/testnextcloudpropagator.cpp

@@ -87,14 +87,16 @@ private slots:
 
         opts._minChunkSize = 
             opts._initialChunkSize = 
-            opts._maxChunkSize = 0;
+            opts._maxChunkSize = 
+            opts._failsafeMaxChunkSize = 0;
 
         propagator.setSyncOptions(opts);
 
         opts = propagator.syncOptions();
         QCOMPARE( opts._minChunkSize, LowestPossibleChunkSize );
         QCOMPARE( opts._initialChunkSize, LowestPossibleChunkSize );
         QCOMPARE( opts._maxChunkSize, LowestPossibleChunkSize );
+        QCOMPARE( opts._failsafeMaxChunkSize, LowestPossibleFailsafeMaxChunkSize );
     }
 
     void testLimitsMaxChunkSizeByAccount()

test/testsyncengine.cpp

@@ -600,12 +600,12 @@ private slots:
         FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
 
         const int minRequestSize = LowestPossibleChunkSize;
-        int maxAllowedRequestSize = 4 * minRequestSize;
+        int maxAllowedRequestSize = LowestPossibleFailsafeMaxChunkSize;
 
         SyncOptions options;
         options._minChunkSize = minRequestSize;
         options._failsafeMaxChunkSize = maxAllowedRequestSize;
-        options._initialChunkSize =  options._maxChunkSize = 4 * maxAllowedRequestSize;
+        options._initialChunkSize = options._maxChunkSize = 4 * maxAllowedRequestSize;
         options._parallelNetworkJobs = 0;
 
         auto &&engine = fakeFolder.syncEngine();
@@ -634,45 +634,56 @@ private slots:
             return reply;
         });
 
+        auto reset = [&]() {
+            nPUT = n413 = 0;
+            SyncOptions::resetJournal(fakeFolder.syncJournal());
+            QCOMPARE(fakeFolder.syncJournal().keyValueStoreGetInt(KeyFailsafeMaxChunkSize, 0), 0);
+        };
+
         // Test initial upload at max size succeeds
+        reset();
         fakeFolder.localModifier().insert("A/big", maxAllowedRequestSize);
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 1);
         QCOMPARE(n413, 0);
         QCOMPARE(fakeFolder.account()->getMaxRequestSize(), -1);
+        QCOMPARE(fakeFolder.syncJournal().keyValueStoreGetInt(KeyFailsafeMaxChunkSize, 0), 0);
 
         // Test _failsafeMaxChunkSize applies on first failure
-        nPUT = n413 = 0;
+        reset();
         fakeFolder.localModifier().insert("A/big1", maxAllowedRequestSize + 1);
         QVERIFY(!fakeFolder.syncOnce() && engine.isAnotherSyncNeeded() == AnotherSyncNeeded::ImmediateFollowUp);
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 2);
         QCOMPARE(n413, 1);
         QCOMPARE(fakeFolder.account()->getMaxRequestSize(), options._failsafeMaxChunkSize);
+        QCOMPARE(fakeFolder.syncJournal().keyValueStoreGetInt(KeyFailsafeMaxChunkSize, 0), options._failsafeMaxChunkSize);
 
         // Test "Connection: Close" applies _failsafeMaxChunkSize as limit
-        nPUT = n413 = 0;
+        reset();
         fakeFolder.account()->setMaxRequestSize(-1);
         fakeFolder.localModifier().insert("A/bigClose", options._failsafeMaxChunkSize + 1);
         QVERIFY(!fakeFolder.syncOnce() && engine.isAnotherSyncNeeded() == AnotherSyncNeeded::ImmediateFollowUp);
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 3);
         QCOMPARE(n413, 0);
         QCOMPARE(fakeFolder.account()->getMaxRequestSize(), options._failsafeMaxChunkSize);
+        QCOMPARE(fakeFolder.syncJournal().keyValueStoreGetInt(KeyFailsafeMaxChunkSize, 0), options._failsafeMaxChunkSize);
 
         // Test _maxChunkSize is reduced until upload succeeds
-        nPUT = n413 = 0;
+        reset();
         maxAllowedRequestSize = options._minChunkSize;
-        fakeFolder.localModifier().insert("A/big2", options._failsafeMaxChunkSize);
+        fakeFolder.localModifier().insert("A/big2", options._minChunkSize * 4);
         QVERIFY(!fakeFolder.syncOnce() && engine.isAnotherSyncNeeded() == AnotherSyncNeeded::ImmediateFollowUp);
         QVERIFY(!fakeFolder.syncOnce() && engine.isAnotherSyncNeeded() == AnotherSyncNeeded::ImmediateFollowUp);
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 4);
         QCOMPARE(n413, 2);
         QCOMPARE(fakeFolder.account()->getMaxRequestSize(), options._minChunkSize);
+        QCOMPARE(fakeFolder.syncJournal().keyValueStoreGetInt(KeyFailsafeMaxChunkSize, 0), options._failsafeMaxChunkSize);
 
         // Test _maxChunkSize is not reduced below _minChunkSize and retry is not requested
-        nPUT = n413 = 0;
+        reset();
         maxAllowedRequestSize = options._minChunkSize / 2;
         fakeFolder.localModifier().insert("A/big3", options._minChunkSize);
         QVERIFY(!fakeFolder.syncOnce() && engine.isAnotherSyncNeeded() == AnotherSyncNeeded::NoFollowUpSync);