Rate limit ingestion per shard to 5 MB/s

quickwit-oss · Nov 9, 2023 · c55261b · c55261b
1 parent f99b3de
commit c55261b
Show file tree

Hide file tree

Showing 7 changed files with 315 additions and 6 deletions.
diff --git a/quickwit/quickwit-common/src/tower/rate.rs b/quickwit/quickwit-common/src/tower/rate.rs
@@ -53,6 +53,11 @@ impl ConstantRate {
         let work = bytes.as_u64();
         Self::new(work, period)
     }
+
+    pub fn bytes_per_sec(bytes: ByteSize) -> Self {
+        let work = bytes.as_u64();
+        Self::new(work, Duration::from_secs(1))
+    }
 }
 
 impl Rate for ConstantRate {

diff --git a/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs b/quickwit/quickwit-ingest/src/ingest_v2/fetch.rs
@@ -626,6 +626,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
@@ -794,6 +795,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));
@@ -860,6 +862,7 @@ mod tests {
         let state = Arc::new(RwLock::new(IngesterState {
             mrecordlog,
             shards: HashMap::new(),
+            rate_limiters: HashMap::new(),
             replication_streams: HashMap::new(),
             replication_tasks: HashMap::new(),
         }));