Revert "[Feature] Mark-and-sweep GC. (#3504)" (#3682)

.config/forest.dic

@@ -99,5 +99,3 @@ VRF
 WebAssembly
 WebSocket
 zstd
-ParityDB
-benchmark/GD

CHANGELOG.md

@@ -31,10 +31,6 @@
 
 ### Changed
 
-- [#3072](https://github.com/ChainSafe/forest/issues/3072) Implemented
-  mark-and-sweep GC, removing GC progress reports along with the corresponding
-  RPC endpoint.
-
 ### Removed
 
 ### Fixed

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "forest-filecoin"
-version = "0.16.0"
+version = "0.15.2"
 authors = ["ChainSafe Systems <[email protected]>"]
 repository = "https://github.com/ChainSafe/forest"
 edition = "2021"
@@ -129,6 +129,7 @@ once_cell = "1.15"
 parity-db = { version = "0.4.6", default-features = false }
 parking_lot = { version = "0.12", features = ["deadlock_detection"] }
 pathfinding = "4.3.1"
+pbr = "1.1"
 pin-project-lite = "0.2"
 positioned-io = "0.3.2"
 pretty_assertions = "1.3.0"
@@ -183,6 +184,7 @@ tracing-loki = { version = "0.2", default-features = false, features = ["compat-
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 unsigned-varint = { version = "0.7", features = ["codec"] }
 url = { version = "2.3", features = ["serde"] }
+uuid = { version = "1.3", features = ['v4'] }
 walkdir = "2"
 zerocopy = { version = "0.7.9", features = ["derive"] }
 zstd = "0.13"
@@ -207,7 +209,6 @@ num-bigint = { version = "0.4", features = ['quickcheck'] }
 predicates = "3.0"
 proc-macro2 = { version = "1.0.68", default-features = false, features = ["span-locations"] }
 quickcheck = "1"
-quickcheck_async = "0.1.1"
 quickcheck_macros = "1"
 ra_ap_syntax = "0.0.183"
 regex-automata = "0.4"

scripts/linters/find_unused_deps.rb

@@ -38,7 +38,7 @@ def excluded?(crates, crate)
   crates.each do |crate|
     pattern = get_pattern(crate)
     unless source_code.any? { |line| line.match?(pattern) } || excluded?(crates, crate)
-      puts "Potentially unused: #{crate} in #{crate_dir}"
+      puts "Protentially unused: #{crate} in #{crate_dir}"
       exit_code = 1
     end
   end

scripts/tests/calibnet_other_check.sh

@@ -13,6 +13,11 @@ forest_init
 echo "Verifying the non calibnet snapshot (./test-snapshots/chain4.car) is being served properly."
 $FOREST_CLI_PATH chain read-obj -c bafy2bzacedjrqan2fwfvhfopi64yickki7miiksecglpeiavf7xueytnzevlu
 
+echo "Running database garbage collection"
+forest_check_db_stats
+$FOREST_CLI_PATH db gc
+forest_check_db_stats
+
 echo "Testing js console"
 $FOREST_CLI_PATH attach --exec 'showPeers()'
 

src/blocks/mod.rs

@@ -6,6 +6,7 @@ pub mod election_proof;
 mod errors;
 pub mod gossip_block;
 pub mod header;
+pub mod persistence;
 pub mod ticket;
 pub mod tipset;
 mod vrf_proof;

src/blocks/persistence.rs

@@ -0,0 +1,37 @@
+// Copyright 2019-2023 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use crate::utils::{db::file_backed_obj::FileBackedObject, encoding::from_slice_with_fallback};
+use fvm_ipld_encoding::to_vec;
+
+use crate::blocks::*;
+
+impl FileBackedObject for TipsetKeys {
+    fn serialize(&self) -> anyhow::Result<Vec<u8>> {
+        Ok(to_vec(self)?)
+    }
+
+    fn deserialize(bytes: &[u8]) -> anyhow::Result<Self> {
+        from_slice_with_fallback(bytes)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::Path;
+
+    use crate::utils::db::file_backed_obj::FileBacked;
+
+    use super::*;
+
+    #[test]
+    fn tipset_keys_round_trip() {
+        let path = Path::new("src/blocks/tests/calibnet/HEAD");
+        let obj1: FileBacked<TipsetKeys> =
+            FileBacked::load_from_file_or_create(path.into(), Default::default).unwrap();
+        let serialized = obj1.inner().serialize().unwrap();
+        let deserialized = TipsetKeys::deserialize(&serialized).unwrap();
+
+        assert_eq!(obj1.inner(), &deserialized);
+    }
+}

src/cli/main.rs

@@ -43,7 +43,7 @@ where
                 Subcommand::Config(cmd) => cmd.run(&mut std::io::stdout()),
                 Subcommand::Send(cmd) => cmd.run(api).await,
                 Subcommand::Info(cmd) => cmd.run(api).await,
-                Subcommand::DB(cmd) => cmd.run().await,
+                Subcommand::DB(cmd) => cmd.run(api).await,
                 Subcommand::Snapshot(cmd) => cmd.run(api).await,
                 Subcommand::Attach(cmd) => cmd.run(api),
                 Subcommand::Shutdown(cmd) => cmd.run(api).await,

src/cli/subcommands/db_cmd.rs

@@ -1,19 +1,63 @@
 // Copyright 2019-2023 ChainSafe Systems
 // SPDX-License-Identifier: Apache-2.0, MIT
 
+use std::sync::Arc;
+
+use crate::rpc_api::progress_api::GetProgressType;
+use crate::rpc_client::ApiInfo;
+use crate::utils::io::ProgressBar;
+use chrono::Utc;
 use clap::Subcommand;
 
 #[derive(Debug, Subcommand)]
 pub enum DBCommands {
-    // This is a noop as the manual GC is no longer available.
-    #[command(hide = true)]
+    /// Run DB garbage collection
     GC,
 }
 
 impl DBCommands {
-    pub async fn run(self) -> anyhow::Result<()> {
+    pub async fn run(self, api: ApiInfo) -> anyhow::Result<()> {
         match self {
-            Self::GC => anyhow::bail!("manual garbage collection has been deprecated"),
+            Self::GC => {
+                let start = Utc::now();
+
+                let bar = Arc::new(tokio::sync::Mutex::new({
+                    let bar = ProgressBar::new(0);
+                    bar.message("Running database garbage collection | blocks ");
+                    bar
+                }));
+                tokio::spawn({
+                    let bar = bar.clone();
+                    let api = api.clone();
+                    async move {
+                        let mut interval =
+                            tokio::time::interval(tokio::time::Duration::from_secs(1));
+                        loop {
+                            interval.tick().await;
+                            if let Ok((progress, total)) = api
+                                .get_progress(GetProgressType::DatabaseGarbageCollection)
+                                .await
+                            {
+                                let bar = bar.lock().await;
+                                if bar.is_finish() {
+                                    break;
+                                }
+                                bar.set_total(total);
+                                bar.set(progress);
+                            }
+                        }
+                    }
+                });
+
+                api.db_gc().await?;
+
+                bar.lock().await.finish_println(&format!(
+                    "Database garbage collection completed. took {}s",
+                    (Utc::now() - start).num_seconds()
+                ));
+
+                Ok(())
+            }
         }
     }
 }

src/cli_shared/cli/client.rs

@@ -8,6 +8,7 @@ use std::{
 };
 
 use crate::rpc_client::DEFAULT_PORT;
+use crate::utils::io::ProgressBarVisibility;
 use chrono::Duration;
 use directories::ProjectDirs;
 use serde::{Deserialize, Serialize};
@@ -71,6 +72,8 @@ pub struct Client {
         |g| Duration::milliseconds(i64::arbitrary(g))
     )))]
     pub token_exp: Duration,
+    /// Display progress bars mode. Auto will display if TTY.
+    pub show_progress_bars: ProgressBarVisibility,
     /// Load actors from the bundle file (possibly generating it if it doesn't exist)
     pub load_actors: bool,
 }
@@ -96,6 +99,7 @@ impl Default for Client {
             metrics_address: FromStr::from_str("0.0.0.0:6116").unwrap(),
             rpc_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), DEFAULT_PORT),
             token_exp: Duration::seconds(5184000), // 60 Days = 5184000 Seconds
+            show_progress_bars: Default::default(),
             load_actors: true,
         }
     }

src/cli_shared/cli/mod.rs

@@ -11,7 +11,7 @@ use std::{
 
 use crate::networks::NetworkChain;
 use crate::utils::{
-    io::{read_file_to_string, read_toml},
+    io::{read_file_to_string, read_toml, ProgressBarVisibility},
     misc::LoggingColor,
 };
 use ahash::HashSet;
@@ -118,6 +118,10 @@ pub struct CliOpts {
     /// Enable or disable colored logging in `stdout`
     #[arg(long, default_value = "auto")]
     pub color: LoggingColor,
+    /// Display progress bars mode [always, never, auto]. Auto will display if
+    /// TTY.
+    #[arg(long)]
+    pub show_progress_bars: Option<ProgressBarVisibility>,
     /// Turn on tokio-console support for debugging
     #[arg(long)]
     pub tokio_console: bool,
@@ -219,6 +223,10 @@ impl CliOpts {
             cfg.client.skip_load = skip_load;
         }
 
+        if let Some(show_progress_bars) = self.show_progress_bars {
+            cfg.client.show_progress_bars = show_progress_bars;
+        }
+
         cfg.network.kademlia = self.kademlia.unwrap_or(cfg.network.kademlia);
         cfg.network.mdns = self.mdns.unwrap_or(cfg.network.mdns);
         if let Some(target_peer_count) = self.target_peer_count {

src/daemon/main.rs

@@ -7,6 +7,7 @@ use crate::cli_shared::{
     logger,
 };
 use crate::daemon::ipc_shmem_conf;
+use crate::utils::io::ProgressBar;
 use crate::utils::version::FOREST_VERSION_STRING;
 use anyhow::Context as _;
 use clap::Parser;
@@ -88,6 +89,7 @@ where
     // subcommand.
 
     let (loki_task, _chrome_flush_guard) = logger::setup_logger(&opts);
+    ProgressBar::set_progress_bars_visibility(cfg.client.show_progress_bars);
 
     if let Some(path) = &path {
         match path {