feat: genesis from DB (#401)

Cargo.Bazel.lock

@@ -1644,6 +1644,29 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "genesis-from-db"
+version = "0.1.5"
+dependencies = [
+ "base64 0.21.2",
+ "clap 3.2.25",
+ "const_format",
+ "hex",
+ "many-error",
+ "many-ledger",
+ "many-modules",
+ "many-types",
+ "merk 2.0.0 (git+https://github.com/liftedinit/merk.git?rev=857bf81963d9282ab03438da5013e1f816bd9da1#857bf81963d9282ab03438da5013e1f816bd9da1)",
+ "minicbor",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "strum 0.24.1",
+ "strum_macros 0.24.3",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.1.16"

Cargo.toml

@@ -2,6 +2,7 @@
 resolver = "2"
 
 members = [
+    "src/genesis-from-db",
     "src/http_proxy",
     "src/idstore-export",
     "src/kvstore",

WORKSPACE.bazel

@@ -49,6 +49,7 @@ crates_repository(
     lockfile = "//:cargo-bazel-lock.json",
     manifests = [
         "//:Cargo.toml",
+        "//src/genesis-from-db:Cargo.toml",
         "//src/http_proxy:Cargo.toml",
         "//src/idstore-export:Cargo.toml",
         "//src/kvstore:Cargo.toml",

cargo-bazel-lock.json

@@ -1,5 +1,5 @@
 {
-  "checksum": "331bb6c08edb42c3d240303fd4b1e238c2564c848db236962c1bf75fbb48bf0d",
+  "checksum": "169e0c6d44395781b16594372cfddae5cd6a6cdf0d1a1b76040c6930631e07e7",
   "crates": {
     "addr2line 0.20.0": {
       "name": "addr2line",
@@ -8680,6 +8680,83 @@
       },
       "license": "MIT"
     },
+    "genesis-from-db 0.1.5": {
+      "name": "genesis-from-db",
+      "version": "0.1.5",
+      "repository": null,
+      "targets": [],
+      "library_target_name": null,
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "base64 0.21.2",
+              "target": "base64"
+            },
+            {
+              "id": "clap 3.2.25",
+              "target": "clap"
+            },
+            {
+              "id": "const_format 0.2.31",
+              "target": "const_format"
+            },
+            {
+              "id": "hex 0.4.3",
+              "target": "hex"
+            },
+            {
+              "id": "merk 2.0.0",
+              "target": "merk"
+            },
+            {
+              "id": "minicbor 0.19.1",
+              "target": "minicbor"
+            },
+            {
+              "id": "serde 1.0.171",
+              "target": "serde"
+            },
+            {
+              "id": "serde_json 1.0.102",
+              "target": "serde_json"
+            },
+            {
+              "id": "strum 0.24.1",
+              "target": "strum"
+            },
+            {
+              "id": "tracing 0.1.37",
+              "target": "tracing"
+            },
+            {
+              "id": "tracing-subscriber 0.3.17",
+              "target": "tracing_subscriber"
+            }
+          ],
+          "selects": {}
+        },
+        "edition": "2021",
+        "proc_macro_deps": {
+          "common": [
+            {
+              "id": "serde_derive 1.0.171",
+              "target": "serde_derive"
+            },
+            {
+              "id": "strum_macros 0.24.3",
+              "target": "strum_macros"
+            }
+          ],
+          "selects": {}
+        },
+        "version": "0.1.5"
+      },
+      "license": null
+    },
     "getrandom 0.1.16": {
       "name": "getrandom",
       "version": "0.1.16",
@@ -29791,6 +29868,7 @@
   },
   "binary_crates": [],
   "workspace_members": {
+    "genesis-from-db 0.1.5": "src/genesis-from-db",
     "http_proxy 0.1.5": "src/http_proxy",
     "idstore-export 0.1.5": "src/idstore-export",
     "kvstore 0.1.5": "src/kvstore",

docker/Makefile.ledger

@@ -1,5 +1,5 @@
-MIGRATIONS ?=
-ABCI_MIGRATIONS ?=
+MIGRATIONS ?= "../staging/ledger_migrations.json"
+ABCI_MIGRATIONS ?= "../staging/abci_migrations.json"
 ALLOW_ADDRS ?= false
 STATE ?= "../staging/ledger_state.json5"
 

src/genesis-from-db/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@crate_index//:defs.bzl", "aliases", "all_crate_deps")
+load("@rules_rust//rust:defs.bzl", "rust_binary")
+
+package(default_visibility = [
+    "//:__pkg__",
+    "//tests/e2e/ledger:__pkg__",
+])
+
+rust_binary(
+    name = "genesis-from-db",
+    srcs = glob(include = ["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    deps = all_crate_deps(
+        normal = True,
+    ) + [
+        "//src/many-error",
+        "//src/many-ledger:many-ledger-lib",
+        "//src/many-modules",
+        "//src/many-types",
+    ],
+)

src/genesis-from-db/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "genesis-from-db"
+version = "0.1.5" # managed by release.sh
+edition = "2021"
+description = "Create new genesis file from existing storage."
+license-file = "../../LICENSE"
+homepage = "https://liftedinit.org/"
+repository = "https://github.com/liftedinit/many-rs.git"
+authors = ["The Lifted Initiative <[email protected]>"]
+
+[[bin]]
+name = "genesis-from-db"
+doc = false
+
+[dependencies]
+base64 = "0.21.2"
+clap = { version = "3.2.25", features = ["derive"] }
+const_format = "0.2.31"
+hex = "0.4.3"
+many-error = { path = "../many-error", version = "0.1.5" } # managed by release.sh
+many-ledger = { path = "../many-ledger", version = "0.1.5" } # managed by release.sh
+many-modules = { path = "../many-modules", version = "0.1.5" } # managed by release.sh
+many-types = { path = "../many-types", version = "0.1.5" } # managed by release.sh
+merk = { git = "https://github.com/liftedinit/merk.git", rev = "857bf81963d9282ab03438da5013e1f816bd9da1" }
+minicbor = "0.19.1"
+serde = "1.0.163"
+serde_derive = "1.0.163"
+serde_json = "1.0.96"
+strum = "0.24.1"
+strum_macros = "0.24.3"
+tracing = "0.1.37"
+tracing-subscriber = "0.3.17"

src/genesis-from-db/README.md

@@ -0,0 +1,57 @@
+# genesis-from-db
+
+This is a tool to extract the genesis data from a RocksDB store.
+This tool requires the persistent storage to have the following migrations activated:
+- Data migration
+- Memo migration
+- Token migration
+
+This tool will extract
+- The IDStore seed
+- The IDStore keys
+- The symbols
+- The token identity
+- The account identity
+- The balances
+- The accounts
+and create a genesis file, i.e., `ledger_state.json`.
+
+It also has the ability to extract
+- Events
+- Multisig transactions
+
+and output the result as JSON.
+
+This tool will NOT extract
+- The data attributes (recalculated at block 1)
+- The data info (recalculated at block 1)
+- The token extended info (not used in the genesis)
+- The next subresource id (recalculated)
+- Kvstore-related data
+
+The new genesis file can be used to start a new ledger with the same state as the original ledger. The following
+migrations will need to be activated from block 0
+- Data migration
+- Memo migration
+- Token migration
+
+## Usage
+
+```sh
+# Create a new genesis file from the database
+$ genesis-from-db storage.db genesis > ledger_state.json
+
+# Extract events from the database
+$ genesis-from-db storage.db events > events.json
+
+# Extract multisig transactions from the database
+$ genesis-from-db storage.db multisig > multisig.json
+```
+
+## Known issues
+- Only Memo containing a single `String` are supported
+- ExtendedInfo are mocked 
+- Only Ledger is supported, i.e., no KvStore, Compute, ...
+- Only the Multisig `Feature` attributes are extracted
+- Unsupported Events/Multisig will trigger a panic
+- Only the MFX token is supported