feat: Add mithril snapshot loader (#341)

* add mithril snapshot loader

* wip

* fix spelling

* bump CI version

* wip

* wip

* wip

* wip

catalyst-gateway/Earthfile

@@ -4,7 +4,7 @@ VERSION --try --global-cache 0.7
 
 # Set up our target toolchains, and copy our files.
 builder:
-   DO github.com/input-output-hk/catalyst-ci/earthly/rust:v3.0.0+SETUP
+   DO github.com/input-output-hk/catalyst-ci/earthly/rust:v2.10.0+SETUP
 
     COPY --dir .cargo .config Cargo.* clippy.toml deny.toml rustfmt.toml bin crates tests .
     COPY --dir ./event-db/queries ./event-db/queries

catalyst-gateway/event-db/Earthfile

@@ -9,7 +9,7 @@ VERSION 0.7
 # Internal: builder is our Event db builder target. Prepares all necessary artifacts.
 #   CI target : dependency
 builder:
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.0.0+BUILDER \
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.10.0+BUILDER \
         --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff
 
     COPY ./../../+repo-config/repo/.sqlfluff .
@@ -24,23 +24,23 @@ builder:
 check:
     FROM +builder
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.0.0+CHECK
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.10.0+CHECK
 
 
 # format all SQL files in the current project.  Local developers tool.
 #   CI target : false
 format:
     LOCALLY
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.0.0+FORMAT --src=$(echo ${PWD}/../../)
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.10.0+FORMAT --src=$(echo ${PWD}/../../)
 
 # build - an event db docker image.
 #   CI target : true
 build:
     FROM +builder
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.0.0+BUILD --image_name=event-db
-    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.0.0+DOCS --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.10.0+BUILD --image_name=event-db
+    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.10.0+DOCS --image_name=event-db
 
 # test the event db database schema
 #   CI target : true

catalyst-gateway/mithril_snapshot_loader/Earthfile

@@ -0,0 +1,16 @@
+VERSION 0.7
+
+# cspell: words tqdm
+
+load-mithril-snapshot:
+    ARG network="preprod"
+
+    FROM github.com/input-output-hk/catalyst-ci/earthly/python:v2.10.0+python-base
+
+    RUN pip3 install requests
+    RUN pip3 install tqdm
+
+    COPY loader.py /scripts
+
+    RUN /scripts/loader.py --network=$network --out="snapshot"
+    SAVE ARTIFACT ./snapshot snapshot

catalyst-gateway/mithril_snapshot_loader/loader.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+# cspell: words tqdm
+
+import argparse
+import rich
+from rich import print
+import os
+import enum
+import requests
+from tqdm import tqdm
+
+# This script loads latest mithril snapshot archive
+
+
+class NetworkType(enum.Enum):
+    Mainnet = "mainnet"
+    Testnet = "testnet"
+    Preprod = "preprod"
+    Preview = "preview"
+
+    def get_aggregator_url(self):
+        match self:
+            case NetworkType.Mainnet:
+                return "https://aggregator.release-mainnet.api.mithril.network/aggregator/artifact/snapshots"
+            case NetworkType.Testnet:
+                return "https://aggregator.testing-preview.api.mithril.network/aggregator/artifact/snapshots"
+            case NetworkType.Preprod:
+                return "https://aggregator.release-preprod.api.mithril.network/aggregator/artifact/snapshots"
+            case NetworkType.Preview:
+                return "https://aggregator.pre-release-preview.api.mithril.network/aggregator/artifact/snapshots"
+
+
+def load_snapshot(network_type: NetworkType, out: str):
+    resp = requests.get(network_type.get_aggregator_url())
+    # getting the latest snapshot from the list, it's always on the first position
+    snapshot_info = resp.json()[0]
+
+    location = snapshot_info["locations"][0]
+    # load archive
+    resp = requests.get(location, stream=True)
+    content_size = int(resp.headers.get("Content-length"))
+    with open(out, "wb") as file:
+        with tqdm(total=content_size) as t:
+            chunk_size = 1024
+            for chunk in resp.iter_content(chunk_size=chunk_size):
+                file.write(chunk)
+                t.update(chunk_size)
+
+
+def main():
+    # Force color output in CI
+    rich.reconfigure(color_system="256")
+
+    parser = argparse.ArgumentParser(description="Mithril snapshot loading.")
+    parser.add_argument(
+        "--network",
+        type=NetworkType,
+        help="Cardano network type, available options: ['mainnet', 'testnet', 'preprod', 'preview']",
+    )
+    parser.add_argument("--out", help="Out file name of the snapshot archive")
+    args = parser.parse_args()
+
+    load_snapshot(args.network, args.out)
+
+
+if __name__ == "__main__":
+    main()

catalyst-gateway/tests/schema-mismatch/Earthfile

@@ -1,10 +1,10 @@
 VERSION --global-cache 0.7
 
 builder:
-    FROM github.com/input-output-hk/catalyst-ci/earthly/python:v2.9.2+python-base
+    FROM github.com/input-output-hk/catalyst-ci/earthly/python:v2.10.0+python-base
 
-    COPY --dir ./schema_mismatch README.md .
-    DO github.com/input-output-hk/catalyst-ci/earthly/python:v2.9.2+BUILDER
+    COPY --dir ./schema_mismatch .
+    DO github.com/input-output-hk/catalyst-ci/earthly/python:v2.10.0+BUILDER
 
 package-tester:
     FROM +builder

docs/Earthfile

@@ -6,7 +6,7 @@ VERSION 0.7
 # Copy all the source we need to build the docs
 src:
     # Common src setup
-    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v3.0.0+SRC
+    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v2.10.0+SRC
 
     # Now copy into that any artifacts we pull from the builds.
     COPY --dir ../+repo-docs/repo /docs/includes
@@ -21,12 +21,12 @@ src:
 docs:
     FROM +src
 
-    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v3.0.0+BUILD
+    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v2.10.0+BUILD
 
 # Make a locally runable container that can serve the docs.
 local:
     # Build a self contained service to show built docs locally.
-    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v3.0.0+PACKAGE
+    DO github.com/input-output-hk/catalyst-ci/earthly/docs:v2.10.0+PACKAGE
 
     # Copy the static pages into the container
     COPY +docs/ /usr/share/nginx/html