feat(dev): add nix-based dev-env (#4726)

Trying to make it easier to spin up a fullnode on localhost. The team
has been focused on writing tooling, like pindexer, and the manual steps
to munge CometBFT configs and setup a localhost postgres database are
not awesome. In addition to pd/cometbft spin-up, the dev-env includes:

  * metrics
  * postgres
  * pindexer

Not yet including grpcui, because the latest published version doesn't
support the new server reflection API. Will tack that on in a dedicated
follow-up.

The dev-env is strongly opinionated and uses nix [0] to manage
cross-platform, polyglot dependencies. There's other great work in our
ecosystem, e.g. [1], and this early push tries to be consistent with
that.

Updates the relevant first-run docs. Much more work should be done on
the developer-focused content in the guide overall, including an
overhaul of discussing RPC endpoints and web-relevant resources, but
those changes can land in the future. This PR aims to establish a basis
for a sane, supported dev-env, providing a pleasant first-run experience
for newcomers (and existing devs too).

[0] https://nixos.org
[1] https://github.com/informalsystems/cosmos.nix

deployments/compose/process-compose-metrics.yml

@@ -0,0 +1,47 @@
+---
+# A process-compose configuration for running a Prometheus/Grafana metrics setup for Penumbra,
+# scraping metrics endpoints for a localhost-based fullnode.
+version: "0.5"
+log_level: info
+is_strict: true
+
+processes:
+  # Run Grafana for displaying metrics in a web UI: http://localhost:3000
+  grafana:
+    command: |-
+      set -e
+      ./deployments/scripts/prep-grafana-env
+      export GF_PATHS_PROVISIONING=~/.penumbra/network_data/grafana/conf/provisioning
+      grafana server --homepath ~/.penumbra/network_data/grafana --config ./deployments/config/grafana/grafana.ini
+    readiness_probe:
+      http_get:
+        host: 127.0.0.1
+        scheme: http
+        path: "/"
+        port: 3000
+      initial_delay_seconds: 10
+      period_seconds: 5
+      failure_threshold: 3
+    availability:
+      restart: exit_on_failure
+    depends_on:
+      prometheus:
+        condition: process_healthy
+
+  # Run Prometheus for scraping local services: http://localhost:9090
+  prometheus:
+    command: |-
+      set -e
+      mkdir -p ~/.penumbra/network_data/prometheus
+      prometheus --config.file ./deployments/config/prometheus/prometheus.yml --storage.tsdb.path ~/.penumbra/network_data/prometheus
+    readiness_probe:
+      http_get:
+        host: 127.0.0.1
+        scheme: http
+        path: "/"
+        port: 9090
+      initial_delay_seconds: 10
+      period_seconds: 5
+      failure_threshold: 3
+    availability:
+      restart: exit_on_failure

deployments/compose/process-compose-postgres.yml

@@ -0,0 +1,76 @@
+---
+# A process-compose configuration for running a PostgreSQL database,
+# specifically for indexing ABCI events as emitted by CometBFT for Penumbra.
+version: "0.5"
+log_level: info
+is_strict: true
+
+processes:
+  # Run postgresql db process
+  postgresql:
+    command: |-
+      set -e
+      ./deployments/scripts/prep-postgres-env
+      postgres -k /tmp -D ~/.penumbra/network_data/postgresql/
+    readiness_probe:
+      exec:
+        command: psql -h localhost -p 5432 -lqt
+      initial_delay_seconds: 5
+      period_seconds: 5
+      failure_threshold: 3
+    shutdown:
+      # Send SIGINT rather than default SIGTERM to get faster postgres shutdown.
+      # More info: https://www.postgresql.org/docs/current/server-shutdown.html
+      # and `man 7 signal`.
+      signal: 2
+    availability:
+      restart: exit_on_failure
+
+  # Set up databases and initial schemas
+  postgresql-init:
+    command: |-
+      set -e
+      if psql -h localhost -p 5432 -lqt | grep penumbra_cometbft ; then
+        >&2 echo "Postgres DB already initialized, skipping schema/grants..."
+        exit 0
+      fi
+      >&2 echo "Creating database schema for CometBFT..."
+      # Create database for CometBFT writes
+      createdb -h localhost -p 5432 penumbra_cometbft || true
+      createuser -h localhost -p 5432 penumbra
+      psql -h localhost -p 5432 -d penumbra_cometbft -f crates/util/cometindex/vendor/schema.sql
+      psql -h localhost -p 5432 -d penumbra_cometbft -c 'GRANT ALL PRIVILEGES ON DATABASE penumbra_cometbft TO penumbra;'
+      psql -h localhost -p 5432 -d penumbra_cometbft -c 'GRANT pg_read_all_data TO penumbra;'
+      psql -h localhost -p 5432 -d penumbra_cometbft -c 'GRANT pg_write_all_data TO penumbra;'
+
+      # Create database for pindexer
+      >&2 echo "Creating database schema for pindexer..."
+      createuser -h localhost -p 5432 penumbra_ro
+      createdb -h localhost -p 5432 penumbra_pindexer || true
+      psql -h localhost -p 5432 -d penumbra_cometbft -c 'GRANT pg_read_all_data TO penumbra_ro;'
+      psql -h localhost -p 5432 -d penumbra_pindexer -c 'GRANT pg_write_all_data TO penumbra_ro;'
+      psql -h localhost -p 5432 -d penumbra_pindexer -c 'GRANT ALL PRIVILEGES ON DATABASE penumbra_pindexer TO penumbra_ro;'
+    availability:
+      restart: exit_on_failure
+    depends_on:
+      postgresql:
+        condition: process_healthy
+
+  # Add rule for CometBFT to wait for db to be ready.
+  cometbft:
+    depends_on:
+      postgresql-init:
+        condition: process_completed_successfully
+
+  # Add rule for CometBFT to wait for db to be ready.
+  pindexer:
+    environment:
+      - "RUST_LOG=debug"
+    command: |-
+      cargo run --release --bin pindexer -- \
+          --src-database-url "postgresql://localhost:5432/penumbra_cometbft?sslmode=disable" \
+          --dst-database-url "postgresql://localhost:5432/penumbra_pindexer?sslmode=disable" \
+          --genesis-json ~/.penumbra/network_data/node0/cometbft/config/genesis.json
+    depends_on:
+      postgresql-init:
+        condition: process_completed_successfully

deployments/compose/process-compose.yml

@@ -0,0 +1,45 @@
+---
+# A process-compose configuration for running a local Penumbra devnet.
+# This isn't used in scripts anywhere (yet?) but serves as a reference point.
+# Potentially could be integrated with e.g. https://www.jetify.com/devbox later on.
+#
+version: "0.5"
+
+# Env vars set here will be accessible to all processes.
+environment:
+  - "RUST_LOG=info,network_integration=debug,pclientd=debug,pcli=info,pd=info,penumbra=info"
+
+log_level: info
+is_strict: true
+# Interleave logs from all services in single file, so it's greppable.
+log_location: deployments/logs/dev-env-combined.log
+
+processes:
+  # Run pd validator based on generated network.
+  pd:
+    command: cargo run --release --bin pd -- start
+    readiness_probe:
+      http_get:
+        host: 127.0.0.1
+        scheme: http
+        path: "/"
+        port: 8080
+      failure_threshold: 2
+      initial_delay_seconds: 5
+      period_seconds: 5
+
+  # Run CometBFT for consensus driver.
+  cometbft:
+    command: cometbft --log_level=debug --home ~/.penumbra/network_data/node0/cometbft start
+    readiness_probe:
+      http_get:
+        host: 127.0.0.1
+        scheme: http
+        path: "/"
+        port: 26657
+      failure_threshold: 2
+      initial_delay_seconds: 5
+      period_seconds: 5
+    depends_on:
+      pd:
+        condition: process_healthy

deployments/config/grafana/dashboards/pd-performance.json

@@ -478,7 +478,7 @@
     ]
   },
   "time": {
-    "from": "now-6h",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {},

deployments/config/grafana/dashboards/sync.json

@@ -809,7 +809,7 @@
     ]
   },
   "time": {
-    "from": "now-7d",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {},

deployments/config/grafana/grafana.ini

@@ -1,15 +1,22 @@
+# Custom Grafana settings file. Compare with defaults at e.g.
+# https://raw.githubusercontent.com/grafana/grafana/main/conf/defaults.ini
+
 [paths]
-provisioning = /etc/grafana/provisioning
+# The filepaths are relative to the git repository, to prioritize
+# dev envs.
+# provisioning = /etc/grafana/provisioning
+provisioning = ./deployments/config/grafana/provisioning
+
+[dashboards]
+# default_home_dashboard_path = /var/lib/grafana/dashboards/pd performance.json
+default_home_dashboard_path = ./deployments/config/grafana/dashboards/pd-performance.json
 
-# grafana config for local development. don't enable accounts & auth, just let
-# me look at the dashboards!
+# Grafana config for local development. Removes all authentication,
+# to make it easier for a developer to view the dashboards.
+# Not appropriate for a production config!
 [auth]
-# this doesn't seem to work but you can just navigate to other URLs...
 disable_login_form = true
 
-[dashboards]
-default_home_dashboard_path = /var/lib/grafana/dashboards/P2P.json
-
 [auth.anonymous]
 # enable anonymous access
 enabled = true

deployments/config/grafana/provisioning/dashboards/all.yml

@@ -3,4 +3,4 @@
   folder: ''
   type: 'file'
   options:
-    folder: '/var/lib/grafana/dashboards'
+    folder: 'deployments/config/grafana/dashboards'

deployments/scripts/check-nix-shell

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Helper script to detect whether a Nix environment is active:
+# if not, warn the user to consult the dev docs for setting it up.
+# Useful for running in developer scripts as a sanity check.
+set -euo pipefail
+
+
+if [[ -z "${IN_NIX_SHELL:-}" ]] ; then
+    >&2 echo "WARNING: nix environment not active, dependencies may be missing. Did you forget to run 'nix develop'?"
+    >&2 printf '\tSee developer docs at https://guide.penumbra.zone for more info\n'
+    # Sleep to ensure warning is visible; presumably we're calling this script in dev tooling,
+    # immediately preceding a TUI-grabbing app like process-compose.
+    sleep_duration_seconds="5"
+    for i in $(seq 0 "$sleep_duration_seconds") ; do
+        printf '\rResuming script in %s... ' "$(( sleep_duration_seconds - i ))"
+        sleep 1
+    done
+    printf '\n'
+fi

deployments/scripts/prep-grafana-env

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Helper script to prepare a localhost grafana instance,
+# provisioned via nix, for displaying Penumbra metrics.
+set -euo pipefail
+
+
+# The Nix env will add the Grafana public directory to XDG_DATA_DIRS.
+# Let's pluck that out of the dir listing env vars, so we can copy files out of it
+# to a writable location.
+grafana_share_dir="$(echo "$XDG_DATA_DIRS" | perl -npE 's/:/\n/g' | grep grafana | tail -n 1)"
+
+if [[ -z "${grafana_share_dir}" ]] ; then
+    >&2 "ERROR: could not find grafana dir in XDG_DATA_DIRS"
+    exit 1
+fi
+
+# Set up a write-friendly directory for grafana state from the local fullnode.
+# The nix store is read-only by default, so add write capability to copy.
+grafana_config_dir="$HOME/.penumbra/network_data/grafana"
+# While debugging it may be useful to nuke the dir between runs.
+# rm -rf "$grafana_config_dir"
+rsync -a "${grafana_share_dir:?}/grafana/" "${grafana_config_dir}/"
+chmod -R u+w "$grafana_config_dir"
+
+# Copy in Penumbra-specific dashboards
+rsync -a --delete-after "${PWD}/deployments/config/grafana/dashboards" "${grafana_config_dir}/conf/dashboards/"
+rsync -a --delete-after "${PWD}/deployments/config/grafana/provisioning/" "${grafana_config_dir}/conf/provisioning/"

deployments/scripts/prep-postgres-env

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Helper script to prepare a localhost postgres instance,
+# provisioned via nix, for ingesting ABCI events from CometBFT for Penumbra.
+set -euo pipefail
+
+
+# The Nix env will add the postgres share directory to XDG_DATA_DIRS.
+# Let's pluck that out of the dir listing env vars, so we can copy files out of it
+# to a writable location.
+postgres_share_dir="$(echo "$XDG_DATA_DIRS" | perl -npE 's/:/\n/g' | grep postgresql | tail -n 1)"
+
+if [[ -z "${postgres_share_dir}" ]] ; then
+    >&2 "ERROR: could not find postgres dir in XDG_DATA_DIRS"
+    exit 1
+fi
+
+# Set up a write-friendly directory for postgres state.
+# The nix store is read-only by default, so add write capability to copy.
+postgres_config_dir="$HOME/.penumbra/network_data/postgresql"
+# While debugging it may be useful to nuke the dir between runs.
+# rm -rf "$postgres_config_dir"
+mkdir -p "${postgres_config_dir}"
+
+# If PG_VERSION exists, then the initdb cmd has already been run.
+if [[ -e "${postgres_config_dir}/PG_VERSION" ]] ; then
+    >&2 echo "Postgres database already configured in '${postgres_config_dir}', skipping initdb..."
+else
+    pg_ctl initdb --pgdata "${postgres_config_dir}"
+fi

deployments/scripts/run-local-devnet.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Dev tooling to spin up a localhost devnet for Penumbra.
+set -euo pipefail
+
+
+repo_root="$(git rev-parse --show-toplevel)"
+# The process-compose file already respects local state and will reuse it.
+# "${repo_root}/deployments/scripts/warn-about-pd-state"
+
+>&2 echo "Building binaries from latest code..."
+cargo build --release --bin pd
+# Also make sure to invoke via `cargo run` so that the process-compose
+# spin-up doesn't block on more building/linking.
+cargo --quiet run --release --bin pd -- --help > /dev/null
+
+# Generate network from latest code, only if network does not already exist.
+if [[ -d ~/.penumbra/network_data ]] ; then
+    >&2 echo "network data exists locally, reusing it"
+else
+    cargo run --release --bin pd -- network generate \
+        --chain-id penumbra-local-devnet \
+        --unbonding-delay 50 \
+        --epoch-duration 50 \
+        --proposal-voting-blocks 50 \
+        --timeout-commit 1s
+    # opt in to cometbft abci indexing to postgres
+    postgresql_db_url="postgresql://penumbra:penumbra@localhost:5432/penumbra_cometbft?sslmode=disable"
+    sed -i -e "s#^indexer.*#indexer = \"psql\"\\npsql-conn = \"$postgresql_db_url\"#" ~/.penumbra/network_data/node0/cometbft/config/config.toml
+fi
+
+# Run the core fullnode config, plus any additional params passed via `$@`.
+process-compose up --no-server --config "${repo_root}/deployments/compose/process-compose.yml" --keep-tui "$@"

deployments/scripts/warn-about-pd-state

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Utility script to check whether a Penumbra state dir exists,
+# and exit non-zero if so. Useful to reuse throughout a variety
+# of CI scripts for the Penumbra monorepo.
+set -euo pipefail
+
+
+# Fail fast if testnet dir exists, otherwise `cargo run ...` will block
+# for a while, masking the error.
+if [[ -d ~/.penumbra/network_data ]] ; then
+    >&2 echo "ERROR: network data directory exists at ~/.penumbra/network_data"
+    >&2 echo "Not removing this directory automatically; to remove, run: pd network unsafe-reset-all"
+    exit 1
+fi