Merge branch 'vincent/beefy-relay-improvements' into ron/skip-mandato…

…ry-commitment

.gitignore

@@ -37,3 +37,4 @@ go.work*
 control/target/
 web/packages/operations/.env.polkadot
 web/packages/operations/.env.rococo
+lodestar

.gitmodules

@@ -13,6 +13,4 @@
 [submodule "contracts/lib/prb-math"]
 	path = contracts/lib/prb-math
 	url = https://github.com/PaulRBerg/prb-math
-[submodule "lodestar"]
-	path = lodestar
-	url = https://github.com/Snowfork/lodestar
+

docs/SUMMARY.md

@@ -38,3 +38,4 @@
 
 * [Contributing to Snowbridge](runbooks/updating-snowbridge-pallets-bridgehub-and-assethub-runtimes.md)
 * [Governance Updates](runbooks/governance-updates.md)
+* [Test Runtime Upgrades](runbooks/test-runtime-upgrades.md)

docs/architecture/components.md

@@ -21,7 +21,7 @@ This [pallet](https://github.com/Snowfork/snowbridge/tree/main/parachain/pallets
 3. At the end of every block, a merkle root of all processed messages is generated and inserted into the parachain header as a [digest item](https://github.com/paritytech/substrate/blob/46136f2a18780d71542ae615565703da754b5348/primitives/runtime/src/generic/digest.rs#L100).
 4. Processed messages are also temporarily held in storage so that they can be queried by offchain message relayers.
 
-The merkle root in (3) is the commitment that needs to verified on the Ethereum side.
+The merkle root in (3) is the commitment that needs to be verified on the Ethereum side.
 
 ### EthereumBeaconClient
 

docs/architecture/fees-and-channels.md

@@ -50,7 +50,7 @@ In both of the scenarios above, there is a common pattern:
 
 Parachain governance therefore has the responsibility to ensure that it has enough funds to cover costs on the destination network.
 
-This can be done by selling collected fees on the source network for currency of the destination network. This currently a manual process, but should only need to be done a few times a year.
+This can be done by selling collected fees on the source network for currency of the destination network. This is currently a manual process, but should only need to be done a few times a year.
 
 Parachains can use the BridgeHub [transfer\_native\_from\_agent](https://github.com/Snowfork/snowbridge/blob/c2142e41b5a2cbd3749a5fd8f22a95abf2b923d9/parachain/pallets/system/src/lib.rs#L503C10-L503C36) API to transfer funds from their agent to some EOA account.
 

docs/architecture/governance.md

@@ -10,7 +10,7 @@ This promotes decentralisation in the following ways:
 
 ## Cross-chain Governance
 
-Our bridge has contracts on the Ethereum, and these contracts need to be able to evolve along with the parachain side. Cross-chain governance will control both configuration and code upgrades on the Ethereum side.
+Our bridge has contracts on the Ethereum side, and these contracts need to be able to evolve along with the parachain side. Cross-chain governance will control both configuration and code upgrades on the Ethereum side.
 
 As a prime example, Polkadot and BEEFY consensus algorithms will change, and so we need to make sure the Ethereum side of the bridge remains compatible. Otherwise locked up collateral will not be redeemable.
 

docs/runbooks/test-runtime-upgrades.md

@@ -0,0 +1,191 @@
+---
+description: How to test upgrades depending on a runtime upgrade not yet executed.
+---
+
+# Test Runtime Upgrades
+
+## Overview
+
+A scenario that frequently occurs is that we need to test a Snowbridge-related runtime upgrade that depends on a system parachain upgrade. Runtime upgrades for system parachains can take up to four weeks to execute. If we wait for the system parachain upgrade to complete first before initiating the Snowbridge upgrades, release cycles could take months. 
+
+Therefore, it is useful to be able to test system parachain upgrades that have not yet executed and then apply Snowbridge upgrades to ensure everything works.
+
+## Steps
+
+In the following scenario, we will simulate execution of the 1.2.0 upgrade: https://github.com/polkadot-fellows/runtimes/releases/tag/v1.2.0.
+
+1. Install [opengov-cli](https://github.com/joepetrowski/opengov-cli)
+2. Build the preimage for the upgrade:
+
+<pre class="language-sh"><code class="lang-sh"><strong>opengov-cli build-upgrade --network polkadot --relay-version 1.2.0 --filename preimage.hex
+</strong></code></pre>
+
+3. Convert the preimage from hex to binary
+
+```sh
+cd upgrade-polkadot-1.2.0 
+xxd -r -p preimage.hex > preimage.bin
+```
+
+4. Determine the size of the of preimage, save as `PREIMAGE_SIZE`
+
+On Linux:
+
+```sh
+$ stat -c%s preimage.bin
+1567371
+$ export PREIMAGE_SIZE=1567371
+```
+
+On Mac:
+
+```sh
+$ stat -f%z preimage.bin
+1567371
+$ export PREIMAGE_SIZE=1567371
+```
+
+5. Compute blake2-256 hash of preimage, save as PREIMAGE\_HASH
+
+```sh
+$ b2sum -l 256 preimage.bin | awk '{print "0x"$1}'
+0x15165c85152568b7f523e374ce1a5172f2aa148721d5dae0441f86c201c1a77b4
+$ export PREIMAGE_HASH=0x15165c85152568b7f523e374ce1a5172f2aa148721d5dae0441f86c201c1a77b4
+```
+
+6. Create a chopsticks configuration file for the Polkadot relay chain, substituting the values generated previously:&#x20;
+
+`polkadot.yml`
+
+```yaml
+endpoint: wss://polkadot-rpc.dwellir.com
+mock-signature-host: true
+block: ${env.POLKADOT_BLOCK_NUMBER}
+db: ./polkadot.sqlite
+
+import-storage:
+  System:
+    Account:
+      - - - 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY
+        - providers: 1
+          data:
+            free: '10000000000000000000'
+  ParasDisputes:
+    $removePrefix: ['disputes'] # those can makes block building super slow
+  Preimage:
+    {
+      PreimageFor:
+        [[[[PREIMAGE_HASH, PREIMAGE_SIZE]], PREIMAGE_WITH_LENGTH_PREFIX]],
+      StatusFor:
+        [[[PREIMAGE_HASH], { Requested: { count: 1, len: PREIMAGE_SIZE } }]],
+    }
+```
+
+7. Use these Chopstics config files for AssetHub and BridgeHub
+
+`polkadot-asset-hub.yml`
+
+```yaml
+endpoint: wss://statemint-rpc.dwellir.com
+mock-signature-host: true
+block: ${env.POLKADOT_ASSET_HUB_BLOCK_NUMBER}
+db: ./assethub.sqlite
+
+import-storage:
+  System:
+    Account:
+      - - - 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY
+        - providers: 1
+          data:
+            free: 1000000000000000
+```
+
+`polkadot-bridge-hub.yml`
+
+```yaml
+endpoint: wss://polkadot-bridge-hub-rpc.dwellir.com
+mock-signature-host: true
+block: ${env.POLKADOT_BRIDGEHUB_BLOCK_NUMBER}
+db: ./bridgehub.sqlite
+
+import-storage:
+  System:
+    Account:
+      - - - 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY
+        - providers: 1
+          data:
+            free: 1000000000000000
+```
+
+8. Run Chopsticks
+
+```sh
+yarn start xcm -r polkadot.yml -p polkadot-asset-hub.yml -p polkadot-bridge-hub.yml
+```
+
+A verification step that can be performed to see if the preimage has been added successfully is to check the `preimage` storage in the chain state. The authorized preimage should be in the list of added preimages.
+
+9. Execute the upgrade on the relay chain using Polkadot-JS:
+
+```rust
+const number = (await api.rpc.chain.getHeader()).number.toNumber()
+
+await api.rpc('dev_setStorage', {
+  Scheduler: {
+    Agenda: [
+      [
+        [number + 1],
+        [
+          {
+            call: {
+              Lookup: {
+                hash: PREIMAGE_HASH,
+                len: PREIMAGE_SIZE,
+              },
+            },
+            origin: {
+              system: 'Root',
+            },
+          },
+        ],
+      ],
+    ],
+  },
+})
+
+await api.rpc('dev_newBlock', { count: 1 })
+```
+
+10. Advance a few blocks on the relay chain
+
+```rust
+await api.rpc('dev_newBlock', { count: 2 })
+```
+
+11. Advance by one block on bridgehub (not sure if necessary, need to experiment)
+
+```rust
+await api.rpc('dev_newBlock', { count: 1 })
+```
+
+12. Now that the upgrade has been authorized on BridgeHub, we can execute the upgrade by calling parachainSystem.enactAuthorizedUpgrade, passing the parachain WASM blob previously generated by opengov-cli:
+
+<figure><img src="../.gitbook/assets/image.png" alt=""><figcaption></figcaption></figure>
+
+12. Advance a few blocks on both bridgehub AND the relay chain
+
+```rust
+await api.rpc('dev_newBlock', { count: 1 })
+```
+
+14. The parachain should now be upgraded.
+
+## Caveats
+
+Some polkadot API endpoints aggressively timeout connections, causing Chopsticks to die: Comment
+
+```sh
+API-WS: disconnected from wss://polkadot-rpc.dwellir.com: 1006:: Abnormal Closure
+```
+
+The usual remedy is to restart chopsticks and pray the API connections don't die again.

flake.nix

@@ -86,6 +86,7 @@
                     export RUSTUP_HOME=$PWD/.rustup
                     export RUST_NIGHTLY_VERSION=nightly-2024-02-08
                     export PATH=$CARGO_HOME/bin:$PATH
+                    export LODESTAR_VERSION=v1.16.0
 
                     eval "$(direnv hook bash)"
 

relayer/cmd/generate_beacon_data.go

@@ -55,6 +55,7 @@ func generateBeaconCheckpointCmd() *cobra.Command {
 	}
 
 	cmd.Flags().String("config", "/tmp/snowbridge/beacon-relay.json", "Path to the beacon relay config")
+	cmd.Flags().Uint64("finalized-slot", 0, "Optional finalized slot to create checkpoint at")
 	cmd.Flags().Bool("export-json", false, "Export Json")
 
 	return cmd
@@ -117,6 +118,7 @@ func generateBeaconCheckpoint(cmd *cobra.Command, _ []string) error {
 		if err != nil {
 			return err
 		}
+		finalizedSlot, _ := cmd.Flags().GetUint64("finalized-slot")
 
 		viper.SetConfigFile(config)
 
@@ -138,7 +140,13 @@ func generateBeaconCheckpoint(cmd *cobra.Command, _ []string) error {
 		client := api.NewBeaconClient(conf.Source.Beacon.Endpoint, conf.Source.Beacon.StateEndpoint)
 		s := syncer.New(client, &store, p)
 
-		checkPointScale, err := s.GetCheckpoint()
+		var checkPointScale scale.BeaconCheckpoint
+		if finalizedSlot == 0 {
+			checkPointScale, err = s.GetCheckpoint()
+		} else {
+			checkPointScale, err = s.GetCheckpointAtSlot(finalizedSlot)
+		}
+
 		if err != nil {
 			return fmt.Errorf("get initial sync: %w", err)
 		}

relayer/cmd/import_beacon_state.go

@@ -106,6 +106,11 @@ func importBeaconState(cmd *cobra.Command, _ []string) error {
 	attestedSlot := attestedState.GetSlot()
 	finalizedSlot := finalizedState.GetSlot()
 
+	err = syncer.ValidatePair(finalizedSlot, attestedSlot, attestedState)
+	if err != nil {
+		return fmt.Errorf("state pair validation failed: %w", err)
+	}
+
 	err = store.WriteEntry(attestedSlot, finalizedSlot, attestedData, finalizedData)
 	if err != nil {
 		return fmt.Errorf("write beacon store entry: %w", err)