Merge branch 'snowbridge' into milagro

.config/lychee.toml

@@ -12,10 +12,10 @@ exclude_all_private = true
 # Treat these codes as success condition:
 accept = [
 	# Ok
-	200,
+	"200",
 
 	# Rate limited - GitHub likes to throw this.
-	429,
+	"429",
 ]
 
 exclude_path = ["./target"]
@@ -47,4 +47,8 @@ exclude = [
 	"https://w3f.github.io/parachain-implementers-guide/node/index.html",
 	"https://w3f.github.io/parachain-implementers-guide/protocol-chain-selection.html",
 	"https://w3f.github.io/parachain-implementers-guide/runtime/session_info.html",
+
+	# Behind a captcha (code 403):
+	"https://iohk.io/en/blog/posts/2023/11/03/partner-chains-are-coming-to-cardano/",
+	"https://www.reddit.com/r/rust/comments/3spfh1/does_collect_allocate_more_than_once_while/",
 ]

.github/review-bot.yml

@@ -20,23 +20,6 @@ rules:
         teams:
           - core-devs
 
-  - name: Audit rules
-    type: basic
-    condition:
-      include:
-        - ^polkadot/runtime/common/.*
-        - ^polkadot/primitives/src\/.+\.rs$
-        - ^substrate/primitives/.*
-        - ^substrate/frame/.*
-      exclude:
-        - ^substrate\/frame\/.+\.md$
-    minApprovals: 1
-    allowedToSkipRule:
-      teams:
-        - core-devs
-    teams:
-      - srlabs
-
   - name: Core developers
     countAuthor: true
     condition:

.github/scripts/check-workspace.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+# Ensures that:
+# - all crates are added to the root workspace
+# - local dependencies are resolved via `path`
+#
+# It does not check that the local paths resolve to the correct crate. This is already done by cargo.
+#
+# Must be called with a folder containing a `Cargo.toml` workspace file.
+
+import os
+import sys
+import toml
+import argparse
+
+def parse_args():
+	parser = argparse.ArgumentParser(description='Check Rust workspace integrity.')
+
+	parser.add_argument('workspace_dir', help='The directory to check', metavar='workspace_dir', type=str, nargs=1)
+	parser.add_argument('--exclude', help='Exclude crate paths from the check', metavar='exclude', type=str, nargs='*', default=[])
+
+	args = parser.parse_args()
+	return (args.workspace_dir[0], args.exclude)
+
+def main(root, exclude):
+	workspace_crates = get_members(root, exclude)
+	all_crates = get_crates(root, exclude)
+	print(f'📦 Found {len(all_crates)} crates in total')
+
+	check_missing(workspace_crates, all_crates)
+	check_links(all_crates)
+
+# Extract all members from a workspace.
+# Return: list of all workspace paths
+def get_members(workspace_dir, exclude):
+	print(f'🔎 Indexing workspace {os.path.abspath(workspace_dir)}')
+
+	root_manifest_path = os.path.join(workspace_dir, "Cargo.toml")
+	if not os.path.exists(root_manifest_path):
+		print(f'❌ No root manifest found at {root_manifest}')
+		sys.exit(1)
+
+	root_manifest = toml.load(root_manifest_path)
+	if not 'workspace' in root_manifest:
+		print(f'❌ No workspace found in root {root_manifest_path}')
+		sys.exit(1)
+
+	if not 'members' in root_manifest['workspace']:
+		return []
+
+	members = []
+	for member in root_manifest['workspace']['members']:
+		if member in exclude:
+			print(f'❌ Excluded member should not appear in the workspace {member}')
+			sys.exit(1)
+		members.append(member)
+
+	return members
+
+# List all members of the workspace.
+# Return: Map name -> (path, manifest)
+def get_crates(workspace_dir, exclude_crates) -> dict:
+	crates = {}
+
+	for root, dirs, files in os.walk(workspace_dir):
+		if "target" in root:
+			continue
+		for file in files:
+			if file != "Cargo.toml":
+				continue
+
+			path = os.path.join(root, file)
+			with open(path, "r") as f:
+				content = f.read()
+				manifest = toml.loads(content)
+
+			if 'workspace' in manifest:
+				if root != workspace_dir:
+					print("⏩ Excluded recursive workspace at %s" % path)
+				continue
+
+			# Cut off the root path and the trailing /Cargo.toml.
+			path = path[len(workspace_dir)+1:-11]
+			name = manifest['package']['name']
+			if path in exclude_crates:
+				print("⏩ Excluded crate %s at %s" % (name, path))
+				continue
+			crates[name] = (path, manifest)
+
+	return crates
+
+# Check that all crates are in the workspace.
+def check_missing(workspace_crates, all_crates):
+	print(f'🔎 Checking for missing crates')
+	if len(workspace_crates) == len(all_crates):
+		print(f'✅ All {len(all_crates)} crates are in the workspace')
+		return
+
+	missing = []
+	# Find out which ones are missing.
+	for name, (path, manifest) in all_crates.items():
+		if not path in workspace_crates:
+			missing.append([name, path, manifest])
+	missing.sort()
+
+	for name, path, _manifest in missing:
+		print("❌ %s in %s" % (name, path))
+	print(f'😱 {len(all_crates) - len(workspace_crates)} crates are missing from the workspace')
+	sys.exit(1)
+
+# Check that all local dependencies are good.
+def check_links(all_crates):
+	print(f'🔎 Checking for broken dependency links')
+	links = []
+	broken = []
+
+	for name, (path, manifest) in all_crates.items():
+		def check_deps(deps):
+			for dep in deps:
+				# Could be renamed:
+				dep_name = dep
+				if 'package' in deps[dep]:
+					dep_name = deps[dep]['package']
+				if dep_name in all_crates:
+					links.append((name, dep_name))
+
+					if not 'path' in deps[dep]:
+						broken.append((name, dep_name, "crate must be linked via `path`"))
+						return
+
+		def check_crate(deps):
+			to_checks = ['dependencies', 'dev-dependencies', 'build-dependencies']
+
+			for to_check in to_checks:
+				if to_check in deps:
+					check_deps(deps[to_check])
+
+		# There could possibly target dependant deps:
+		if 'target' in manifest:
+			# Target dependant deps can only have one level of nesting:
+			for _, target in manifest['target'].items():
+				check_crate(target)
+
+		check_crate(manifest)
+
+
+
+	links.sort()
+	broken.sort()
+
+	if len(broken) > 0:
+		for (l, r, reason) in broken:
+			print(f'❌ {l} -> {r} ({reason})')
+
+		print("💥 %d out of %d links are broken" % (len(broken), len(links)))
+		sys.exit(1)
+	else:
+		print("✅ All %d internal dependency links are correct" % len(links))
+
+if __name__ == "__main__":
+	args = parse_args()
+	main(args[0], args[1])

.github/workflows/build-and-attach-release-runtimes.yml

@@ -35,7 +35,7 @@ jobs:
 
     - name: Build ${{ matrix.runtime.name }} ${{ matrix.build_config.type }}
       id: srtool_build
-      uses: chevdor/[email protected].1
+      uses: chevdor/[email protected].2
       env:
         BUILD_OPTS: ${{ matrix.build_config.opts }}
       with:

.github/workflows/check-features.yml

@@ -0,0 +1,19 @@
+name: Check Features
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  check-features:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch latest code
+        uses: actions/checkout@v4
+      - name: Check
+        uses: hack-ink/cargo-featalign-action@bea88a864d6ca7d0c53c26f1391ce1d431dc7f34 # v0.1.1
+        with:
+          crate: substrate/bin/node/runtime
+          features: std,runtime-benchmarks,try-runtime
+          ignore: sc-executor
+          default-std: true

.github/workflows/check-links.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Restore lychee cache
-        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 (7. Sep 2023)
+        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.2 (7. Sep 2023)
         with:
           path: .lycheecache
           key: cache-lychee-${{ github.sha }}
@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
 
       - name: Lychee link checker
-        uses: lycheeverse/lychee-action@2ac9f030ccdea0033e2510a23a67da2a2da98492 # for v1.8.0 (15. May 2023)
+        uses: lycheeverse/lychee-action@c3089c702fbb949e3f7a8122be0c33c017904f9b # for v1.9.1 (10. Jan 2024)
         with:
           args: >-
             --config .config/lychee.toml

.github/workflows/check-markdown.yml

@@ -31,4 +31,5 @@ jobs:
         env:
           CONFIG: .github/.markdownlint.yaml
         run: |
+          echo "Checking markdown formatting. More info: docs/contributor/markdown_linting.md"
           markdownlint --config "$CONFIG" --ignore target .

.github/workflows/check-publish.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Rust Cache
-        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
+        uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
         with:
           cache-on-failure: true
 

.github/workflows/check-workspace.yml

@@ -0,0 +1,23 @@
+name: Check workspace
+
+on:
+  pull_request:
+    paths:
+      - "*.toml"
+  merge_group:
+
+jobs:
+  check-workspace:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.0 (22. Sep 2023)
+
+      - name: install python deps
+        run: pip3 install toml
+
+      - name: check integrity
+        run: >
+            python3 .github/scripts/check-workspace.py .
+            --exclude
+            "substrate/frame/contracts/fixtures/build" 
+            "substrate/frame/contracts/fixtures/contracts/common"

.github/workflows/claim-crates.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Rust Cache
-        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
+        uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
         with:
           cache-on-failure: true
 

.github/workflows/release-50_publish-docker.yml

@@ -104,7 +104,7 @@ jobs:
           fetch_release_artifacts
 
       - name: Cache the artifacts
-        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
         with:
           key: artifacts-${{ env.BINARY }}-${{ github.sha }}
           path: |
@@ -121,7 +121,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Get artifacts from cache
-        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
         with:
           key: artifacts-${{ env.BINARY }}-${{ github.sha }}
           fail-on-cache-miss: true
@@ -250,7 +250,7 @@ jobs:
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
 
       - name: Cache Docker layers
-        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+        uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
         with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/review-trigger.yml

@@ -10,7 +10,6 @@ on:
       - review_request_removed
       - ready_for_review
   pull_request_review:
-  merge_group:
 
 jobs:
   trigger-review-bot:

.github/workflows/srtool.yml

@@ -73,7 +73,7 @@ jobs:
 
       - name: Srtool build
         id: srtool_build
-        uses: chevdor/[email protected].1
+        uses: chevdor/[email protected].2
         with:
           chain: ${{ matrix.chain }}
           runtime_dir: ${{ matrix.runtime_dir }}

.gitlab-ci.yml

@@ -275,16 +275,6 @@ cancel-pipeline-test-linux-stable3:
   needs:
     - job: "test-linux-stable 3/3"
 
-cancel-pipeline-test-linux-stable-additional-tests:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable-additional-tests"
-
-cancel-pipeline-test-linux-stable-slow:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable-slow"
-
 cancel-pipeline-cargo-check-benches1:
   extends: .cancel-pipeline-template
   needs:

.gitlab/pipeline/build.yml

@@ -382,3 +382,21 @@ build-subkey-linux:
 #   after_script: [""]
 #   tags:
 #     - osx
+
+# bridges
+
+# we need some non-binary artifacts in our bridges+zombienet image
+prepare-bridges-zombienet-artifacts:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  before_script:
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/bridges
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/cumulus/zombienet
+  script:
+    - cp -r bridges/zombienet ./artifacts/bridges-polkadot-sdk/bridges/zombienet
+    - cp -r cumulus/scripts ./artifacts/bridges-polkadot-sdk/cumulus/scripts
+    - cp -r cumulus/zombienet/bridge-hubs ./artifacts/bridges-polkadot-sdk/cumulus/zombienet/bridge-hubs