wip: debugging k8s network transfer

.github/workflows/ci.yml

@@ -56,6 +56,7 @@ jobs:
       noir: ${{ steps.filter.outputs.noir }}
       noir-projects: ${{ steps.filter.outputs.noir-projects }}
       txe: ${{ steps.filter.outputs.txe }}
+      yarn-project: ${{ steps.filter.outputs.yarn-project }}
       l1-contracts: ${{ steps.filter.outputs.l1-contracts }}
       non-docs: ${{ steps.filter.outputs.non-docs }}
       non-misc-ci: ${{ steps.filter.outputs.non-misc-ci }}
@@ -81,6 +82,8 @@ jobs:
               - 'l1-contracts/**'
             noir-projects:
               - 'noir-projects/**'
+            yarn-project:
+              - 'yarn-project/**'
             txe:
               - 'yarn-project/txe/**'
             non-barretenberg-cpp:
@@ -746,6 +749,7 @@ jobs:
       - changes
       - build
       - e2e
+      - deploy-test
       # - bench-e2e # non-blocking
       # - acir-bench # non-blocking
       # - bench-summary # non-blocking
@@ -791,6 +795,32 @@ jobs:
         if: github.event.pull_request.draft
         run: echo "Can't merge drafts." && exit 1
 
+  deploy-test:
+    needs: [build, changes]
+    runs-on: ubuntu-20.04
+    if: needs.changes.outputs.yarn-project == 'true' || needs.changes.outputs.l1-contracts == 'true' || contains(github.event.pull_request.labels.*.name, 'e2e-all')
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 45
+        with:
+          runner_type: '16core-tester-x86'
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 40
+          run: |
+            set -eux
+            ./spartan/scripts/setup_local_k8s.sh
+            cd yarn-project/end-to-end
+            NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=3-validators.yaml ./scripts/network_test.sh ./src/spartan/smoke.test.ts
+
   rerun-check:
     runs-on: ubuntu-20.04
     permissions:
@@ -801,6 +831,7 @@ jobs:
       - changes
       - build
       - e2e
+      - deploy-test
       # - bench-e2e # non-blocking
       # - acir-bench # non-blocking
       # - bench-summary # non-blocking

yarn-project/aztec.js/src/rpc_clients/node/index.ts

@@ -48,9 +48,9 @@ async function axiosFetch(host: string, rpcMethod: string, body: any, useApiEndp
   if (isOK) {
     return resp.data;
   } else if (resp.status >= 400 && resp.status < 500) {
-    throw new NoRetryError('(JSON-RPC PROPAGATED) ' + resp.data.error.message);
+    throw new NoRetryError(`(JSON-RPC PROPAGATED) ' (${resp.status}) ${resp.data.error.message}`);
   } else {
-    throw new Error('(JSON-RPC PROPAGATED) ' + resp.data.error.message);
+    throw new Error(`(JSON-RPC PROPAGATED) ' (${resp.status}) ${resp.data.error.message}`);
   }
 }
 

yarn-project/circuit-types/src/aztec_node/rpc/aztec_node_client.ts

@@ -19,6 +19,7 @@ import { PublicDataWitness } from '../../public_data_witness.js';
 import { SiblingPath } from '../../sibling_path/index.js';
 import { PublicSimulationOutput, Tx, TxHash, TxReceipt } from '../../tx/index.js';
 import { TxEffect } from '../../tx_effect.js';
+import { EpochProofQuote } from '../../prover_coordination/epoch_proof_quote.js';
 
 /**
  * Creates a JSON-RPC client to remotely talk to an Aztec Node.
@@ -33,6 +34,7 @@ export function createAztecNodeClient(url: string, fetch = defaultFetch): AztecN
       AztecAddress,
       Buffer32,
       EthAddress,
+      EpochProofQuote,
       EventSelector,
       ExtendedUnencryptedL2Log,
       Fr,

yarn-project/circuit-types/src/prover_coordination/epoch_proof_quote.ts

@@ -4,10 +4,13 @@ import { Signature } from '@aztec/foundation/eth-signature';
 import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
 import { type FieldsOf } from '@aztec/foundation/types';
 
+
+
 import { Gossipable } from '../p2p/gossipable.js';
 import { TopicType, createTopicString } from '../p2p/topic_type.js';
 import { EpochProofQuotePayload } from './epoch_proof_quote_payload.js';
 
+
 export class EpochProofQuote extends Gossipable {
   static override p2pTopic: string = createTopicString(TopicType.epoch_proof_quote);
 
@@ -27,6 +30,21 @@ export class EpochProofQuote extends Gossipable {
     return serializeToBuffer(...EpochProofQuote.getFields(this));
   }
 
+  /**
+   * Returns a string representation of the EpochProofQuote object.
+   */
+  override toString(): string {
+    return this.toBuffer().toString('hex');
+  }
+  /**
+   * Deserializes a EpochProofQuote object from a string.
+   * @param str - String to deserialize.
+   * @returns An instance of EpochProofQuote.
+   */
+  static fromString(str: string) {
+    return EpochProofQuote.fromBuffer(Buffer.from(str, 'hex'));
+  }
+
   static fromBuffer(buf: Buffer | BufferReader): EpochProofQuote {
     const reader = BufferReader.asReader(buf);
     return new EpochProofQuote(reader.readObject(EpochProofQuotePayload), reader.readObject(Signature));
@@ -57,4 +75,4 @@ export class EpochProofQuote extends Gossipable {
       signature: this.signature.toViemSignature(),
     };
   }
-}
+}

yarn-project/end-to-end/scripts/helm_deploy.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Usage: ./helm_deploy.sh
+# Required environment variables:
+#   NAMESPACE
+# Optional environment variables:
+#   VALUES_FILE (default: "default.yaml")
+#   CHAOS_VALUES (default: "")
+#   FRESH_INSTALL (default: "false")
+#   AZTEC_DOCKER_TAG (default: current git commit)
+
+set -eux
+
+# Default values for environment variables
+VALUES_FILE="${VALUES_FILE:-default.yaml}"
+CHAOS_VALUES="${CHAOS_VALUES:-}"
+FRESH_INSTALL="${FRESH_INSTALL:-false}"
+AZTEC_DOCKER_TAG=${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}
+
+# Check required environment variable
+if [ -z "${NAMESPACE:-}" ]; then
+  echo "Environment variable NAMESPACE is required."
+  exit 1
+fi
+
+if ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "aztecprotocol/aztec:$AZTEC_DOCKER_TAG"; then
+  echo "Aztec docker image not found. It needs to be built with 'earthly ./yarn-project/+export-aztec' or otherwise an image named aztecprotocol/aztec:$AZTEC_DOCKER_TAG needs to exist, or AZTEC_DOCKER_TAG passed with a tag that exists."
+  exit 1
+fi
+
+# Load the Docker images into kind
+kind load docker-image aztecprotocol/aztec:$AZTEC_DOCKER_TAG
+
+# If FRESH_INSTALL is true, delete the namespace
+if [ "$FRESH_INSTALL" = "true" ]; then
+  kubectl delete namespace "$NAMESPACE" --ignore-not-found=true --wait=true --now --timeout=10m
+fi
+
+function show_status_until_pxe_ready() {
+  set +x # don't spam with our commands
+  sleep 15 # let helm upgrade start
+  for i in {1..200} ; do
+    if kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=20s >/dev/null 2>/dev/null ; then
+      break # we are up, stop showing status
+    fi
+    # show startup status
+    kubectl get pods -n "$NAMESPACE"
+  done
+}
+
+show_status_until_pxe_ready &
+
+# Install the Helm chart
+helm upgrade --install spartan "$(git rev-parse --show-toplevel)/spartan/aztec-network/" \
+      --namespace "$NAMESPACE" \
+      --create-namespace \
+      --values "$(git rev-parse --show-toplevel)/spartan/aztec-network/values/$VALUES_FILE" \
+      --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
+      --wait \
+      --wait-for-jobs=true \
+      --timeout=30m
+
+kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m

yarn-project/end-to-end/scripts/network_test.sh

@@ -1,87 +1,46 @@
 #!/bin/bash
 
 # Usage: ./network_test.sh <test>
-# Required environment variables:
-#   NAMESPACE
 # Optional environment variables:
+#   END_TO_END_DOCKER_TAG (default: AZTEC_DOCKER_TAG, if not set then current git commit)
+# Required environment variables passed to helm_deploy.sh:
+#   NAMESPACE
+# Optional environment variables passed to helm_deploy.sh:
 #   VALUES_FILE (default: "default.yaml")
 #   CHAOS_VALUES (default: "")
 #   FRESH_INSTALL (default: "false")
-#   AZTEC_DOCKER_TAG (default: current git commit)
 
 set -eux
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
 # Main positional parameter
 TEST="$1"
 
-# Default values for environment variables
-VALUES_FILE="${VALUES_FILE:-default.yaml}"
-CHAOS_VALUES="${CHAOS_VALUES:-}"
-FRESH_INSTALL="${FRESH_INSTALL:-false}"
 AZTEC_DOCKER_TAG=${AZTEC_DOCKER_TAG:-$(git rev-parse HEAD)}
+END_TO_END_DOCKER_TAG=${END_TO_END_DOCKER_TAG:-$AZTEC_DOCKER_TAG}
 
-# Check required environment variable
-if [ -z "${NAMESPACE:-}" ]; then
-  echo "Environment variable NAMESPACE is required."
-  exit 1
-fi
-
-if ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "aztecprotocol/aztec:$AZTEC_DOCKER_TAG" || \
-   ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG"; then
-  echo "Docker images not found. They need to be built with 'earthly ./yarn-project/+export-e2e-test-images' or otherwise tagged with aztecprotocol/aztec:$AZTEC_DOCKER_TAG and aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG."
-  exit 1
-fi
-
-# Load the Docker images into kind
-kind load docker-image aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG
-kind load docker-image aztecprotocol/aztec:$AZTEC_DOCKER_TAG
+# Ensure our aztec helm chart is deployed
+$(git rev-parse --show-toplevel)/yarn-project/end-to-end/scripts/helm_deploy.sh
 
-# If FRESH_INSTALL is true, delete the namespace
-if [ "$FRESH_INSTALL" = "true" ]; then
-  kubectl delete namespace "$NAMESPACE" --ignore-not-found=true --wait=true --now --timeout=10m
-fi
-
-function show_status_until_pxe_ready() {
-  set +x # don't spam with our commands
-  sleep 15 # let helm upgrade start
-  for i in {1..100} ; do
-    if kubectl wait pod -l app==pxe --for=condition=Ready -n "transfer" --timeout=20s >/dev/null 2>/dev/null ; then
-      break # we are up, stop showing status
-    fi
-    # show startup status
-    kubectl get pods -n "$NAMESPACE"
-  done
+function k8s_pxe_port_forward() {
+  # Clear any existing port forward
+  ps aux | grep "kubectl port-forward" | grep 9080 | awk '{print $2}' | xargs kill || true
+  # tunnel in to get access directly to our PXE service in k8s
+  kubectl port-forward -n "$NAMESPACE" svc/spartan-aztec-network-pxe 9080:8080 || true
 }
 
-show_status_until_pxe_ready &
-
-# Install the Helm chart
-helm upgrade --install spartan "$(git rev-parse --show-toplevel)/spartan/aztec-network/" \
-      --namespace "$NAMESPACE" \
-      --create-namespace \
-      --values "$(git rev-parse --show-toplevel)/spartan/aztec-network/values/$VALUES_FILE" \
-      --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
-      --set ingress.enabled=true \
-      --wait \
-      --wait-for-jobs=true \
-      --timeout=30m
-
-kubectl wait pod -l app==pxe --for=condition=Ready -n "$NAMESPACE" --timeout=10m
-
-function forward_pxe_k8s_port() {
-  # NOTE we fail silently, and work in the background
-  kubectl port-forward --namespace transfer svc/spartan-aztec-network-pxe 9082:8080 2>/dev/null >/dev/null || true
-}
-# tunnel in to get access directly to our PXE service in k8s
-(kubectl port-forward --namespace transfer svc/spartan-aztec-network-pxe 9082:8080 2>/dev/null >/dev/null || true) &
+# Make sure the port forwarding outlives the script
+nohup k8s_pxe_port_forward >/dev/null 2>/dev/null &
 
 # run our test in the host network namespace (so we can access the above with localhost)
+if ! docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "aztecprotocol/aztec:$END_TO_END_DOCKER_TAG"; then
+  echo "Test docker image not found. It needs to be built with 'earthly ./yarn-project/+export-end-to-end' or otherwise an image named aztecprotocol/aztec:$AZTEC_DOCKER_TAG needs to exist, or AZTEC_DOCKER_TAG passed with a tag that exists."
+  exit 1
+fi
+
 docker run --rm --network=host \
   -e SCENARIO=default \
-  -e PXE_URL=http://localhost:9082 \
+  -e PXE_URL=http://localhost:9080 \
   -e DEBUG="aztec:*" \
   -e LOG_LEVEL=debug \
   -e LOG_JSON=1 \
-  aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG $TEST
+  aztecprotocol/end-to-end:$END_TO_END_DOCKER_TAG $TEST