fix(examples): backoff trusted-sync invalid payload retries (#411)

examples/trusted-sync/src/cli.rs

@@ -54,6 +54,13 @@ pub struct Cli {
         help = "Parameterized threshold amount of blocks to drift from before fast-forwarding or walking back"
     )]
     pub drift_threshold: u64,
+    /// Number of failed payload validation retries before recording an invalid payload attributes.
+    #[clap(
+        long,
+        default_value_t = 3,
+        help = "Number of failed payload validation retries before recording an invalid payload attributes"
+    )]
+    pub invalid_payload_retries: u64,
 }
 
 impl Cli {

examples/trusted-sync/src/main.rs

@@ -92,13 +92,15 @@ async fn sync(cli: cli::Cli) -> Result<()> {
     // Reset metrics so they can be queried.
     metrics::FAILED_PAYLOAD_DERIVATION.reset();
     metrics::DRIFT_WALKBACK.set(0);
+    metrics::RETRIES.reset();
     metrics::DRIFT_WALKBACK_TIMESTAMP.set(0);
     metrics::DERIVED_ATTRIBUTES_COUNT.reset();
     metrics::FAST_FORWARD_BLOCK.set(0);
     metrics::FAST_FORWARD_TIMESTAMP.set(0);
 
     // Continuously step on the pipeline and validate payloads.
     let mut advance_cursor_flag = false;
+    let mut retries = 0;
     loop {
         // Update the reference l2 head.
         match l2_provider.latest_block_number().await {
@@ -226,6 +228,20 @@ async fn sync(cli: cli::Cli) -> Result<()> {
                 Ok((true, _)) => trace!(target: LOG_TARGET, "Validated payload attributes"),
                 Ok((false, expected)) => {
                     error!(target: LOG_TARGET, "Failed payload validation. Derived payload attributes: {:?}, Expected: {:?}", attributes, expected);
+                    // Attempt to re-validate payload attributes if we haven't reached the retry
+                    // limit. Since validation didn't error, either this is hit
+                    // because:
+                    // - The payload attributes are actually invalid
+                    // - Validation returned a flakey result (e.g. `debug_getRawTransaction` returns
+                    //   empty bytes which has been seen on multiple occurances)
+                    if retries < cli.invalid_payload_retries {
+                        retries += 1;
+                        metrics::RETRIES.inc();
+                        // Back-off for a few seconds before retrying.
+                        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+                        continue;
+                    }
+                    retries = 0;
                     metrics::FAILED_PAYLOAD_DERIVATION.inc();
                     let _ = pipeline.next(); // Take the attributes and continue
                     continue;
@@ -240,6 +256,7 @@ async fn sync(cli: cli::Cli) -> Result<()> {
             debug!(target: LOG_TARGET, "No attributes to validate");
             continue;
         };
+        retries = 0;
 
         // Take the next attributes from the pipeline since they're valid.
         let attributes = if let Some(attributes) = pipeline.next() {

examples/trusted-sync/src/metrics.rs

@@ -72,6 +72,10 @@ lazy_static! {
         let opts = opts!("trusted_sync_pipeline_steps", "Number of pipeline steps");
         register_gauge_vec!(opts, &["status"]).expect("Failed to register pipeline steps metric")
     };
+
+    /// Tracks the number of retries for invalid payloads.
+    pub static ref RETRIES: IntCounter =
+        register_int_counter!("trusted_sync_validation_retries", "Invalid Payload Retries").expect("Failed to register retries metric");
 }
 
 /// Starts the metrics server.