fix(lambda): configuring log retention fails on 70+ Lambdas

When the Log Retention Lambda runs massively parallel (on 70+ Lambdas
at the same time), it can run into throttling problems and fail.

Raise the retry count and delays:

- Raise the default amount of retries from 5 -> 10
- Raise the sleep base from 100ms to 1s.
- Change the sleep calculation to apply the 10s limit *after* jitter instead
  of before (previously, we would take a fraction of 10s; now we're
  taking a fraction of the accumulated wait time, and after calculating
  that limit it to 10s).

Fixes #31338.

packages/@aws-cdk/custom-resource-handlers/lib/aws-logs/log-retention-handler/index.ts

@@ -11,7 +11,7 @@ interface LogRetentionEvent extends Omit<AWSLambda.CloudFormationCustomResourceE
     SdkRetry?: {
       maxRetries?: string;
     };
-    RemovalPolicy?: string
+    RemovalPolicy?: string;
   };
 }
 
@@ -91,7 +91,7 @@ export async function handler(event: LogRetentionEvent, context: AWSLambda.Conte
     const logGroupRegion = event.ResourceProperties.LogGroupRegion;
 
     // Parse to AWS SDK retry options
-    const maxRetries = parseIntOptional(event.ResourceProperties.SdkRetry?.maxRetries) ?? 5;
+    const maxRetries = parseIntOptional(event.ResourceProperties.SdkRetry?.maxRetries) ?? 10;
     const withDelay = makeWithDelay(maxRetries);
 
     const sdkConfig: Logs.CloudWatchLogsClientConfig = {
@@ -188,8 +188,8 @@ function parseIntOptional(value?: string, base = 10): number | undefined {
 
 function makeWithDelay(
   maxRetries: number,
-  delayBase: number = 100,
-  delayCap = 10 * 1000, // 10s
+  delayBase: number = 1_000,
+  delayCap = 60_000, // 60s
 ): (block: () => Promise<void>) => Promise<void> {
   // If we try to update the log group, then due to the async nature of
   // Lambda logging there could be a race condition when the same log group is
@@ -224,5 +224,5 @@ function makeWithDelay(
 }
 
 function calculateDelay(attempt: number, base: number, cap: number): number {
-  return Math.round(Math.random() * Math.min(cap, base * 2 ** attempt));
+  return Math.min(Math.round(Math.random() * base * 2 ** attempt), cap);
 }