feat(sdk): stabilizing tf-azure target (#6648)

fixes: #6619 fixes: #5111 fixes: #5123 fixes #5965 edit: fixes [#6652](#6652) as well - Fixed the Azure functions, and failing bucket tests - Skipped math in the cloud SDK tests - Reduce amount of tests on util and expect folders - **Added tf-azure to the stable targets** and removed from the un-stable ones (then disable the unstable testing being run after each publish) - increase the chances of the tf-azure tests to pass terraform (by adding the parallelism flag on apply and delete, and prevent test failure if the destroying failed) ## Checklist - [ ] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted) - [ ] Description explains motivation and solution - [ ] Tests added (always) - [ ] Docs updated (only required for features) - [ ] Added `pr/e2e-full` label if this feature requires end-to-end testing *By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*.
winglang · Jun 16, 2024 · ec28099 · ec28099
1 parent 0e04fa6
commit ec28099
Show file tree

Hide file tree

Showing 35 changed files with 434 additions and 613 deletions.
diff --git a/.github/workflows/sdk-spec-test.yml b/.github/workflows/sdk-spec-test.yml
@@ -1,9 +1,9 @@
 name: SDK Spec Tests
 run-name: SDK Spec Tests (${{ inputs.repo || github.repository }}/${{inputs.ref || github.ref}})
 on:
-  release:
-    types:
-      - published # runs only unstable targets
+  # release: #TODO: uncomment to enable unstable targets, at this moment there are none
+  # types:
+  # - published # runs only unstable targets
   workflow_call: {}
   workflow_dispatch:
     inputs:
@@ -68,8 +68,8 @@ jobs:
       - name: Get list of directories and save them to the output
         id: setdirs
         shell: bash
-        run: | # TODO: skipping std and external folders, when https://github.com/winglang/wing/issues/3168 is resolve- we'll skip only the external folder.
-          dirs=$(ls -d examples/tests/sdk_tests/*/ | sed 's/\/$//' | grep -v "external\|std" | jq -R -s -c 'split("\n")[:-1]')
+        run: | # TODO: skipping std, math and external folders, when https://github.com/winglang/wing/issues/3168 is resolve- we'll skip only the external folder.
+          dirs=$(ls -d examples/tests/sdk_tests/*/ | sed 's/\/$//' | grep -v "external\|std\|math" | jq -R -s -c 'split("\n")[:-1]')
           processed_dirs=$(echo "{ \"directory\": $dirs }" | jq -c '[ .directory[] | {directory: ., name: (split("/") | last)}]')
           wrapped_dirs=$(echo $processed_dirs | jq -c .)
           echo "dirs=$wrapped_dirs" >> $GITHUB_OUTPUT
@@ -79,9 +79,9 @@ jobs:
           if [ "${{env.TARGET}}" = "all" ]; then
             target='["tf-aws", "tf-azure"]'
           elif [ "${{env.TARGET}}" = "all-stable" ]; then
-            target='["tf-aws", "sim"]'
+            target='["tf-aws", "sim", "tf-azure"]'
           elif [ "${{env.TARGET}}" = "all-unstable" ]; then
-            target='["tf-azure"]'
+            target='[]'
           else
            target='["${{env.TARGET}}"]'
           fi
@@ -162,32 +162,39 @@ jobs:
         uses: azure/login@v2
         with:
           creds: ${{ secrets.AZURE_CREDENTIALS }}
-      - name: Execute wing test in matrix directory
+      - name: pnpm insall
+        if: ${{ env.MANUAL == 'true' }}
         uses: nick-fields/retry@v3
         env:
           TF_LOG: info
           TF_LOG_PATH: ${{ runner.workspace }}/terraform.log
         with:
           max_attempts: 3
           retry_on: error
-          timeout_minutes: 60
+          timeout_minutes: 30
           command: |
-            if  ${{ env.MANUAL == 'true' }}
-            then
-              pnpm install
-              pnpm turbo compile -F=winglang
-              WING_CLI=$(realpath apps/wing/bin/wing)
-            elif ${{ env.LOCAL_BUILD == 'false'}}
-            then 
-              WING_CLI=$(which wing)
-              # COMPATIBILITY="-t @winglang/compatibility-spy" //TODO: will be handled in a following PR
-            else
-              WING_CLI=$(realpath localwing/node_modules/.bin/wing)
-              COMPATIBILITY="-t  ../../../../localwing/node_modules/@winglang/compatibility-spy/lib"
-              echo $COMPATIBILITY
-            fi
-            cd ${{ matrix.test.directory }}
-            $WING_CLI test --snapshots=deploy -t ${{ matrix.target }} -p 10 $COMPATIBILITY *.test.w -o ../../../../out/${{ matrix.test.name }}-${{ matrix.target }}.json
+            pnpm install
+            pnpm turbo compile -F=winglang
+
+      - name: Execute wing test in matrix directory
+        env:
+          TF_LOG: info
+          TF_LOG_PATH: ${{ runner.workspace }}/terraform.log
+        run: |
+          if  ${{ env.MANUAL == 'true' }}
+          then
+            WING_CLI=$(realpath apps/wing/bin/wing)
+          elif ${{ env.LOCAL_BUILD == 'false'}}
+          then 
+            WING_CLI=$(which wing)
+            # COMPATIBILITY="-t @winglang/compatibility-spy" //TODO: will be handled in a following PR
+          else
+            WING_CLI=$(realpath localwing/node_modules/.bin/wing)
+            COMPATIBILITY="-t  ../../../../localwing/node_modules/@winglang/compatibility-spy/lib"
+            echo $COMPATIBILITY
+          fi
+          cd ${{ matrix.test.directory }}
+          $WING_CLI test --snapshots=deploy -t ${{ matrix.target }} -p ${{ (matrix.target == 'tf-azure' && 2 ) || 10 }} --retry 3 $COMPATIBILITY *.test.w -o ../../../../out/${{ matrix.test.name }}-${{ matrix.target }}.json
 
       - name: Upload Artifacts
         if: ${{ env.LOCAL_BUILD ==  'true' }}

diff --git a/apps/wing/src/commands/test/test.test.ts b/apps/wing/src/commands/test/test.test.ts
@@ -356,7 +356,7 @@ describe("retry and parallel options", () => {
     });
 
     const retryLogs = logSpy.mock.calls.filter((args) => args[0].includes("Retrying"));
-    expect(retryLogs.length).toBe(3);
+    expect(retryLogs.length).toBe(2);
   });
 
   test("wing test --parallel [batch]", async () => {

diff --git a/apps/wing/src/commands/test/test.ts b/apps/wing/src/commands/test/test.ts
@@ -26,6 +26,8 @@ const log = debug("wing:test");
 const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS = "WING_TEST_RUNNER_FUNCTION_IDENTIFIERS";
 const ENV_WING_TEST_RUNNER_FUNCTION_IDENTIFIERS_AWSCDK = "WingTestRunnerFunctionArns";
 
+const PARALLELISM = { [BuiltinPlatform.TF_AZURE]: 5 };
+
 /**
  * Options for the `test` command.
  */
@@ -43,7 +45,7 @@ export interface TestOptions extends CompileOptions {
    */
   readonly testFilter?: string;
   /**
-   * How many times failed tests should be retried.
+   * How many times failed tests should be retried. default is one
    */
   readonly retry?: number;
   /**
@@ -113,17 +115,27 @@ export async function test(entrypoints: string[], options: TestOptions): Promise
   const startTime = Date.now();
   const results: SingleTestResult[] = [];
   process.env.WING_TARGET = determineTargetFromPlatforms(options.platform ?? []);
-  const testFile = async (entrypoint: string) => {
+  const testFile = async (
+    entrypoint: string,
+    retries: number = options.retry || 1
+  ): Promise<void> => {
     const testName = renderTestName(entrypoint);
     try {
       const singleTestResults = await testOne(testName, entrypoint, options);
+      if (singleTestResults.results.some((t) => !t.pass) && retries > 1) {
+        console.log(`Retrying failed tests. ${retries - 1} retries left.`);
+        return await testFile(entrypoint, retries - 1);
+      }
       results.push(singleTestResults);
     } catch (error: any) {
       console.log(error.message);
+      if (retries > 1) {
+        console.log(`Retrying failed tests. ${retries - 1} retries left.`);
+        return await testFile(entrypoint, retries - 1);
+      }
       const snapshot = error.message?.startsWith(SNAPSHOT_ERROR_PREFIX)
         ? SnapshotResult.MISMATCH
         : SnapshotResult.SKIPPED;
-
       results.push({
         testName,
         snapshot,
@@ -144,7 +156,7 @@ export async function test(entrypoints: string[], options: TestOptions): Promise
 
   await PromisePool.withConcurrency(options.parallel || selectedEntrypoints.length)
     .for(selectedEntrypoints)
-    .process(testFile);
+    .process((entrypointFile) => testFile(entrypointFile));
 
   const testDuration = Date.now() - startTime;
   printResults(results, testDuration);
@@ -340,34 +352,15 @@ export function filterTests(tests: Array<string>, regexString?: string): Array<s
   }
 }
 
-async function runTestsWithRetry(
+async function runTests(
   testRunner: std.ITestRunnerClient,
-  tests: string[],
-  retries: number
+  tests: string[]
 ): Promise<std.TestResult[]> {
-  let runCount = retries + 1;
-  let remainingTests = tests;
   const results: std.TestResult[] = [];
 
-  while (runCount > 0 && remainingTests.length > 0) {
-    const failedTests: string[] = [];
-
-    for (const testPath of remainingTests) {
-      const result = await testRunner.runTest(testPath);
-      results.push(result);
-
-      if (!result.pass) {
-        failedTests.push(testPath);
-      }
-    }
-
-    remainingTests = failedTests;
-
-    if (remainingTests.length > 0 && runCount > 1) {
-      console.log(`Retrying failed tests. ${runCount - 1} retries left.`);
-    }
-
-    runCount--;
+  for (const testPath of tests) {
+    const result = await testRunner.runTest(testPath);
+    results.push(result);
   }
 
   return results;
@@ -448,7 +441,7 @@ function shouldSkipTrace(trace: std.Trace): boolean {
 
 async function testSimulator(synthDir: string, options: TestOptions) {
   const s = new simulator.Simulator({ simfile: synthDir });
-  const { clean, testFilter, retry } = options;
+  const { clean, testFilter } = options;
 
   let outputStream: SpinnerStream | undefined;
   let traceProcessor: TraceProcessor | undefined;
@@ -514,7 +507,7 @@ async function testSimulator(synthDir: string, options: TestOptions) {
   const tests = await testRunner.listTests();
   const filteredTests = filterTests(tests, testFilter);
 
-  const results = await runTestsWithRetry(testRunner, filteredTests, retry ?? 0);
+  const results = await runTests(testRunner, filteredTests);
 
   await s.stop();
 
@@ -547,7 +540,8 @@ async function testSimulator(synthDir: string, options: TestOptions) {
 }
 
 async function testTf(synthDir: string, options: TestOptions): Promise<std.TestResult[]> {
-  const { clean, testFilter, retry, platform = [BuiltinPlatform.SIM] } = options;
+  const { clean, testFilter, platform = [BuiltinPlatform.SIM] } = options;
+  let tfParallelism = PARALLELISM[platform[0]];
 
   try {
     const installed = await isTerraformInstalled(synthDir);
@@ -559,7 +553,7 @@ async function testTf(synthDir: string, options: TestOptions): Promise<std.TestR
 
     await withSpinner("terraform init", async () => terraformInit(synthDir));
 
-    await withSpinner("terraform apply", () => terraformApply(synthDir));
+    await withSpinner("terraform apply", () => terraformApply(synthDir, tfParallelism));
 
     const [testRunner, tests] = await withSpinner("Setting up test runner...", async () => {
       const target = determineTargetFromPlatforms(platform);
@@ -575,7 +569,7 @@ async function testTf(synthDir: string, options: TestOptions): Promise<std.TestR
     });
 
     const results = await withSpinner("Running tests...", async () => {
-      return runTestsWithRetry(testRunner, tests, retry ?? 0);
+      return runTests(testRunner, tests);
     });
 
     const testReport = await renderTestReport(synthDir, results);
@@ -598,15 +592,15 @@ async function testTf(synthDir: string, options: TestOptions): Promise<std.TestR
     return [{ pass: false, path: "", error: (err as Error).message, traces: [] }];
   } finally {
     if (clean) {
-      await cleanupTf(synthDir);
+      await cleanupTf(synthDir, tfParallelism);
     } else {
       noCleanUp(synthDir);
     }
   }
 }
 
 async function testAwsCdk(synthDir: string, options: TestOptions): Promise<std.TestResult[]> {
-  const { clean, testFilter, retry } = options;
+  const { clean, testFilter } = options;
   try {
     await isAwsCdkInstalled(synthDir);
 
@@ -632,7 +626,7 @@ async function testAwsCdk(synthDir: string, options: TestOptions): Promise<std.T
     });
 
     const results = await withSpinner("Running tests...", async () => {
-      return runTestsWithRetry(testRunner, tests, retry ?? 0);
+      return runTests(testRunner, tests);
     });
 
     const testReport = await renderTestReport(synthDir, results);
@@ -697,9 +691,13 @@ const targetFolder: Record<string, string> = {
   [BuiltinPlatform.TF_GCP]: "shared-gcp",
 };
 
-async function cleanupTf(synthDir: string) {
-  await withSpinner("terraform destroy", () => terraformDestroy(synthDir));
-  rmSync(synthDir, { recursive: true, force: true });
+async function cleanupTf(synthDir: string, parallelism?: number) {
+  try {
+    await withSpinner("terraform destroy", () => terraformDestroy(synthDir, parallelism));
+    rmSync(synthDir, { recursive: true, force: true });
+  } catch (e) {
+    console.error(e);
+  }
 }
 
 async function isTerraformInstalled(synthDir: string) {
@@ -711,12 +709,20 @@ export async function terraformInit(synthDir: string) {
   return execCapture("terraform init", { cwd: synthDir });
 }
 
-async function terraformApply(synthDir: string) {
-  return execCapture("terraform apply -auto-approve", { cwd: synthDir });
+async function terraformApply(synthDir: string, parallelism?: number) {
+  return execCapture(
+    `terraform apply -auto-approve ${parallelism ? `-parallelism=${parallelism}` : ""}`,
+    { cwd: synthDir }
+  );
 }
 
-async function terraformDestroy(synthDir: string) {
-  return execCapture("terraform destroy -auto-approve", { cwd: synthDir });
+async function terraformDestroy(synthDir: string, parallelism?: number) {
+  return execCapture(
+    `terraform destroy -auto-approve ${parallelism ? `-parallelism=${parallelism}` : ""}`,
+    {
+      cwd: synthDir,
+    }
+  );
 }
 
 async function terraformOutput(synthDir: string, name: string) {

diff --git a/examples/tests/sdk_tests/bucket/get.test.w b/examples/tests/sdk_tests/bucket/get.test.w
@@ -17,6 +17,8 @@ test "get range of an object" {
 
   b.put("test1.txt", "12345");
 
+  expect.equal(b.get("test1.txt"), "12345");
+
   expect.equal(b.get("test1.txt", startByte: 1, endByte: 3), "234");
   expect.equal(b.get("test1.txt", startByte: 1), "2345");
   expect.equal(b.get("test1.txt", endByte: 3), "1234");
@@ -28,9 +30,8 @@ test "get range of an object" {
   assertThrows("The encoded data was not valid for encoding utf-8", () => {
     b.get("test2.txt", startByte: 0, endByte: 2);
   });
-}
 
-test "get empty object" {
+// "get empty object" 
   b.put("empty.txt", "");
 
   expect.equal(b.get("empty.txt"), "");

diff --git a/examples/tests/sdk_tests/counter/dec.test.w b/examples/tests/sdk_tests/counter/dec.test.w
@@ -26,23 +26,22 @@ test "dec()" {
   let r3 = counter1.dec(0);
   expect.equal(r3, -2);
   expect.equal(counter1.peek(), -2);
-}
 
-test "dec() with custom key" {
+// "dec() with custom key" 
   let key = "custom-key";
 
   // explicit decrement (positive int)
-  let r1 = counter2.dec(5, key);
-  expect.equal(r1, -1);
+  let r4 = counter2.dec(5, key);
+  expect.equal(r4, -1);
   expect.equal(counter2.peek(key), -6);
 
   // explicit decrement (negative int)
-  let r2 = counter2.dec(-4, key);
-  expect.equal(r2, -6);
+  let r5 = counter2.dec(-4, key);
+  expect.equal(r5, -6);
   expect.equal(counter2.peek(key), -2);
 
   // explicit decrement (-0)
-  let r3 = counter2.dec(0, key);
-  expect.equal(r3, -2);
+  let r6 = counter2.dec(0, key);
+  expect.equal(r6, -2);
   expect.equal(counter2.peek(key), -2);
 }
diff --git a/examples/tests/sdk_tests/counter/inc.test.w b/examples/tests/sdk_tests/counter/inc.test.w
@@ -26,23 +26,22 @@ test "inc()" {
   let r3 = counter1.inc(0);
   expect.equal(r3, 2);
   expect.equal(counter1.peek(), 2);
-}
 
-test "inc() with custom key" {
+// "inc() with custom key" 
   let key = "custom-key";
 
   // explicit increment (positive int)
-  let r1 = counter2.inc(5, key);
-  expect.equal(r1, -1);
+  let r4 = counter2.inc(5, key);
+  expect.equal(r4, -1);
   expect.equal(counter2.peek(key), 4);
 
   // explicit increment (negative int)
-  let r2 = counter2.inc(-4, key);
-  expect.equal(r2, 4);
+  let r5 = counter2.inc(-4, key);
+  expect.equal(r5, 4);
   expect.equal(counter2.peek(key), 0);
 
   // explicit increment (+0)
-  let r3 = counter2.inc(0, key);
-  expect.equal(r3, 0);
+  let r6 = counter2.inc(0, key);
+  expect.equal(r6, 0);
   expect.equal(counter2.peek(key), 0);
 }