adding back tests-nightly workflow (#4827)

.changeset/rotten-garlics-perform.md

@@ -0,0 +1,5 @@
+---
+"saleor-dashboard": minor
+---
+
+QA Adding back tests-nightly workflow

.github/actions/combineReportsFromE2E/action.yml

@@ -0,0 +1,51 @@
+name: combine-e2e-results
+description: "Combines reports from matrix and upload them as one"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version-file: ".nvmrc"
+        cache: npm
+    - name: Cache node modules
+      uses: actions/cache@v3
+      env:
+        cache-name: cache-node-modules
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-qa-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-qa-${{ env.cache-name }}-
+          ${{ runner.os }}-qa-
+          ${{ runner.os }}-
+    - name: Install Dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      shell: bash
+      run: NODE_OPTIONS=--max_old_space_size=4096 npm ci
+    - run: npm ci
+      shell: bash
+      working-directory: .github/workflows
+    - name: Download reports artifacts
+      uses: actions/download-artifact@v3
+      with:
+        path: ./cypress/reports
+    - name: Create reports dir
+      shell: bash
+      continue-on-error: true
+      run: npm run qa:create-artifacts-dirs
+    - name: Merge report files
+      shell: bash
+      continue-on-error: true
+      run: npm run qa:generate-html-report
+    - name: Move artifacts screenshots into reports dir
+      shell: bash
+      continue-on-error: true
+      run: npm run qa:artifact-move-screenshots
+    - name: Upload reports
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: combined-report
+        path: ./cypress/reports
+        retention-days: 5
+        if-no-files-found: ignore

.github/actions/e2e/action.yml

@@ -0,0 +1,92 @@
+name: e2e-parallel
+description: "Trigger e2e tests and create reports"
+inputs:
+  apiUrl:
+    description: "API URL from PR or staging"
+    required: true
+  baseUrl:
+    description: "FE URL from PR or staging"
+    required: true
+  commitInfoMessage:
+    description: "Info displayed in commit"
+    required: false
+  cypressGrepTags:
+    description: "Passed tags that should be executed"
+    required: true
+  split:
+    description: "E2E tests containers number"
+    required: true
+  splitIndex:
+    description: "E2E tests container index"
+    required: true
+  install:
+    description: "Triggers cypress installation"
+    required: false
+  browser:
+    description: "Browser which e2e should test"
+    required: false
+  appMountUri:
+    description: "URL for Apps"
+    required: true
+  userName:
+    description: "Name of user which e2e uses"
+    required: true
+  secondUserName:
+    description: "Name of the second user which e2e uses"
+    required: true
+  userPassword:
+    description: "User password for e2e"
+    required: true
+  permissionsUserPassword:
+    description: "User with limited permissions password"
+    required: true
+  mailpitUrl:
+    description: "Mail provider URL"
+    required: true
+  stripeSecretKey:
+    description: "Payments providers secret"
+    required: false
+  stripePublicKey:
+    description: "Payments providers secret"
+    required: false
+runs:
+  using: "composite"
+  steps:
+    - name: e2e with reports
+      uses: cypress-io/github-action@v6
+      with:
+        install: ${{inputs.install}}
+        browser: ${{inputs.browser}}
+      env:
+        API_URI: ${{inputs.apiUrl}}
+        APP_MOUNT_URI: ${{ inputs.appMountUri }}
+        BASE_URL: ${{inputs.baseUrl}}
+        CYPRESS_USER_NAME: ${{ inputs.userName }}
+        CYPRESS_SECOND_USER_NAME: ${{ inputs.secondUserName }}
+        CYPRESS_USER_PASSWORD: ${{ inputs.userPassword }}
+        CYPRESS_PERMISSIONS_USERS_PASSWORD: ${{ inputs.permissionsUserPassword }}
+        CYPRESS_MAILPITURL: ${{ inputs.mailpitUrl }}
+        CYPRESS_grepTags: ${{ inputs.cypressGrepTags }}
+        SPLIT: ${{ inputs.split }}
+        SPLIT_INDEX: ${{ inputs.splitIndex}}
+        STRIPE_SECRET_KEY: ${{inputs.stripeSecretKey}}
+        STRIPE_PUBLIC_KEY: ${{inputs.stripePublicKey}}
+        COMMIT_INFO_MESSAGE: ${{inputs.commitInfoMessage}}
+    - name: Create reports dir
+      shell: bash
+      continue-on-error: true
+      if: ${{ ! cancelled() }}
+      run: npm run qa:create-artifacts-dirs
+    - name: Move screenshots into reports dir
+      shell: bash
+      continue-on-error: true
+      if: ${{ ! cancelled() }}
+      run: npm run qa:move-screenshots
+    - name: Upload reports
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: report-${{ strategy.job-index }}
+        path: ./cypress/reports
+        retention-days: 5
+        if-no-files-found: ignore

.github/workflows/cypress-repeat.yml

@@ -0,0 +1,68 @@
+name: Cypress repeat
+
+on:
+  workflow_dispatch:
+    inputs:
+      spec:
+        required: false
+        description: "Spec to run. Leave empty if you prefer to filter tests using grep"
+      grep:
+        required: false
+        description: "Run all tests which titles contain text. Leave empty if you prefer to filter tests using spec."
+      dashboard_url:
+        required: true
+        default: "https://automation-dashboard.staging.saleor.cloud/dashboard"
+        description: "Dashboard url"
+      API_url:
+        required: true
+        default: "https://automation-dashboard.staging.saleor.cloud/graphql/"
+        description: "API url"
+      repeat:
+        required: true
+        default: "10"
+        description: "How many times run tests"
+
+jobs:
+  run-tests:
+    if: ${{ github.event.inputs.spec }} || ${{ github.event.inputs.grep }} ## Do not run if spec or grep not provided
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: Package
+        run: npm ci
+
+      - name: Install cypress-repeat
+        run: npm i -D cypress-repeat
+
+      - name: Cypress run
+        id: cypress
+        uses: cypress-io/github-action@v4
+        env:
+          API_URI: ${{ github.event.inputs.API_url}}
+          APP_MOUNT_URI: ${{ secrets.APP_MOUNT_URI }}
+          BASE_URL: ${{ github.event.inputs.dashboard_url}}
+          CYPRESS_USER_NAME: ${{ secrets.CYPRESS_USER_NAME }}
+          CYPRESS_SECOND_USER_NAME: ${{ secrets.CYPRESS_SECOND_USER_NAME }}
+          CYPRESS_USER_PASSWORD: ${{ secrets.CYPRESS_USER_PASSWORD }}
+          CYPRESS_PERMISSIONS_USERS_PASSWORD: ${{ secrets.CYPRESS_PERMISSIONS_USERS_PASSWORD }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CYPRESS_MAILPITURL: ${{ secrets.CYPRESS_MAILPITURL }}
+          COMMIT_INFO_MESSAGE: Cypress repeat with - ${{ github.event.inputs.spec }} ${{ github.event.inputs.grep }} ${{ github.event.inputs.dashboard_url}}
+          CYPRESS_grep: ${{ github.event.inputs.grep }}
+          STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }}
+          STRIPE_PUBLIC_KEY: ${{ secrets.STRIPE_PUBLIC_KEY }}
+        with:
+          parallel: true
+          group: "UI - Chrome"
+          record: true
+          tag: Repeat
+          spec: ${{ github.event.inputs.spec }}
+          command: npx cypress-repeat -n ${{ github.event.inputs.repeat}}

.github/workflows/cypressTestsHelpers/approveAndMergeReleasePR.js

@@ -0,0 +1,173 @@
+const { Octokit } = require("@octokit/core");
+const { Command } = require("commander");
+const { GraphQLClient } = require("graphql-request");
+const { statusAndID } = require("./getTestsResults");
+const { failedTestCases } = require("./getTestsResults");
+
+const program = new Command();
+const client = new GraphQLClient("https://dashboard.cypress.io/graphql");
+
+const repo = "saleor-cloud-deployments";
+const owner = "saleor";
+
+program
+  .name("Approve PR")
+  .description("Approve and merge PR if patch release")
+  .option("--version <version>", "version of a project")
+  .option("--pull_request_number <pull_request_number>", "Pull Request number")
+  .option("--auto_release", "is auto release")
+  .option("--dashboard_url <dashboard_url>", "Cypress dashboard url")
+  .action(async options => {
+    const octokit = new Octokit({
+      auth: process.env.GITHUB_TOKEN,
+    });
+
+    const pullNumber = options.pull_request_number;
+
+    const pullRequest = await octokit.request(
+      "GET /repos/{owner}/{repo}/pulls/{pull_number}",
+      {
+        owner,
+        repo,
+        pull_number: pullNumber,
+      },
+    );
+
+    const commitId = pullRequest.data.merge_commit_sha;
+
+    const data = await statusAndID(options.dashboard_url);
+
+    let testsStatus = data.status;
+
+    let requestBody = `Cypress tests passed. See results at ${options.dashboard_url}`;
+
+    if (testsStatus === "FAILED") {
+      const failedNewTests = [];
+      const listOfTestIssues = await getListOfTestsIssues(octokit);
+      const testCases = await failedTestCases(data.runId);
+      testCases.forEach(testCase => {
+        if (testCase.titleParts) {
+          const issue = issueOnGithub(listOfTestIssues, testCase.titleParts[1]);
+          if (issue) {
+            const knownBug = isIssueAKnownBugForReleaseVersion(
+              issue,
+              options.version,
+            );
+            if (!knownBug) {
+              failedNewTests.push({
+                title: testCase.titleParts[1],
+                url: issue.html_url,
+                spec: testCase.titleParts[0],
+              });
+            }
+          } else {
+            failedNewTests.push({
+              title: testCase.titleParts[1],
+              spec: testCase.titleParts[0],
+            });
+          }
+        }
+      });
+
+      if (failedNewTests.length === 0) {
+        requestBody = `All failed tests are known bugs, can be merged. See results at ${options.dashboard_url}`;
+        testsStatus = "PASSED";
+      } else if (failedNewTests.length > 10) {
+        //If there are more than 10 new bugs it's probably caused by something else. Server responses with 500, or test user was deleted, etc.
+
+        requestBody =
+          "There is more than 10 new bugs, check results manually and create issues for them if necessary";
+      } else {
+        requestBody = `New bugs found, results at: ${options.dashboard_url}. List of issues to check: `;
+        for (const newBug of failedNewTests) {
+          if (!newBug.url) {
+            requestBody += `\n${newBug.title}`;
+          } else {
+            requestBody += `\n${newBug.title} - ${newBug.url}`;
+          }
+        }
+        requestBody += `\nIf this bugs won't be fixed in next patch release for this version mark them as known issues`;
+      }
+    } else if (testsStatus === "ERRORED") {
+      requestBody = `Tests ERRORED! Check log at ${options.dashboard_url}`;
+    }
+
+    const event = "COMMENT";
+
+    await octokit.request(
+      "POST /repos/{owner}/{repo}/pulls/{pull_number}/reviews",
+      {
+        owner,
+        repo,
+        pull_number: pullNumber,
+        commit_id: commitId,
+        body: requestBody,
+        event,
+        comments: [],
+      },
+    );
+
+    if (
+      options.auto_release &&
+      isPatchRelease(options.version) &&
+      testsStatus === "PASSED"
+    ) {
+      await octokit.request(
+        "PUT /repos/{owner}/{repo}/pulls/{pull_number}/merge",
+        {
+          owner,
+          repo,
+          pull_number: pullNumber,
+          merge_method: "squash",
+        },
+      );
+    }
+  })
+  .parse();
+
+function isPatchRelease(version) {
+  const regex = /\d+\.\d+\.[1-9]/;
+  return version.match(regex) ? true : false;
+}
+
+async function getListOfTestsIssues(octokit) {
+  const result = await octokit.request(
+    "GET /repos/{owner}/saleor-dashboard/issues?labels=tests",
+    {
+      owner,
+    },
+  );
+  return result.data;
+}
+
+function issueOnGithub(listOfTestIssues, testCaseTitle) {
+  if (listOfTestIssues.length > 0) {
+    return listOfTestIssues.find(issue => {
+      return issue.title.includes(testCaseTitle);
+    });
+  }
+}
+
+function isIssueAKnownBugForReleaseVersion(issue, releaseVersion) {
+  const issueBody = issue.body;
+  const regex = /Known bug for versions:([\s\S]*)Additional/;
+  const lines = issueBody.match(regex)[1].split("\n");
+  const lineContainReleaseVersionRegex = /v(\d{2,3}).*(true|false)/;
+  const releaseVersionLine = lines.find(line => {
+    if (line.match(lineContainReleaseVersionRegex)) {
+      const version = line.match(lineContainReleaseVersionRegex)[1];
+      if (version === getFormattedVersion(releaseVersion)) {
+        return line;
+      }
+    }
+  });
+  const knownBugOnReleaseVersion = releaseVersionLine
+    ? releaseVersionLine.match(lineContainReleaseVersionRegex)[2]
+    : false;
+  return knownBugOnReleaseVersion === "true" ? true : false;
+}
+
+function getFormattedVersion(version) {
+  const regex = /^\d+\.\d+\./;
+  return version.match(regex)[0].replace(/\./g, "");
+}