diff --git a/.github/workflows/test-e2e.yaml b/.github/workflows/test-e2e.yaml index 7de9c67b72..0dfe860c98 100644 --- a/.github/workflows/test-e2e.yaml +++ b/.github/workflows/test-e2e.yaml @@ -32,18 +32,26 @@ jobs: - name: Build Maestro CLI run: ./gradlew :maestro-cli:distZip - - uses: actions/upload-artifact@v4 + - name: Upload zipped Maestro CLI artifact + uses: actions/upload-artifact@v4 with: name: maestro-cli-jdk${{ matrix.java-version }}-run_id${{ github.run_id }} path: maestro-cli/build/distributions/maestro.zip retention-days: 1 - test-local-android: + - name: Upload build/Products to artifacts + uses: actions/upload-artifact@v4 + with: + name: build__Products-jdk${{ matrix.java-version }} + path: build/Products + retention-days: 1 + + test-android: name: Test on Android - timeout-minutes: 30 runs-on: ubuntu-latest needs: build - + timeout-minutes: 60 + env: ANDROID_HOME: /home/runner/androidsdk ANDROID_SDK_ROOT: /home/runner/androidsdk @@ -148,27 +156,30 @@ jobs: - name: Install apps working-directory: ${{ github.workspace }}/e2e - run: ./install_apps + run: ./install_apps android - name: Start screen recording of AVD run: | adb shell screenrecord /sdcard/screenrecord.mp4 & + echo $! > ~/screenrecord.pid - name: Run tests working-directory: ${{ github.workspace }}/e2e - run: ./run_tests + timeout-minutes: 10 + run: ./run_tests android - name: Stop screen recording of AVD + if: success() || failure() run: | - adb shell pkill -SIGINT screenrecord || echo "exited with code $?" && exit 0 + kill -SIGINT "$(cat ~/screenrecord.pid)" || echo "failed to kill screenrecord: code $?" && exit 0 sleep 5 # prevent video file corruption - adb pull /sdcard/screenrecord.mp4 ~/maestro_screenrecord.mp4 + adb pull /sdcard/screenrecord.mp4 ~/screenrecord.mp4 - name: Upload ~/.maestro artifacts uses: actions/upload-artifact@v4 if: success() || failure() with: - name: maestro-e2e-output + name: maestro-root-dir-android path: ~/.maestro retention-days: 7 include-hidden-files: true @@ -177,6 +188,152 @@ jobs: uses: actions/upload-artifact@v4 if: success() || failure() with: - name: maestro_screenrecord.mp4 - path: ~/maestro_screenrecord.mp4 + name: maestro-screenrecord-android.mp4 + path: ~/screenrecord.mp4 retention-days: 7 + + test-ios: + name: Test on iOS + runs-on: macos-latest + needs: build + timeout-minutes: 120 + if: ${{ false }} + + env: + MAESTRO_DRIVER_STARTUP_TIMEOUT: 240000 # 240s + MAESTRO_CLI_LOG_PATTERN_CONSOLE: '%d{HH:mm:ss.SSS} [%5level] %logger.%method: %msg%n' + + steps: + - name: Clone repository (only needed for the e2e directory) + uses: actions/checkout@v4 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 8 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: maestro-cli-jdk11-run_id${{ github.run_id }} + + - name: Add Maestro CLI executable to PATH + run: | + unzip maestro.zip -d maestro_extracted + echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH + + - name: Check if Maestro CLI executable starts up + run: | + maestro --help + maestro --version + + - name: Run simulator + run: xcrun simctl boot iPhone\ 15 + + - name: Wait for simulator to boot + run: xcrun simctl bootstatus iPhone\ 15 + + - name: Download apps + working-directory: ${{ github.workspace }}/e2e + run: ./download_apps + + - name: Install apps + working-directory: ${{ github.workspace }}/e2e + run: ./install_apps ios + + - name: Start screen recording + run: | + xcrun simctl io booted recordVideo --codec h264 ~/screenrecord.mp4 & + echo $! > ~/screenrecord.pid + + - name: Run tests + working-directory: ${{ github.workspace }}/e2e + timeout-minutes: 120 + run: ./run_tests ios + + - name: Stop screen recording + if: success() || failure() + run: kill -SIGINT "$(cat ~/screenrecord.pid)" + + - name: Upload ~/.maestro artifacts + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: maestro-root-dir-ios + path: ~/.maestro + retention-days: 7 + include-hidden-files: true + + - name: Upload xc test runner logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: xctest_runner_logs + path: ~/Library/Logs/maestro/xctest_runner_logs + retention-days: 7 + include-hidden-files: true + + - name: Upload screen recording of AVD + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: maestro-screenrecord-ios.mp4 + path: ~/screenrecord.mp4 + retention-days: 7 + + test-ios-xctest-runner: + name: Test on iOS (XCTest Runner only) + runs-on: macos-latest + needs: build + timeout-minutes: 30 + + steps: + - name: Clone repository (only needed for the e2e directory) + uses: actions/checkout@v4 + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 8 + + - name: Download Maestro artifact + uses: actions/download-artifact@v4 + with: + name: maestro-cli-jdk11-run_id${{ github.run_id }} + + - name: Download build/Products artifact + uses: actions/download-artifact@v4 + with: + name: build__Products-jdk11 + path: build/Products + + - name: Add Maestro CLI executable to PATH + run: | + unzip maestro.zip -d maestro_extracted + echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH + + - name: Check if Maestro CLI executable starts up + run: | + maestro --help + maestro --version + + - name: Run simulator + run: xcrun simctl boot iPhone\ 15 + + - name: Wait for simulator to boot + run: xcrun simctl bootstatus iPhone\ 15 + + - name: Run tests + timeout-minutes: 15 + run: ./maestro-ios-xctest-runner/test-maestro-ios-runner.sh + + - name: Upload xc test runner logs + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: test-ios-xctest-runner__xctest_runner_logs + path: ~/Library/Logs/maestro/xctest_runner_logs + retention-days: 7 + include-hidden-files: true diff --git a/e2e/install_apps b/e2e/install_apps index 9903efc6bd..f633deb947 100755 --- a/e2e/install_apps +++ b/e2e/install_apps @@ -9,18 +9,23 @@ set -eu [ "$(basename "$PWD")" = "e2e" ] || { echo "must be run from e2e directory" && exit 1; } +platform="${1:-}" +if [ "$platform" != "android" ] && [ "$platform" != "ios" ]; then + echo "usage: $0 " + exit 1 +fi + command -v adb >/dev/null 2>&1 || { echo "adb is required" && exit 1; } for file in ./apps/*; do filename="$(basename "$file")" - echo "install $filename" extension="${file##*.}" - if [ "$extension" = "apk" ]; then + if [ "$platform" = android ] && [ "$extension" = "apk" ]; then + echo "install $filename" adb install -r "$file" >/dev/null || echo "adb: could not install $filename" - elif [ "$extension" = "app" ] && [ "$(uname)" = "Darwin" ]; then + elif [ "$platform" = ios ] && [ "$extension" = "app" ] && [ "$(uname)" = "Darwin" ]; then + echo "install $filename" xcrun simctl install booted "$file" || echo "xcrun: could not install $filename" - else - echo "ignored unsupported file extension $extension" fi done diff --git a/e2e/run_tests b/e2e/run_tests index 672221e476..8c80723238 100755 --- a/e2e/run_tests +++ b/e2e/run_tests @@ -14,13 +14,23 @@ _h1() { } _h2() { - printf "==> $1\n" + printf "==> [$1] $2\n" } _h3() { - printf "===> $1\n" + printf "==> [$1] [$2] => $3\n" } +platform="${1:-}" +if [ "$platform" = "android" ]; then + exclude_tags="ios,ai" +elif [ "$platform" = "ios" ]; then + exclude_tags="android,ai" +else + echo "usage: $0 " + exit 1 +fi + mkfifo pipe trap 'rm -f pipe' EXIT @@ -28,25 +38,24 @@ trap 'rm -f pipe' EXIT for workspace_dir in ./workspaces/*; do WORKSPACE_PASS=true app_name="$(basename "$workspace_dir")" - _h1 "run tests for app $app_name" + _h1 "run tests for app \"$app_name\" on platform \"$platform\"" ### ### Run passing tests ### - _h2 "[$app_name] run passing tests" + _h2 "$app_name" "run passing tests" while IFS= read -r line; do - _h3 "[$app_name] $line" + _h3 "$app_name" "passing" "$line" done < pipe & - 2>&1 maestro test --include-tags passing --exclude-tags ios,ai "$workspace_dir" > pipe || WORKSPACE_PASS=false + maestro --verbose --platform "$platform" test --include-tags passing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 || WORKSPACE_PASS=false if [ "$WORKSPACE_PASS" = "false" ]; then - _h2 "[$app_name] FAIL! Expected to pass, but it failed instead" + _h2 "$app_name" "FAIL! Expected all pass, but at least some failed instead" ALL_PASS=false fi - ### ### Run failing tests ### @@ -56,16 +65,16 @@ for workspace_dir in ./workspaces/*; do fi WORKSPACE_PASS=true # Reset for failing flows - _h2 "[$app_name] run failing tests" + _h2 "$app_name" "run failing tests" while IFS= read -r line; do - _h3 "[$app_name] $line" + _h3 "$app_name" "failing" "$line" done < pipe & - maestro test --include-tags failing --exclude-tags ios,ai "$workspace_dir" > pipe && WORKSPACE_PASS=false + maestro --verbose --platform "$platform" test --include-tags failing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 && WORKSPACE_PASS=false if [ "$WORKSPACE_PASS" = "false" ]; then - _h2 "[$app_name] FAIL! Expected to fail, but it passed instead" + _h2 "$app_name" "FAIL! Expected all to fail, but at least some passed instead" ALL_PASS=false fi done diff --git a/e2e/workspaces/demo_app/ai.yaml b/e2e/workspaces/demo_app/ai.yaml index 82d72b6e3f..ca1336dea0 100644 --- a/e2e/workspaces/demo_app/ai.yaml +++ b/e2e/workspaces/demo_app/ai.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - failing - ai --- diff --git a/e2e/workspaces/demo_app/commands_optional_tournee.yaml b/e2e/workspaces/demo_app/commands_optional_tournee.yaml index 988dd6dc55..a50c3444d0 100644 --- a/e2e/workspaces/demo_app/commands_optional_tournee.yaml +++ b/e2e/workspaces/demo_app/commands_optional_tournee.yaml @@ -1,7 +1,6 @@ # This flow is to ensure that commands with optional flag are not failing the flow. appId: com.example.example tags: - - android - passing --- - launchApp: diff --git a/e2e/workspaces/demo_app/fail_fast.yaml b/e2e/workspaces/demo_app/fail_fast.yaml index 28c91cbb24..ee601ab9ee 100644 --- a/e2e/workspaces/demo_app/fail_fast.yaml +++ b/e2e/workspaces/demo_app/fail_fast.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - failing --- - launchApp: diff --git a/e2e/workspaces/demo_app/fail_not_found.yaml b/e2e/workspaces/demo_app/fail_not_found.yaml index 2bec5b4fae..4fd5a7dd4f 100644 --- a/e2e/workspaces/demo_app/fail_not_found.yaml +++ b/e2e/workspaces/demo_app/fail_not_found.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - failing --- - launchApp: diff --git a/e2e/workspaces/demo_app/fill_form.yaml b/e2e/workspaces/demo_app/fill_form.yaml index 8be6dc1df4..773ff41fbc 100644 --- a/e2e/workspaces/demo_app/fill_form.yaml +++ b/e2e/workspaces/demo_app/fill_form.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - passing --- - launchApp: diff --git a/e2e/workspaces/demo_app/long_input_text.yaml b/e2e/workspaces/demo_app/long_input_text.yaml index 853af71db3..9c2b51e11c 100644 --- a/e2e/workspaces/demo_app/long_input_text.yaml +++ b/e2e/workspaces/demo_app/long_input_text.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - passing --- - launchApp: diff --git a/e2e/workspaces/demo_app/relatives.yaml b/e2e/workspaces/demo_app/relatives.yaml index e3f17df724..56a07d9429 100644 --- a/e2e/workspaces/demo_app/relatives.yaml +++ b/e2e/workspaces/demo_app/relatives.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - passing --- - launchApp: diff --git a/e2e/workspaces/demo_app/swipe.yaml b/e2e/workspaces/demo_app/swipe.yaml index 5551cb6f1d..97a2966fb4 100644 --- a/e2e/workspaces/demo_app/swipe.yaml +++ b/e2e/workspaces/demo_app/swipe.yaml @@ -1,6 +1,5 @@ appId: com.example.example tags: - - android - passing --- - launchApp: diff --git a/e2e/workspaces/nowinandroid/bookmarks.yaml b/e2e/workspaces/nowinandroid/bookmarks.yaml index 43e14bcd9a..758cfe8fc8 100644 --- a/e2e/workspaces/nowinandroid/bookmarks.yaml +++ b/e2e/workspaces/nowinandroid/bookmarks.yaml @@ -1,6 +1,7 @@ appId: com.google.samples.apps.nowinandroid.demo.debug name: Bookmarks tags: + - android - passing --- - launchApp: diff --git a/e2e/workspaces/nowinandroid/fail.yaml b/e2e/workspaces/nowinandroid/fail.yaml index c36d3897cb..c6528c5c76 100644 --- a/e2e/workspaces/nowinandroid/fail.yaml +++ b/e2e/workspaces/nowinandroid/fail.yaml @@ -1,6 +1,7 @@ appId: com.google.samples.apps.nowinandroid.demo.debug name: Fail tags: + - android - failing --- - launchApp: diff --git a/maestro-ios-driver/src/main/kotlin/xcuitest/installer/LocalXCTestInstaller.kt b/maestro-ios-driver/src/main/kotlin/xcuitest/installer/LocalXCTestInstaller.kt index 8033db3c28..8e3894ec1c 100644 --- a/maestro-ios-driver/src/main/kotlin/xcuitest/installer/LocalXCTestInstaller.kt +++ b/maestro-ios-driver/src/main/kotlin/xcuitest/installer/LocalXCTestInstaller.kt @@ -73,6 +73,8 @@ class LocalXCTestInstaller( } override fun start(): XCTestClient? { + logger.info("start()") + if (useXcodeTestRunner) { logger.info("USE_XCODE_TEST_RUNNER is set. Will wait for XCTest runner to be started manually") @@ -112,7 +114,13 @@ class LocalXCTestInstaller( } private fun ensureOpen(): Boolean { - return MaestroTimer.retryUntilTrue(10_000, 200) { isChannelAlive() } + val timeout = 120_000L + logger.info("ensureOpen(): Will spend $timeout ms waiting for the channel to become alive") + val result = MaestroTimer.retryUntilTrue(timeout, 200, onException = { + logger.error("ensureOpen() failed with exception: $it") + }) { isChannelAlive() } + logger.info("ensureOpen() finished, is channel alive?: $result") + return result } private fun xcTestDriverStatusCheck(): Boolean { diff --git a/maestro-ios-xctest-runner/test-maestro-ios-runner.sh b/maestro-ios-xctest-runner/test-maestro-ios-runner.sh new file mode 100755 index 0000000000..3cbe3bf988 --- /dev/null +++ b/maestro-ios-xctest-runner/test-maestro-ios-runner.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$(basename "$PWD")" != "maestro" ]; then + echo "This script must be run from the maestro root directory" + exit 1 +fi + +if [ ! -d ./build/Products/Debug-iphonesimulator/maestro-driver-iosUITests-Runner.app ]; then + echo "XCTest runner app not found in ./build/Products/Debug-iphonesimulator/maestro-driver-iosUITests-Runner.app" + exit 1 +fi + +if [ ! -d ./build/Products/Debug-iphonesimulator/maestro-driver-ios.app ]; then + echo "Dummy test app not found in ./build/Products/Debug-iphonesimulator/maestro-driver-ios.app" + exit 1 +fi + +if [ -z "$(ls ./build/Products/*.xctestrun 2>/dev/null)" ]; then + echo "xctestrun file not found in ./build/Products/" + exit 1 +fi + +echo "Will run the XCTest runner in the background and redirect its output" +mkfifo pipe +trap 'rm -f pipe' EXIT + +while IFS= read -r line; do + printf "==> XCTestRunner: %s\n" "$line" +done < pipe & + +./maestro-ios-xctest-runner/run-maestro-ios-runner.sh 1>pipe 2>&1 & +echo "XCTest runner started in background, PID: $!" + +sleep 5 + +request_successful=false +while [ "$request_successful" = false ]; do + echo "Will curl the /deviceInfo endpoint to check if the XCTest runner is ready" + if ! test_upload_response="$(curl --fail-with-body -sS -X GET "http://localhost:22087/deviceInfo")"; then + echo "Error: failed to GET /deviceInfo endpoint" + echo "$test_upload_response" + echo "Will wait 5 seconds and try again" + sleep 5 + else + request_successful=true + echo "GET /deviceInfo endpoint successful" + fi +done diff --git a/maestro-utils/src/main/kotlin/MaestroTimer.kt b/maestro-utils/src/main/kotlin/MaestroTimer.kt index 951e811b32..4f8d999a72 100644 --- a/maestro-utils/src/main/kotlin/MaestroTimer.kt +++ b/maestro-utils/src/main/kotlin/MaestroTimer.kt @@ -23,7 +23,12 @@ object MaestroTimer { return null } - fun retryUntilTrue(timeoutMs: Long, delayMs: Long? = null, block: () -> Boolean): Boolean { + fun retryUntilTrue( + timeoutMs: Long, + delayMs: Long? = null, + onException: (Exception) -> Unit = {}, + block: () -> Boolean, + ): Boolean { val endTime = System.currentTimeMillis() + timeoutMs do { try { @@ -34,7 +39,7 @@ object MaestroTimer { return true } } catch (ignored: Exception) { - // Try again + onException(ignored) } } while (System.currentTimeMillis() < endTime)