Add E2E test for iOS (#1885)

As of now, it's disabled because of a bug that causes it to always fail on GHA. See PR for more.

* LocalXCTestInstaller: increase ensureOpen() timeout to 60 seconds

* add test that makes requests to standalone XCTest runner

.github/workflows/test-e2e.yaml

@@ -32,18 +32,26 @@ jobs:
       - name: Build Maestro CLI
         run: ./gradlew :maestro-cli:distZip
 
-      - uses: actions/upload-artifact@v4
+      - name: Upload zipped Maestro CLI artifact
+        uses: actions/upload-artifact@v4
         with:
           name: maestro-cli-jdk${{ matrix.java-version }}-run_id${{ github.run_id }}
           path: maestro-cli/build/distributions/maestro.zip
           retention-days: 1
 
-  test-local-android:
+      - name: Upload build/Products to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build__Products-jdk${{ matrix.java-version }}
+          path: build/Products
+          retention-days: 1
+
+  test-android:
     name: Test on Android
-    timeout-minutes: 30
     runs-on: ubuntu-latest
     needs: build
-
+    timeout-minutes: 60
+
     env:
       ANDROID_HOME: /home/runner/androidsdk
       ANDROID_SDK_ROOT: /home/runner/androidsdk
@@ -148,27 +156,30 @@ jobs:
 
       - name: Install apps
         working-directory: ${{ github.workspace }}/e2e
-        run: ./install_apps
+        run: ./install_apps android
 
       - name: Start screen recording of AVD
         run: |
           adb shell screenrecord /sdcard/screenrecord.mp4 &
+          echo $! > ~/screenrecord.pid
 
       - name: Run tests
         working-directory: ${{ github.workspace }}/e2e
-        run: ./run_tests
+        timeout-minutes: 10
+        run: ./run_tests android
 
       - name: Stop screen recording of AVD
+        if: success() || failure()
         run: |
-          adb shell pkill -SIGINT screenrecord || echo "exited with code $?" && exit 0
+          kill -SIGINT "$(cat ~/screenrecord.pid)" || echo "failed to kill screenrecord: code $?" && exit 0
           sleep 5 # prevent video file corruption
-          adb pull /sdcard/screenrecord.mp4 ~/maestro_screenrecord.mp4
+          adb pull /sdcard/screenrecord.mp4 ~/screenrecord.mp4
 
       - name: Upload ~/.maestro artifacts
         uses: actions/upload-artifact@v4
         if: success() || failure()
         with:
-          name: maestro-e2e-output
+          name: maestro-root-dir-android
           path: ~/.maestro
           retention-days: 7
           include-hidden-files: true
@@ -177,6 +188,152 @@ jobs:
         uses: actions/upload-artifact@v4
         if: success() || failure()
         with:
-          name: maestro_screenrecord.mp4
-          path: ~/maestro_screenrecord.mp4
+          name: maestro-screenrecord-android.mp4
+          path: ~/screenrecord.mp4
           retention-days: 7
+
+  test-ios:
+    name: Test on iOS
+    runs-on: macos-latest
+    needs: build
+    timeout-minutes: 120
+    if: ${{ false }}
+
+    env:
+      MAESTRO_DRIVER_STARTUP_TIMEOUT: 240000 # 240s
+      MAESTRO_CLI_LOG_PATTERN_CONSOLE: '%d{HH:mm:ss.SSS} [%5level] %logger.%method: %msg%n'
+
+    steps:
+      - name: Clone repository (only needed for the e2e directory)
+        uses: actions/checkout@v4
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: maestro-cli-jdk11-run_id${{ github.run_id }}
+
+      - name: Add Maestro CLI executable to PATH
+        run: |
+          unzip maestro.zip -d maestro_extracted
+          echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH
+
+      - name: Check if Maestro CLI executable starts up
+        run: |
+          maestro --help
+          maestro --version
+
+      - name: Run simulator
+        run: xcrun simctl boot iPhone\ 15
+
+      - name: Wait for simulator to boot
+        run: xcrun simctl bootstatus iPhone\ 15
+
+      - name: Download apps
+        working-directory: ${{ github.workspace }}/e2e
+        run: ./download_apps
+
+      - name: Install apps
+        working-directory: ${{ github.workspace }}/e2e
+        run: ./install_apps ios
+
+      - name: Start screen recording
+        run: |
+          xcrun simctl io booted recordVideo --codec h264 ~/screenrecord.mp4 &
+          echo $! > ~/screenrecord.pid
+
+      - name: Run tests
+        working-directory: ${{ github.workspace }}/e2e
+        timeout-minutes: 120
+        run: ./run_tests ios
+
+      - name: Stop screen recording
+        if: success() || failure()
+        run: kill -SIGINT "$(cat ~/screenrecord.pid)"
+
+      - name: Upload ~/.maestro artifacts
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: maestro-root-dir-ios
+          path: ~/.maestro
+          retention-days: 7
+          include-hidden-files: true
+
+      - name: Upload xc test runner logs
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: xctest_runner_logs
+          path: ~/Library/Logs/maestro/xctest_runner_logs
+          retention-days: 7
+          include-hidden-files: true
+
+      - name: Upload screen recording of AVD
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: maestro-screenrecord-ios.mp4
+          path: ~/screenrecord.mp4
+          retention-days: 7
+
+  test-ios-xctest-runner:
+    name: Test on iOS (XCTest Runner only)
+    runs-on: macos-latest
+    needs: build
+    timeout-minutes: 30
+
+    steps:
+      - name: Clone repository (only needed for the e2e directory)
+        uses: actions/checkout@v4
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: 8
+
+      - name: Download Maestro artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: maestro-cli-jdk11-run_id${{ github.run_id }}
+
+      - name: Download build/Products artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: build__Products-jdk11
+          path: build/Products
+
+      - name: Add Maestro CLI executable to PATH
+        run: |
+          unzip maestro.zip -d maestro_extracted
+          echo "$PWD/maestro_extracted/maestro/bin" >> $GITHUB_PATH
+
+      - name: Check if Maestro CLI executable starts up
+        run: |
+          maestro --help
+          maestro --version
+
+      - name: Run simulator
+        run: xcrun simctl boot iPhone\ 15
+
+      - name: Wait for simulator to boot
+        run: xcrun simctl bootstatus iPhone\ 15
+
+      - name: Run tests
+        timeout-minutes: 15
+        run: ./maestro-ios-xctest-runner/test-maestro-ios-runner.sh
+
+      - name: Upload xc test runner logs
+        uses: actions/upload-artifact@v4
+        if: success() || failure()
+        with:
+          name: test-ios-xctest-runner__xctest_runner_logs
+          path: ~/Library/Logs/maestro/xctest_runner_logs
+          retention-days: 7
+          include-hidden-files: true

e2e/install_apps

@@ -9,18 +9,23 @@ set -eu
 
 [ "$(basename "$PWD")" = "e2e" ] || { echo "must be run from e2e directory" && exit 1; }
 
+platform="${1:-}"
+if [ "$platform" != "android" ] && [ "$platform" != "ios" ]; then
+	echo "usage: $0 <android|ios>"
+	exit 1
+fi
+
 command -v adb >/dev/null 2>&1 || { echo "adb is required" && exit 1; }
 
 for file in ./apps/*; do
 	filename="$(basename "$file")"
-	echo "install $filename"
 
 	extension="${file##*.}"
-	if [ "$extension" = "apk" ]; then
+	if [ "$platform" = android ] && [ "$extension" = "apk" ]; then
+		echo "install $filename"
 		adb install -r "$file" >/dev/null || echo "adb: could not install $filename"
-	elif [ "$extension" = "app" ] && [ "$(uname)" = "Darwin" ]; then
+	elif [ "$platform" = ios ] && [ "$extension" = "app" ] && [ "$(uname)" = "Darwin" ]; then
+		echo "install $filename"
 		xcrun simctl install booted "$file" || echo "xcrun: could not install $filename"
-	else
-		echo "ignored unsupported file extension $extension"
 	fi
 done

e2e/run_tests

@@ -14,39 +14,48 @@ _h1() {
 }
 
 _h2() {
-	printf "==> $1\n"
+	printf "==> [$1] $2\n"
 }
 
 _h3() {
-	printf "===> $1\n"
+	printf "==> [$1] [$2] => $3\n"
 }
 
+platform="${1:-}"
+if [ "$platform" = "android" ]; then
+	exclude_tags="ios,ai"
+elif [ "$platform" = "ios" ]; then
+	exclude_tags="android,ai"
+else
+	echo "usage: $0 <android|ios>"
+	exit 1 
+fi
+
 mkfifo pipe
 trap 'rm -f pipe' EXIT
 
 # Run passing tests
 for workspace_dir in ./workspaces/*; do
 	WORKSPACE_PASS=true
 	app_name="$(basename "$workspace_dir")"
-	_h1 "run tests for app $app_name"
+	_h1 "run tests for app \"$app_name\" on platform \"$platform\""
 
 	###
 	### Run passing tests
 	###
-	_h2 "[$app_name] run passing tests"
+	_h2 "$app_name" "run passing tests"
 
 	while IFS= read -r line; do
-	    _h3 "[$app_name] $line"
+	    _h3 "$app_name" "passing" "$line"
 	done < pipe &
 
-	2>&1 maestro test --include-tags passing --exclude-tags ios,ai "$workspace_dir" > pipe || WORKSPACE_PASS=false
+	maestro --verbose --platform "$platform" test --include-tags passing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 || WORKSPACE_PASS=false
 
 	if [ "$WORKSPACE_PASS" = "false" ]; then
-        _h2 "[$app_name] FAIL! Expected to pass, but it failed instead"
+        _h2 "$app_name" "FAIL! Expected all pass, but at least some failed instead"
 		ALL_PASS=false
 	fi
 
-
 	###
 	### Run failing tests
 	###
@@ -56,16 +65,16 @@ for workspace_dir in ./workspaces/*; do
 	fi
 
 	WORKSPACE_PASS=true # Reset for failing flows
-	_h2 "[$app_name] run failing tests"
+	_h2 "$app_name" "run failing tests"
 
 	while IFS= read -r line; do
-	    _h3 "[$app_name] $line"
+	    _h3 "$app_name" "failing" "$line"
 	done < pipe &
 
-	maestro test --include-tags failing --exclude-tags ios,ai "$workspace_dir" > pipe && WORKSPACE_PASS=false
+	maestro --verbose --platform "$platform" test --include-tags failing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 && WORKSPACE_PASS=false
 
 	if [ "$WORKSPACE_PASS" = "false" ]; then
-		_h2 "[$app_name] FAIL! Expected to fail, but it passed instead"
+		_h2 "$app_name" "FAIL! Expected all to fail, but at least some passed instead"
 		ALL_PASS=false
     fi
 done

e2e/workspaces/demo_app/ai.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - failing
     - ai
 ---

e2e/workspaces/demo_app/commands_optional_tournee.yaml

@@ -1,7 +1,6 @@
 # This flow is to ensure that commands with optional flag are not failing the flow.
 appId: com.example.example
 tags:
-    - android
     - passing
 ---
 - launchApp:

e2e/workspaces/demo_app/fail_fast.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - failing
 ---
 - launchApp:

e2e/workspaces/demo_app/fail_not_found.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - failing
 ---
 - launchApp:

e2e/workspaces/demo_app/fill_form.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - passing
 ---
 - launchApp:

e2e/workspaces/demo_app/long_input_text.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - passing
 ---
 - launchApp:

e2e/workspaces/demo_app/relatives.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - passing
 ---
 - launchApp:

e2e/workspaces/demo_app/swipe.yaml

@@ -1,6 +1,5 @@
 appId: com.example.example
 tags:
-    - android
     - passing
 ---
 - launchApp:

e2e/workspaces/nowinandroid/bookmarks.yaml

@@ -1,6 +1,7 @@
 appId: com.google.samples.apps.nowinandroid.demo.debug
 name: Bookmarks
 tags:
+  - android
   - passing
 ---
 - launchApp: