ci: Add mobile UI Tests

build/ci/.azure-pipelines.yml

@@ -66,6 +66,9 @@ stages:
   # Don't trigger this stage if only docs files are changed
   condition: ne(dependencies.Determine_Changes.outputs['evaluate_changes.DetermineChanges.docsOnly'], 'true')
   jobs:
-  - template: stage-build-uitests-wasm.yml
+  # Temporary disabled to make build faster
+  #- template: stage-build-uitests-wasm.yml
+  #- template: stage-build-uitests-android.yml
+  - template: stage-build-uitests-ios.yml
   # Disabled waiting on uno fix: https://github.com/unoplatform/uno/pull/17668
   # - template: stage-build-runtimetests-skia.yml

build/ci/scripts/android-uitest-run.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+# echo commands
+set -x
+
+export UNO_UITEST_SCREENSHOT_PATH=$BUILD_ARTIFACTSTAGINGDIRECTORY/screenshots/android
+export UNO_UITEST_PLATFORM=Android
+export UNO_UITEST_ANDROIDAPK_PATH=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness/bin/Release/net8.0-android/android-x64/com.companyname.TestHarness-Signed.apk
+export UNO_UITEST_PROJECT=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness.UITest
+export UNO_UITEST_ANDROID_PROJECT=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness
+export UNO_UITEST_BINARY=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness.UITest/bin/Release/net47/TestHarness.UITest.dll
+export UNO_EMULATOR_INSTALLED=$BUILD_SOURCESDIRECTORY/build/.emulator_started
+export UITEST_TEST_TIMEOUT=60m
+export ANDROID_SIMULATOR_APILEVEL=28
+
+# Override Android SDK tooling
+export ANDROID_HOME=$BUILD_SOURCESDIRECTORY/build/android-sdk
+export ANDROID_SDK_ROOT=$BUILD_SOURCESDIRECTORY/build/android-sdk
+export CMDLINETOOLS=commandlinetools-mac-8512546_latest.zip
+mkdir -p $ANDROID_HOME
+wget https://dl.google.com/android/repository/$CMDLINETOOLS
+unzip $CMDLINETOOLS -d $ANDROID_HOME/cmdline-tools
+rm $CMDLINETOOLS
+mv $ANDROID_SDK_ROOT/cmdline-tools/cmdline-tools $ANDROID_SDK_ROOT/cmdline-tools/latest
+
+if [[ -f $ANDROID_HOME/platform-tools/platform-tools/adb ]]
+then
+	# It appears that the platform-tools 29.0.6 are extracting into an incorrect path
+	mv $ANDROID_HOME/platform-tools/platform-tools/* $ANDROID_HOME/platform-tools
+fi
+
+# Install Android SDK emulators and SDKs
+if [ ! -f "$UNO_EMULATOR_INSTALLED" ];
+then
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install 'tools'| tr '\r' '\n' | uniq
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install 'platform-tools'  | tr '\r' '\n' | uniq
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install 'build-tools;34.0.0' | tr '\r' '\n' | uniq
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install "platforms;android-$ANDROID_SIMULATOR_APILEVEL" | tr '\r' '\n' | uniq
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install 'platforms;android-34' | tr '\r' '\n' | uniq
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install 'extras;android;m2repository' | tr '\r' '\n' | uniq
+
+	# Install AVD files
+	echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install "system-images;android-$ANDROID_SIMULATOR_APILEVEL;google_apis_playstore;x86_64"
+
+	# Create emulator
+	echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager create avd -n xamarin_android_emulator --abi "x86_64" -k "system-images;android-$ANDROID_SIMULATOR_APILEVEL;google_apis_playstore;x86_64" --force
+
+	echo $ANDROID_HOME/emulator/emulator -list-avds
+
+	echo "Starting emulator"
+
+	# Start emulator in background
+	nohup $ANDROID_HOME/emulator/emulator -avd xamarin_android_emulator -skin 1280x800 -memory 4096 -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim > /dev/null 2>&1 &
+
+	touch "$UNO_EMULATOR_INSTALLED"
+fi
+
+# Build the sample, while the emulator is starting
+cd $UNO_UITEST_ANDROID_PROJECT
+dotnet publish -f net8.0-android -p:UnoTargetFrameworkOverride=net8.0-android -c Release -p:RuntimeIdentifier=android-x64 /p:IsUiAutomationMappingEnabled=True /p:AndroidUseSharedRuntime=false /p:AotAssemblies=false  -bl:$BUILD_ARTIFACTSTAGINGDIRECTORY/android-app.binlog
+
+cd $BUILD_SOURCESDIRECTORY/build
+
+mkdir -p $UNO_UITEST_SCREENSHOT_PATH
+
+cp $UNO_UITEST_ANDROIDAPK_PATH $UNO_UITEST_SCREENSHOT_PATH
+
+# Wait for the emulator to finish booting
+$ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done; input keyevent 82'
+
+$ANDROID_HOME/platform-tools/adb devices
+
+echo "Emulator started"
+
+cd $UNO_UITEST_PROJECT
+
+dotnet test \
+	-c Release \
+	-l:"console;verbosity=normal" \
+	--logger "nunit;LogFileName=$BUILD_SOURCESDIRECTORY/build/TestResult.xml" \
+	--blame-hang-timeout $UITEST_TEST_TIMEOUT \
+	-v m \
+	|| true

build/ci/scripts/ios-uitest-run.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+export UNO_UITEST_PLATFORM=iOS
+export UNO_UITEST_IOSBUNDLE_PATH=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness/bin/Release/net8.0-ios/iossimulator-x64/TestHarnessApp.app
+export UNO_UITEST_SCREENSHOT_PATH=$BUILD_ARTIFACTSTAGINGDIRECTORY/screenshots/ios
+export UNO_UITEST_PROJECT=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness.UITest
+export UNO_UITEST_LOGFILE=$BUILD_ARTIFACTSTAGINGDIRECTORY/screenshots/ios/nunit-log.txt
+export UNO_UITEST_IOS_PROJECT=$BUILD_SOURCESDIRECTORY/testing/TestHarness/TestHarness
+export UITEST_TEST_TIMEOUT=90m
+export UNO_UITEST_SIMULATOR_VERSION="com.apple.CoreSimulator.SimRuntime.iOS-17-2"
+export UNO_UITEST_SIMULATOR_NAME="iPad Pro (12.9-inch) (6th generation)"
+
+echo "Listing iOS simulators"
+xcrun simctl list devices --json
+
+/Applications/Xcode.app/Contents/Developer/Applications/Simulator.app/Contents/MacOS/Simulator &
+
+# Prime the output directory
+mkdir -p $UNO_UITEST_SCREENSHOT_PATH/_logs
+
+cd $UNO_UITEST_IOS_PROJECT
+dotnet build -p:UnoTargetFrameworkOverride=net8.0-ios -r iossimulator-x64 -c Release -p:IsUiAutomationMappingEnabled=True -bl:$BUILD_ARTIFACTSTAGINGDIRECTORY/ios-app.binlog
+
+##
+## Pre-install the application to avoid https://github.com/microsoft/appcenter/issues/2389
+##
+
+## Install iOS 17.2 simulators
+xcodes runtimes install --keep-archive 'iOS 17.2' || true
+
+# Wait while ios runtime 17.2 is not having simulators. The install process may
+# take a few seconds and "simctl list devices" may not return devices.
+while true; do
+	export UITEST_IOSDEVICE_ID=`xcrun simctl list -j | jq -r --arg sim "$UNO_UITEST_SIMULATOR_VERSION" --arg name "$UNO_UITEST_SIMULATOR_NAME" '.devices[$sim] | .[] | select(.name==$name) | .udid'`
+	export UITEST_IOSDEVICE_DATA_PATH=`xcrun simctl list -j | jq -r --arg sim "$UNO_UITEST_SIMULATOR_VERSION" --arg name "$UNO_UITEST_SIMULATOR_NAME" '.devices[$sim] | .[] | select(.name==$name) | .dataPath'`
+
+	if [ -n "$UITEST_IOSDEVICE_ID" ]; then
+		break
+	fi
+
+	echo "Waiting for the simulator to be available"
+	sleep 5
+done
+
+echo "Simulator Data Path: $UITEST_IOSDEVICE_DATA_PATH"
+cp "$UITEST_IOSDEVICE_DATA_PATH/../device.plist" $UNO_UITEST_SCREENSHOT_PATH/_logs
+
+echo "Starting simulator: [$UITEST_IOSDEVICE_ID] ($UNO_UITEST_SIMULATOR_VERSION / $UNO_UITEST_SIMULATOR_NAME)"
+
+# check for the presence of idb, and install it if it's not present
+export PATH=$PATH:~/.local/bin
+
+if ! command -v idb &> /dev/null
+then
+	echo "Installing idb"
+	brew install pipx
+	# # https://github.com/microsoft/appcenter/issues/2605#issuecomment-1854414963
+	brew tap facebook/fb
+	brew install idb-companion
+	pipx install fb-idb
+else
+	echo "Using idb from:" `command -v idb`
+fi
+
+xcrun simctl boot "$UITEST_IOSDEVICE_ID" || true
+
+idb install --udid "$UITEST_IOSDEVICE_ID" "$UNO_UITEST_IOSBUNDLE_PATH"
+
+
+# Run the tests
+
+cd $UNO_UITEST_PROJECT
+
+dotnet test \
+	-c Release \
+	-l:"console;verbosity=normal" \
+	--logger:"nunit;LogFileName=$BUILD_SOURCESDIRECTORY/build/TestResult.xml" \
+	--blame-hang-timeout $UITEST_TEST_TIMEOUT \
+	-v m \
+	|| true
+
+
+cp $BUILD_SOURCESDIRECTORY/build/TestResult.xml" $UNO_UITEST_SCREENSHOT_PATH
+
+# export the simulator logs
+export LOG_FILEPATH=$UNO_UITEST_SCREENSHOT_PATH/_logs
+export TMP_LOG_FILEPATH=/tmp/DeviceLog-`date +"%Y%m%d%H%M%S"`.logarchive
+export LOG_FILEPATH_FULL=$LOG_FILEPATH/DeviceLog-`date +"%Y%m%d%H%M%S"`.txt
+
+# xcrun simctl spawn booted log collect --output $TMP_LOG_FILEPATH
+# log show --style syslog $TMP_LOG_FILEPATH > $LOG_FILEPATH_FULL

build/ci/stage-build-uitests-android.yml

@@ -0,0 +1,43 @@
+jobs:
+- job: Android_Tests
+  displayName: 'Android UI Tests'
+  timeoutInMinutes: 90
+  variables:
+    CI_Build: true
+    SourceLinkEnabled: false
+
+  pool:
+    vmImage: 'macos-14'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - template: templates/dotnet-install-mac.yml
+  - template: templates/canary-updater.yml
+
+  - bash: |
+      chmod +x $(build.sourcesdirectory)/build/ci/scripts/android-uitest-run.sh
+      $(build.sourcesdirectory)/build/ci/scripts/android-uitest-run.sh
+    displayName: Build and Run Android Tests
+    env:
+      BUILD_SOURCESDIRECTORY: "$(build.sourcesdirectory)"
+      BUILD_ARTIFACTSTAGINGDIRECTORY: "$(build.artifactstagingdirectory)"
+
+  - task: PublishTestResults@2
+    condition: always()
+    retryCountOnTaskFailure: 2
+    inputs:
+      testRunTitle: 'Android Test Run'
+      testResultsFormat: 'NUnit'
+      testResultsFiles: '$(build.sourcesdirectory)/build/TestResult.xml'
+      failTaskOnFailedTests: true
+      failTaskOnMissingResultsFile: true
+
+  - task: PublishBuildArtifacts@1
+    condition: always()
+    retryCountOnTaskFailure: 2
+    inputs:
+      PathtoPublish: $(build.artifactstagingdirectory)
+      ArtifactName: uno-uitest-tests
+      ArtifactType: Container

build/ci/stage-build-uitests-ios.yml

@@ -0,0 +1,56 @@
+jobs:
+- job: iOS_Tests
+  displayName: 'iOS UI Tests'
+  timeoutInMinutes: 90
+  variables:
+    CI_Build: true
+    SourceLinkEnabled: false
+
+  pool:
+    vmImage: 'macos-14'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - template: templates/dotnet-install-mac.yml
+
+  - template: templates/xcode-select.yml
+    parameters:
+      xCodeRoot: '/Applications/Xcode_15.2.app'
+
+  - template: templates/canary-updater.yml
+
+  - bash: |
+      chmod +x $(build.sourcesdirectory)/build/ci/scripts/ios-uitest-run.sh
+      $(build.sourcesdirectory)/build/ci/scripts/ios-uitest-run.sh
+    displayName: Build and Run iOS Test
+    env:
+      BUILD_SOURCESDIRECTORY: "$(build.sourcesdirectory)"
+      BUILD_ARTIFACTSTAGINGDIRECTORY: "$(build.artifactstagingdirectory)"
+
+  - task: PublishTestResults@2
+    condition: always()
+    retryCountOnTaskFailure: 2
+    inputs:
+      testRunTitle: 'iOS Test Run'
+      testResultsFormat: 'NUnit'
+      testResultsFiles: '$(build.sourcesdirectory)/build/TestResult.xml'
+      failTaskOnFailedTests: true
+      failTaskOnMissingResultsFile: true
+
+  - task: PublishBuildArtifacts@1
+    condition: always()
+    retryCountOnTaskFailure: 2
+    inputs:
+      PathtoPublish: $(build.artifactstagingdirectory)
+      ArtifactName: uno-uitest-tests
+      ArtifactType: Container
+
+  - task: PublishBuildArtifacts@1
+    condition: always()
+    retryCountOnTaskFailure: 2
+    inputs:
+      PathtoPublish: $(build.sourcesdirectory)/testing/TestHarness/TestHarness/bin/Release/net8.0-ios/iossimulator-x64/TestHarnessApp.app
+      ArtifactName: uno-uitest-ios-bin
+      ArtifactType: Container

build/ci/stage-build-uitests-wasm.yml

@@ -36,6 +36,7 @@ jobs:
       testResultsFormat: 'NUnit'
       testResultsFiles: '$(build.sourcesdirectory)/build/TestResult.xml'
       failTaskOnFailedTests: true
+      failTaskOnMissingResultsFile: true
 
   - task: PublishBuildArtifacts@1
     condition: always()

build/ci/templates/canary-updater.yml

@@ -14,3 +14,21 @@ steps:
   - template: canary-update-template.yml
     parameters:
       SolutionToUpdate: 'testing/TestHarness/TestHarness.sln'
+
+  - pwsh: |
+      gci -r -include "info.plist" |
+      foreach-object {
+          $a = $_.fullname; ( get-content $a ) |
+          foreach-object { $_ -replace "com.companyname.TestHarness","com.companyname.TestHarness-canary" } |
+          set-content $a }
+    displayName: 'Adjust info.plist bundle name'
+    condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/canaries')
+
+  - pwsh: |
+      gci -r -include AndroidManifest.xml,android-uitest-run.sh |
+      foreach-object {
+          $a = $_.fullname; ( get-content $a ) |
+          foreach-object { $_ -replace "com.companyname.TestHarness","com.companyname.TestHarness.canary" } |
+          set-content $a }
+    displayName: 'Adjust AndroidManifest bundle name'
+    condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/canaries')

build/ci/templates/xcode-select.yml

@@ -0,0 +1,10 @@
+parameters:
+  xCodeRoot: ''
+
+steps:
+  - bash: |
+      echo 'Xcode Root to ${{parameters.xCodeRoot}}'
+      echo '##vso[task.setvariable variable=MD_APPLE_SDK_ROOT;]'${{parameters.xCodeRoot}}
+      sudo xcode-select --switch ${{parameters.xCodeRoot}}/Contents/Developer
+
+    displayName: Select Xcode