PoC gui test automation app launch test

Create generic keyword for launching and closing
any app with gui test automation.
Add example test cases for chromium and firefox.

Signed-off-by: Samuli Leivo <[email protected]>

Robot-Framework/lib/gui_testing.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+from pyscreeze import locate, center
+import logging
+
+
+def locate_image(image):
+    screenshot = "./screenshot.png"
+    image_box = locate(image, screenshot)
+    image_center = center(image_box)
+    logging.info(image_box)
+    logging.info(image_center)
+    image_center_in_mouse_coordinates = convert_resolution(image_center)
+    logging.info(image_center_in_mouse_coordinates)
+    return image_center_in_mouse_coordinates
+
+def convert_resolution(coordinates):
+    # Screenshot image resolution is 1920x1200 but ydotool mouse movement resolution was tested to be 960x600.
+    # Let's have hard coded resolution scaling factor for now.
+    # However, if ghaf resolution is changed this might not work anymore.
+    scaling_factor = 2
+    mouse_coordinates = {
+        'x': coordinates[0] / scaling_factor,
+        'y': coordinates[1] / scaling_factor
+    }
+    return mouse_coordinates

Robot-Framework/resources/common_keywords.resource

@@ -19,8 +19,13 @@ Check that the application was started
     Should Not Be Empty  ${app_pids}  ${app_name} is not started
     Log To Console       ${app_name} is started
 
+Check that the application is not running
+    [Arguments]          ${app_name}
+    ${found_status}  ${pids}  Run Keyword And Ignore Error   Find pid by name    ${app_name}
+    Should Be Empty   ${pids}
+
 Check If Ping Fails
     [Documentation]  Check that ping is not getting response from host
     # ${out}   Run and Return RC   ping ${DEVICE_IP_ADDRESS} -c 1
     ${result}  Run Process  ping  ${DEVICE_IP_ADDRESS}  -c1  timeout=1s
-    Should Not Be Equal   ${result.rc}  ${0}
+    Should Not Be Equal   ${result.rc}  ${0}

Robot-Framework/test-suites/bat-tests/apps.robot

@@ -7,6 +7,9 @@ Force Tags          apps
 Resource            ../../resources/ssh_keywords.resource
 Resource            ../../config/variables.robot
 Resource            ../../resources/common_keywords.resource
+Library             ../../lib/gui_testing.py
+Library             Collections
+Library             BuiltIn
 Suite Teardown      Close All Connections
 
 

Robot-Framework/test-suites/bat-tests/gui-tests.robot

@@ -0,0 +1,139 @@
+# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII)
+# SPDX-License-Identifier: Apache-2.0
+
+*** Settings ***
+Documentation       Testing launching applications
+Force Tags          gui
+Resource            ../../resources/ssh_keywords.resource
+Resource            ../../config/variables.robot
+Resource            ../../resources/common_keywords.resource
+Library             ../../lib/gui_testing.py
+Library             Collections
+Suite Teardown      Close All Connections
+
+
+*** Variables ***
+@{app_pids}         ${EMPTY}
+
+
+*** Test Cases ***
+
+Start and close chromium via GUI on LenovoX1
+    [Documentation]   Start Chromium via GUI test automation and verify related process started
+    ...               Close Chromium via GUI test automation and verify related process stopped
+    [Tags]            bat   SP-T97-1   lenovo-x1
+    Start and close app via GUI on LenovoX1   chromium-vm  chromium
+
+Start Firefox via GUI
+    [Documentation]   Passing this test requires that display is connected to the target device
+    ...
+    ...               Start Firefox via GUI test automation and verify related process started
+    ...               Close Firefox via GUI test automation and verify related process stopped
+    [Tags]            SP-T45-1  experimental
+    Start and close app via GUI   firefox   agx
+
+
+*** Keywords ***
+
+Start and close app via GUI on LenovoX1
+    [Documentation]    Start Application via GUI test automation and verify related process started
+    ...                Close Application via GUI test automation and verify related process stopped
+    [Arguments]        ${app-vm}=chromium-vm
+    ...                ${app}=chromium
+    ...                ${launch_icon}=../gui-ref-images/${app}/launch_icon.png
+    ...                ${close_button}=../gui-ref-images/${app}/close_button.png
+
+    Verify service status  range=15  service=microvm@${app-vm}.service  expected_status=active  expected_state=running
+    Connect to netvm
+    Connect to VM       ${GUI_VM}
+    Check if ssh is ready on vm    ${app-vm}
+
+    Run Keyword And Ignore Error  Execute Command   -b /run/current-system/sw/bin/ydotoold --socket-path /tmp/.ydotool_socket  sudo=True  sudo_password=${PASSWORD}  timeout=3
+
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ../gui-ref-images/start_menu_x1.png
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+    BuiltIn.Sleep   1
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${launch_icon}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+    # Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Execute Command   ydotool mousemove --absolute -x 10 -y 10  sudo=True  sudo_password=${PASSWORD}
+
+    Connect to VM       ${app-vm}
+    Check that the application was started    ${app}
+
+    Connect to VM      ${GUI_VM}
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${close_button}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+    # Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Execute Command   ydotool mousemove --absolute -x 10 -y 10  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+    Connect to VM       ${app-vm}
+    Check that the application is not running    ${app}
+
+    # In case closing the app via GUI failed
+    [Teardown]  Kill process  @{app_pids}
+    ...         Connect to VM     ${GUI_VM}
+    ...         Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+
+Start and close app via GUI
+    [Documentation]    Start Application via GUI test automation and verify related process started
+    ...                Close Application via GUI test automation and verify related process stopped
+    ...                Only for ghaf builds where desktop is running on ghaf-host
+    [Arguments]        ${app}=firefox   ${target}=agx
+    ...                ${launch_icon}=../gui-ref-images/${app}/launch_icon.png
+    ...                ${close_button}=../gui-ref-images/${app}/close_button.png
+
+    Connect
+
+    Execute Command   export YDOTOOL_SOCKET=/tmp/.ydotools_socket
+    Run Keyword And Ignore Error  Execute Command   -b /run/current-system/sw/bin/ydotoold  sudo=True  sudo_password=${PASSWORD}  timeout=3
+
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ../gui-ref-images/start_menu_${target}.png
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+    BuiltIn.Sleep   1
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${launch_icon}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+    # Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Execute Command   ydotool mousemove --absolute -x 10 -y 10  sudo=True  sudo_password=${PASSWORD}
+
+    BuiltIn.Sleep  1
+    Check that the application was started    ${app}
+
+    ${mouse_x}  ${mouse_y}  Locate image on screen  ${close_button}
+    Execute Command   ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y}  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   ydotool click 0xC0  sudo=True  sudo_password=${PASSWORD}
+
+    # Move the cursor to the upper left corner so that it will not block searching further gui screenshots
+    Execute Command   ydotool mousemove --absolute -x 10 -y 10  sudo=True  sudo_password=${PASSWORD}
+    Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+    Check that the application is not running    ${app}
+
+    # In case closing the app via GUI failed
+    [Teardown]  Kill process  @{app_pids}
+    ...         Execute Command   pkill ydotoold  sudo=True  sudo_password=${PASSWORD}
+
+Locate image on screen
+    [Documentation]    Take a screenshot. Locate given image on the screenshot.
+    ...                Return center coordinates of the image in mouse coordinate system
+    [Arguments]        ${image_to_be_searched}
+    Log To Console    Taking screenshot
+    Execute Command   grim screenshot.png
+    Log To Console    Saving screenshot
+    SSHLibrary.Get File  screenshot.png   screenshot.png
+    Log To Console    Locating image on screenshot
+    ${coordinates}  Locate image  ${image_to_be_searched}
+    Log To Console    Coordinates: ${coordinates}
+    ${mouse_x}  Get From Dictionary   ${coordinates}  x
+    ${mouse_y}  Get From Dictionary   ${coordinates}  y
+    RETURN  ${mouse_x}  ${mouse_y}

flake.nix

@@ -56,6 +56,7 @@
               python-kasa
               pytz
               pandas
+              pyscreeze
             ]))
         ];
       };