From 9a8003f43a4d413e492ec5d3c0251bbb081a1261 Mon Sep 17 00:00:00 2001 From: Samuli Leivo Date: Thu, 19 Sep 2024 10:20:26 +0300 Subject: [PATCH] PoC gui test automation app launch test Create generic keyword for launching and closing any app with gui test automation. Add example test cases for chromium and firefox. Signed-off-by: Samuli Leivo --- .../gui-ref-images/chromium/close_button.png | Bin 0 -> 270 bytes .../gui-ref-images/chromium/launch_icon.png | Bin 0 -> 1224 bytes .../gui-ref-images/firefox/close_button.png | Bin 0 -> 349 bytes .../gui-ref-images/firefox/launch_icon.png | Bin 0 -> 1209 bytes .../gui-ref-images/start_menu_agx.png | Bin 0 -> 872 bytes .../gui-ref-images/start_menu_x1.png | Bin 0 -> 878 bytes Robot-Framework/lib/gui_testing.py | 27 ++++ .../resources/common_keywords.resource | 7 +- .../test-suites/bat-tests/apps.robot | 3 + .../test-suites/bat-tests/gui-tests.robot | 139 ++++++++++++++++++ flake.nix | 1 + 11 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 Robot-Framework/gui-ref-images/chromium/close_button.png create mode 100644 Robot-Framework/gui-ref-images/chromium/launch_icon.png create mode 100644 Robot-Framework/gui-ref-images/firefox/close_button.png create mode 100644 Robot-Framework/gui-ref-images/firefox/launch_icon.png create mode 100644 Robot-Framework/gui-ref-images/start_menu_agx.png create mode 100644 Robot-Framework/gui-ref-images/start_menu_x1.png create mode 100644 Robot-Framework/lib/gui_testing.py create mode 100644 Robot-Framework/test-suites/bat-tests/gui-tests.robot diff --git a/Robot-Framework/gui-ref-images/chromium/close_button.png b/Robot-Framework/gui-ref-images/chromium/close_button.png new file mode 100644 index 0000000000000000000000000000000000000000..3b2fda13a72b0bdfd9ae86578bd6a958b12833a2 GIT binary patch literal 270 zcmeAS@N?(olHy`uVBq!ia0vp^qChOp!2~3;_f9?qq&N#aB8wRq_>O=u<5X=vX`tW% zPZ!4!i{50PwA9oE3+XW@g8S z*)3%{yW5rqzo&oBeCK@UKj-`BoFM~)SMkU}J$lLiO_f>6b~phnpUz#C(JW9jG3&J(W(Gcg-V9V zK#JLwqGf>qP!;m>Q}nOb%saV;!lX&%?A#`OKA2!=0-$W*)Or2*IlWMJAIrm&{r&9T zK9*+ypjyQbe;Ie@-N|&~nRi`ZYznbQ1`{5E29V8L9~>|Jc*W4oMl*zN;?k5kHfi>4 zVLUt`1o0r5$yyb|h3sBAgy7?YF?(S%n)Tk-#fjP09sw|!t_~e5W;6?cz>-}l;!2}f z&Sh)&^m@BHBmiXt!zWAjD62$9vksps*(pYRbW(Lmj0(s6jwV8I=#`)ypBU3`++9_c z%Q11MdioFDj&~$n2$a|7k>m%eqB9{RqXGb}WStn(*D7h_o3pxIJbsxD%FcDB07PQ} zENkEWq-Y1fPneAy+Y*aqYxH`}dz*Yt6H)+pk@E%RrKXb=E>Qq9&616gvG$^BUbRE6VN z0MgQfu4K1_R(NksSn8fHI1Ax{6_>rDM-kHZ%JyS2^iZnF zP;Kyh(dkO(K1X7~i{m*v9!Ra>%wLIJDcTNhr;9CAd4c`&mR?t-J^VV;m3i`PU3EI^ zR<8~$QlVS3c>sz)_V=^vQ+;EAm3aanlPeZXskJIA$|FC`+VQ?Vc3az;Z*-k8>h-5< z(;fNl7mgQOs{*)^mak1~05}H2dj(Zl!#an(WO%P&pJW#F&+nM=A z9e}EkFWweAW2?$|Yn1xzfN0lDZeII&obf1ul{C8dwGUnU>+oD2pL)JWr6?kCWX}_< z$dQRV)kfC|nI~`U=7wJ4y&?f%7}cYvZ-(0v6zx(7Sx;*K literal 0 HcmV?d00001 diff --git a/Robot-Framework/gui-ref-images/firefox/close_button.png b/Robot-Framework/gui-ref-images/firefox/close_button.png new file mode 100644 index 0000000000000000000000000000000000000000..2026bc6770a182aa7bef98464e8431c98cf6d987 GIT binary patch literal 349 zcmV-j0iyniP)l$|wP*nVP&M-<2ZRn;@yGt+E?2Mm}DHkjCq7h;DbMp7{ZQ zwjP>c@&DBC^?R>=UDZ|f2>&*WLy8c{O)JLR4<_%bW?-KXeyM9D(lx4Sft6VI+>1+! zCV&3rkKr?m)oWV{W8Yn`KP(D>Qn6l$^>)ms)rN%9> zeDL3ajDH?D~zDokDv%YgFvBM$dnLQkvA&yn1|Q3&*$FyYw?q$8geb| z6O}IpemL2<3c#|3(b=2VZViF(Dl}Ut62rwHywoKmZy>55p`oIVuilnhma|>s(Z1k0}_SkLc@kZPQjP=(fwm#Em^wY>t?Un2LZ0y}@m_nqm(}Z$4T4fH08~ zMhGK>FzR90XF<2?*=%j2<8nx#D^C|bqY~S4-&&g}nq}+! zkN@s@h2wJoki??Hww$lm1)9tN*dGNzPB%3s(AVMsP~*=!?A?EtMfik{l5!Bry5_B; zUO}rCbC3w^*3h!U<=MqD!$>?{eyoa;H&D>8oEd%{l(wd&fRuN*#4VIHhM6o({SYVs z7BlZe9={IC)$>D@u7CglDT;9ug^g|ZK_~^Y){4H{HF!LZ6AjVrJ7`k^fuABlFa{)7 zcfUCZ1;Bjt#?^m1Li0!Epwznc{x77WF=+JQg@7@F_zJk~h(EX_=fd-;J2xJ5w>&=r z2;l?$_fIuBn>R1RydTdrY%IZc>}fUAQ0@BIk!g6RCO*4n&c!BvdwhqZ@@}dSqvod{ z{_I#9PLPw3mXL};-P^U0;%XN;LT^hMLT&ll@bpUeqp6Qerh8-1*0eOxf4|(n0f0tP z@Sz|gpFtsBF&SlLU$xWNxuc+zF{#8BVKGyDj~&?IYn3dWN+tiXX`XE zfD(f)K=(qop`@XtVY{~nrJ|gj?yF&5qJ48gl=H4Z)kGOY8KMk91oMJ25DEac1#ZJK zVCpaw=*8`0!&R2y-I%;mhdKfXskw`xo~MFz#3R+Vpk|?F500!-!)6<^ks4l02mt&K X0UyM?e}DS_00000NkvXXu0mjf!AnF( literal 0 HcmV?d00001 diff --git a/Robot-Framework/gui-ref-images/start_menu_agx.png b/Robot-Framework/gui-ref-images/start_menu_agx.png new file mode 100644 index 0000000000000000000000000000000000000000..f61b2f0314d056c36f8c85b61a108835fefadfbe GIT binary patch literal 872 zcmV-u1DE`XP)H&d#Q#GypL5 z%*e<{`=xfX*-RLW@%GMkC=~km;iIM??M^)xJa68-&6INS!fpvGWP#N{pF7#~`qgW{ z-wy!caM zPxb$2ODT$CG#GUWBQgR&sZ=U2Fqw2QlgXqgN|hWOr1I32HW;xZ=2& zl2Y;CaR5lAQg`p%-QC^IWHJZ48jT*0hh^FI_4TsEFbu;m4W}AbUaVlf*iY^E5AE`C<%GZ)eBx12xK9?s7ZK0h`r^DeOiY&{Uo15EP z+eE1<6hg>gs8SA1bwwRUOW1C=hnK>-=Sn0JJ-s~|*6Z&TMX{DqRdr!;fo0jMUpxR)i4G^IKb6k0&2zv)Rf_J32Zh?oSL24=pY% z9%MQ_Go4PS8NC=C8^svA+%ETdH^w+H8qmcunauRev^Flcl-X>SWx4kH03e^w0{~6a zq%8l!NQIBxGsPHJe)zS4E1sEdAcP20HWdVcW!ZYRKDD1ZM7vxrRaFH+KnSU->U275 z9qpW*W!ZQ>S!{KOaYYQ)hhlhp%VE);B&#j(lG%AWB zVJM1fX=$Npy6bY+&)81@Fj5o+0Lf%>_2sI~X0uoZuV(57&7z`>U1pwC8)+9+{ yPBL|Mb%ZGvi=QH&0)fDf??0+CJ=!=vE%g^WDvWK#y?~00009a7bBm000ie z000ie0hKEb8vpCl|M`qaTLejuPt1!e~#;66A*}L>i{GWBMD++OiggJ zCISITAPkVO79xWzIMfY-upyD8u241*i3BGJQeCj2xKtqpdOc`+l-J6?V0RYMC*x1;xzWzC$5<=(4DV$bly^6eLtX%uF z*51(p0E-@v-EQ~!d?zO-3PTWt&1Pd5#{S5@z5OGTqr>3<1jFAa{z%EZDxOx`J6j(c z40?o6YisM+tFg(+H+hUsNACUJb2uDXMU%-SBol>l01!>`2qA`HWS3>xLPii0JkQI_ zb?Eh#?VfZxtyZav5^AIdfK)1#`9Y&8ifJ^ObUK|?^!xpWQr(m4hkIv-62=OrRhK?6 z--+?DqoWW2#A31T=iSH0$MJYPJEdCP(9l5B^v=!>0DzIAj1*OVqkQ?pGA?G(ShT0N zN5*=3dm@oYQEX{xi797{6oqncimIxcH8nN4=OG*p2Lb^>NGK7Vjekvu|fzuU%fRSCS-UKD}N)@?xZ-qN2904k5Jb_XmT)@rm)Xv$NbBo12?QUyk+< z^v};PWOAIEo;p1}mB)GC(C`q(*lM%dZrd=%&LL+}EFO{HYPCv|#Ih_x zNRlLr#ZoB8FeaL&L!l5TuY%2H-T1O02!h}5KR-WLP_x-wUtjN8^z`=+0KnYbyS8U- zKA(@{IE5hyqPDh{B*~WhErGxY$-12TYozC6epV=HQ(XPYp`9Br^0;740&l#L*dH?_b07*qoM6N<$ Ef(R6ng#Z8m literal 0 HcmV?d00001 diff --git a/Robot-Framework/lib/gui_testing.py b/Robot-Framework/lib/gui_testing.py new file mode 100644 index 0000000..8e7fcc5 --- /dev/null +++ b/Robot-Framework/lib/gui_testing.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +from pyscreeze import locate, center +import logging + + +def locate_image(image): + screenshot = "./screenshot.png" + image_box = locate(image, screenshot) + image_center = center(image_box) + logging.info(image_box) + logging.info(image_center) + image_center_in_mouse_coordinates = convert_resolution(image_center) + logging.info(image_center_in_mouse_coordinates) + return image_center_in_mouse_coordinates + +def convert_resolution(coordinates): + # Screenshot image resolution is 1920x1200 but ydotool mouse movement resolution was tested to be 960x600. + # Let's have hard coded resolution scaling factor for now. + # However, if ghaf resolution is changed this might not work anymore. + scaling_factor = 2 + mouse_coordinates = { + 'x': coordinates[0] / scaling_factor, + 'y': coordinates[1] / scaling_factor + } + return mouse_coordinates \ No newline at end of file diff --git a/Robot-Framework/resources/common_keywords.resource b/Robot-Framework/resources/common_keywords.resource index 2cac671..cfdf1e2 100644 --- a/Robot-Framework/resources/common_keywords.resource +++ b/Robot-Framework/resources/common_keywords.resource @@ -19,8 +19,13 @@ Check that the application was started Should Not Be Empty ${app_pids} ${app_name} is not started Log To Console ${app_name} is started +Check that the application is not running + [Arguments] ${app_name} + ${found_status} ${pids} Run Keyword And Ignore Error Find pid by name ${app_name} + Should Be Empty ${pids} + Check If Ping Fails [Documentation] Check that ping is not getting response from host # ${out} Run and Return RC ping ${DEVICE_IP_ADDRESS} -c 1 ${result} Run Process ping ${DEVICE_IP_ADDRESS} -c1 timeout=1s - Should Not Be Equal ${result.rc} ${0} \ No newline at end of file + Should Not Be Equal ${result.rc} ${0} diff --git a/Robot-Framework/test-suites/bat-tests/apps.robot b/Robot-Framework/test-suites/bat-tests/apps.robot index 99f75a0..9b7bb4d 100644 --- a/Robot-Framework/test-suites/bat-tests/apps.robot +++ b/Robot-Framework/test-suites/bat-tests/apps.robot @@ -7,6 +7,9 @@ Force Tags apps Resource ../../resources/ssh_keywords.resource Resource ../../config/variables.robot Resource ../../resources/common_keywords.resource +Library ../../lib/gui_testing.py +Library Collections +Library BuiltIn Suite Teardown Close All Connections diff --git a/Robot-Framework/test-suites/bat-tests/gui-tests.robot b/Robot-Framework/test-suites/bat-tests/gui-tests.robot new file mode 100644 index 0000000..ed431c9 --- /dev/null +++ b/Robot-Framework/test-suites/bat-tests/gui-tests.robot @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2022-2024 Technology Innovation Institute (TII) +# SPDX-License-Identifier: Apache-2.0 + +*** Settings *** +Documentation Testing launching applications +Force Tags gui +Resource ../../resources/ssh_keywords.resource +Resource ../../config/variables.robot +Resource ../../resources/common_keywords.resource +Library ../../lib/gui_testing.py +Library Collections +Suite Teardown Close All Connections + + +*** Variables *** +@{app_pids} ${EMPTY} + + +*** Test Cases *** + +Start and close chromium via GUI on LenovoX1 + [Documentation] Start Chromium via GUI test automation and verify related process started + ... Close Chromium via GUI test automation and verify related process stopped + [Tags] bat SP-T97-1 lenovo-x1 + Start and close app via GUI on LenovoX1 chromium-vm chromium + +Start Firefox via GUI + [Documentation] Passing this test requires that display is connected to the target device + ... + ... Start Firefox via GUI test automation and verify related process started + ... Close Firefox via GUI test automation and verify related process stopped + [Tags] SP-T45-1 experimental + Start and close app via GUI firefox agx + + +*** Keywords *** + +Start and close app via GUI on LenovoX1 + [Documentation] Start Application via GUI test automation and verify related process started + ... Close Application via GUI test automation and verify related process stopped + [Arguments] ${app-vm}=chromium-vm + ... ${app}=chromium + ... ${launch_icon}=../gui-ref-images/${app}/launch_icon.png + ... ${close_button}=../gui-ref-images/${app}/close_button.png + + Verify service status range=15 service=microvm@${app-vm}.service expected_status=active expected_state=running + Connect to netvm + Connect to VM ${GUI_VM} + Check if ssh is ready on vm ${app-vm} + + Run Keyword And Ignore Error Execute Command -b /run/current-system/sw/bin/ydotoold --socket-path /tmp/.ydotool_socket sudo=True sudo_password=${PASSWORD} timeout=3 + + ${mouse_x} ${mouse_y} Locate image on screen ../gui-ref-images/start_menu_x1.png + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + BuiltIn.Sleep 1 + ${mouse_x} ${mouse_y} Locate image on screen ${launch_icon} + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + + # Move the cursor to the upper left corner so that it will not block searching further gui screenshots + Execute Command ydotool mousemove --absolute -x 10 -y 10 sudo=True sudo_password=${PASSWORD} + + Connect to VM ${app-vm} + Check that the application was started ${app} + + Connect to VM ${GUI_VM} + ${mouse_x} ${mouse_y} Locate image on screen ${close_button} + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + + # Move the cursor to the upper left corner so that it will not block searching further gui screenshots + Execute Command ydotool mousemove --absolute -x 10 -y 10 sudo=True sudo_password=${PASSWORD} + Execute Command pkill ydotoold sudo=True sudo_password=${PASSWORD} + + Connect to VM ${app-vm} + Check that the application is not running ${app} + + # In case closing the app via GUI failed + [Teardown] Kill process @{app_pids} + ... Connect to VM ${GUI_VM} + ... Execute Command pkill ydotoold sudo=True sudo_password=${PASSWORD} + + +Start and close app via GUI + [Documentation] Start Application via GUI test automation and verify related process started + ... Close Application via GUI test automation and verify related process stopped + ... Only for ghaf builds where desktop is running on ghaf-host + [Arguments] ${app}=firefox ${target}=agx + ... ${launch_icon}=../gui-ref-images/${app}/launch_icon.png + ... ${close_button}=../gui-ref-images/${app}/close_button.png + + Connect + + Execute Command export YDOTOOL_SOCKET=/tmp/.ydotools_socket + Run Keyword And Ignore Error Execute Command -b /run/current-system/sw/bin/ydotoold sudo=True sudo_password=${PASSWORD} timeout=3 + + ${mouse_x} ${mouse_y} Locate image on screen ../gui-ref-images/start_menu_${target}.png + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + BuiltIn.Sleep 1 + ${mouse_x} ${mouse_y} Locate image on screen ${launch_icon} + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + + # Move the cursor to the upper left corner so that it will not block searching further gui screenshots + Execute Command ydotool mousemove --absolute -x 10 -y 10 sudo=True sudo_password=${PASSWORD} + + BuiltIn.Sleep 1 + Check that the application was started ${app} + + ${mouse_x} ${mouse_y} Locate image on screen ${close_button} + Execute Command ydotool mousemove --absolute -x ${mouse_x} -y ${mouse_y} sudo=True sudo_password=${PASSWORD} + Execute Command ydotool click 0xC0 sudo=True sudo_password=${PASSWORD} + + # Move the cursor to the upper left corner so that it will not block searching further gui screenshots + Execute Command ydotool mousemove --absolute -x 10 -y 10 sudo=True sudo_password=${PASSWORD} + Execute Command pkill ydotoold sudo=True sudo_password=${PASSWORD} + + Check that the application is not running ${app} + + # In case closing the app via GUI failed + [Teardown] Kill process @{app_pids} + ... Execute Command pkill ydotoold sudo=True sudo_password=${PASSWORD} + +Locate image on screen + [Documentation] Take a screenshot. Locate given image on the screenshot. + ... Return center coordinates of the image in mouse coordinate system + [Arguments] ${image_to_be_searched} + Log To Console Taking screenshot + Execute Command grim screenshot.png + Log To Console Saving screenshot + SSHLibrary.Get File screenshot.png screenshot.png + Log To Console Locating image on screenshot + ${coordinates} Locate image ${image_to_be_searched} + Log To Console Coordinates: ${coordinates} + ${mouse_x} Get From Dictionary ${coordinates} x + ${mouse_y} Get From Dictionary ${coordinates} y + RETURN ${mouse_x} ${mouse_y} \ No newline at end of file diff --git a/flake.nix b/flake.nix index 3f82f55..62ee988 100644 --- a/flake.nix +++ b/flake.nix @@ -56,6 +56,7 @@ python-kasa pytz pandas + pyscreeze ])) ]; };