From 199654f369d66228b3127cd6011fe37d4b848fb5 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Thu, 17 Aug 2023 16:11:39 -0700 Subject: [PATCH] Enable evergreen test on RDK platform Added launcher infrastructure for AH212 RDK (v5). Note: RDK launcher isn't always reliable to run Cobalt from cmd, so Cobalt needs to be launched outside the container. It can be achieved by modifying /etc/rfcdefaults/rfcdefaults.ini Details about this operation is documented in b/293172465 --- starboard/build/platforms.py | 1 + starboard/evergreen/shared/launcher.py | 37 ++++++ starboard/raspi/rdk/gyp_configuration.py | 25 ++++ starboard/raspi/rdk/test_filters.py | 28 +++++ starboard/raspi/shared/launcher.py | 143 +++++++++++++++++++++-- starboard/tools/testing/test_runner.py | 13 ++- 6 files changed, 234 insertions(+), 13 deletions(-) create mode 100644 starboard/raspi/rdk/gyp_configuration.py create mode 100644 starboard/raspi/rdk/test_filters.py diff --git a/starboard/build/platforms.py b/starboard/build/platforms.py index 696453e633f6..eabfbdaf09c9 100644 --- a/starboard/build/platforms.py +++ b/starboard/build/platforms.py @@ -36,6 +36,7 @@ 'android-x86': 'starboard/android/x86', 'raspi-2': 'starboard/raspi/2', 'raspi-2-skia': 'starboard/raspi/2/skia', + 'raspi-rdk': 'starboard/raspi/rdk', 'evergreen-x64': 'starboard/evergreen/x64', 'evergreen-x86': 'starboard/evergreen/x86', 'evergreen-arm-hardfp': 'starboard/evergreen/arm/hardfp', diff --git a/starboard/evergreen/shared/launcher.py b/starboard/evergreen/shared/launcher.py index cd427f433230..8c150991a371 100644 --- a/starboard/evergreen/shared/launcher.py +++ b/starboard/evergreen/shared/launcher.py @@ -175,6 +175,8 @@ def _StageTargetsAndContents(self): # TODO(b/267568637): Make the Linux launcher run from the install_directory. if 'linux' in self.loader_platform: self._StageTargetsAndContentsLinux() + elif 'rdk' in self.loader_platform: + self._StageTargetsAndContentsRdk() else: self._StageTargetsAndContentsRaspi() @@ -271,6 +273,41 @@ def _StageTargetsAndContentsRaspi(self): os.makedirs(os.path.join(target_staging_dir, 'lib')) shutil.copy(target_binary_src, target_binary_dst) + def _StageTargetsAndContentsRdk(self): + """Stage targets and their contents for GN builds for RDK platforms.""" + content_subdir = os.path.join('usr', 'share', 'cobalt') + #target_name should be self.target_name. However because loader is hardcoded + #to call "libcobalt.so" in path "cobalt", it has to use this name for now. + #b/296633713 + target_name = 'cobalt' + + # Copy target content and binary + target_install_path = os.path.join(self.out_directory, 'install') + target_staging_dir = os.path.join(self.staging_directory, 'content', 'app', + target_name) + os.makedirs(target_staging_dir) + + # TODO(b/218889313): Reset the content path for the evergreen artifacts. + content_subdir = os.path.join('usr', 'share', 'cobalt') + target_content_src = os.path.join(target_install_path, content_subdir) + target_content_dst = os.path.join(target_staging_dir, 'content') + shutil.copytree(target_content_src, target_content_dst) + + shlib_name = f'lib{self.target_name}' + shlib_name += '.lz4' if self.use_compressed_library else '.so' + + target_binary_src = os.path.join(self.out_directory, shlib_name) + if target_name in shlib_name: + target_binary_dst = os.path.join(target_staging_dir, 'lib', shlib_name) + else: + rdk_shlib_name = f'lib{target_name}' + rdk_shlib_name += '.lz4' if self.use_compressed_library else '.so' + target_binary_dst = os.path.join(target_staging_dir, 'lib', + rdk_shlib_name) + + os.makedirs(os.path.join(target_staging_dir, 'lib')) + shutil.copy(target_binary_src, target_binary_dst) + def SupportsSuspendResume(self): return self.launcher.SupportsSuspendResume() diff --git a/starboard/raspi/rdk/gyp_configuration.py b/starboard/raspi/rdk/gyp_configuration.py new file mode 100644 index 000000000000..4bc787eb0a9c --- /dev/null +++ b/starboard/raspi/rdk/gyp_configuration.py @@ -0,0 +1,25 @@ +# Copyright 2017 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starboard RDK platform configuration.""" + +from starboard.raspi.shared import gyp_configuration as shared_configuration + + +class RaspiRdkPlatformConfig(shared_configuration.RaspiPlatformConfig): + """Starboard raspi-rdk platform configuration.""" + pass + + +def CreatePlatformConfig(): + return RaspiRdkPlatformConfig('raspi-rdk') diff --git a/starboard/raspi/rdk/test_filters.py b/starboard/raspi/rdk/test_filters.py new file mode 100644 index 000000000000..baf977fabeed --- /dev/null +++ b/starboard/raspi/rdk/test_filters.py @@ -0,0 +1,28 @@ +# Copyright 2022 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starboard RDK Platform Test Filters.""" + +from starboard.raspi.shared import test_filters as shared_test_filters + + +def CreateTestFilters(): + return RaspiRdkTestFilters() + + +class RaspiRdkTestFilters(shared_test_filters.TestFilters): + """Starboard RDK Platform Test Filters.""" + + def GetTestFilters(self): + filters = super().GetTestFilters() + return filters diff --git a/starboard/raspi/shared/launcher.py b/starboard/raspi/shared/launcher.py index 8368fa75930c..5bdc9ef9782f 100644 --- a/starboard/raspi/shared/launcher.py +++ b/starboard/raspi/shared/launcher.py @@ -57,11 +57,17 @@ class Launcher(abstract_launcher.AbstractLauncher): _STARTUP_TIMEOUT_SECONDS = 1800 + #RASPI-2 specific parameters _RASPI_USERNAME = 'pi' _RASPI_PASSWORD = 'raspberry' + _RASPI_PROMPT = 'pi@raspberrypi:' + #RDK specific parameters + _RDK_USERNAME = 'root' + _RDK_PASSWORD = '' + _RDK_PROMPT = 'root@AmlogicFirebolt:' + _SSH_LOGIN_SIGNAL = 'cobalt-launcher-login-success' _SSH_SLEEP_SIGNAL = 'cobalt-launcher-done-sleeping' - _RASPI_PROMPT = 'pi@raspberrypi:' # pexpect times out each second to allow Kill to quickly stop a test run _PEXPECT_TIMEOUT = 1 @@ -101,6 +107,7 @@ def __init__(self, platform, target_name, config, device_id, **kwargs): env = os.environ.copy() env.update(self.env_variables) self.full_env = env + self.platform = platform if not self.device_id: self.device_id = self.full_env.get('RASPI_ADDR') @@ -112,7 +119,11 @@ def __init__(self, platform, target_name, config, device_id, **kwargs): self.startup_timeout_seconds = Launcher._STARTUP_TIMEOUT_SECONDS self.pexpect_process = None - self._InitPexpectCommands() + + if 'rdk' in platform: + self._InitPexpectCommandsRdk() + else: + self._InitPexpectCommands() self.run_inactive = threading.Event() self.run_inactive.set() @@ -178,6 +189,47 @@ def _InitPexpectCommands(self): self.test_command = (f'{test_base_command} {test_success_output} ' f'{test_failure_output}') + def _InitPexpectCommandsRdk(self): + """Initializes all of the pexpect commands needed for running the test.""" + + # Ensure no trailing slashes + self.out_directory = self.out_directory.rstrip('/') + + rdk_user_hostname = Launcher._RDK_USERNAME + '@' + self.device_id + rdk_test_dir = '/usr/share/content/data/app' + + # scp command setup + options = '-rOCq ' + rsa_options = '-o \"LogLevel ERROR\" \ + -o \"UserKnownHostsFile=/dev/null\" -o StrictHostKeyChecking=no ' + + source = os.path.join(self.out_directory, 'content', 'app', 'cobalt') + destination = f'{rdk_user_hostname}:{rdk_test_dir}/' + self.scp_command = 'scp ' + options + rsa_options + \ + ' ' + source + ' ' + destination + + # ssh command setup + self.ssh_command = 'ssh -t ' + rsa_options + rdk_user_hostname + \ + ' TERM=dumb bash -l' + + # test output tags + self.test_complete_tag = 'test suites ran.' + self.test_failure_tag = 'tests, listed below' + self.test_failure_list_tag = 'FAILED TESTS' + + # test command setup + # note: If RDK has ResidentApp feature turned on (default), the log + # goes to a buffer that can be retrieved by command + # journalctl -f -u dobby + # Otherwise, the container is off, and the log is dumped into + # file /opt/logs/wpeframework.log. + # tail -f /opt/logs/wpeframework.log + self.test_command = 'curl -X POST http://127.0.0.1:9998/jsonrpc -d \ + \'{\"jsonrpc\":\"2.0\", \"id\":3, \ + \"method\":\"org.rdk.RDKShell.1.launch\", \ + \"params\":{\"callsign\":\"YouTube\"}}\'; \ + tail -f /opt/logs/wpeframework.log' + # pylint: disable=no-method-argument def _CommandBackoff(): time.sleep(Launcher._INTER_COMMAND_DELAY_SECONDS) @@ -230,6 +282,46 @@ def _inner(): _inner() + def _PexpectSpawnAndConnectRdk(self, command): + """Spawns a process with pexpect and connect to the RDK. + + Args: + command: The command to use when spawning the pexpect process. + """ + + logging.info('executing: %s', command) + kwargs = {} if six.PY2 else {'encoding': 'utf-8'} + self.pexpect_process = pexpect.spawn( + command, timeout=Launcher._PEXPECT_TIMEOUT, **kwargs) + # Let pexpect output directly to our output stream + self.pexpect_process.logfile_read = self.output_file + expected_prompts = [ + r'.*Are\syou\ssure.*', # Fingerprint verification + r'.* password:', # Password prompt + '.*[a-zA-Z]+.*', # Any other text input + ] + + # pylint: disable=unnecessary-lambda + @retry.retry( + exceptions=Launcher._RETRY_EXCEPTIONS, + retries=Launcher._PEXPECT_PASSWORD_TIMEOUT_MAX_RETRIES, + backoff=lambda: self._ShutdownBackoff(), + wrap_exceptions=False) + def _inner(): + i = self.pexpect_process.expect(expected_prompts) + if i == 0: + self._PexpectSendLine('yes') + elif i == 1: + self._PexpectSendLine(Launcher._RDK_PASSWORD) + else: + # If any other input comes in, maybe we've logged in with rsa key or + # raspi does not have password. Check if we've logged in by echoing + # a special sentence and expect it back. + self._PexpectSendLine('echo ' + Launcher._SSH_LOGIN_SIGNAL) + i = self.pexpect_process.expect([Launcher._SSH_LOGIN_SIGNAL]) + + _inner() + @retry.retry( exceptions=_RETRY_EXCEPTIONS, retries=_PEXPECT_SENDLINE_RETRIES, @@ -265,6 +357,34 @@ def _readloop(): _readloop() + def _PexpectReadLinesRdk(self): + """Reads all lines from the pexpect process.""" + # pylint: disable=unnecessary-lambda + @retry.retry( + exceptions=Launcher._RETRY_EXCEPTIONS, + retries=Launcher._PEXPECT_READLINE_TIMEOUT_MAX_RETRIES, + backoff=lambda: self.shutdown_initiated.is_set(), + wrap_exceptions=False) + def _readloop(): + test_completed = 0 + while True: + # Sanitize the line to remove ansi color codes. + line = Launcher._PEXPECT_SANITIZE_LINE_RE.sub( + '', self.pexpect_process.readline()) + self.output_file.flush() + if not line: + return + # Check for the test complete tag. It will be followed by either a + # success or failure tag. + if line.find(self.test_complete_tag) != -1: + test_completed = 1 + if test_completed and line.find(self.test_failure_tag) != -1: + self.return_value = 1 + if self.return_value and line.find(self.test_failure_list_tag) != -1: + return + + _readloop() + def _Sleep(self, val): self._PexpectSendLine(f'sleep {val};echo {Launcher._SSH_SLEEP_SIGNAL}') self.pexpect_process.expect([Launcher._SSH_SLEEP_SIGNAL]) @@ -347,15 +467,21 @@ def Run(self): # Notify other threads that the run is now active self.run_inactive.clear() - # rsync the test files to the raspi + # rsync the test files to the raspi / RDK if not self.shutdown_initiated.is_set(): - self._PexpectSpawnAndConnect(self.rsync_command) + if 'rdk' in self.platform: + self._PexpectSpawnAndConnectRdk(self.scp_command) + else: + self._PexpectSpawnAndConnect(self.rsync_command) if not self.shutdown_initiated.is_set(): self._PexpectReadLines() - # ssh into the raspi and run the test + # ssh into the raspi/RDK and run the test if not self.shutdown_initiated.is_set(): - self._PexpectSpawnAndConnect(self.ssh_command) + if 'rdk' in self.platform: + self._PexpectSpawnAndConnectRdk(self.ssh_command) + else: + self._PexpectSpawnAndConnect(self.ssh_command) self._Sleep(self._INTER_COMMAND_DELAY_SECONDS) # Execute debugging commands on the first run first_run_commands = [] @@ -383,7 +509,10 @@ def _readline(): if not self.shutdown_initiated.is_set(): self._PexpectSendLine(self.test_command) - self._PexpectReadLines() + if 'rdk' in self.platform: + self._PexpectReadLinesRdk() + else: + self._PexpectReadLines() except retry.RetriesExceeded: logging.exception('Command retry exceeded (cmd: %s)', diff --git a/starboard/tools/testing/test_runner.py b/starboard/tools/testing/test_runner.py index b6fa3e14314c..71a45043b4d0 100755 --- a/starboard/tools/testing/test_runner.py +++ b/starboard/tools/testing/test_runner.py @@ -40,11 +40,11 @@ # pylint: disable=consider-using-f-string _FLAKY_RETRY_LIMIT = 4 -_TOTAL_TESTS_REGEX = re.compile(r"^\[==========\] (.*) tests? from .*" +_TOTAL_TESTS_REGEX = re.compile(r"\[==========\] (.*) tests? from .*" r"test suites? ran. \(.* ms total\)") -_TESTS_PASSED_REGEX = re.compile(r"^\[ PASSED \] (.*) tests?") -_TESTS_FAILED_REGEX = re.compile(r"^\[ FAILED \] (.*) tests?, listed below:") -_SINGLE_TEST_FAILED_REGEX = re.compile(r"^\[ FAILED \] (.*)") +_TESTS_PASSED_REGEX = re.compile(r"\[ PASSED \] (.*) tests?") +_TESTS_FAILED_REGEX = re.compile(r"\[ FAILED \] (.*) tests?, listed below:") +_SINGLE_TEST_FAILED_REGEX = re.compile(r"\[ FAILED \] (.*)") _NATIVE_CRASHPAD_TARGET = "native_target/crashpad_handler" _LOADER_TARGET = "elf_loader_sandbox" @@ -265,7 +265,8 @@ def __init__(self, _VerifyConfig(self._platform_config, self._platform_test_filters.GetTestFilters()) - if self.loader_platform: + if self.loader_platform and self.loader_platform.find("rdk") == -1: + print(" self.loader_platform = ", self.loader_platform) _EnsureBuildDirectoryExists(self.loader_out_directory) _VerifyConfig(self._loader_platform_config, self._loader_platform_test_filters.GetTestFilters()) @@ -786,7 +787,7 @@ def BuildAllTargets(self, ninja_flags): # The loader is not built with the same platform configuration as our # tests so we need to build it separately. - if self.loader_platform: + if self.loader_platform and self.loader_platform.find("rdk") == -1: target_list = [_LOADER_TARGET, _NATIVE_CRASHPAD_TARGET] build_tests.BuildTargets( target_list, self.loader_out_directory, self.dry_run,