Add blackbox test for logTrace and watchdog violations APIs. (#2930)

Add blackbox tests for several use cases including Cobalt crash and restart. b/334122726 Change-Id: I932eda36f724d38f8b8d7eac514fef625a3edd50 (cherry picked from commit 27c6edd)
youtube · Apr 24, 2024 · c85def1 · c85def1
1 parent cf57615
commit c85def1
Show file tree

Hide file tree

Showing 7 changed files with 315 additions and 0 deletions.
diff --git a/cobalt/black_box_tests/black_box_tests.py b/cobalt/black_box_tests/black_box_tests.py
@@ -69,6 +69,7 @@
     'default_site_can_load',
     'disable_eval_with_csp',
     'h5vcc_storage_write_verify_test',
+    'h5vcc_watchdog_api_test',
     'http_cache',
     'javascript_profiler',
     'persistent_cookie',

diff --git a/cobalt/black_box_tests/testdata/h5vcc_logevent_api_test.html b/cobalt/black_box_tests/testdata/h5vcc_logevent_api_test.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+  <title>h5vcc logEvent() API test</title>
+</head>
+<body>
+<script src='black_box_js_test_utils.js'></script>
+<script src='h5vcc_logevent_api_test.js'></script>
+</body>
+</html>
diff --git a/cobalt/black_box_tests/testdata/h5vcc_logevent_api_test.js b/cobalt/black_box_tests/testdata/h5vcc_logevent_api_test.js
@@ -0,0 +1,124 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+function failTest() {
+  notReached();
+  onEndTest();
+}
+
+function canCallH5vccLogEventTest() {
+  if (!h5vcc.crashLog) {
+    console.log("h5vcc.crashLog does not exist");
+    return;
+  }
+
+  if (!h5vcc.crashLog.logEvent) {
+    console.log("h5vcc.crashLog.logEvent does not exist");
+    return;
+  }
+
+  h5vcc.crashLog.logEvent("test-frame");
+}
+
+function canCallH5vccGetLogTraceTest() {
+  h5vcc.crashLog.logEvent("frame");
+}
+
+function getLogTraceReturnsLastLoggedEventTest() {
+  if (!h5vcc.crashLog) {
+    console.log("h5vcc.crashLog does not exist");
+    return;
+  }
+
+  if (!h5vcc.crashLog.logEvent) {
+    console.log("h5vcc.crashLog.logEvent does not exist");
+    return;
+  }
+
+  h5vcc.crashLog.logEvent(
+      "yt.ui.uix.Abc.efG_ (http://www.youtube.com/yts/jsbin/a-b-c.js:14923:34)");
+  h5vcc.crashLog.logEvent(
+      "yy.ttt.Aaa.b_ (http://www.youtube.com/yts/jsbin/a-b-c.js:123:45)");
+  h5vcc.crashLog.logEvent(
+      "yy.ttt.Ccc.d_ (http://www.youtube.com/yts/jsbin/a-b-c.js:678:90)");
+
+  let logTrace = h5vcc.crashLog.getLogTrace();
+  if (logTrace[logTrace.length - 1]
+      !== "yy.ttt.Ccc.d_ (http://www.youtube.com/yts/jsbin/a-b-c.js:678:90)") {
+    failTest();
+  }
+}
+
+function clearLogWorks() {
+  if (!h5vcc.crashLog.clearLog) {
+    console.log("h5vcc.crashLog.clearLog does not exist");
+    return;
+  }
+
+  h5vcc.crashLog.clearLog();
+
+  h5vcc.crashLog.logEvent("1");
+  h5vcc.crashLog.logEvent("2");
+  h5vcc.crashLog.logEvent("3");
+
+  h5vcc.crashLog.clearLog();
+
+  h5vcc.crashLog.logEvent("4");
+  h5vcc.crashLog.logEvent("5");
+
+  let logTrace = h5vcc.crashLog.getLogTrace();
+
+  if (logTrace.length !== 2) {
+    failTest();
+  }
+
+  if (logTrace[0] !== "4") {
+    failTest();
+  }
+  if (logTrace[1] !== "5") {
+    failTest();
+  }
+}
+
+function canReadLogtraceFromWatchdogViolation() {
+  h5vcc.crashLog.register("test-client", "", "started", 500, 0, 'all',);
+  h5vcc.crashLog.ping("test-client", "");
+
+  h5vcc.crashLog.clearLog();
+  h5vcc.crashLog.logEvent("1");
+  h5vcc.crashLog.logEvent("2");
+
+  // sleep to generate violation
+  let start = Date.now();
+  while ((Date.now() - start) < 2000) {
+  }
+
+  let violationsJson = h5vcc.crashLog.getWatchdogViolations();
+  if (!violationsJson) {
+    failTest();
+  }
+
+  let violations = JSON.parse(violationsJson);
+  if (violations['test-client']['violations'][0]['logTrace'].length !== 2) {
+    failTest();
+  }
+}
+
+canCallH5vccLogEventTest();
+canCallH5vccGetLogTraceTest();
+getLogTraceReturnsLastLoggedEventTest();
+clearLogWorks();
+canReadLogtraceFromWatchdogViolation();
+onEndTest();
diff --git a/cobalt/black_box_tests/testdata/h5vcc_watchdog_violation_test.html b/cobalt/black_box_tests/testdata/h5vcc_watchdog_violation_test.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+  <title>h5vcc Watchdog API test</title>
+</head>
+<body>
+<script src='black_box_js_test_utils.js'></script>
+<script src='h5vcc_watchdog_violation_test.js'></script>
+</body>
+</html>
diff --git a/cobalt/black_box_tests/testdata/h5vcc_watchdog_violation_test.js b/cobalt/black_box_tests/testdata/h5vcc_watchdog_violation_test.js
@@ -0,0 +1,102 @@
+// Copyright 2024 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the 'License');
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an 'AS IS' BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+'use strict';
+
+function failTest() {
+  notReached();
+  onEndTest();
+}
+
+function doBusyLoopFor(msToSleep) {
+  let start = Date.now();
+  while ((Date.now() - start) < msToSleep) {
+  }
+}
+
+function canDetectViolation() {
+  h5vcc.crashLog.register('test-client', '', 'started', 500, 0, 'all');
+  // clear whatever violations are in the cache
+  h5vcc.crashLog.getWatchdogViolations();
+
+  h5vcc.crashLog.ping('test-client', '');
+  doBusyLoopFor(2000)
+
+  let violationsJson = h5vcc.crashLog.getWatchdogViolations();
+  if (!violationsJson) {
+    failTest();
+  }
+
+  let violations = JSON.parse(violationsJson);
+  if (violations['test-client']['violations'].length !== 1) {
+    failTest();
+  }
+}
+
+function reportsRepitativeViolations() {
+  h5vcc.crashLog.register('test-client', '', 'started', 500, 0, 'all');
+  // clear whatever violations are in the cache
+  h5vcc.crashLog.getWatchdogViolations();
+
+  h5vcc.crashLog.ping('test-client', '');
+  doBusyLoopFor(2000);
+  h5vcc.crashLog.ping('test-client', '');
+  doBusyLoopFor(2000);
+
+  let violationsJson = h5vcc.crashLog.getWatchdogViolations();
+  if (!violationsJson) {
+    failTest();
+  }
+
+  let violations = JSON.parse(violationsJson);
+  if (violations['test-client']['violations'].length !== 2) {
+    failTest();
+  }
+}
+
+function reportsLogtraceWithViolations() {
+  h5vcc.crashLog.register('test-client', '', 'started', 500, 0, 'all');
+  // clear whatever violations are in the cache
+  h5vcc.crashLog.getWatchdogViolations();
+  h5vcc.crashLog.logEvent('frame_1');
+  h5vcc.crashLog.logEvent('frame_2');
+
+  h5vcc.crashLog.ping('test-client', '');
+  h5vcc.crashLog.logEvent('frame_3');
+  h5vcc.crashLog.logEvent('frame_4');
+
+  doBusyLoopFor(2000);
+
+  let violationsJson = h5vcc.crashLog.getWatchdogViolations();
+  if (!violationsJson) {
+    failTest();
+  }
+
+  let violations = JSON.parse(violationsJson);
+  let logTrace = violations['test-client']['violations'][0]['logTrace'];
+  if (logTrace.length !== 4) {
+    failTest();
+
+  }
+
+  if (logTrace[0] !== 'frame_1' || logTrace[1] !== 'frame_2' || logTrace[2]
+      !== 'frame_3' || logTrace[3] !== 'frame_4') {
+    failTest();
+  }
+}
+
+canDetectViolation();
+reportsRepitativeViolations();
+reportsLogtraceWithViolations();
+
+onEndTest();
diff --git a/cobalt/black_box_tests/tests/h5vcc_logevent_api_test.py b/cobalt/black_box_tests/tests/h5vcc_logevent_api_test.py
@@ -0,0 +1,31 @@
+# Copyright 2024 The Cobalt Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+"""Test H5vcc logEvent() API"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from cobalt.black_box_tests import black_box_tests
+from cobalt.black_box_tests.threaded_web_server import ThreadedWebServer
+
+
+class H5vccLogEventApiTest(black_box_tests.BlackBoxTestCase):
+
+  def test_h5vcc_logevent_api_test(self):
+    with ThreadedWebServer(binding_address=self.GetBindingAddress()) as server:
+      url = server.GetURL(file_name='testdata/h5vcc_logevent_api_test.html')
+      with self.CreateCobaltRunner(url=url) as runner:
+        runner.WaitForActiveElement()
+        self.assertTrue(runner.JSTestsSucceeded())
diff --git a/cobalt/black_box_tests/tests/h5vcc_watchdog_api_test.py b/cobalt/black_box_tests/tests/h5vcc_watchdog_api_test.py
@@ -0,0 +1,35 @@
+# Copyright 2024 The Cobalt Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+"""Test H5vcc logEvent() API"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from cobalt.black_box_tests import black_box_tests
+from cobalt.black_box_tests.threaded_web_server import ThreadedWebServer
+from cobalt.tools.automated_testing import webdriver_utils
+
+keys = webdriver_utils.import_selenium_module('webdriver.common.keys')
+
+
+class H5vccWatchdogApiTest(black_box_tests.BlackBoxTestCase):
+
+  def test_h5vcc_watchdog_api_test(self):
+    with ThreadedWebServer(binding_address=self.GetBindingAddress()) as server:
+      url = server.GetURL(
+          file_name='testdata/h5vcc_watchdog_violation_test.html')
+      with self.CreateCobaltRunner(url=url) as runner:
+        runner.WaitForJSTestsSetup()
+        self.assertTrue(runner.JSTestsSucceeded())