Add EXP-4561 - Update iOS test suite to work with github actions flow (…

…#20360)

* Add EXP-4561 - Update iOS test suite to work with github actions flow

* Fix linting errors.

* Fix value error.

* Revert change to exception catch.

firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests.xctestplan

@@ -9,6 +9,39 @@
     }
   ],
   "defaultOptions" : {
+    "codeCoverage" : false,
+    "commandLineArgumentEntries" : [
+      {
+        "argument" : "FIREFOX_SKIP_INTRO"
+      },
+      {
+        "argument" : "FIREFOX_TEST"
+      },
+      {
+        "argument" : "FIREFOX_CLEAR_PROFILE"
+      },
+      {
+        "argument" : "\"GCDWEBSERVER_PORT:7777\""
+      },
+      {
+        "argument" : "DISABLE_ANIMATIONS"
+      }
+    ],
+    "environmentVariableEntries" : [
+      {
+        "key" : "WEBSERVER_PORT",
+        "value" : "7777"
+      },
+      {
+        "key" : "EXPERIMENT_NAME",
+        "value" : "${EXPERIMENT_NAME}"
+      }
+    ],
+    "targetForVariableExpansion" : {
+      "containerPath" : "container:Client.xcodeproj",
+      "identifier" : "F84B21BD1A090F8100AAB793",
+      "name" : "Client"
+    },
     "testTimeoutsEnabled" : true
   },
   "testTargets" : [
@@ -17,7 +50,6 @@
         "ActivityStreamTest",
         "AuthenticationTest",
         "BaseTestCase",
-        "BookmarksTests",
         "BrowsingPDFTests",
         "ClipBoardTests",
         "CreditCardsTests",
@@ -45,8 +77,6 @@
         "JumpBackInTests",
         "LibraryTestsIpad",
         "LoginTest",
-        "NavigationTest",
-        "NewTabSettingsTest",
         "NightModeTests",
         "OnboardingTests",
         "OpeningScreenTests",
@@ -60,20 +90,16 @@
         "ReportSiteTests",
         "ScreenGraphTest",
         "SearchSettingsUITests",
-        "SearchTests",
         "SettingsTests",
         "SiteLoadTest",
         "SyncUITests",
         "TabCounterTests",
         "ThirdPartySearchTest",
         "ToolbarMenuTests",
-        "ToolbarTests",
-        "TopTabsTest",
         "TopTabsTestIpad",
         "TopTabsTestIphone",
         "TrackingProtectionTests",
         "URLValidationTests",
-        "UrlBarTests",
         "WhatsNewTest"
       ],
       "target" : {

firefox-ios/firefox-ios-tests/Tests/ExperimentIntegrationTests/conftest.py

@@ -24,7 +24,7 @@
 KLAATU_SERVER_URL = "http://localhost:1378"
 KLAATU_LOCAL_SERVER_URL = "http://localhost:1378"
 
-here = Path("./")
+here = Path().cwd()
 
 
 def pytest_addoption(parser):
@@ -34,6 +34,32 @@ def pytest_addoption(parser):
     parser.addoption(
         "--stage", action="store_true", default=None, help="Use the stage server"
     )
+    parser.addoption(
+        "--build-dev", action="store_true", default=False, help="Build the developer edition of Firefox"
+    )
+    parser.addoption(
+        "--feature", action="store", help="Feature name you want to test against"
+    )
+    parser.addoption(
+        "--experiment-branch", action="store", default="control", help="Experiment Branch you want to test on"
+    )
+
+
+def pytest_runtest_setup(item):
+    envnames = [mark.name for mark in item.iter_markers()]
+    if envnames:
+        if item.config.getoption("--feature") not in envnames:
+            pytest.skip("test does not match feature name")
+
+
+@pytest.fixture(name="nimbus_cli_args")
+def fixture_nimbus_cli_args():
+    return "FIREFOX_SKIP_INTRO FIREFOX_TEST DISABLE_ANIMATIONS 'GCDWEBSERVER_PORT:7777'"
+
+@pytest.fixture(name="experiment_branch")
+def fixture_experiment_branch(request):
+    return request.config.getoption("--experiment-branch")
+
 
 @pytest.fixture(name="load_branches")
 def fixture_load_branches(experiment_url):
@@ -57,57 +83,78 @@ def fixture_load_branches(experiment_url):
     return branches
 
 
-@pytest.fixture
-def xcodebuild_log(request, tmpdir):
-    xcodebuild_log = str(tmpdir.join("xcodebuild.log"))
+@pytest.fixture()
+def xcodebuild_log(request, tmp_path_factory):
+    xcodebuild_log = tmp_path_factory.mktemp("logs") / "xcodebuild.log"
+    logging.info(f"Logs stored at: {xcodebuild_log}")
     request.config._xcodebuild_log = xcodebuild_log
     yield xcodebuild_log
 
 
 @pytest.fixture(scope="session", autouse=True)
-def fixture_build_fennec():
-    xcrun = XCRun()
-    hold = None
-    command = "xcodebuild build -project Client.xcodeproj -scheme Fennec -configuration Fennec -sdk iphonesimulator"
+def fixture_build_fennec(request):
+    if not request.config.getoption("--build-dev"):
+        return
+    command = "xcodebuild build-for-testing -project Client.xcodeproj -scheme Fennec -configuration Fennec -sdk iphonesimulator -destination 'platform=iOS Simulator,name=iPhone 15,OS=17.2'"
     try:
+        logging.info("Building app")
         subprocess.check_output(
             command,
-            cwd=os.chdir("../../"),
+            cwd=here.parents[2],
             stderr=subprocess.STDOUT,
             universal_newlines=True,
             shell=True)
     except subprocess.CalledProcessError:
         raise
 
 
-@pytest.fixture
+@pytest.fixture()
 def xcodebuild(xcodebuild_log):
     yield XCodeBuild(xcodebuild_log, scheme="Fennec", test_plan="ExperimentIntegrationTests")
 
 
+@pytest.fixture(scope="session")
+def xcrun():
+    return XCRun()
+
+
+@pytest.fixture(name="device_control", scope="module", autouse=True)
+def fixture_device_control(xcrun):
+    xcrun.boot()
+    yield
+    xcrun.erase()
+
+
 @pytest.fixture(name="start_app")
-def fixture_start_app():
+def fixture_start_app(nimbus_cli_args):
     def _():
-        command = f"nimbus-cli --app firefox_ios --channel developer open"
+        command = f"nimbus-cli --app firefox_ios --channel developer open -- {nimbus_cli_args}"
         out = subprocess.check_output(
             command,
-            cwd=os.path.join(here, os.pardir),
+            cwd=here.parent,
             stderr=subprocess.STDOUT,
             universal_newlines=True,
             shell=True,
         )
-        print(out)
+        logging.debug(out)
 
     return _
 
 
 @pytest.fixture(name="experiment_data")
-def fixture_experiment_data(experiment_url):
+def fixture_experiment_data(experiment_url, request):
     data = requests.get(experiment_url).json()
-    for item in data["branches"][0]["features"][0]["value"]["messages"].values():
-        for count, trigger in enumerate(item["trigger"]):
-            if "USER_EN_SPEAKER" not in trigger:
-                del item["trigger"][count]
+    match request.config.getoption("--feature"):
+        case "messaging_survey":
+            for branch in data.get("branches"):
+                for feature in branch.get("features"):
+                    for item in feature["value"]["messages"].values():
+                        if "USER_EN-US_SPEAKER"in item["trigger-if-all"]:
+                            item["trigger-if-all"] = ["ALWAYS"]
+        case _:
+            pass
+
+    logging.debug(f"JSON Data used for this test: {data}")
     return [data]
 
 
@@ -116,9 +163,9 @@ def fixture_experiment_url(request, variables):
     if slug := request.config.getoption("--experiment"):
         # Build URL from slug
         if request.config.getoption("--stage"):
-            url = f"{variables['urls']['stage_server']}/api/v6/experiments/{slug}"
+            url = f"{variables['urls']['stage_server']}/api/v6/experiments/{slug}/"
         else:
-            url = f"{variables['urls']['prod_server']}/api/v6/experiments/{slug}"
+            url = f"{variables['urls']['prod_server']}/api/v6/experiments/{slug}/"
     else:
         try:
             data = requests.get(f"{KLAATU_SERVER_URL}/experiment").json()
@@ -156,19 +203,26 @@ def fixture_experiment_slug(experiment_data):
     return experiment_data[0]["slug"]
 
 
-@pytest.fixture(name="send_test_results", autouse=True)
-def fixture_send_test_results():
+@pytest.fixture(name="send_test_results", scope="session")
+def fixture_send_test_results(xcrun):
     yield
-    here = Path()
+    xcrun.shutdown()
+    here = Path().cwd()
 
-    with open(f"{here.resolve()}/Tests/ExperimentIntegrationTests/results/index.html", "rb") as f:
+    with open(f"{here.parent}/ExperimentIntegrationTests/results/index.html", "rb") as f:
         files = {"file": f}
         try:
             requests.post(f"{KLAATU_SERVER_URL}/test_results", files=files)    
         except requests.exceptions.ConnectionError:
             pass
 
 
+@pytest.fixture(name="set_env_variables", autouse=True)
+def fixture_set_env_variables(experiment_data):
+    """Set any env variables XCUITests might need"""
+    os.environ["EXPERIMENT_NAME"] = experiment_data[0]["userFacingName"]
+
+
 @pytest.fixture(name="check_ping_for_experiment")
 def fixture_check_ping_for_experiment(experiment_slug, variables):
     def _check_ping_for_experiment(
@@ -209,16 +263,20 @@ def _check_ping_for_experiment(
 
 
 @pytest.fixture(name="setup_experiment")
-def setup_experiment(experiment_slug, json_data, request):
-    def _setup_experiment(branch):
-        logging.info(f"Testing experiment {experiment_slug}, BRANCH: {branch[0]}")
-        command = f"nimbus-cli --app firefox_ios --channel developer enroll {experiment_slug} --branch {branch[0]} --file {json_data} --reset-app"
+def setup_experiment(experiment_slug, json_data, request, experiment_branch, nimbus_cli_args):
+    def _setup_experiment():
+        logging.info(f"Testing experiment {experiment_slug}, BRANCH: {experiment_branch}")
+        command = f"nimbus-cli --app firefox_ios --channel developer enroll {experiment_slug} --branch {experiment_branch} --file {json_data} --reset-app -- {nimbus_cli_args}"
+        logging.info(f"Nimbus CLI Command: {command}\n")
         out = subprocess.check_output(
             command,
             cwd=os.path.join(here, os.pardir),
             stderr=subprocess.STDOUT,
             universal_newlines=True,
             shell=True,
         )
+
 
     return _setup_experiment
+
+# boot device xcrun simctl boot "iPhone 15 Pro Mox"