Clean up scripts and re-use common functionality

.github/workflows/ci.yml

@@ -108,7 +108,8 @@ jobs:
   run-integration-tests:
     name: Run Integration Test
     runs-on: ubuntu-22.04
-    container: ghcr.io/eclipse-velocitas/devcontainer-base-images/${{ matrix.language }}:v0.3
+    env:
+      packageInstallPath: .velocitas/packages/package_under_test/@main
     strategy:
       matrix:
         language: ["python", "cpp"]
@@ -123,46 +124,39 @@ jobs:
       fail-fast: false
 
     steps:
-      - name: Checkout repository
+      - name: Checkout template repository
         uses: actions/checkout@v4
         with:
+          repository: eclipse-velocitas/vehicle-app-${{ matrix.language }}-template
           fetch-depth: 0
 
-      - name: Add safe directory
-        run: |
-          git config --global --add safe.directory $(pwd)
-
-      - name: Checkout template repository
+      - name: Checkout package repository
         uses: actions/checkout@v4
         with:
-          repository: eclipse-velocitas/vehicle-app-${{ matrix.language }}-template
-          path: test/${{ matrix.language }}/repo
-
-      - name: Replace project config
-        run: |
-          cp -f test/${{ matrix.language }}/.velocitas.json test/${{ matrix.language }}/repo
+          path: ${{ env.packageInstallPath }}
+          fetch-depth: 0
 
       - name: Add safe directory
         run: |
-          git config --global --add safe.directory $(pwd)/test/${{ matrix.language }}/repo
+          git config --global --add safe.directory "*"
 
-      - name: Setup local package version
-        shell: bash
-        run: |
-          shopt -s dotglob
-          export PACKAGE_PATH=~/.velocitas/packages/package_under_test/@main
-          mkdir -p $PACKAGE_PATH
-          cp -r * $PACKAGE_PATH
-          git init $PACKAGE_PATH
-          ls -al $PACKAGE_PATH
-
-      - name: Run integration tests
+      - name: Replace project config
         run: |
-          export THIS_REPO_PATH=$(pwd)
-          export VELOCITAS_TEST_LANGUAGE=${{ matrix.language }}
-          export VELOCITAS_TEST_ROOT=test/${{ matrix.language }}/repo
-          export VELOCITAS_COMPONENT_PATH=${THIS_REPO_PATH}/${{ matrix.component }}
-
-          python -m pip install -r ./${{ matrix.component }}/test/requirements.txt
-          python -m pip install -r ./${{ matrix.component }}/requirements.txt
-          pytest -s ./${{ matrix.component }}/test/integration
+          cp -f ${{ env.packageInstallPath }}/test/${{ matrix.language }}/.velocitas.json .
+
+      - name: Run devContainer
+        uses: devcontainers/[email protected]
+        with:
+          env: VELOCITAS_HOME=/workspaces/devenv-devcontainer-setup
+          runCmd: |
+            CWD=$(pwd)
+
+            export VELOCITAS_TEMPLATE_REPO_PATH=$CWD
+            export VELOCITAS_PACKAGE_REPO_PATH=$CWD/${{ env.packageInstallPath }}
+            export VELOCITAS_TEST_LANGUAGE=${{ matrix.language }}
+            export VELOCITAS_COMPONENT_PATH=${VELOCITAS_PACKAGE_REPO_PATH}/${{ matrix.component }}
+
+            cd $VELOCITAS_PACKAGE_REPO_PATH
+            python -m pip install -r ./${{ matrix.component }}/test/requirements.txt
+            python -m pip install -r ./${{ matrix.component }}/requirements.txt
+            pytest -s ./${{ matrix.component }}/test/integration

build-system/cpp-cmake-conan/test/integration/conftest.py

@@ -33,11 +33,4 @@ def pytest_sessionstart(session):
         )
     )
 
-    if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp":
-        # FIXME: The C++ base image does not install conan globally
-        # but just for the vscode user, hence we have to download
-        # conan manually here. Can be removed once conan is installed
-        # globally.
-        subprocess.check_call(["python", "-m", "pip", "install", "conan==1.63.0"])
-
     subprocess.check_call(["velocitas", "init", "-v"], stdin=subprocess.PIPE)

build-system/cpp-cmake-conan/test/integration/data/cpp/minimal_project/.velocitas.json

@@ -9,5 +9,5 @@
         "language": "cpp",
         "repoType": "app"
     },
-    "cliVersion": "v0.11.0"
+    "cliVersion": "v0.12.0"
 }

build-system/cpp-cmake-conan/test/integration/test_integration_cpp.py

@@ -32,12 +32,19 @@ def test_normal_build_successful():
 
 def test_cross_build_successful():
     subprocess.check_call(
-        ["apt", "update"],
+        ["sudo", "apt", "update"],
         stdin=subprocess.PIPE,
     )
 
     subprocess.check_call(
-        ["apt", "install", "-y", "gcc-aarch64-linux-gnu", "g++-aarch64-linux-gnu"],
+        [
+            "sudo",
+            "apt",
+            "install",
+            "-y",
+            "gcc-aarch64-linux-gnu",
+            "g++-aarch64-linux-gnu",
+        ],
         stdin=subprocess.PIPE,
     )
 

grpc-interface-support/test/integration/conftest.py

@@ -14,7 +14,6 @@
 
 import os
 import shutil
-import subprocess
 
 
 def pytest_sessionstart(session):
@@ -26,26 +25,26 @@ def pytest_sessionstart(session):
     client_path = "/tmp/service_client"
     server_path = "/tmp/service_server"
 
-    shutil.copytree(os.environ["VELOCITAS_TEST_ROOT"], client_path)
-    shutil.copytree(os.environ["VELOCITAS_TEST_ROOT"], server_path)
+    shutil.copytree(os.environ["VELOCITAS_TEMPLATE_REPO_PATH"], client_path)
+    shutil.copytree(os.environ["VELOCITAS_TEMPLATE_REPO_PATH"], server_path)
 
     shutil.copytree(
-        f"test/{os.environ['VELOCITAS_TEST_LANGUAGE']}/service_client",
+        f"{os.environ['VELOCITAS_PACKAGE_REPO_PATH']}/test/{os.environ['VELOCITAS_TEST_LANGUAGE']}/service_client",
         client_path,
         dirs_exist_ok=True,
     )
     shutil.copytree(
-        f"test/{os.environ['VELOCITAS_TEST_LANGUAGE']}/service_server",
+        f"{os.environ['VELOCITAS_PACKAGE_REPO_PATH']}/test/{os.environ['VELOCITAS_TEST_LANGUAGE']}/service_server",
         server_path,
         dirs_exist_ok=True,
     )
     shutil.copytree(
-        "test/common/proto",
+        f"{os.environ['VELOCITAS_PACKAGE_REPO_PATH']}/test/common/proto",
         server_path,
         dirs_exist_ok=True,
     )
     shutil.copytree(
-        "test/common/proto",
+        f"{os.environ['VELOCITAS_PACKAGE_REPO_PATH']}/test/common/proto",
         client_path,
         dirs_exist_ok=True,
     )
@@ -62,10 +61,3 @@ def pytest_sessionstart(session):
 
     os.environ["SERVICE_CLIENT_ROOT"] = client_path
     os.environ["SERVICE_SERVER_ROOT"] = server_path
-
-    if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp":
-        # FIXME: The C++ base image does not install conan globally
-        # but just for the vscode user, hence we have to download
-        # conan manually here. Can be removed once conan is installed
-        # globally.
-        subprocess.check_call(["python", "-m", "pip", "install", "conan==1.63.0"])

grpc-interface-support/test/integration/test_integration_cpp.py

@@ -27,8 +27,9 @@ def get_subdirs(path: str) -> List[str]:
 
 
 def get_project_cache_dir() -> str:
-    project_caches = os.path.join(os.path.expanduser("~"), ".velocitas", "projects")
-    return get_subdirs(project_caches)[0]
+    return subprocess.check_output(
+        ["velocitas", "cache", "get", "--path"], encoding="utf-8"
+    ).strip()
 
 
 def get_dependency_count(service_name: str) -> int:

grpc-interface-support/test/integration/test_integration_python.py

@@ -42,19 +42,14 @@ def get_subdirs(path: str) -> List[str]:
 
 
 def get_project_cache_dir() -> str:
-    project_caches = os.path.join(os.path.expanduser("~"), ".velocitas", "projects")
-    return get_subdirs(project_caches)[0]
+    return subprocess.check_output(
+        ["velocitas", "cache", "get", "--path"], encoding="utf-8"
+    ).strip()
 
 
-def start_app(
-    python_file_name: str, env=os.environ
-) -> subprocess.CompletedProcess[bytes]:
+def start_app(python_file_name: str, env=os.environ) -> subprocess.Popen[bytes]:
     python_file = f"app/src/{python_file_name}.py"
-    return subprocess.run(
-        args=["python", python_file],
-        env=env,
-        shell=True,
-    )
+    return subprocess.Popen(["python", python_file], env=env)
 
 
 def assert_python_package_generated(
@@ -85,6 +80,9 @@ def test_pip_package_is_generated():
     assert_python_package_generated("vcsmotortrqmngservice", "motorcontrol")
 
 
+@pytest.mark.skip(
+    "Skip b/c a running MQTT broker is required for the test at the moment. See: https://github.com/eclipse-velocitas/vehicle-app-python-sdk/issues/143"
+)
 def test_pip_package_is_usable():
     envs = os.environ.copy()
     envs["SDV_SEATS_ADDRESS"] = "127.0.0.1:1234"
@@ -98,15 +96,15 @@ def test_pip_package_is_usable():
 
     assert subprocess.check_call(["velocitas", "init", "-v"]) == 0
 
-    launcher = start_app("launcher", envs)
-
-    assert launcher.returncode == 0
+    server_process = start_app("launcher", envs)
 
     print("============= TEST CLIENTS ===================")
     os.chdir(os.environ["SERVICE_CLIENT_ROOT"])
 
     assert subprocess.check_call(["velocitas", "init", "-v"]) == 0
 
-    launcher = start_app("launcher", envs)
+    client_process = start_app("launcher", envs)
 
-    assert launcher.returncode == 0
+    client_code = client_process.wait()
+    server_process.kill()
+    assert client_code == 0

sdk-installer/test/integration/conftest.py

@@ -13,8 +13,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-import shutil
-import subprocess
 
 
 def pytest_sessionstart(session):
@@ -23,12 +21,4 @@ def pytest_sessionstart(session):
     before performing collection and entering the run test loop.
     """
 
-    os.chdir(os.environ["VELOCITAS_TEST_ROOT"])
-    shutil.copy("../../common/AppManifest.json", "./app/AppManifest.json")
-
-    # FIXME: The C++ base image does not install conan globally
-    # but just for the vscode user, hence we have to download
-    # conan manually here. Can be removed once conan is installed
-    # globally.
-    if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp":
-        subprocess.check_call(["python", "-m", "pip", "install", "conan==1.63.0"])
+    os.chdir(os.environ["VELOCITAS_TEMPLATE_REPO_PATH"])

sdk-installer/test/integration/test_integration_cpp.py

@@ -17,17 +17,29 @@
 
 import pytest
 
+PACKAGE_NAME = "vehicle-app-sdk"
+
 if not os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp":
     pytest.skip("skipping C++ only tests", allow_module_level=True)
 
 
+@pytest.fixture(autouse=True)
+def remove_preinstalled_package():
+    print("Removing old packages")
+    remove_package(PACKAGE_NAME)
+
+
 def is_package_installed(package_name: str) -> bool:
     output = subprocess.check_output(
         ["conan", "search", package_name], encoding="utf-8"
     )
     return output.find("Existing package recipes:") != -1
 
 
+def remove_package(package_name: str):
+    subprocess.check_call(["conan", "remove", "-f", package_name])
+
+
 def test_no_sdk_reference_found__nothing_installed():
     conanfile_contents = """
 [requires]
@@ -37,7 +49,7 @@ def test_no_sdk_reference_found__nothing_installed():
         conanfile.write(conanfile_contents)
 
     subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE)
-    assert not is_package_installed("vehicle-app-sdk")
+    assert not is_package_installed(PACKAGE_NAME)
 
 
 def test_sdk_reference_found__sdk_installed():

sdk-installer/test/integration/test_integration_python.py

@@ -23,6 +23,9 @@
     pytest.skip("skipping Python only tests", allow_module_level=True)
 
 
+PACKAGE_NAME = "velocitas_sdk"
+
+
 @pytest.fixture(autouse=True)
 def clean_velocitas_download_directory():
     # return early if project directory does not yet exist
@@ -36,20 +39,30 @@ def clean_velocitas_download_directory():
     )
 
 
+@pytest.fixture(autouse=True)
+def remove_preinstalled_package():
+    remove_package(PACKAGE_NAME)
+
+
 def get_subdirs(path: str) -> List[str]:
     return [f.path for f in os.scandir(path) if f.is_dir()]
 
 
 def get_project_cache_dir() -> str:
-    project_caches = os.path.join(os.path.expanduser("~"), ".velocitas", "projects")
-    return get_subdirs(project_caches)[0]
+    return subprocess.check_output(
+        ["velocitas", "cache", "get", "--path"], encoding="utf-8"
+    ).strip()
 
 
 def is_package_installed(package_name: str) -> bool:
     return_code = subprocess.call(["pip", "show", package_name])
     return return_code == 0
 
 
+def remove_package(package_name: str):
+    subprocess.call(["pip", "uninstall", "-y", package_name])
+
+
 def can_import_and_use_vehicleapp() -> bool:
     try:
         from velocitas_sdk.vehicle_app import VehicleApp
@@ -75,8 +88,8 @@ def test_no_sdk_reference_found__nothing_installed():
         requirements.write(requirements_contents)
 
     subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE)
-    assert not is_package_installed("velocitas_sdk") and not is_package_installed(
-        "velocitas-sdk"
+    assert not is_package_installed(PACKAGE_NAME) and not is_package_installed(
+        PACKAGE_NAME.replace("_", "-")
     )
 
 
@@ -90,7 +103,7 @@ def test_sdk_reference_found__sdk_installed():
         requirements.write(requirements_contents)
 
     subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE)
-    assert is_package_installed("velocitas_sdk") or is_package_installed(
-        "velocitas-sdk"
+    assert is_package_installed(PACKAGE_NAME) or is_package_installed(
+        PACKAGE_NAME.replace("_", "-")
     )
     assert can_import_and_use_vehicleapp()