Merge branch 'master' of ssh://github.com/commaai/openpilot into pand…

…ad_single_thread

.github/workflows/jenkins-pr-trigger.yaml

@@ -0,0 +1,45 @@
+name: jenkins scan
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+jobs:
+  # TODO: gc old branches in a separate job in this workflow
+  scan-comments:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.issue.pull_request }}
+    steps:
+    - name: Check for trigger phrase
+      id: check_comment
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const triggerPhrase = "trigger-jenkins";
+          const comment = context.payload.comment.body;
+          const commenter = context.payload.comment.user.login;
+
+          const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            username: commenter
+          });
+
+          const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin';
+
+          return (hasWriteAccess && comment.includes(triggerPhrase));
+        result-encoding: json
+
+    - name: Checkout repository
+      if: steps.check_comment.outputs.result == 'true'
+      uses: actions/checkout@v4
+      with:
+        ref: refs/pull/${{ github.event.issue.number }}/head
+
+    - name: Push to tmp-jenkins branch
+      if: steps.check_comment.outputs.result == 'true'
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git checkout -b tmp-jenkins-${{ github.event.issue.number }}
+        GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}

.github/workflows/selfdrive_tests.yaml

@@ -74,8 +74,11 @@ jobs:
   build:
     strategy:
       matrix:
-        arch: ${{ fromJson('["x86_64"]') }} # TODO: Re-add build test for aarch64 once we switched to ubuntu-2404
-    runs-on: ubuntu-latest
+        arch: ${{ fromJson(
+           ((github.repository == 'commaai/openpilot') &&
+              ((github.event_name != 'pull_request') ||
+               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }}
     steps:
     - uses: actions/checkout@v4
       with:
@@ -84,7 +87,7 @@ jobs:
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        : # (TODO: Re-add this once we test other archs) echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
@@ -118,6 +121,23 @@ jobs:
     - name: Building openpilot
       run: . .venv/bin/activate && scons -j$(nproc)
 
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        export PUSH_IMAGE=true
+        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
+
   static_analysis:
     name: static analysis
     runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
@@ -335,8 +355,8 @@ jobs:
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report-${{ inputs.run_number }}
-          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}
+          name: report-${{ inputs.run_number || '1' }}
+          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number || '1' }}
       - name: Get changes to selfdrive/ui
         if: ${{ github.event_name == 'pull_request' }}
         id: changed-files
@@ -359,7 +379,7 @@ jobs:
           git checkout -b openpilot/pr-${{ github.event.pull_request.number }}
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
-          sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report/screenshots/* .
+          sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report_1/screenshots/* .
           git add .
           git commit -m "screenshots for PR #${{ github.event.pull_request.number }}"
           git push origin openpilot/pr-${{ github.event.pull_request.number }} --force

.importlinter

@@ -0,0 +1,44 @@
+[importlinter]
+root_packages =
+    openpilot
+
+[importlinter:contract:1]
+name = Forbid imports from openpilot.selfdrive.car to openpilot.system
+type = forbidden
+source_modules =
+    openpilot.selfdrive.car
+forbidden_modules =
+    openpilot.system
+    openpilot.body
+    openpilot.docs
+    openpilot.msgq
+    openpilot.panda
+    openpilot.rednose
+    openpilot.release
+    openpilot.teleoprtc
+    openpilot.tinygrad
+ignore_imports =
+    openpilot.selfdrive.car.card -> openpilot.common.realtime
+    openpilot.selfdrive.car.card -> openpilot.selfdrive.controls.lib.events
+    openpilot.selfdrive.car.interfaces -> openpilot.selfdrive.controls.lib.events
+    openpilot.selfdrive.car.tests.test_models -> openpilot.tools.lib.logreader
+    openpilot.selfdrive.car.tests.test_models -> openpilot.selfdrive.car.card
+    openpilot.selfdrive.car.tests.test_models -> openpilot.tools.lib.route
+    openpilot.selfdrive.car.tests.test_models -> openpilot.system.hardware.hw
+    openpilot.selfdrive.car.tests.test_models -> openpilot.selfdrive.test.helpers
+    openpilot.selfdrive.car.isotp_parallel_query -> openpilot.common.swaglog
+    openpilot.selfdrive.car.fw_versions -> openpilot.common.swaglog
+    openpilot.selfdrive.car.disable_ecu -> openpilot.common.swaglog
+    openpilot.selfdrive.car.vin -> openpilot.common.swaglog
+    openpilot.selfdrive.car.ecu_addrs -> openpilot.common.swaglog
+    openpilot.selfdrive.car.car_helpers -> openpilot.common.swaglog
+    openpilot.selfdrive.car.car_helpers -> openpilot.system.version
+    openpilot.selfdrive.car.ford.carcontroller -> openpilot.selfdrive.controls.lib.drive_helpers
+    openpilot.selfdrive.car.gm.interface -> openpilot.selfdrive.controls.lib.drive_helpers
+    openpilot.selfdrive.car.honda.carcontroller -> openpilot.selfdrive.controls.lib.drive_helpers
+    openpilot.selfdrive.car.interfaces -> openpilot.selfdrive.controls.lib.drive_helpers
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_angle
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.longcontrol
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_torque
+    openpilot.selfdrive.car.tests.test_car_interfaces -> openpilot.selfdrive.controls.lib.latcontrol_pid
+unmatched_ignore_imports_alerting = warn

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.5.4
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
@@ -93,6 +93,13 @@ repos:
         pass_filenames: false
         files: '^selfdrive/ui/translations/'
 -   repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.28.6
+    rev: 0.29.0
     hooks:
     -   id: check-github-workflows
+-   repo: local
+    hooks:
+    -   id: import-linter
+        name: import linter
+        entry: lint-imports
+        language: system
+        pass_filenames: false

Jenkinsfile

@@ -82,7 +82,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
 
     lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
       docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
-        timeout(time: 20, unit: 'MINUTES') {
+        timeout(time: 35, unit: 'MINUTES') {
           retry (3) {
             device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }

README.md

@@ -19,7 +19,7 @@ To start using openpilot in a car
 To use openpilot in a car, you need four things:
 1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
 2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
-3. **Supported Car:** Ensure that you have one of [the 250+ supported cars](docs/CARS.md).
+3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
 4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car.
 
 We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.

SConstruct

@@ -102,7 +102,6 @@ if arch == "larch64":
 
   libpath = [
     "/usr/local/lib",
-    "/usr/lib",
     "/system/vendor/lib64",
     f"#third_party/acados/{arch}/lib",
   ]

cereal/log.capnp

@@ -137,8 +137,6 @@ struct FrameData {
   requestId @28 :UInt32;
   encodeId @1 :UInt32;
 
-  frameType @7 :FrameType;
-
   # Timestamps
   timestampEof @2 :UInt64;
   timestampSof @8 :UInt64;
@@ -158,7 +156,7 @@ struct FrameData {
 
   temperaturesC @24 :List(Float32);
 
-  enum FrameType {
+  enum FrameTypeDEPRECATED {
     unknown @0;
     neo @1;
     chffrAndroid @2;
@@ -175,6 +173,7 @@ struct FrameData {
 
   frameLengthDEPRECATED @3 :Int32;
   globalGainDEPRECATED @5 :Int32;
+  frameTypeDEPRECATED @7 :FrameTypeDEPRECATED;
   androidCaptureResultDEPRECATED @9 :AndroidCaptureResult;
   lensPosDEPRECATED @11 :Int32;
   lensSagDEPRECATED @12 :Float32;
@@ -1252,6 +1251,38 @@ struct LiveLocationKalman {
   }
 }
 
+
+struct LivePose {
+  # More info on reference frames:
+  # https://github.com/commaai/openpilot/tree/master/common/transformations
+  orientationNED @0 :XYZMeasurement;
+  velocityDevice @1 :XYZMeasurement;
+  accelerationDevice @2 :XYZMeasurement;
+  angularVelocityDevice @3 :XYZMeasurement;
+
+  inputsOK @4 :Bool = false;
+  posenetOK @5 :Bool = false;
+  sensorsOK @6 :Bool = false;
+
+  filterState @7 :FilterState;
+
+  struct XYZMeasurement {
+    x @0 :Float32;
+    y @1 :Float32;
+    z @2 :Float32;
+    xStd @3 :Float32;
+    yStd @4 :Float32;
+    zStd @5 :Float32;
+    valid @6 :Bool;
+  }
+
+  struct FilterState {
+    value @0 : List(Float64);
+    std @1 : List(Float64);
+    valid @2 : Bool;
+  }
+}
+
 struct ProcLog {
   cpuTimes @0 :List(CPUTimes);
   mem @1 :Mem;
@@ -2293,6 +2324,7 @@ struct Event {
     carParams @69: Car.CarParams;
     driverMonitoringState @71: DriverMonitoringState;
     liveLocationKalman @72 :LiveLocationKalman;
+    livePose @129 :LivePose;
     modelV2 @75 :ModelDataV2;
     drivingModelData @128 :DrivingModelData;
     driverStateV2 @92 :DriverStateV2;

cereal/messaging/__init__.py

@@ -2,7 +2,8 @@
 from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
                                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
 from msgq.ipc_pyx import MultiplePublishersError, IpcError
-from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw, context
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
+import msgq
 
 import os
 import capnp
@@ -17,6 +18,10 @@
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
+def reset_context():
+  msgq.context = Context()
+
+
 def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
   with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
     return msg

cereal/services.py

@@ -38,7 +38,7 @@ def __init__(self, should_log: bool, frequency: float, decimation: Optional[int]
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
-  "longitudinalPlan": (True, 20., 5),
+  "longitudinalPlan": (True, 20., 10),
   "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
@@ -47,9 +47,10 @@ def __init__(self, should_log: bool, frequency: float, decimation: Optional[int]
   "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
-  "liveLocationKalman": (True, 20., 5),
+  "livePose": (True, 20., 4),
+  "liveLocationKalman": (True, 20.),
   "liveParameters": (True, 20., 5),
-  "cameraOdometry": (True, 20., 5),
+  "cameraOdometry": (True, 20., 10),
   "thumbnail": (True, 0.2, 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),

pyproject.toml

@@ -52,7 +52,8 @@ dependencies = [
   "websocket_client",
 
   # acados deps
-  "casadi",
+  "casadi @ https://github.com/commaai/casadi/releases/download/nightly-release-3.6.6/casadi-3.6.6-cp312-none-manylinux2014_aarch64.whl ; (python_version == '3.12' and platform_machine == 'aarch64')", # TODO: Go back to pypi casadi when they fix aarch64 for python312
+  "casadi; platform_machine != 'aarch64' or python_version != '3.12'",
   "future-fstrings",
 
   # these should be removed
@@ -73,6 +74,7 @@ docs = [
 testing = [
   "coverage",
   "hypothesis ==6.47.*",
+  "import-linter",
   "mypy",
   "pre-commit",
   "pytest",
@@ -99,31 +101,28 @@ dev = [
   "inputs",
   "lru-dict",
   "matplotlib",
-  "metadrive-simulator; platform_machine != 'aarch64'",
+  "metadrive-simulator@git+https://github.com/commaai/metadrive@opencv_headless ; platform_machine != 'aarch64'",
   "mpld3",
   "myst-parser",
   "natsort",
   "opencv-python-headless",
   "parameterized >=0.8, <0.9",
   #pprofile = "*"
   "pyautogui",
-  "pygame",
   "pyopencl; platform_machine != 'aarch64'", # broken on arm64
   "pywinctl",
   "pyprof2calltree",
   "rerun-sdk",
   "tabulate",
   "types-requests",
   "types-tabulate",
+  "hexdump",
 
   # this is only pinned since 5.15.11 is broken
   "pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
 
 ]
 
-[tool.uv.sources]
-metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "opencv_headless" }
-
 [project.urls]
 Homepage = "https://comma.ai"
 
@@ -134,6 +133,9 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = [ "." ]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = "--ignore=openpilot/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/  -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"