Merge remote-tracking branch 'upstream/master' into lta-bit

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: tests
 on: [push, pull_request]
 
 env:
-  RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS=error --shm-size 1G --rm opendbc /bin/bash -c
+  RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS="error,default::DeprecationWarning" --shm-size 1G --rm opendbc /bin/bash -c
   BUILD: |
       docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile) || true
       docker pull ghcr.io/commaai/opendbc:latest || true

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         - --check-hidden
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.4.1
     hooks:
     -   id: mypy
 -   repo: https://github.com/PyCQA/flake8

Dockerfile

@@ -34,13 +34,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.8.10
-RUN pyenv global 3.8.10
+RUN pyenv install 3.11.4
+RUN pyenv global 3.11.4
 RUN pyenv rehash
 
 COPY requirements.txt /tmp/
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
-RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.5.2
+RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.17.4
 
 ENV PYTHONPATH=/project
 

can/parser_pyx.pyx

@@ -41,12 +41,17 @@ cdef class CANParser:
     self.vl_all = {}
     self.ts_nanos = {}
     msg_name_to_address = {}
+    msg_address_to_signals = {}
 
     for i in range(self.dbc[0].msgs.size()):
       msg = self.dbc[0].msgs[i]
       name = msg.name.decode("utf8")
 
       msg_name_to_address[name] = msg.address
+      msg_address_to_signals[msg.address] = set()
+      for sig in msg.sigs:
+        msg_address_to_signals[msg.address].add(sig.name.decode("utf8"))
+
       self.address_to_msg_name[msg.address] = name
       self.vl[msg.address] = {}
       self.vl[name] = self.vl[msg.address]
@@ -58,12 +63,13 @@ cdef class CANParser:
     # Convert message names into addresses
     for i in range(len(signals)):
       s = signals[i]
-      if not isinstance(s[1], numbers.Number):
-        if s[1] not in msg_name_to_address:
-          print(msg_name_to_address)
-          raise RuntimeError(f"could not find message {repr(s[1])} in DBC {self.dbc_name}")
-        s = (s[0], msg_name_to_address[s[1]])
-        signals[i] = s
+      address = s[1] if isinstance(s[1], numbers.Number) else msg_name_to_address.get(s[1])
+      if address not in msg_address_to_signals:
+        raise RuntimeError(f"could not find message {repr(s[1])} in DBC {self.dbc_name}")
+      if s[0] not in msg_address_to_signals[address]:
+        raise RuntimeError(f"could not find signal {repr(s[0])} in {repr(s[1])}, DBC {self.dbc_name}")
+
+      signals[i] = (s[0], address)
 
     for i in range(len(checks)):
       c = checks[i]

can/tests/test_packer_parser.py

@@ -2,6 +2,7 @@
 import os
 import unittest
 import random
+from functools import partial
 
 import cereal.messaging as messaging
 from opendbc.can.parser import CANParser
@@ -317,6 +318,24 @@ def test_timestamp_nanos(self):
       ts_nanos = parser.ts_nanos["POWERTRAIN_DATA"].values()
       self.assertEqual(set(ts_nanos), {0})
 
+  def test_undefined_signals(self):
+    # Ensure we don't allow messages or signals not in the DBC
+    existing_signals = {
+      "STEERING_CONTROL": ["STEER_TORQUE_REQUEST", "SET_ME_X00_2", "COUNTER"],
+      228: ["STEER_TORQUE_REQUEST", "SET_ME_X00_2", "COUNTER"],
+      "CAN_FD_MESSAGE": ["SIGNED", "64_BIT_LE", "64_BIT_BE", "COUNTER"],
+      245: ["SIGNED", "64_BIT_LE", "64_BIT_BE", "COUNTER"],
+    }
+
+    for msg, sigs in existing_signals.items():
+      for sig in sigs:
+        CANParser(TEST_DBC, [(sig, msg)], [(msg, 0)])
+        new_msg = msg + "1" if isinstance(msg, str) else msg + 1
+        self.assertRaises(RuntimeError, partial(CANParser, TEST_DBC, [(sig + "1", msg)], [(msg, 0)]))
+        self.assertRaises(RuntimeError, partial(CANParser, TEST_DBC, [(sig, new_msg)], [(msg, 0)]))
+        self.assertRaises(RuntimeError, partial(CANParser, TEST_DBC, [(sig, msg)], [(new_msg, 0)]))
+        self.assertRaises(RuntimeError, partial(CANParser, TEST_DBC, [(sig, new_msg)], [(new_msg, 0)]))
+
 
 if __name__ == "__main__":
   unittest.main()

generator/chrysler/_stellantis_common_ram.py

@@ -32,7 +32,7 @@
   chrysler_path = os.path.dirname(os.path.realpath(__file__))
 
   for out, addr_lookup in chrysler_to_ram.items():
-    with open(os.path.join(chrysler_path, src)) as in_f, open(os.path.join(chrysler_path, out), 'w') as out_f:
+    with open(os.path.join(chrysler_path, src), encoding='utf-8') as in_f, open(os.path.join(chrysler_path, out), 'w', encoding='utf-8') as out_f:
       out_f.write(f'CM_ "Generated from {src}"\n\n')
 
       wrote_addrs = set()

generator/generator.py

@@ -11,7 +11,7 @@
 
 
 def read_dbc(src_dir: str, filename: str) -> str:
-  with open(os.path.join(src_dir, filename)) as file_in:
+  with open(os.path.join(src_dir, filename), encoding='utf-8') as file_in:
     return file_in.read()
 
 
@@ -23,7 +23,7 @@ def create_dbc(src_dir: str, filename: str, output_path: str):
   output_filename = filename.replace('.dbc', generated_suffix)
   output_file_location = os.path.join(output_path, output_filename)
 
-  with open(output_file_location, 'w') as dbc_file_out:
+  with open(output_file_location, 'w', encoding='utf-8') as dbc_file_out:
     dbc_file_out.write('CM_ "AUTOGENERATED FILE, DO NOT EDIT";\n')
 
     for include_filename in includes:

generator/hyundai/hyundai_kia_mando_corner_radar.py

@@ -5,7 +5,7 @@
 if __name__ == "__main__":
   dbc_name = os.path.basename(__file__).replace(".py", ".dbc")
   hyundai_path = os.path.dirname(os.path.realpath(__file__))
-  with open(os.path.join(hyundai_path, dbc_name), "w") as f:
+  with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f:
     f.write("""
 VERSION ""
 

generator/hyundai/hyundai_kia_mando_front_radar.py

@@ -4,7 +4,7 @@
 if __name__ == "__main__":
   dbc_name = os.path.basename(__file__).replace(".py", ".dbc")
   hyundai_path = os.path.dirname(os.path.realpath(__file__))
-  with open(os.path.join(hyundai_path, dbc_name), "w") as f:
+  with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f:
     f.write("""
 VERSION ""
 

generator/toyota/_toyota_2017.dbc

@@ -241,7 +241,7 @@ BO_ 1042 LKAS_HUD: 8 DSU
  SG_ LDA_UNAVAILABLE : 16|1@0+ (1,0) [0|1] "" XXX
  SG_ LDA_SENSITIVITY : 18|2@0+ (1,0) [0|3] "" XXX
  SG_ LDA_SA_TOGGLE : 20|2@0+ (1,0) [0|3] "" XXX
- SG_ LDA_SPEED_TOO_LOW : 21|1@0+ (1,0) [0|1] "" XXX
+ SG_ LDA_MESSAGES : 23|3@0+ (1,0) [0|1] "" XXX
  SG_ LDA_ON_MESSAGE : 31|2@0+ (1,0) [0|3] "" XXX
  SG_ REPEATED_BEEPS : 32|1@0+ (1,0) [0|1] "" XXX
  SG_ LANE_SWAY_TOGGLE : 43|1@0+ (1,0) [0|1] "" XXX
@@ -423,7 +423,7 @@ CM_ SG_ 1042 LDA_SENSITIVITY "LDA Sensitivity";
 CM_ SG_ 1042 LDA_ON_MESSAGE "Display LDA Turned ON message";
 CM_ SG_ 1042 REPEATED_BEEPS "LDA audible warning";
 CM_ SG_ 1042 LDA_UNAVAILABLE_QUIET "LDA toggles and sensitivity settings are greyed out if set to 1";
-CM_ SG_ 1042 LDA_SPEED_TOO_LOW "length is 3 bits in the leaked DBC, displays LDA unavailable below approx 50 km/h if set to 1";
+CM_ SG_ 1042 LDA_MESSAGES "Various LDA Messages";
 CM_ SG_ 1042 LDA_FRONT_CAMERA_BLOCKED "originally LDAFCVB, LDA related settings are greyed out if set to 1";
 CM_ SG_ 1042 TAKE_CONTROL "Please Control Steering Wheel warning";
 CM_ SG_ 1042 LANE_SWAY_TOGGLE "Lane Sway Warning System SWS Switch";
@@ -487,7 +487,7 @@ VAL_ 1042 LEFT_LINE 3 "orange" 2 "faded" 1 "solid" 0 "none";
 VAL_ 1042 LDA_ON_MESSAGE 2 "Lane Departure Alert Turned ON, Steering Assist Inactive" 1 "Lane Departure Alert Turned ON, Steering Assist Active" 0 "clear";
 VAL_ 1042 LDA_SA_TOGGLE  2 "steering assist off" 1 "steering assist on";
 VAL_ 1042 LDA_SENSITIVITY 2 "standard" 1 "high" 0 "undefined";
-VAL_ 1042 LDA_SPEED_TOO_LOW 1 "lda unavailable, speed too low" 0 "ok";
+VAL_ 1042 LDA_MESSAGES 4 "lda unavailable at this speed" 1 "lda unavailable below approx 50km/h" 0 "ok";
 VAL_ 1042 LDA_FRONT_CAMERA_BLOCKED 1 "lda unavailable" 0 "ok";
 VAL_ 1042 TAKE_CONTROL 1 "take control" 0 "ok";
 VAL_ 1042 LANE_SWAY_WARNING 3 "ok" 2 "orange please take a break" 1 "prompt would you like to take a break" 0 "ok";

hyundai_kia_generic.dbc

@@ -698,6 +698,7 @@ BO_ 1365 FPCM11: 8 FPCM
 
 BO_ 871 LVR12: 8 LVR
  SG_ CF_Lvr_CruiseSet : 0|8@1+ (1.0,0.0) [0.0|255.0] "" CLU,TCU
+ SG_ CF_Lvr_IsgState : 8|2@1+ (1.0,0.0) [0.0|3.0] "" CLU,TCU
  SG_ CF_Lvr_Gear : 32|4@1+ (1.0,0.0) [0.0|15.0] ""  CLU,TCU
 
 BO_ 872 LVR11: 8 LVR
@@ -1641,9 +1642,11 @@ BO_ 1348 Navi_HU: 8 XXX
  SG_ SpeedLim_Nav_Clu : 7|8@0+ (1,0) [0|255] "" XXX
 
 CM_ "BO_ E_EMS11: All (plug-in) hybrids use this gas signal: CR_Vcu_AccPedDep_Pos, and all EVs use the Accel_Pedal_Pos signal. See hyundai/values.py for a specific car list";
+CM_ SG_ 871 CF_Lvr_IsgState "Idle Stop and Go";
 CM_ SG_ 1348 SpeedLim_Nav_Clu "Speed limit displayed on Nav, Cluster and HUD";
 
 VAL_ 274 CUR_GR 1 "D" 2 "D" 3 "D" 4 "D" 5 "D" 6 "D" 7 "D" 8 "D" 14 "R" 0 "P";
+VAL_ 871 CF_Lvr_IsgState 0 "enabled" 1 "activated" 2 "unknown" 3 "disabled";
 VAL_ 871 CF_Lvr_Gear 12 "T" 5 "D" 8 "S" 6 "N" 7 "R" 0 "P";
 VAL_ 882 Elect_Gear_Shifter 5 "D" 8 "S" 6 "N" 7 "R" 0 "P";
 VAL_ 905 ACCMode 0 "off" 1 "enabled" 2 "driver_override" 3 "off_maybe_fault" 4 "cancelled";

requirements.txt

@@ -1,8 +1,8 @@
-Cython==0.29.34
-flake8==6.0.0
-Jinja2==3.1.2
-numpy==1.24.2
-pycapnp==1.3.0
-pylint==2.17.2
-pyyaml==6.0
+Cython
+flake8
+Jinja2
+numpy
+pycapnp
+pylint==2.17.4
+pyyaml
 scons

toyota_new_mc_pt_generated.dbc

@@ -286,7 +286,7 @@ BO_ 1042 LKAS_HUD: 8 DSU
  SG_ LDA_UNAVAILABLE : 16|1@0+ (1,0) [0|1] "" XXX
  SG_ LDA_SENSITIVITY : 18|2@0+ (1,0) [0|3] "" XXX
  SG_ LDA_SA_TOGGLE : 20|2@0+ (1,0) [0|3] "" XXX
- SG_ LDA_SPEED_TOO_LOW : 21|1@0+ (1,0) [0|1] "" XXX
+ SG_ LDA_MESSAGES : 23|3@0+ (1,0) [0|1] "" XXX
  SG_ LDA_ON_MESSAGE : 31|2@0+ (1,0) [0|3] "" XXX
  SG_ REPEATED_BEEPS : 32|1@0+ (1,0) [0|1] "" XXX
  SG_ LANE_SWAY_TOGGLE : 43|1@0+ (1,0) [0|1] "" XXX
@@ -468,7 +468,7 @@ CM_ SG_ 1042 LDA_SENSITIVITY "LDA Sensitivity";
 CM_ SG_ 1042 LDA_ON_MESSAGE "Display LDA Turned ON message";
 CM_ SG_ 1042 REPEATED_BEEPS "LDA audible warning";
 CM_ SG_ 1042 LDA_UNAVAILABLE_QUIET "LDA toggles and sensitivity settings are greyed out if set to 1";
-CM_ SG_ 1042 LDA_SPEED_TOO_LOW "length is 3 bits in the leaked DBC, displays LDA unavailable below approx 50 km/h if set to 1";
+CM_ SG_ 1042 LDA_MESSAGES "Various LDA Messages";
 CM_ SG_ 1042 LDA_FRONT_CAMERA_BLOCKED "originally LDAFCVB, LDA related settings are greyed out if set to 1";
 CM_ SG_ 1042 TAKE_CONTROL "Please Control Steering Wheel warning";
 CM_ SG_ 1042 LANE_SWAY_TOGGLE "Lane Sway Warning System SWS Switch";
@@ -532,7 +532,7 @@ VAL_ 1042 LEFT_LINE 3 "orange" 2 "faded" 1 "solid" 0 "none";
 VAL_ 1042 LDA_ON_MESSAGE 2 "Lane Departure Alert Turned ON, Steering Assist Inactive" 1 "Lane Departure Alert Turned ON, Steering Assist Active" 0 "clear";
 VAL_ 1042 LDA_SA_TOGGLE  2 "steering assist off" 1 "steering assist on";
 VAL_ 1042 LDA_SENSITIVITY 2 "standard" 1 "high" 0 "undefined";
-VAL_ 1042 LDA_SPEED_TOO_LOW 1 "lda unavailable, speed too low" 0 "ok";
+VAL_ 1042 LDA_MESSAGES 4 "lda unavailable at this speed" 1 "lda unavailable below approx 50km/h" 0 "ok";
 VAL_ 1042 LDA_FRONT_CAMERA_BLOCKED 1 "lda unavailable" 0 "ok";
 VAL_ 1042 TAKE_CONTROL 1 "take control" 0 "ok";
 VAL_ 1042 LANE_SWAY_WARNING 3 "ok" 2 "orange please take a break" 1 "prompt would you like to take a break" 0 "ok";

toyota_nodsu_pt_generated.dbc

@@ -286,7 +286,7 @@ BO_ 1042 LKAS_HUD: 8 DSU
  SG_ LDA_UNAVAILABLE : 16|1@0+ (1,0) [0|1] "" XXX
  SG_ LDA_SENSITIVITY : 18|2@0+ (1,0) [0|3] "" XXX
  SG_ LDA_SA_TOGGLE : 20|2@0+ (1,0) [0|3] "" XXX
- SG_ LDA_SPEED_TOO_LOW : 21|1@0+ (1,0) [0|1] "" XXX
+ SG_ LDA_MESSAGES : 23|3@0+ (1,0) [0|1] "" XXX
  SG_ LDA_ON_MESSAGE : 31|2@0+ (1,0) [0|3] "" XXX
  SG_ REPEATED_BEEPS : 32|1@0+ (1,0) [0|1] "" XXX
  SG_ LANE_SWAY_TOGGLE : 43|1@0+ (1,0) [0|1] "" XXX
@@ -468,7 +468,7 @@ CM_ SG_ 1042 LDA_SENSITIVITY "LDA Sensitivity";
 CM_ SG_ 1042 LDA_ON_MESSAGE "Display LDA Turned ON message";
 CM_ SG_ 1042 REPEATED_BEEPS "LDA audible warning";
 CM_ SG_ 1042 LDA_UNAVAILABLE_QUIET "LDA toggles and sensitivity settings are greyed out if set to 1";
-CM_ SG_ 1042 LDA_SPEED_TOO_LOW "length is 3 bits in the leaked DBC, displays LDA unavailable below approx 50 km/h if set to 1";
+CM_ SG_ 1042 LDA_MESSAGES "Various LDA Messages";
 CM_ SG_ 1042 LDA_FRONT_CAMERA_BLOCKED "originally LDAFCVB, LDA related settings are greyed out if set to 1";
 CM_ SG_ 1042 TAKE_CONTROL "Please Control Steering Wheel warning";
 CM_ SG_ 1042 LANE_SWAY_TOGGLE "Lane Sway Warning System SWS Switch";
@@ -532,7 +532,7 @@ VAL_ 1042 LEFT_LINE 3 "orange" 2 "faded" 1 "solid" 0 "none";
 VAL_ 1042 LDA_ON_MESSAGE 2 "Lane Departure Alert Turned ON, Steering Assist Inactive" 1 "Lane Departure Alert Turned ON, Steering Assist Active" 0 "clear";
 VAL_ 1042 LDA_SA_TOGGLE  2 "steering assist off" 1 "steering assist on";
 VAL_ 1042 LDA_SENSITIVITY 2 "standard" 1 "high" 0 "undefined";
-VAL_ 1042 LDA_SPEED_TOO_LOW 1 "lda unavailable, speed too low" 0 "ok";
+VAL_ 1042 LDA_MESSAGES 4 "lda unavailable at this speed" 1 "lda unavailable below approx 50km/h" 0 "ok";
 VAL_ 1042 LDA_FRONT_CAMERA_BLOCKED 1 "lda unavailable" 0 "ok";
 VAL_ 1042 TAKE_CONTROL 1 "take control" 0 "ok";
 VAL_ 1042 LANE_SWAY_WARNING 3 "ok" 2 "orange please take a break" 1 "prompt would you like to take a break" 0 "ok";

toyota_tnga_k_pt_generated.dbc

@@ -286,7 +286,7 @@ BO_ 1042 LKAS_HUD: 8 DSU
  SG_ LDA_UNAVAILABLE : 16|1@0+ (1,0) [0|1] "" XXX
  SG_ LDA_SENSITIVITY : 18|2@0+ (1,0) [0|3] "" XXX
  SG_ LDA_SA_TOGGLE : 20|2@0+ (1,0) [0|3] "" XXX
- SG_ LDA_SPEED_TOO_LOW : 21|1@0+ (1,0) [0|1] "" XXX
+ SG_ LDA_MESSAGES : 23|3@0+ (1,0) [0|1] "" XXX
  SG_ LDA_ON_MESSAGE : 31|2@0+ (1,0) [0|3] "" XXX
  SG_ REPEATED_BEEPS : 32|1@0+ (1,0) [0|1] "" XXX
  SG_ LANE_SWAY_TOGGLE : 43|1@0+ (1,0) [0|1] "" XXX
@@ -468,7 +468,7 @@ CM_ SG_ 1042 LDA_SENSITIVITY "LDA Sensitivity";
 CM_ SG_ 1042 LDA_ON_MESSAGE "Display LDA Turned ON message";
 CM_ SG_ 1042 REPEATED_BEEPS "LDA audible warning";
 CM_ SG_ 1042 LDA_UNAVAILABLE_QUIET "LDA toggles and sensitivity settings are greyed out if set to 1";
-CM_ SG_ 1042 LDA_SPEED_TOO_LOW "length is 3 bits in the leaked DBC, displays LDA unavailable below approx 50 km/h if set to 1";
+CM_ SG_ 1042 LDA_MESSAGES "Various LDA Messages";
 CM_ SG_ 1042 LDA_FRONT_CAMERA_BLOCKED "originally LDAFCVB, LDA related settings are greyed out if set to 1";
 CM_ SG_ 1042 TAKE_CONTROL "Please Control Steering Wheel warning";
 CM_ SG_ 1042 LANE_SWAY_TOGGLE "Lane Sway Warning System SWS Switch";
@@ -532,7 +532,7 @@ VAL_ 1042 LEFT_LINE 3 "orange" 2 "faded" 1 "solid" 0 "none";
 VAL_ 1042 LDA_ON_MESSAGE 2 "Lane Departure Alert Turned ON, Steering Assist Inactive" 1 "Lane Departure Alert Turned ON, Steering Assist Active" 0 "clear";
 VAL_ 1042 LDA_SA_TOGGLE  2 "steering assist off" 1 "steering assist on";
 VAL_ 1042 LDA_SENSITIVITY 2 "standard" 1 "high" 0 "undefined";
-VAL_ 1042 LDA_SPEED_TOO_LOW 1 "lda unavailable, speed too low" 0 "ok";
+VAL_ 1042 LDA_MESSAGES 4 "lda unavailable at this speed" 1 "lda unavailable below approx 50km/h" 0 "ok";
 VAL_ 1042 LDA_FRONT_CAMERA_BLOCKED 1 "lda unavailable" 0 "ok";
 VAL_ 1042 TAKE_CONTROL 1 "take control" 0 "ok";
 VAL_ 1042 LANE_SWAY_WARNING 3 "ok" 2 "orange please take a break" 1 "prompt would you like to take a break" 0 "ok";