Merge branch 'upstream/opendbc/master' into palisade-2023-port-1

.github/workflows/repo.yml

@@ -0,0 +1,28 @@
+name: repo
+
+on:
+  schedule:
+    - cron: "0 15 * * 2"
+  workflow_dispatch:
+
+jobs:
+  pre-commit-autoupdate:
+    name: pre-commit autoupdate
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/commaai/opendbc:latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: pre-commit autoupdate
+      run: |
+        git config --global --add safe.directory '*'
+        pre-commit autoupdate
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
+      with:
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
+        commit-message: Update pre-commit hook versions
+        title: 'pre-commit: autoupdate hooks'
+        branch: pre-commit-updates
+        base: master
+        delete-branch: true

.github/workflows/tests.yml

@@ -3,32 +3,55 @@ name: tests
 on: [push, pull_request]
 
 env:
-  RUN: docker run --shm-size 1G --rm opendbc /bin/sh -c
+  RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS=error --shm-size 1G --rm opendbc /bin/bash -c
   BUILD: |
       docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile) || true
       docker pull ghcr.io/commaai/opendbc:latest || true
       docker build --cache-from ghcr.io/commaai/opendbc:latest -t opendbc -f Dockerfile .
+  PYTHONWARNINGS: error
 
 jobs:
-  test:
+  unit-tests:
+    name: unit tests
     runs-on: ubuntu-20.04
+    #strategy:
+    #  fail-fast: false
+    #  matrix:
+    #    run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
     steps:
-      - uses: actions/checkout@v2
-      - name: Build Docker image
-        run: eval "$BUILD"
-      - name: Static analysis
-        run: |
-          docker run opendbc bash -c "cd opendbc && git init && git add -A && pre-commit run --all"
-      - name: Generator test
-        run: |
-          docker run opendbc bash -c "cd opendbc/generator && ./test_generator.py"
-      - name: Unit tests
-        run: |
-          docker run opendbc bash -c "python -m unittest discover opendbc"
-      - name: Push to dockerhub
-        if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/opendbc'
-        run: |
-          docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-          docker tag opendbc ghcr.io/commaai/opendbc:latest
-          docker push ghcr.io/commaai/opendbc:latest
+    - uses: actions/checkout@v3
+    - name: Build Docker image
+      run: eval "$BUILD"
+    - name: Build opendbc
+      run: ${{ env.RUN }} "cd ../ && scons -j$(nproc)"
+    - name: Unit tests
+      run: ${{ env.RUN }} "python -m unittest discover ."
 
+  static-analysis:
+    name: static analysis
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build Docker image
+      run: eval "$BUILD"
+    - name: Build opendbc
+      run: ${{ env.RUN }} "cd ../ && scons -j$(nproc)"
+    - name: pre-commit
+      # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
+      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
+    - name: Generator test
+      run: ${{ env.RUN }} "generator/test_generator.py"
+
+  docker-push:
+    name: docker push
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/opendbc'
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build Docker image
+      run: eval "$BUILD"
+    - name: Push to dockerhub
+      run: |
+        docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+        docker tag opendbc ghcr.io/commaai/opendbc:latest
+        docker push ghcr.io/commaai/opendbc:latest

.gitignore

@@ -9,6 +9,7 @@
 .sconsign.dblite
 
 can/*.so
+can/*.a
 can/build/
 can/obj/
 can/packer_pyx.cpp

.pre-commit-config.yaml

@@ -1,25 +1,25 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     -   id: check-ast
     -   id: check-yaml
     -   id: check-merge-conflict
     -   id: check-symlinks
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.1
+    rev: v2.2.4
     hooks:
     -   id: codespell
         exclude: '\.dbc$'
         args:
         - --check-hidden
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.971
+    rev: v1.2.0
     hooks:
     -   id: mypy
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         exclude: 'site_scons/'

Dockerfile

@@ -44,16 +44,14 @@ RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.5.2
 
 ENV PYTHONPATH=/project
 
+RUN git config --global --add safe.directory '*'
+
 WORKDIR /project
-# TODO: Add tag to cereal
 RUN git clone https://github.com/commaai/cereal.git /project/cereal && \
     cd /project/cereal && \
-    git checkout 959ff79963b80829be9902d146c31fda44dbbd20
+    git checkout 959ff79963b80829be9902d146c31fda44dbbd20 && \
+    rm -rf .git && \
+    scons -j$(nproc)
 
 COPY SConstruct .
 COPY ./site_scons /project/site_scons
-COPY . /project/opendbc
-
-RUN rm -rf /project/opendbc/.git && \
-    rm -rf /project/cereal/.git
-RUN scons -c && scons -j$(nproc)

acura_ilx_2016_can_generated.dbc

@@ -255,7 +255,7 @@ BO_ 829 LKAS_HUD: 5 ADAS
  SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
  SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
  SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ BOH_2 : 23|2@0+ (1,0) [0|4] "" BDY
  SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
  SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
  SG_ LDW_ON : 28|1@0+ (1,0) [0|1] "" BDY

acura_rdx_2018_can_generated.dbc

@@ -255,7 +255,7 @@ BO_ 829 LKAS_HUD: 5 ADAS
  SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
  SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
  SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ BOH_2 : 23|2@0+ (1,0) [0|4] "" BDY
  SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
  SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
  SG_ LDW_ON : 28|1@0+ (1,0) [0|1] "" BDY

acura_rdx_2020_can_generated.dbc

@@ -357,7 +357,7 @@ BO_ 13274 LKAS_HUD_A: 5 ADAS
  SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
  SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
  SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ BOH_2 : 23|2@0+ (1,0) [0|4] "" BDY
  SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
  SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
  SG_ SET_ME_X01 : 20|1@0+ (1,0) [0|1] "" BDY
@@ -378,7 +378,7 @@ BO_ 13275 LKAS_HUD_B: 8 ADAS
  SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
  SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
  SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ BOH_2 : 23|2@0+ (1,0) [0|4] "" BDY
  SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
  SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
  SG_ SET_ME_X01 : 20|1@0+ (1,0) [0|1] "" BDY
@@ -436,7 +436,7 @@ BO_ 829 LKAS_HUD: 5 ADAS
  SG_ SOLID_LANES : 10|1@0+ (1,0) [0|1] "" BDY
  SG_ LDW_RIGHT : 9|1@0+ (1,0) [0|1] "" BDY
  SG_ STEERING_REQUIRED : 8|1@0+ (1,0) [0|1] "" BDY
- SG_ BOH : 23|2@0+ (1,0) [0|4] "" BDY
+ SG_ BOH_2 : 23|2@0+ (1,0) [0|4] "" BDY
  SG_ LDW_PROBLEM : 21|1@0+ (1,0) [0|1] "" BDY
  SG_ BEEP : 17|2@0+ (1,0) [0|1] "" BDY
  SG_ LDW_ON : 28|1@0+ (1,0) [0|1] "" BDY

can/SConscript

@@ -5,7 +5,14 @@ import os
 envDBC = env.Clone()
 dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("..").abspath)
 envDBC['CXXFLAGS'] += [dbc_file_path]
-libdbc = envDBC.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"], LIBS=[common, "capnp", "kj", "zmq"])
+src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]
+libs = [common, "capnp", "kj", "zmq"]
+
+# shared library for openpilot
+libdbc = envDBC.SharedLibrary('libdbc', src, LIBS=libs)
+
+# static library for tools like cabana
+envDBC.Library('libdbc_static', src, LIBS=libs)
 
 # Build packer and parser
 lenv = envCython.Clone()

can/common.h

@@ -37,6 +37,7 @@ unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vect
 
 class MessageState {
 public:
+  std::string name;
   uint32_t address;
   unsigned int size;
 
@@ -80,11 +81,12 @@ class CANParser {
   CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
   #ifndef DYNAMIC_CAPNP
   void update_string(const std::string &data, bool sendcan);
+  void update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan);
   void UpdateCans(uint64_t sec, const capnp::List<cereal::CanData>::Reader& cans);
   #endif
   void UpdateCans(uint64_t sec, const capnp::DynamicStruct::Reader& cans);
   void UpdateValid(uint64_t sec);
-  std::vector<SignalValue> query_latest();
+  void query_latest(std::vector<SignalValue> &vals, uint64_t last_ts = 0);
 };
 
 class CANPacker {

can/common.pxd

@@ -78,9 +78,8 @@ cdef extern from "common.h":
     bool can_valid
     bool bus_timeout
     CANParser(int, string, vector[MessageParseOptions], vector[SignalParseOptions])
-    void update_string(string, bool)
-    vector[SignalValue] query_latest()
+    void update_strings(vector[string]&, vector[SignalValue]&, bool)
 
   cdef cppclass CANPacker:
    CANPacker(string)
-   vector[uint8_t] pack(uint32_t, vector[SignalPackValue])
+   vector[uint8_t] pack(uint32_t, vector[SignalPackValue]&)

can/common_dbc.h

@@ -84,6 +84,6 @@ typedef struct ChecksumState {
 } ChecksumState;
 
 DBC* dbc_parse(const std::string& dbc_path);
-DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, ChecksumState *checksum = nullptr);
+DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, ChecksumState *checksum = nullptr, bool allow_duplicate_msg_name=false);
 const DBC* dbc_lookup(const std::string& dbc_name);
 std::vector<std::string> get_dbc_names();

can/dbc.cc

@@ -98,10 +98,11 @@ void set_signal_type(Signal& s, ChecksumState* chk, const std::string& dbc_name,
   }
 }
 
-DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, ChecksumState *checksum) {
+DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, ChecksumState *checksum, bool allow_duplicate_msg_name) {
   uint32_t address = 0;
   std::set<uint32_t> address_set;
   std::set<std::string> msg_name_set;
+  std::map<uint32_t, std::set<std::string>> signal_name_sets;
   std::map<uint32_t, std::vector<Signal>> signals;
   DBC* dbc = new DBC;
   dbc->name = dbc_name;
@@ -134,8 +135,11 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch
       // check for duplicates
       DBC_ASSERT(address_set.find(address) == address_set.end(), "Duplicate message address: " << address << " (" << msg.name << ")");
       address_set.insert(address);
-      DBC_ASSERT(msg_name_set.find(msg.name) == msg_name_set.end(), "Duplicate message name: " << msg.name);
-      msg_name_set.insert(msg.name);
+
+      if (!allow_duplicate_msg_name) {
+        DBC_ASSERT(msg_name_set.find(msg.name) == msg_name_set.end(), "Duplicate message name: " << msg.name);
+        msg_name_set.insert(msg.name);
+      }
     } else if (startswith(line, "SG_ ")) {
       // new signal
       int offset = 0;
@@ -162,6 +166,10 @@ DBC* dbc_parse_from_stream(const std::string &dbc_name, std::istream &stream, Ch
         sig.msb = sig.start_bit;
       }
       DBC_ASSERT(sig.lsb < (64 * 8) && sig.msb < (64 * 8), "Signal out of bounds: " << line);
+
+      // Check for duplicate signal names
+      DBC_ASSERT(signal_name_sets[address].find(sig.name) == signal_name_sets[address].end(), "Duplicate signal name: " << sig.name);
+      signal_name_sets[address].insert(sig.name);
     } else if (startswith(line, "VAL_ ")) {
       // new signal value/definition
       bool ret = std::regex_search(line, match, val_regexp);