remove dependence on cereal

SConstruct

@@ -6,8 +6,6 @@ import numpy as np
 zmq = 'zmq'
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 
-cereal_dir = Dir('.')
-
 python_path = sysconfig.get_paths()['include']
 cpppath = [
   '#',
@@ -55,11 +53,6 @@ env = Environment(
 common = ''
 Export('env', 'zmq', 'arch', 'common')
 
-cereal = [File('#cereal/libcereal.a')]
-messaging = [File('#cereal/libmessaging.a')]
-Export('cereal', 'messaging')
-
-
 envCython = env.Clone()
 envCython["CPPPATH"] += [np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
@@ -79,6 +72,4 @@ envCython["LIBS"] = python_libs
 
 Export('envCython')
 
-
-SConscript(['cereal/SConscript'])
 SConscript(['opendbc/can/SConscript'])

can/SConscript

@@ -1,12 +1,12 @@
-Import('env', 'envCython', 'cereal', 'common')
+Import('env', 'envCython', 'common')
 
 import os
 
 envDBC = env.Clone()
 dbc_file_path = '-DDBC_FILE_PATH=\'"%s"\'' % (envDBC.Dir("..").abspath)
 envDBC['CXXFLAGS'] += [dbc_file_path]
 src = ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]
-libs = [common, "capnp", "kj", "zmq"]
+libs = [common, "zmq"]
 
 # shared library for openpilot
 libdbc = envDBC.SharedLibrary('libdbc', src, LIBS=libs)

can/common.h

@@ -6,13 +6,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include <capnp/dynamic.h>
-#include <capnp/serialize.h>
-
-#ifndef DYNAMIC_CAPNP
-#include "cereal/gen/cpp/log.capnp.h"
-#endif
-
 #include "opendbc/can/common_dbc.h"
 
 #define INFO printf
@@ -35,6 +28,17 @@ unsigned int xor_checksum(uint32_t address, const Signal &sig, const std::vector
 unsigned int hkg_can_fd_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 unsigned int pedal_checksum(uint32_t address, const Signal &sig, const std::vector<uint8_t> &d);
 
+struct CanFrame {
+  long src;
+  long address;
+  std::vector<uint8_t> dat;
+};
+
+struct CanData {
+  uint64_t nanos;
+  std::vector<CanFrame> frames;
+};
+
 class MessageState {
 public:
   std::string name;
@@ -61,8 +65,6 @@ class MessageState {
 class CANParser {
 private:
   const int bus;
-  kj::Array<capnp::word> aligned_buf;
-
   const DBC *dbc = NULL;
   std::unordered_map<uint32_t, MessageState> message_states;
 
@@ -78,14 +80,12 @@ class CANParser {
   CANParser(int abus, const std::string& dbc_name,
             const std::vector<std::pair<uint32_t, int>> &messages);
   CANParser(int abus, const std::string& dbc_name, bool ignore_checksum, bool ignore_counter);
-  #ifndef DYNAMIC_CAPNP
-  void update_string(const std::string &data, bool sendcan);
-  void update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan);
-  void UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans);
-  #endif
-  void UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cans);
-  void UpdateValid(uint64_t nanos);
+  void update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals);
   void query_latest(std::vector<SignalValue> &vals, uint64_t last_ts = 0);
+
+protected:
+  void update_can(const CanData &can);
+  void UpdateValid(uint64_t nanos);
 };
 
 class CANPacker {

can/common.pxd

@@ -64,11 +64,20 @@ cdef extern from "common_dbc.h":
 cdef extern from "common.h":
   cdef const DBC* dbc_lookup(const string) except +
 
+  cdef struct CanFrame:
+    long src
+    long address
+    vector[uint8_t] dat
+
+  cdef struct CanData:
+    uint64_t nanos
+    vector[CanFrame] frames
+
   cdef cppclass CANParser:
     bool can_valid
     bool bus_timeout
     CANParser(int, string, vector[pair[uint32_t, int]]) except +
-    void update_strings(vector[string]&, vector[SignalValue]&, bool) except +
+    void update(vector[CanData]&, vector[SignalValue]&) except +
 
   cdef cppclass CANPacker:
    CANPacker(string)

can/parser.cc

@@ -94,7 +94,7 @@ bool MessageState::update_counter_generic(int64_t v, int cnt_size) {
 
 
 CANParser::CANParser(int abus, const std::string& dbc_name, const std::vector<std::pair<uint32_t, int>> &messages)
-  : bus(abus), aligned_buf(kj::heapArray<capnp::word>(1024)) {
+  : bus(abus) {
   dbc = dbc_lookup(dbc_name);
   assert(dbc);
   init_crc_lookup_tables();
@@ -171,106 +171,57 @@ CANParser::CANParser(int abus, const std::string& dbc_name, bool ignore_checksum
   }
 }
 
-#ifndef DYNAMIC_CAPNP
-void CANParser::update_string(const std::string &data, bool sendcan) {
-  // format for board, make copy due to alignment issues.
-  const size_t buf_size = (data.length() / sizeof(capnp::word)) + 1;
-  if (aligned_buf.size() < buf_size) {
-    aligned_buf = kj::heapArray<capnp::word>(buf_size);
-  }
-  memcpy(aligned_buf.begin(), data.data(), data.length());
-
-  // extract the messages
-  capnp::FlatArrayMessageReader cmsg(aligned_buf.slice(0, buf_size));
-  cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
-  if (first_nanos == 0) {
-    first_nanos = event.getLogMonoTime();
-  }
-  last_nanos = event.getLogMonoTime();
-
-  auto cans = sendcan ? event.getSendcan() : event.getCan();
-  UpdateCans(last_nanos, cans);
-
-  UpdateValid(last_nanos);
-}
-
-void CANParser::update_strings(const std::vector<std::string> &data, std::vector<SignalValue> &vals, bool sendcan) {
+void CANParser::update(const std::vector<CanData> &can_data, std::vector<SignalValue> &vals) {
   uint64_t current_nanos = 0;
-  for (const auto &d : data) {
-    update_string(d, sendcan);
+  for (const auto &c : can_data) {
+    if (first_nanos == 0) {
+      first_nanos = c.nanos;
+    }
     if (current_nanos == 0) {
-      current_nanos = last_nanos;
+      current_nanos = c.nanos;
     }
+    last_nanos = c.nanos;
+
+    update_can(c);
+    UpdateValid(last_nanos);
   }
+
   query_latest(vals, current_nanos);
 }
 
-void CANParser::UpdateCans(uint64_t nanos, const capnp::List<cereal::CanData>::Reader& cans) {
-  //DEBUG("got %d messages\n", cans.size());
-
+void CANParser::update_can(const CanData &can) {
   bool bus_empty = true;
-
-  // parse the messages
-  for (const auto cmsg : cans) {
-    if (cmsg.getSrc() != bus) {
+  for (const auto &frame : can.frames) {
+    if (frame.src != bus) {
       // DEBUG("skip %d: wrong bus\n", cmsg.getAddress());
       continue;
     }
     bus_empty = false;
 
-    auto state_it = message_states.find(cmsg.getAddress());
+    auto state_it = message_states.find(frame.address);
     if (state_it == message_states.end()) {
       // DEBUG("skip %d: not specified\n", cmsg.getAddress());
       continue;
     }
-
-    auto dat = cmsg.getDat();
-
-    if (dat.size() > 64) {
-      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", cmsg.getAddress(), dat.size());
+    if (frame.dat.size() > 64) {
+      DEBUG("got message longer than 64 bytes: 0x%X %zu\n", frame.address, frame.dat.size());
       continue;
     }
 
     // TODO: this actually triggers for some cars. fix and enable this
-    //if (dat.size() != state_it->second.size) {
+    // if (dat.size() != state_it->second.size) {
     //  DEBUG("got message with unexpected length: expected %d, got %zu for %d", state_it->second.size, dat.size(), cmsg.getAddress());
     //  continue;
     //}
 
-    std::vector<uint8_t> data(dat.size(), 0);
-    memcpy(data.data(), dat.begin(), dat.size());
-    state_it->second.parse(nanos, data);
+    state_it->second.parse(can.nanos, frame.dat);
   }
 
   // update bus timeout
   if (!bus_empty) {
-    last_nonempty_nanos = nanos;
+    last_nonempty_nanos = can.nanos;
   }
-  bus_timeout = (nanos - last_nonempty_nanos) > bus_timeout_threshold;
-}
-#endif
-
-void CANParser::UpdateCans(uint64_t nanos, const capnp::DynamicStruct::Reader& cmsg) {
-  // assume message struct is `cereal::CanData` and parse
-  assert(cmsg.has("address") && cmsg.has("src") && cmsg.has("dat") && cmsg.has("busTime"));
-
-  if (cmsg.get("src").as<uint8_t>() != bus) {
-    DEBUG("skip %d: wrong bus\n", cmsg.get("address").as<uint32_t>());
-    return;
-  }
-
-  auto state_it = message_states.find(cmsg.get("address").as<uint32_t>());
-  if (state_it == message_states.end()) {
-    DEBUG("skip %d: not specified\n", cmsg.get("address").as<uint32_t>());
-    return;
-  }
-
-  auto dat = cmsg.get("dat").as<capnp::Data>();
-  if (dat.size() > 64) return; // shouldn't ever happen
-  std::vector<uint8_t> data(dat.size(), 0);
-  memcpy(data.data(), dat.begin(), dat.size());
-  state_it->second.parse(nanos, data);
+  bus_timeout = (can.nanos - last_nonempty_nanos) > bus_timeout_threshold;
 }
 
 void CANParser::UpdateValid(uint64_t nanos) {

can/parser_pyx.pyx

@@ -9,9 +9,11 @@ from libcpp.unordered_set cimport unordered_set
 from libc.stdint cimport uint32_t
 
 from .common cimport CANParser as cpp_CANParser
-from .common cimport dbc_lookup, SignalValue, DBC
+from .common cimport dbc_lookup, SignalValue, DBC, CanData, CanFrame
 
 import numbers
+import capnp
+from cereal import log
 from collections import defaultdict
 
 
@@ -75,10 +77,30 @@ cdef class CANParser:
       for l in v.values():  # no-cython-lint
         l.clear()
 
+    cdef CanFrame* frame
+    cdef CanData* can_data
+    cdef vector[CanData] can_data_array
+
+    can_data_array.reserve(len(strings))
+    try:
+      for s in strings:
+        with log.Event.from_bytes(s) as msg:
+          can_event = msg.sendCan if sendcan else msg.can
+          can_data = &(can_data_array.emplace_back())
+          can_data.nanos = msg.logMonoTime
+          can_data.frames.reserve(len(can_event))
+          for c in can_event:
+            frame = &(can_data.frames.emplace_back())
+            frame.src = c.src
+            frame.address = c.address
+            frame.dat = c.dat
+    except capnp.lib.capnp.KjException as ex:
+      raise RuntimeError(str(ex))
+
     cdef vector[SignalValue] new_vals
     cdef unordered_set[uint32_t] updated_addrs
 
-    self.can.update_strings(strings, new_vals, sendcan)
+    self.can.update(can_data_array, new_vals)
     cdef vector[SignalValue].iterator it = new_vals.begin()
     cdef SignalValue* cv
     while it != new_vals.end():