Upgrade to python 3.9 and TF 2.9 (#1119)

* Updating TF to 2.9 and python to 3.9
* On Jetson 5.0.X use system python 3.8 with Nvidia's TF 2.9
* Tensorrt models cannot be build on the PC any longer, but can be created at runtime on the Jetson during first inference. This might be slow.
* Update gh actions accordingly
* Small updates in the UI
* On RPi use 64-bit Bullseye using Picamera2
* Switch to using savedmodel format by default instead of legacy h5
* Factor out predict method into interpreter base class, as now TF interfaces between different interpreters are aligned and can all be called by input dictionary.
* Complying with pep-440 for version numbering of dev versions

.github/workflows/python-package-conda.yml

@@ -22,11 +22,11 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-      - name: Create python 3.7 conda env
+      - name: Create python 3.9 conda env
         uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.7
-          mamba-version: "*"
+          python-version: 3.9
+          mamba-version: 1.3 # "*"
           activate-environment: donkey
           environment-file: ${{matrix.ENV_FILE}}
           auto-activate-base: false

donkeycar/__init__.py

@@ -14,8 +14,8 @@
 print(f.renderText('Donkey Car'))
 print(f'using donkey v{__version__} ...')
 
-if sys.version_info.major < 3 or sys.version_info.minor < 6:
-    msg = f'Donkey Requires Python 3.6 or greater. You are using {sys.version}'
+if sys.version_info.major < 3 or sys.version_info.minor < 8:
+    msg = f'Donkey Requires Python 3.8 or greater. You are using {sys.version}'
     raise ValueError(msg)
 
 # The default recursion limits in CPython are too small.

donkeycar/management/base.py

@@ -480,8 +480,8 @@ def plot_predictions(self, cfg, tub_paths, model_path, limit, model_type,
             pilot_angles.append(pilot_angle)
             pilot_throttles.append(pilot_throttle)
             bar.next()
-        print()  # to break the line after progress bar finishes.
 
+        bar.finish()
         angles_df = pd.DataFrame({'user_angle': user_angles,
                                   'pilot_angle': pilot_angles})
         throttles_df = pd.DataFrame({'user_throttle': user_throttles,

donkeycar/management/kivy_ui.py

@@ -716,7 +716,7 @@ def on_model_type(self, obj, model_type):
                 if 'tflite' in self.model_type:
                     self.filters = ['*.tflite']
                 elif 'tensorrt' in self.model_type:
-                    self.filters = ['*.trt']
+                    self.filters = ['*.trt', '*.savedmodel']
                 else:
                     self.filters = ['*.h5', '*.savedmodel']
 
@@ -971,7 +971,7 @@ class DataFrameLabel(Label):
 
 class TransferSelector(BoxLayout, FileChooserBase):
     """ Class to select transfer model"""
-    filters = ['*.h5']
+    filters = ['*.h5', '*.savedmodel']
 
 
 class TrainScreen(Screen):
@@ -980,34 +980,57 @@ class TrainScreen(Screen):
     database = ObjectProperty()
     pilot_df = ObjectProperty(force_dispatch=True)
     tub_df = ObjectProperty(force_dispatch=True)
+    train_checker = False
 
-    def train_call(self, model_type, *args):
-        # remove car directory from path
+    def train_call(self, *args):
         tub_path = tub_screen().ids.tub_loader.tub.base_path
         transfer = self.ids.transfer_spinner.text
+        model_type = self.ids.train_spinner.text
         if transfer != 'Choose transfer model':
-            transfer = os.path.join(self.config.MODELS_PATH, transfer + '.h5')
+            h5 = os.path.join(self.config.MODELS_PATH, transfer + '.h5')
+            sm = os.path.join(self.config.MODELS_PATH, transfer + '.savedmodel')
+            if os.path.exists(sm):
+                transfer = sm
+            elif os.path.exists(h5):
+                transfer = h5
+            else:
+                transfer = None
+                self.ids.status.text = \
+                    f'Could find neither {sm} nor {trans_h5} - training ' \
+                    f'without transfer'
         else:
             transfer = None
         try:
             history = train(self.config, tub_paths=tub_path,
                             model_type=model_type,
                             transfer=transfer,
                             comment=self.ids.comment.text)
-            self.ids.status.text = f'Training completed.'
-            self.ids.comment.text = 'Comment'
-            self.ids.transfer_spinner.text = 'Choose transfer model'
-            self.reload_database()
         except Exception as e:
             Logger.error(e)
             self.ids.status.text = f'Train failed see console'
-        finally:
-            self.ids.train_button.state = 'normal'
 
-    def train(self, model_type):
+    def train(self):
         self.config.SHOW_PLOT = False
-        Thread(target=self.train_call, args=(model_type,)).start()
-        self.ids.status.text = f'Training started.'
+        t = Thread(target=self.train_call)
+        self.ids.status.text = 'Training started.'
+
+        def func(dt):
+            t.start()
+
+        def check_training_done(dt):
+            if not t.is_alive():
+                self.train_checker.cancel()
+                self.ids.comment.text = 'Comment'
+                self.ids.transfer_spinner.text = 'Choose transfer model'
+                self.ids.train_button.state = 'normal'
+                self.ids.status.text = 'Training completed.'
+                self.ids.train_button.disabled = False
+                self.reload_database()
+
+        # schedules the call after the current frame
+        Clock.schedule_once(func, 0)
+        # checks if training finished and updates the window if
+        self.train_checker = Clock.schedule_interval(check_training_done, 0.5)
 
     def set_config_attribute(self, input):
         try:
@@ -1197,19 +1220,20 @@ def connected(self, event):
             self.is_connected = False
             if return_val is None:
                 # command still running, do nothing and check next time again
-                status = 'Awaiting connection...'
+                status = 'Awaiting connection to...'
                 self.ids.connected.color = 0.8, 0.8, 0.0, 1
             else:
                 # command finished, check if successful and reset connection
                 if return_val == 0:
-                    status = 'Connected'
+                    status = 'Connected to'
                     self.ids.connected.color = 0, 0.9, 0, 1
                     self.is_connected = True
                 else:
-                    status = 'Disconnected'
+                    status = 'Disconnected from'
                     self.ids.connected.color = 0.9, 0, 0, 1
                 self.connection = None
-            self.ids.connected.text = status
+            self.ids.connected.text \
+                = f'{status} {getattr(self.config, "PI_HOSTNAME")}'
 
     def drive(self):
         model_args = ''

donkeycar/management/makemovie.py

@@ -104,7 +104,8 @@ def draw_user_input(self, record, img, img_drawon):
         user_angle = float(record["user/angle"])
         user_throttle = float(record["user/throttle"])
         green = (0, 255, 0)
-        self.draw_line_into_image(user_angle, user_throttle, False, img_drawon, green)
+        self.draw_line_into_image(user_angle, user_throttle, False,
+                                  img_drawon, green)
 
     def draw_model_prediction(self, img, img_drawon):
         """
@@ -114,7 +115,7 @@ def draw_model_prediction(self, img, img_drawon):
         if self.keras_part is None:
             return
 
-        expected = tuple(self.keras_part.get_input_shapes()[0][1:])
+        expected = tuple(self.keras_part.get_input_shape('img_in')[1:])
         actual = img.shape
 
         # if model expects grey-scale but got rgb, covert

donkeycar/management/ui.kv

@@ -588,8 +588,9 @@
             ToggleButton:
                 id: train_button
                 text: 'Training running...' if self.state == 'down' else 'Train'
-                on_press: root.train(train_spinner.text)
-
+                on_press:
+                    root.train()
+                    self.disabled = True
         ScrollableLabel:
             DataFrameLabel:
                 id: scroll_pilots
@@ -604,7 +605,7 @@
             spacing: layout_pad_x
             MySpinner:
                 id: delete_spinner
-                text: 'Pilot'
+                text_autoupdate: True
             Button:
                 id: delete_btn
                 on_press:

donkeycar/parts/camera.py

@@ -9,29 +9,37 @@
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
+
 class CameraError(Exception):
     pass
 
+
 class BaseCamera:
 
     def run_threaded(self):
         return self.frame
 
-class PiCamera(BaseCamera):
-    def __init__(self, image_w=160, image_h=120, image_d=3, framerate=20, vflip=False, hflip=False):
-        from picamera.array import PiRGBArray
-        from picamera import PiCamera
 
-        resolution = (image_w, image_h)
-        # initialize the camera and stream
-        self.camera = PiCamera() #PiCamera gets resolution (height, width)
-        self.camera.resolution = resolution
-        self.camera.framerate = framerate
-        self.camera.vflip = vflip
-        self.camera.hflip = hflip
-        self.rawCapture = PiRGBArray(self.camera, size=resolution)
-        self.stream = self.camera.capture_continuous(self.rawCapture,
-            format="rgb", use_video_port=True)
+class PiCamera(BaseCamera):
+    """
+    RPi Camera class based on Bullseye's python class Picamera2.
+    """
+    def __init__(self, image_w=160, image_h=120, image_d=3,
+                 vflip=False, hflip=False):
+        from picamera2 import Picamera2
+        from libcamera import Transform
+
+        # it's weird but BGR returns RGB images
+        config_dict = {"size": (image_w, image_h), "format": "BGR888"}
+        transform = Transform(hflip=hflip, vflip=vflip)
+        self.camera = Picamera2()
+        config = self.camera.create_preview_configuration(
+            config_dict, transform=transform)
+        self.camera.align_configuration(config)
+        self.camera.configure(config)
+        # try min / max frame rate as 0.1 / 1 ms (it will be slower though)
+        self.camera.set_controls({"FrameDurationLimits": (100, 1000)})
+        self.camera.start()
 
         # initialize the frame and the variable used to indicate
         # if the thread should be stopped
@@ -40,31 +48,23 @@ def __init__(self, image_w=160, image_h=120, image_d=3, framerate=20, vflip=Fals
         self.image_d = image_d
 
         # get the first frame or timeout
-        logger.info('PiCamera loaded...')
-        if self.stream is not None:
-            logger.info('PiCamera opened...')
-            warming_time = time.time() + 5  # quick after 5 seconds
-            while self.frame is None and time.time() < warming_time:
-                logger.info("...warming camera")
-                self.run()
-                time.sleep(0.2)
+        logger.info('PiCamera opened...')
+        warming_time = time.time() + 5  # quick after 5 seconds
+        while self.frame is None and time.time() < warming_time:
+            logger.info("...warming camera")
+            self.run()
+            time.sleep(0.2)
+
+        if self.frame is None:
+            raise CameraError("Unable to start PiCamera.")
 
-            if self.frame is None:
-                raise CameraError("Unable to start PiCamera.")
-        else:
-            raise CameraError("Unable to open PiCamera.")
         logger.info("PiCamera ready.")
 
     def run(self):
-        # grab the frame from the stream and clear the stream in
-        # preparation for the next frame
-        if self.stream is not None:
-            f = next(self.stream)
-            if f is not None:
-                self.frame = f.array
-                self.rawCapture.truncate(0)
-                if self.image_d == 1:
-                    self.frame = rgb2gray(self.frame)
+        # grab the next frame from the camera buffer
+        self.frame = self.camera.capture_array("main")
+        if self.image_d == 1:
+            self.frame = rgb2gray(self.frame)
 
         return self.frame
 
@@ -78,16 +78,13 @@ def shutdown(self):
         self.on = False
         logger.info('Stopping PiCamera')
         time.sleep(.5)
-        self.stream.close()
-        self.rawCapture.close()
         self.camera.close()
-        self.stream = None
-        self.rawCapture = None
         self.camera = None
 
 
 class Webcam(BaseCamera):
-    def __init__(self, image_w=160, image_h=120, image_d=3, framerate = 20, camera_index = 0):
+    def __init__(self, image_w=160, image_h=120, image_d=3,
+                 framerate=20, camera_index=0):
         #
         # pygame is not installed by default.
         # Installation on RaspberryPi (with env activated):
@@ -270,18 +267,20 @@ def shutdown(self):
         self.running = False
         logger.info('Stopping CSICamera')
         time.sleep(.5)
-        del(self.camera)
+        del self.camera
 
 
 class V4LCamera(BaseCamera):
     '''
-    uses the v4l2capture library from this fork for python3 support: https://github.com/atareao/python3-v4l2capture
+    uses the v4l2capture library from this fork for python3 support:
+    https://github.com/atareao/python3-v4l2capture
     sudo apt-get install libv4l-dev
     cd python3-v4l2capture
     python setup.py build
     pip install -e .
     '''
-    def __init__(self, image_w=160, image_h=120, image_d=3, framerate=20, dev_fn="/dev/video0", fourcc='MJPG'):
+    def __init__(self, image_w=160, image_h=120, image_d=3, framerate=20,
+                 dev_fn="/dev/video0", fourcc='MJPG'):
 
         self.running = True
         self.frame = None

donkeycar/parts/datastore.py

@@ -219,7 +219,7 @@ def put_record(self, data):
             elif typ in ['str', 'float', 'int', 'boolean', 'vector']:
                 json_data[key] = val
 
-            elif typ is 'image':
+            elif typ == 'image':
                 path = self.make_file_path(key)
                 val.save(path)
                 json_data[key]=path