diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index c338be517..0c784027e 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -15,7 +15,6 @@ jobs: include: - os: macos-latest ENV_FILE: install/envs/mac.yml - - os: ubuntu-latest ENV_FILE: install/envs/ubuntu.yml fail-fast: false diff --git a/.github/workflows/superlinter.yml b/.github/workflows/superlinter.yml index 591e164c0..46eaa26d5 100644 --- a/.github/workflows/superlinter.yml +++ b/.github/workflows/superlinter.yml @@ -1,7 +1,7 @@ name: Super-Linter # Run this workflow every time a new commit pushed to your repository -on: push +on: [push, pull_request] jobs: # Set the job key. The key is displayed as the job name diff --git a/docs/assets/RC_receiver.jpg b/docs/assets/RC_receiver.jpg new file mode 100644 index 000000000..9e01d8ea3 Binary files /dev/null and b/docs/assets/RC_receiver.jpg differ diff --git a/docs/assets/cuthere.jpg b/docs/assets/cuthere.jpg new file mode 100644 index 000000000..3f5c297ad Binary files /dev/null and b/docs/assets/cuthere.jpg differ diff --git a/docs/assets/driveshaft.jpg b/docs/assets/driveshaft.jpg new file mode 100644 index 000000000..8d06efafb Binary files /dev/null and b/docs/assets/driveshaft.jpg differ diff --git a/docs/assets/encoder1.jpg b/docs/assets/encoder1.jpg new file mode 100644 index 000000000..0893e92d7 Binary files /dev/null and b/docs/assets/encoder1.jpg differ diff --git a/docs/assets/encoder2.jpg b/docs/assets/encoder2.jpg new file mode 100644 index 000000000..f0f3ad345 Binary files /dev/null and b/docs/assets/encoder2.jpg differ diff --git a/docs/assets/encoder_inplace.jpg b/docs/assets/encoder_inplace.jpg new file mode 100644 index 000000000..03f81dece Binary files /dev/null and b/docs/assets/encoder_inplace.jpg differ diff --git a/docs/assets/encoder_wiring.jpg b/docs/assets/encoder_wiring.jpg new file mode 100644 index 000000000..66a709b76 Binary files /dev/null and b/docs/assets/encoder_wiring.jpg differ diff --git a/docs/assets/lidar.jpg b/docs/assets/lidar.jpg new file mode 100644 index 000000000..a39d2f5a3 Binary files /dev/null and b/docs/assets/lidar.jpg differ diff --git a/docs/assets/lidar_angle.png b/docs/assets/lidar_angle.png new file mode 100644 index 000000000..1625bfa85 Binary files /dev/null and b/docs/assets/lidar_angle.png differ diff --git a/docs/assets/rc.jpg b/docs/assets/rc.jpg new file mode 100644 index 000000000..8145511fb Binary files /dev/null and b/docs/assets/rc.jpg differ diff --git a/docs/assets/rc_wiring.jpg b/docs/assets/rc_wiring.jpg new file mode 100644 index 000000000..cf05841c0 Binary files /dev/null and b/docs/assets/rc_wiring.jpg differ diff --git a/docs/assets/ui-car-connector-1.png b/docs/assets/ui-car-connector-1.png new file mode 100644 index 000000000..5d33a61c4 Binary files /dev/null and b/docs/assets/ui-car-connector-1.png differ diff --git a/docs/assets/ui-pilot-arena.png b/docs/assets/ui-pilot-arena.png new file mode 100644 index 000000000..f8323eaf9 Binary files /dev/null and b/docs/assets/ui-pilot-arena.png differ diff --git a/docs/assets/ui-trainer.png b/docs/assets/ui-trainer.png new file mode 100644 index 000000000..96864f5f3 Binary files /dev/null and b/docs/assets/ui-trainer.png differ diff --git a/docs/assets/ui-tub-manager-2.png b/docs/assets/ui-tub-manager-2.png new file mode 100644 index 000000000..1bfbda19b Binary files /dev/null and b/docs/assets/ui-tub-manager-2.png differ diff --git a/docs/assets/ui-tub-manager.png b/docs/assets/ui-tub-manager.png new file mode 100644 index 000000000..09bf4cb08 Binary files /dev/null and b/docs/assets/ui-tub-manager.png differ diff --git a/docs/guide/install_software.md b/docs/guide/install_software.md index 8fd68a744..6e5e0a755 100644 --- a/docs/guide/install_software.md +++ b/docs/guide/install_software.md @@ -41,6 +41,10 @@ This guide will help you to setup the software to run Donkeycar on your Raspberr * Setup [Jetson Nano](robot_sbc/setup_jetson_nano.md) ![donkey](/assets/logos/nvidia_logo.png) +## [Optional] Use the Intel Realsense T265 localization sensor instead of a RPi camera + +Read [this](/guide/robot_sbc/intelt265) for more information. + ## [Optional] Use TensorRT on the Jetson Nano Read [this](/guide/robot_sbc/tensorrt_jetson_nano) for more information. diff --git a/docs/guide/robot_sbc/intelt265.md b/docs/guide/robot_sbc/intelt265.md new file mode 100644 index 000000000..bce7d1e6d --- /dev/null +++ b/docs/guide/robot_sbc/intelt265.md @@ -0,0 +1,45 @@ +# A Guide to using the Intel Realsense T265 sensor with Donkeycar + +---- + +* **Note** Although the Realsense T265 can be used with a Nvidia Jetson Nano, it's a bit easier to set up with a Raspberry Pi (we recommend the RPi 4, with at least 4GB memory). Also, the Intel Realsense D4XX series can also be used with Donkeycar as a regular camera (with the use of its depth sensing data coming soon), and we'll add instructions for that when it's ready. + +Original T265 path follower code by [Tawn Kramer](https://github.com/tawnkramer/donkey) +---- + +## Step 1: Setup Donkeycar + +## Step 2: Setup Librealsense on Ubuntu Machine + +Using the latest version of Raspian (tested with Raspian Buster) on the RPi, follow [these instructions](https://github.com/IntelRealSense/librealsense/blob/master/doc/installation_raspbian.md) to set up Intel's Realsense libraries (Librealsense) and dependencies. + +## Step 3: Setup TensorRT on your Jetson Nano + +After you’ve done that, set up the directory with this: + +```donkey createcar --path ~/follow --template path_follower + +Running +``` cd ~/follow +python3 manage.py drive``` + +Once it’s running, open a browser on your laptop and enter this in the URL bar: http://:8887 + +The rest of the instructions from Tawn’s repo: + +When you drive, this will draw a red line for the path, a green circle for the robot location. +Mark a nice starting spot for your robot. Be sure to put it right back there each time you start. +Drive the car in some kind of loop. You see the red line show the path. +Hit X on the PS3/4 controller to save the path. +Put the bot back at the start spot. +Then hit the “select” button (on a PS3 controller) or “share” (on a PS4 controller) twice to go to pilot mode. This will start driving on the path. If you want it go faster or slower, change this line in the myconfig.py file: THROTTLE_FORWARD_PWM = 530 +Check the bottom of myconfig.py for some settings to tweak. PID values, map offsets and scale. things like that. You might want to start by downloading and using the myconfig.py file from my repo, which has some known-good settings and is otherwise a good place to start. +Some tips: + +When you start, the green dot will be in the top left corner of the box. You may prefer to have it in the center. If so, change PATH_OFFSET = (0, 0) in the myconfig.py file to PATH_OFFSET = (250, 250) + +For a small course, you may find that the path is too small to see well. In that case, change PATH_SCALE = 5.0 to PATH_SCALE = 10.0 (or more, if necessary) + +If you’re not seeing the red line, that means that a path file has already been written. Delete “donkey_path.pkl” (rm donkey_path.pkl) and the red line should show up + +It defaults to recording a path point every 0.3 meters. If you want it to be smoother, you can change to a smaller number in myconfig.py with this line: PATH_MIN_DIST = 0.3 \ No newline at end of file diff --git a/docs/guide/robot_sbc/setup_raspberry_pi.md b/docs/guide/robot_sbc/setup_raspberry_pi.md index 98af1a392..284ea63ad 100644 --- a/docs/guide/robot_sbc/setup_raspberry_pi.md +++ b/docs/guide/robot_sbc/setup_raspberry_pi.md @@ -220,10 +220,11 @@ cd donkeycar git checkout master pip install -e .[pi] pip install numpy --upgrade -wget "https://raw.githubusercontent.com/PINTO0309/Tensorflow-bin/master/tensorflow-2.3.1-cp37-none-linux_armv7l_download.sh" -chmod u+x tensorflow-2.3.1-cp37-none-linux_armv7l_download.sh -./tensorflow-2.3.1-cp37-none-linux_armv7l_download.sh -pip install tensorflow-2.3.1-cp37-none-linux_armv7l.whl + +curl -sc /tmp/cookie "https://drive.google.com/uc?export=download&id=1DCfoSwlsdX9X4E3pLClE1z0fvw8tFESP" > /dev/null +CODE="$(awk '/_warning_/ {print $NF}' /tmp/cookie)" +curl -Lb /tmp/cookie "https://drive.google.com/uc?export=download&confirm=${CODE}&id=1DCfoSwlsdX9X4E3pLClE1z0fvw8tFESP" -o tensorflow-2.2.0-cp37-cp37m-linux_armv7l.whl +pip install tensorflow-2.2.0-cp37-cp37m-linux_armv7l.whl ``` You can validate your tensorflow install with diff --git a/docs/parts/controllers.md b/docs/parts/controllers.md index 1639be8ed..5fc16ef52 100644 --- a/docs/parts/controllers.md +++ b/docs/parts/controllers.md @@ -11,6 +11,36 @@ The default controller to drive the car with your phone or browser. This has a w > Note: Recently iOS has [disabled default Safari](https://www.macrumors.com/2019/02/04/ios-12-2-safari-motion-orientation-access-toggle/) access to motion control. +## RC Controller +If you bought an RC car then it might have come with a standard 2.4GHz car radio and receiver as shown in picture below. This can be used to drive the car. +![wiring diagram](../assets/rc.jpg) + +* Hardware setup + +Using female-to-female jumper cables connect the following pins from your RC receiver to your RPi GPIO row as shown the diagram below +![wiring diagram](../assets/rc_wiring.jpg) + +Any of the RC receiver's + pin should go to any of the RPi's 3v pins. Any of the receiver's - pins can go to any RPi ground pin. + +For the three RC channels, CH-1 is for steering, CH-2 for throttle and CH-3 is linked to a press button on the remote control. The default connections are steering: GPIO 26, throttle: GPIO 20, channel 3 (for record deletion): GPIO 19. + +* Software setup + +You must have `pigpio` installed. Do so with these commands: `sudo apt update && sudo apt install python3-pigpio && sudo systemctl enable pigpiod & sudo systemctl start pigpiod` + +The `basic` template which you install with `donkey createcar --path ~/mycar --template basic` has an additional function `manage.py calibrate` which you should use to zero your angle and throttle PWM signal. + +> Note: The PWM signal drifts over time. Hence check your calibration regularly before starting recording. + +To use RC control, change 'USE_RC' to 'True' in your myconfig.py file in your mycar directory. If you used different GPIO pins than the above, you can set them here, too. + +```#RC CONTROL +USE_RC = True +STEERING_RC_GPIO = 26 +THROTTLE_RC_GPIO = 20 +DATA_WIPER_RC_GPIO = 19 +``` + ## Joystick Controller Many people find it easier to control the car using a game controller. There are several parts that provide this option. diff --git a/docs/parts/keras.md b/docs/parts/keras.md index d9de8e2fa..f7467fd9e 100644 --- a/docs/parts/keras.md +++ b/docs/parts/keras.md @@ -1,3 +1,4 @@ + # Keras Parts These parts encapsulate models defined using the [Keras](https://keras.io/) high level api. They are intended to be used with the Tensorflow backend. The parts are designed to use the trained artificial neural network to reproduce the steering and throttle given the image the camera sees. They are created by using the [train command](/guide/train_autopilot/). diff --git a/docs/parts/lidar.md b/docs/parts/lidar.md new file mode 100644 index 000000000..3079a9f68 --- /dev/null +++ b/docs/parts/lidar.md @@ -0,0 +1,38 @@ +# Lidar + +A Lidar sensor can be used with Donkeycar to provide obstacle avoidance or to help navigate on tracks with walls. It records data along with the camera during training and this can be used for training + +![Donkey lidar](../assets/lidar.jpg) +## Supported Lidars + +We currently only support the RPLidar series of sensors, but will be adding support for the similar YDLidar series soon. + +We recommend the [$99 A1M8](https://amzn.to/3vCabyN) (12m range) + + +## Hardware Setup + +Mount the Lidar underneath the camera canopy as shown above (the RPLidar A2M8 is used there, but the A1M8 mounting is the same). You can velcro the USB adapter under the Donkey plate and use a short USB cable to connect to one of your RPi or Nano USB ports. It can be powered by the USB port so there's no need for an additional power supply. + +## Software Setup + +Lidar requires the glob library to be installed. If you don't already have that, install it with `pip3 install glob2` + +Also install the Lidar driver: `pip3 install rplidar` + +Right now Lidar is only supported with the basic template. Install it as follows: + +`donkey createcar --path ~/lidarcar --template basic` + +Then go to the lidarcar directory and edit the myconfig.py file to ensure that the Lidar is turned on. The upper and lower limits should be set to reflect the areas you want your Lidar to "look at", omitting the areas that are blocked by parts of the car body. An example is shown below. For the RPLidar series, 0 degrees is in the direction of the motor (in the case of the A1M8) or cable (in the case of the A2M8) + +``` +# LIDAR +USE_LIDAR = True +LIDAR_TYPE = 'RP' #(RP|YD) +LIDAR_LOWER_LIMIT = 44 # angles that will be recorded. Use this to block out obstructed areas on your car and/or to avoid looking backwards. Note that for the RP A1M8 Lidar, "0" is in the direction of the motor +LIDAR_UPPER_LIMIT = 136 +``` +![Lidar limits](../assets/lidar_angle.png) + + diff --git a/docs/parts/odometry.md b/docs/parts/odometry.md new file mode 100644 index 000000000..1175b8dcf --- /dev/null +++ b/docs/parts/odometry.md @@ -0,0 +1,62 @@ +## Odometry + +Odometry is a way to calculate the speed and distance travelled of the car by measuring the rotation of its wheels using a sensor called an rotary encoder. This encoder can be on the motor, on the main drive shaft or on individual wheels. The advantage of using an encoder is that it "closes the loop" with your throttle, so your car can reliably command an actual velocity rather than just issuing a motor control which will produce a faster or slower velocity depending on the slope of the track, the surface or mechanical friction in your drive train while turning. In short, an encoder gives you much better control over your speed. + +Encoders come in various forms: +* Quadrature encoders use hall-effect sensors to measure magnetic pulses as the shaft turns, and have the advantage of being very precise as well as being able to tell the difference between forward and reverse rotation +* Single-output encoders are like quadrature encoders but they can not determine the direction of motion +* Optical encoders are typicall a LED/Light sensor combo with a disk that has slots cut in in-between them. As the disk rotates, the light is interruprted and those pulses are counted. These sensors are cheap and easy to install but cannot determine the direction of rotation + +There are several ways to read encoders with Donkey: +* Directly with the RaspberryPi's GPIO pins. This is best for optical encoders, since they don't generate as many pulses as a quadrature encoder and the RPi will miss fewer of them as it task-swaps between the various Donkeycar parts +* With an Arduino or Teensy. This is best for quadrature encoders, since the Arduino/Teensy is 100% devoted to counting pulses. It transmits the count to the RPi via the USB serial port when requested by Donkeycar, which lightens the processing load for the Rpi +* With an Astar board. This is just a fancy Arduino, but if you have one, it makes for a neat installation + + +## Supported Encoders + +Examples of rotary encoders that are supported: + +* Optical encoder sensors and discs [Available from many sources on Amazon](https://amzn.to/3s05QmG) +* Quadrature encoders. [Larger, cheaper](https://amzn.to/3liBUjj), [Smaller, more expensive](https://www.sparkfun.com/products/10932) + +## Hardware Setup + +How you attach your encoder is up to you and which kind of encoder you're using. For example, [here's](https://diyrobocars.com/2020/01/31/how-to-add-an-encoder-to-the-donkeycar-chassis/) one way to put a quadrature encoder on the main drive shaft. [Here](https://guitar.ucsd.edu/maeece148/index.php/Project_encoders) is a more complex setup with dual encoders. + +But this is the easiest way to do it, with a cheap and simple optical encoder on the main drive shaft of a standard Donkeycar chassis (if your chassis is different, the same overall approach should work, although you may have to find a different place to mount the sensor): + +First, unscrew the plate over the main drive shaft. Tilt the rear wheels back a bit and you should be able to remove the shaft. + +![drive shaft](../assets/driveshaft.jpg) + +Now enlarge the hole in the optical encoder disc that came with your sensor (use a drill or Dremel grinding stone) so you can slip it onto the shaft. Stretch a rubber grommet (you can use the sort typically included with servos to mount them, but any one of the right size will do) over the shaft and push it into the encoder disc hole. If you don't have a grommet, you can wrap tape around the shaft until it's large enough to hold the disc firmly. Once you've ensured it's in the right place, use a few drops of superglue or hot glue to hold it in place) + +![drive shaft](../assets/encoder1.jpg) + +![drive shaft](../assets/encoder2.jpg) + +Cut out a small notch (marked in pencil here) in the plate covering the drive shaft, so you can mount the encoder sensor there, ensuring that the disc can turn freely in the gap in front of the steering servo + +![drive plate](../assets/cuthere.jpg) + +Now replace the plate and drill two holes so you can screw in the encoder sensor. Slide the disc along the shaft so that it doesn't bind on the sensor. + +![drive plate](../assets/encoder_inplace.jpg) + +Use three female-to-female jumper cables and connect the sensor to your RPi GPIO pins as follows. Connect the GND, V+ (which might say 5V or 3.3V) and data pin (which will say "Out or "D0") to the RPi 5V, Ground and GPIO 13 as shown here (if your sensor encoder has four pins, ignore the one that says "A0"): +![wiring diagram](../assets/encoder_wiring.jpg) + + +## Software Setup + +Enable odometry in `myconfig.py`. + +```HAVE_ODOM = True # Do you have an odometer/encoder +ENCODER_TYPE = 'GPIO' # What kind of encoder? GPIO|Arduino|Astar +MM_PER_TICK = 12.7625 # How much travel with a single tick, in mm. Roll you car a meter and divide total ticks measured by 1,000 +ODOM_PIN = 13 # if using GPIO, which GPIO board mode pin to use as input +ODOM_DEBUG = False # Write out values on vel and distance as it runs +``` + +If you are using an Arduino or Teensy to read your encoder, select 'Arduino' in the myconfig.py file libe above. The microcontroller should be flashed using the Arduino IDE with [this sketch](https://github.com/zlite/donkeycar/tree/master/donkeycar/parts/encoder/encoder). Make sure you check the sketch using the "test_encoder.py code in the Donkeycar tests folder to make sure you've got your encoder plugged into the right pins, or edit it to reflect the pins you are using. \ No newline at end of file diff --git a/docs/parts/path_following.md b/docs/parts/path_following.md new file mode 100644 index 000000000..eb464dde3 --- /dev/null +++ b/docs/parts/path_following.md @@ -0,0 +1,86 @@ + +# Path Following with the Intel Realsense T265 sensor + +Rather than using a standard camera and training a network to drive, Donkeycar supports using the [Intel Realsense T265 "tracking camera"](https://www.intelrealsense.com/tracking-camera-t265/) to follow a path instead. In this application, you simply drive a path once manually, and Donkeycar will "remember" that path and repeat it autonomously. + +The Intel T265 uses a combination of stereo cameras and an internal Inertial Measurement Unit (IMU) plus its own Myriad X processor to do Visual Inertial Odometry, which is a fancy way of saying that it knows where it is by looking at the scene around it as it moves and correlating that with the IMU's sensing to localize itself, outputting an X,Y,Z position to Donkey, much as a GPS sensor would (but ideally much more accurately, to a precision of centemeters) + +--------------- +* **Note** Although the Realsense T265 can be used with a Nvidia Jetson Nano, it's a bit easier to set up with a Raspberry Pi (we recommend the RPi 4, with at least 4GB memory). Also, the Intel Realsense D4XX series can also be used with Donkeycar as a regular camera (with the use of its depth sensing data coming soon), and we'll add instructions for that when it's ready. + +Original T265 path follower code by [Tawn Kramer](https://github.com/tawnkramer/donkey) +---- + + +## Step 1: Setup Librealsense on Ubuntu Machine + +Using the latest version of Raspian (tested with Raspian Buster) on the RPi, follow [these instructions](https://github.com/acrobotic/Ai_Demos_RPi/wiki/Raspberry-Pi-4-and-Intel-RealSense-D435) to set up Intel's Realsense libraries (Librealsense) and dependencies. Although those instructions discuss another Realsense sensor, they work equally well for the T265. There are also [video instructions](https://www.youtube.com/watch?v=LBIBUntnxp8) + +## Step 2: Setup Donkeycar + +Follow the standard instructions [here](https://docs.donkeycar.com/guide/install_software/). With the Path Follower, there is no need to install Tensorflow for this particular Donkeycar configuration however do install numpy/upgrade before running "pip install -e .[pi]" + +## Step 3: Create the Donkeycar path follower app + +```donkey createcar --path ~/follow --template path_follow + +## Step 4: Check/change your config settings + +```cd ~follow``` +```sudo nano myconfig.py``` + +Make sure you agree with the default values or adjust them to your liking (ie. "throttle", "steering", PIDs, etc.). Uncomment (remove the #) for any line you've changed. In Nano press cntrl-o to save the file and cntrl-x to exit. + +## Step 5: Run the Donkeycar path follower app + +Running +``ssh pi@`` +``` cd ~/follow``` +```python3 manage.py drive``` + +Keep the terminal open to see the printed output of the app while it is running. + +If you get an error saying that it can't find the T265, unplug the sensor, plug it back in and try again. Ensure that your gamepad is on and connected, too (blue light is on the controller) + +Once it’s running, open a browser on your laptop and enter this in the URL bar: http://:8890 + +When you drive, the Web interface will draw a red line for the path, a green circle for the robot location. If you're seeing the green dot but not the red line, that means that a path file has already been written. Delete “donkey_path.pkl” (rm donkey_path.pkl), restart and the red line should show up + + +PS4 Gamepad controls are as follows: ++------------------+--------------------------+ +| control | action | ++------------------+--------------------------+ +| share | toggle auto/manual mode | +| circle | save_path | +| triangle | erase_path | +| cross | emergency_stop | +| L1 | increase_max_throttle | +| R1 | decrease_max_throttle | +| options | toggle_constant_throttle | +| square | reset_origin | +| L2 | dec_pid_d | +| R2 | inc_pid_d | +| left_stick_horz | set_steering | +| right_stick_vert | set_throttle | ++------------------+--------------------------+ + +## Step 6: Driving instructions + +1) Mark a nice starting spot for your robot. Be sure to put it right back there each time you start. +2) Drive the car in some kind of loop. You see the red line show the path. +3) Hit circle on the PS3/4 controller to save the path. +4) Put the bot back at the start spot. +5) Then hit the “select” button (on a PS3 controller) or “share” (on a PS4 controller) twice to go to pilot mode. This will start driving on the path. If you want it go faster or slower, change this line in the myconfig.py file: ```THROTTLE_FORWARD_PWM = 400``` + +Check the bottom of myconfig.py for some settings to tweak. PID values, map offsets and scale. things like that. You might want to start by downloading and using the myconfig.py file from my repo, which has some known-good settings and is otherwise a good place to start. + +Some tips: + +When you start, the green dot will be in the top left corner of the box. You may prefer to have it in the center. If so, change PATH_OFFSET = (0, 0) in the myconfig.py file to PATH_OFFSET = (250, 250) + +For a small course, you may find that the path is too small to see well. In that case, change PATH_SCALE = 5.0 to PATH_SCALE = 10.0 (or more, if necessary) + +When you're running in auto mode, the green dot will change to blue + +It defaults to recording a path point every 0.3 meters. If you want it to be smoother, you can change to a smaller number in myconfig.py with this line: PATH_MIN_DIST = 0.3 diff --git a/docs/utility/donkey.md b/docs/utility/donkey.md index 2aaf1962e..a4f46571d 100644 --- a/docs/utility/donkey.md +++ b/docs/utility/donkey.md @@ -66,11 +66,29 @@ donkey tubclean * Hit `Ctrl + C` to exit ## Train the model -**Note:** _This section only applies to version >= 4.1_ +**Note:** _This section only applies to version >= 4.1.0_ This command trains the model. + ```bash -donkey train --tub= [--config=] [--model=] [--model_type=(linear|categorical|inferred)] -``` +donkey train --tub= [--config=] [--model=] +[--model_type=(linear|categorical|inferred)] +``` +* Uses the data from the `--tub` datastore +* Uses the config file from the `--config` path (optionally) +* Saves the model into `--model` +* Uses the model type `--type` +* Supports filtering of records using a function defined in the variable + `TRAIN_FILTER` in the `my_config.py` file. For example: + + ```python + def filter_record(record): + return record['user/throttle'] > 0 + + TRAIN_FILTER = filter_record + ``` + only uses records with positive throttle in training. + + The `createcar` command still creates a `train.py` file for backward compatibility, but it's not required for training. @@ -95,36 +113,6 @@ donkey makemovie --tub= [--out=] [--config=] * optional `--start` and/or `--end` can specify a range of frame numbers to use. * scale will cause ouput image to be scaled by this amount -## Check Tub - -This command allows you to see how many records are contained in any/all tubs. It will also open each record and ensure that the data is readable and intact. If not, it will allow you to remove corrupt records. - -> Note: This should be moved from manage.py to donkey command - -Usage: - -```bash -donkey tubcheck [--fix] -``` - -* Run on the host computer or the robot -* It will print summary of record count and channels recorded for each tub -* It will print the records that throw an exception while reading -* The optional `--fix` will delete records that have problems - -## Augment Tub - -This command allows you to perform the data augmentation on a tub or set of tubs directly. The augmentation is also available in training via the `--aug` flag. Preprocessing the tub can speed up the training as the augmentation can take some time. Also you can train with the unmodified tub and the augmented tub joined together. - -Usage: - -```bash -donkey tubaugment [--inplace] -``` - -* Run on the host computer or the robot -* The optional `--inplace` will replace the original tub images when provided. Otherwise `tub_XY_YY-MM-DD` will be copied to a new tub `tub_XX_aug_YY-MM-DD` and the original data remains unchanged - ## Histogram @@ -240,3 +228,27 @@ Example: ```bash donkey cnnactivations --model models/model.h5 --image data/tub/1_cam-image_array_.jpg ``` + +## Tub manager UI + +**Note:** _This section only applies to version >= 4.2.0_ + + +Usage: + +```bash +donkey ui +``` + +This opens a UI to analyse tub data supporting following features: + +* show selected data fields live as values and graphical bars +* delete or un-delete records +* try filters for data selection +* plot data of selected data fields + +The UI is an alternative to the web based `donkey tubclean`. + +![Tub UI](../assets/ui-tub-manager.png) + +A full documentation of the UI is [here.](./ui.md) \ No newline at end of file diff --git a/docs/utility/ui.md b/docs/utility/ui.md new file mode 100644 index 000000000..12a19a00c --- /dev/null +++ b/docs/utility/ui.md @@ -0,0 +1,116 @@ +# Donkey UI + +The Donkey UI currently contains three screens supporting the following workflows: + +1. The tub manager - this is a replacement for the web-based application launched through `donkey tubclean` + +1. The trainer - this is a UI based alternative to train the pilot. Note, for longer trainings containing larger tubs or batches it is recommended to perform these in the shell using the `donkey train` command. The UI based training is geared towards an experimental and rapid analysis cycle consisting of: + * data manipulation / selection + * training + * pilot benchmarking + +1. The pilot arena - here you can test two pilots' performance against each other. + +**_Note_:** Under linux the app depends on `xclip`, if this is not installed, then please run: +```bash +sudo apt-get install xclip +``` + +## The tub manager +![Tub_manager UI](../assets/ui-tub-manager.png) + +In the tub manager screen you have to select the car directory that contains the config file `config.py` first, using the `Load car directory` button. Then select the tub you want to be working with using `Load tub`, the tub needs to be inside the car directory. The application remembers the last loaded config and tub. + +The drop-down menu `Add/remove' in the data panel to the left of the image allows to select the record fields, like `user/angle`, `user/throttle`, etc. + +**Note:** if your tub contains more data than the standard `user/angle`, `user/throttle` and you want the progress bars to correctly show the values of these fields, you need to add an entry into the `.donkeyrc` file in your home directory. This file is automatically created by the Donkey UI app. Here is an example: +```yaml +field_mapping: +- centered: true + field: car/accel + max_value_id: IMU_ACCEL_NORM +``` + +This data entry into the `field_mapping` list contains the name of the tub field, a switch, if the data is centered around 0 and the name of the maximum value of that data field which has to be provided in the `myconfig.py` file. For example, the data above represents the IMU acceleration of the IMU6050 which ranges between +/- 2g, i.e. ~ +/-20 m/s2. With an IMU_ACCEL_NORM of 20 the progress bar can display these values. Therefore, the `myconfig.py` should contain: +```python +IMU_ACCEL_NORM = 20 +``` + +**Note**: Vectors, i.e. list / arrays are being decomposed by the UI into their components automatically. + +Here is an example of a tub that has `car/accel` and `car/gyro` arrays that hold IMU data, as well as `car/distance` and `car/m_in_lap`. The first two show a progress bar because there is a corresponding entry in the `field_mapping` list as explained above. +![Tub_manager UI_more_data](../assets/ui-tub-manager-2.png) + +The control panel allows moving forward and backward in single steps using <, > and scrolling continuously with <<, >>. These buttons are also linked to the keyboard keys < left >, < right >, < space >. + +To delete unwanted records press `Set left` / `Set right` buttons to determine the range and hit `Delete` to remove such records from training. To see the impact on the current tub press `Reload tub`. If you want to resurrect accidentally deleted records, just choose a left/right value outside the deleted range and press `Restore`. + +**Note:** The left/right values are invertible, i.e. left > right operates on all records outside [left, right). + +In the filter section you can suppress records, say you want to restrict the next training on only right curves, then add `user/angle > 0` to select those records. +**Note:** the filter on only for display in the tub manager. If you want to apply this in training you have to write the predicate as explained in [utility](../utility/donkey.md) + +The lower panel contains a graph with the currently selected data from the data panel. If nothing is selected, all fields from the record are displayed. The display scales the data between the minimum and maximum value of each record field, hence there are no absolute measurements possible. For more advanced data graphing capabilities, press `Browser Graph` which opens a plotly history graph in the web browser. + +## The trainer +![Trainer UI](../assets/ui-trainer.png) + +The trainer screen allows to train a model to the tub data. In the `Overwrite config` section you can set any config parameter by typing an updated value into the text field on the right and hitting return. + +To train a pilot use the model type dropdown, enter a comment and hit `Train`. After training, the pilot will appear in the pilot database which is shown underneath. You can also choose a transfer model if you don't want to train from scratch. Note, tensorflow saves the model with the optimiser state, so training will commence where it stopped in the saved state. + +Pilots might be trained on multiple tubs, this is currently not supported in the trainer. However, if multiple tubs are passed in `donkey train` then these will show in the database too. In order to not clutter up the view and group different tubs, you can use the `Group multiple tubs` button to group all tub groups of two and more, and show a group alias instead. The group alias mapping is shown in the lower area of the window then. + +## The pilot arena +![Pilot Arena UI](../assets/ui-pilot-arena.png) + +Here you can benchmark two pilots against each other. Use this panel to test if changes in the training through optimiser parameters or model types, or through deletion of certain records, or augmentations to images have made the pilot better or worse. The last selected pilots will be remembered in the app. + +Choose a pilot by selecting the `Model type` and loading the keras model using the file chooser by pressing `Choose pilot`. The control panel is the same as in the tub manager. The lower right data panel shows the tub record's data. You can select the throttle field as some folks train on car speed instead of throttle values. In such case, the corresponding field name must be added into the `.donkeyrc` file in the section, see an example here. + +```yaml +user_pilot_map: + car/speed: pilot/speed + user/angle: pilot/angle + user/throttle: pilot/throttle +``` + +The 'user/angle' and 'user/throttle' mappings are automatically loaded by the app. In order to show the variable `car/speed` and compare it to the AI produced `pilot/speed` the map has to contain the corresponding entry. + +Under the two pilots there are sliders with pre-defined image augmentations, here `Brightness` and `Blur`. You can mix brightness and blur values into the images and compare how well the pilots are reacting to such a modification of the testing data. Press the buttons to activate the sliders for enabling this feature. + +The application will remember the last two selected pilots. + +## The car connection +![Car_Connector_UI](../assets/ui-car-connector-1.png) + +**_Note_**: This screen will only work on linux / OSX as it makes use of `ssh` and `rsync` in the background. SSH needs to be configured to allow login from the PC to the car without password. This can be done by running on the PC: +```bash +ssh-keygen +``` +When being asked for a passphrase just hit <return>. This creates a public and private key in your `./ssh` directory. Then copy the key over to the car with the command below. Here we assume your car's hostname is `donkeypi` - otherwise replace it with the corresponding hostname. +```bash +ssh-copy-id -i ~/.ssh/id_rsa.pub pi@donkeypi.local +``` +Log into your car using: +```bash +ssh pi@donkeypi.local +``` +If SSH asks you if that host should be added to the list of known hosts, hit <return> and you are done. From now on, you can ssh into the car without being prompted for the password again. **The login-free setup is required for the screen to work.** + +With the car connector you can transfer the tub data from the car to your PC and transfer the pilots back to the car. + +* Under the `Car directory` enter the car folder and hit return. This should populate the `Select tub` drop down. Most likely you want to select the `data/` directory but you might have tubs in subfolders. In that case use `~/mycar/data` in the `Car directory`, select the tub you want to pull and enable `Create new folder` on the button. This will copy a tub on the car like `~/mycar/data/tub_21-04-09_11` into the same location on your PC. Without the `Create new folder` it would copy the content of the car's tub folder into `~/mycar/data` of your PC, possibly overwriting other tub data that might be there. + +* Press `Pull tub data/` to copy the tub from the car +* Press `Send pilots` to sync your local `models/` folder with the `models/` folder on the car. This command syncs all pilots that are locally stored. +* In the `Drive car` section you can start the car and also select a model for autonomous driving. After starting you have to use the controller, either web or joystick as usual. + + + +### Future plans +1. Moving the car connector screen to a web interface, supported on all OS. +1. Handling of multiple tubs. +1. The ability to also use the filter in training without the need to edit the `myconfig.py` file. +1. Migration of the `~/.donkeyrc` file to the kivy internal settings. +1. Support using only a single pilot (or more than two) in the pilot arena. diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 99fe7984e..2714c29a8 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -1,7 +1,7 @@ import sys from pyfiglet import Figlet -__version__ = '4.1.0' +__version__ = '4.2.0' f = Figlet(font='speed') print(f.renderText('Donkey Car')) diff --git a/donkeycar/benchmarks/tub_v2.py b/donkeycar/benchmarks/tub_v2.py index 492c1110a..48087096a 100644 --- a/donkeycar/benchmarks/tub_v2.py +++ b/donkeycar/benchmarks/tub_v2.py @@ -25,9 +25,7 @@ def benchmark(): tub.write_record(record) deletions = np.random.randint(0, write_count, 100) - for index in deletions: - index = int(index) - tub.delete_record(index) + tub.delete_records(deletions) for record in tub: print('Record %s' % record) diff --git a/donkeycar/config.py b/donkeycar/config.py index 4f0919139..f5a559e84 100644 --- a/donkeycar/config.py +++ b/donkeycar/config.py @@ -69,6 +69,7 @@ def load_config(config_path=None, myconfig="myconfig.py"): if hasattr(cfg, 'IMAGE_H') and hasattr(cfg, 'IMAGE_W'): cfg.TARGET_H = cfg.IMAGE_H cfg.TARGET_W = cfg.IMAGE_W - cfg.TARGET_D = cfg.IMAGE_DEPTH + if hasattr(cfg, 'IMAGE_DEPTH'): + cfg.TARGET_D = cfg.IMAGE_DEPTH return cfg diff --git a/donkeycar/contrib/coordconv b/donkeycar/contrib/coordconv deleted file mode 160000 index 88ef7c416..000000000 --- a/donkeycar/contrib/coordconv +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 88ef7c416c2dc94898df45ebd303676f6d78479b diff --git a/donkeycar/contrib/get_coorconv.sh b/donkeycar/contrib/get_coorconv.sh deleted file mode 100644 index 666bf13c1..000000000 --- a/donkeycar/contrib/get_coorconv.sh +++ /dev/null @@ -1 +0,0 @@ -git clone https://github.com/titu1994/keras-coordconv coordconv \ No newline at end of file diff --git a/donkeycar/management/__init__.py b/donkeycar/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/donkeycar/management/base.py b/donkeycar/management/base.py index ede7f4052..127a16d82 100644 --- a/donkeycar/management/base.py +++ b/donkeycar/management/base.py @@ -6,7 +6,6 @@ import stat import sys from socket import * -from pathlib import Path from progress.bar import IncrementalBar import donkeycar as dk @@ -17,6 +16,8 @@ PACKAGE_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) TEMPLATES_PATH = os.path.join(PACKAGE_PATH, 'templates') +HELP_CONFIG = 'location of config file to use. default: ./config.py' + def make_dir(path): real_path = os.path.expanduser(path) @@ -65,7 +66,7 @@ def run(self, args): args = self.parse_args(args) self.create_car(path=args.path, template=args.template, overwrite=args.overwrite) - def create_car(self, path, template='basic', overwrite=False): + def create_car(self, path, template='complete', overwrite=False): """ This script sets up the folder structure for donkey to work. It must run without donkey installed so that people installing with @@ -74,7 +75,7 @@ def create_car(self, path, template='basic', overwrite=False): # these are neeeded incase None is passed as path path = path or '~/mycar' - template = template or 'basic' + template = template or 'complete' print("Creating car folder: {}".format(path)) path = make_dir(path) @@ -173,8 +174,10 @@ def run(self, args): print("Finding your car's IP address...") cmd = "sudo nmap -sP " + ip + "/24 | awk '/^Nmap/{ip=$NF}/B8:27:EB/{print ip}'" + cmdRPi4 = "sudo nmap -sP " + ip + "/24 | awk '/^Nmap/{ip=$NF}/DC:A6:32/{print ip}'" print("Your car's ip address is:" ) os.system(cmd) + os.system(cmdRPi4) class CalibrateCar(BaseCommand): @@ -245,7 +248,7 @@ def parse_args(self, args): parser = argparse.ArgumentParser(prog='makemovie') parser.add_argument('--tub', help='The tub to make movie from') parser.add_argument('--out', default='tub_movie.mp4', help='The movie filename to create. default: tub_movie.mp4') - parser.add_argument('--config', default='./config.py', help='location of config file to use. default: ./config.py') + parser.add_argument('--config', default='./config.py', help=HELP_CONFIG) parser.add_argument('--model', default=None, help='the model to use to show control outputs') parser.add_argument('--type', default=None, required=False, help='the model type to load') parser.add_argument('--salient', action="store_true", help='should we overlay salient map showing activations') @@ -329,7 +332,7 @@ def parse_args(self, args): parser = argparse.ArgumentParser(prog='cnnactivations', usage='%(prog)s [options]') parser.add_argument('--image', help='path to image') parser.add_argument('--model', default=None, help='path to model') - parser.add_argument('--config', default='./config.py', help='location of config file to use. default: ./config.py') + parser.add_argument('--config', default='./config.py', help=HELP_CONFIG) parsed_args = parser.parse_args(args) return parsed_args @@ -411,7 +414,7 @@ def parse_args(self, args): parser.add_argument('--model', default=None, help='model for predictions') parser.add_argument('--limit', type=int, default=1000, help='how many records to process') parser.add_argument('--type', default=None, help='model type') - parser.add_argument('--config', default='./config.py', help='location of config file to use. default: ./config.py') + parser.add_argument('--config', default='./config.py', help=HELP_CONFIG) parsed_args = parser.parse_args(args) return parsed_args @@ -425,13 +428,23 @@ def run(self, args): class Train(BaseCommand): def parse_args(self, args): + HELP_FRAMEWORK = 'the AI framework to use (tensorflow|pytorch). ' \ + 'Defaults to config.DEFAULT_AI_FRAMEWORK' parser = argparse.ArgumentParser(prog='train', usage='%(prog)s [options]') parser.add_argument('--tub', nargs='+', help='tub data for training') parser.add_argument('--model', default=None, help='output model name') parser.add_argument('--type', default=None, help='model type') - parser.add_argument('--config', default='./config.py', help='location of config file to use. default: ./config.py') - parser.add_argument('--framework', choices=['tensorflow', 'pytorch', None], required=False, help='the AI framework to use (tensorflow|pytorch). Defaults to config.DEFAULT_AI_FRAMEWORK') - parser.add_argument('--checkpoint', type=str, help='location of checkpoint to resume training from') + parser.add_argument('--config', default='./config.py', help=HELP_CONFIG) + parser.add_argument('--framework', + choices=['tensorflow', 'pytorch', None], + required=False, + help=HELP_FRAMEWORK) + parser.add_argument('--checkpoint', type=str, + help='location of checkpoint to resume training from') + parser.add_argument('--transfer', type=str, help='transfer model') + parser.add_argument('--comment', type=str, + help='comment added to model database - use ' + 'double quotes for multiple words') parsed_args = parser.parse_args(args) return parsed_args @@ -439,17 +452,26 @@ def run(self, args): args = self.parse_args(args) args.tub = ','.join(args.tub) cfg = load_config(args.config) - framework = args.framework if args.framework else cfg.DEFAULT_AI_FRAMEWORK + framework = args.framework if args.framework \ + else getattr(cfg, 'DEFAULT_AI_FRAMEWORK', 'tensorflow') if framework == 'tensorflow': from donkeycar.pipeline.training import train - train(cfg, args.tub, args.model, args.type) + train(cfg, args.tub, args.model, args.type, args.transfer, + args.comment) elif framework == 'pytorch': from donkeycar.parts.pytorch.torch_train import train train(cfg, args.tub, args.model, args.type, checkpoint_path=args.checkpoint) else: - print("Unrecognized framework: {}. Please specify one of 'tensorflow' or 'pytorch'".format(framework)) + print(f"Unrecognized framework: {framework}. Please specify one of " + f"'tensorflow' or 'pytorch'") + + +class Gui(BaseCommand): + def run(self, args): + from donkeycar.management.kivy_ui import main + main() def execute_from_command_line(): @@ -467,6 +489,7 @@ def execute_from_command_line(): 'cnnactivations': ShowCnnActivations, 'update': UpdateCar, 'train': Train, + 'ui': Gui, } args = sys.argv[:] @@ -478,7 +501,7 @@ def execute_from_command_line(): else: dk.utils.eprint('Usage: The available commands are:') dk.utils.eprint(list(commands.keys())) - + if __name__ == "__main__": execute_from_command_line() diff --git a/donkeycar/management/graph.py b/donkeycar/management/graph.py new file mode 100644 index 000000000..b492c733b --- /dev/null +++ b/donkeycar/management/graph.py @@ -0,0 +1,189 @@ +from kivy.properties import ListProperty, ObjectProperty, StringProperty, \ + NumericProperty +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.button import Button +from kivy.uix.label import Label +from kivy.lang import Builder +from kivy.graphics import Color, Line +from kivy.app import App + +import numpy as np +import sys +from random import random +import pandas as pd +from kivy.uix.widget import Widget + +Builder.load_string(''' +#: import platform sys.platform +: + orientation: 'horizontal' + Label: + size_hint_x: 0.25 + canvas.before: + Color: + hsv: root.hsv + [1] + Line: + width: 1.5 + points: [self.x, self.y + self.size[1]/2, self.x + self.size[0], self.y + self.size[1]/2] + Label: + text: root.text + text_size: self.size + font_size: sp(12) + halign: 'center' + valign: 'middle' + +: + + +: + orientation: 'vertical' + BoxLayout: + orientation: 'horizontal' + BoxLayout: + orientation: 'vertical' + PlotArea: + id: plot + size_hint_y: 8.0 + on_size: self.draw_axes() + BoxLayout: + id: x_ticks + orientation: 'horizontal' + Label: + id: index_label + size_hint_y: None + halign: 'center' + valign: 'top' + font_size: 12 if platform == 'linux' else 24 + size: self.texture_size + text: 'index' + BoxLayout: + id: legend + size_hint_x: 0.15 + orientation: 'vertical' +''') + + +class PlotArea(Widget): + """ The graph area of the time series plot """ + offset = [50, 20] + bounding_box = ListProperty() + + def make_bounding_box(self): + return [[self.x + self.offset[0], + self.y + self.offset[1]], + [self.x + self.size[0] - self.offset[0], + self.y + self.size[1] - self.offset[1]]] + + def draw_axes(self): + self.canvas.clear() + self.bounding_box = self.make_bounding_box() + bb = self.bounding_box + with self.canvas: + Color(.9, .9, .9, 1.) + points = [bb[0][0], bb[1][1], bb[0][0], bb[0][1], bb[1][0], bb[0][1]] + Line(width=1 if sys.platform == 'linux' else 1.5, points=points) + + def draw_x_ticks(self, num_ticks): + x_length = self.bounding_box[1][0] - self.bounding_box[0][0] + for i in range(num_ticks + 1): + i_len = x_length * i / num_ticks + top = [self.bounding_box[0][0] + i_len, self.bounding_box[0][1]] + bottom = [top[0], top[1] - self.offset[1] / 2] + with self.canvas: + Color(.9, .9, .9, 1.) + Line(width=1 if sys.platform == 'linux' else 1.5, points=bottom+top) + + def get_x(self, num_points): + x_scale = (self.bounding_box[1][0] - self.bounding_box[0][0]) \ + / (num_points - 1) + x_trafo = x_scale * np.array(range(num_points)) + self.bounding_box[0][0] + return x_trafo + + def transform_y(self, y): + y_scale = (self.bounding_box[1][1] - self.bounding_box[0][1]) \ + / (y.max() - y.min()) + y_trafo = y_scale * (y - y.min()) + self.y + self.offset[1] + return y_trafo + + def add_line(self, y_points, len, hsv): + x_transformed = self.get_x(len) + y_transformed = self.transform_y(y_points) + xy_points = list() + for x, y, in zip(x_transformed, y_transformed): + if not xy_points or x > xy_points[-2] + 1: + xy_points += [x, y] + with self.canvas: + Color(*hsv, mode='hsv') + Line(points=xy_points, width=1 if sys.platform == 'linux' else 1.5) + + +class TsPlot(BoxLayout): + """ Time series plot. Can be integrated as a widget into another kivy + app.""" + len = 0 + x_ticks = 10 + df = ObjectProperty(force_dispatch=True, allownone=True) + + def draw_axes(self): + self.ids.x_ticks.clear_widgets() + self.ids.legend.clear_widgets() + self.len = 0 + self.ids.plot.draw_axes() + + def draw_x_ticks(self): + if self.len == 0: + return + self.ids.plot.draw_x_ticks(self.x_ticks) + for i in range(self.x_ticks + 1): + tick_label = Label(text=str(int(i * self.len / self.x_ticks)), + font_size=12 if sys.platform == 'linux' else 24) + self.ids.x_ticks.add_widget(tick_label) + + def add_line(self, y_points, idx): + if self.len == 0: + self.len = len(y_points) + self.draw_x_ticks() + hsv = idx / len(self.df.columns), 0.7, 0.8 + self.ids.plot.add_line(y_points, self.len, hsv) + l = LegendLabel(text=self.df.columns[idx], hsv=hsv) + self.ids.legend.add_widget(l) + + def on_df(self, e=None, z=None): + self.draw_axes() + if self.df is not None: + self.ids.index_label.text = self.df.index.name or 'index' + else: + return + self.len = 0 + for i, col in enumerate(self.df.columns): + y = self.df[col] + self.add_line(y, i) + + def set_df(self, e): + n = int(random() * 20) + 1 + cols = ['very very long line ' + str(i) for i in range(n)] + df = pd.DataFrame(np.random.randn(20, n), columns=cols) + df.index.name = 'My Index' + self.df = df + + +class LegendLabel(BoxLayout): + hsv = ListProperty() + text = StringProperty() + pass + + +class GraphApp(App): + """ Test app for time series graph. """ + def build(self): + b = BoxLayout(orientation='vertical') + ts_plot = TsPlot() + b.add_widget(ts_plot) + btn = Button(text='Create random series', size_hint_y=0.1) + btn.bind(on_press=ts_plot.set_df) + b.add_widget(btn) + return b + + +if __name__ == '__main__': + GraphApp().run() diff --git a/donkeycar/management/kivy_ui.py b/donkeycar/management/kivy_ui.py new file mode 100644 index 000000000..aad9cd3ac --- /dev/null +++ b/donkeycar/management/kivy_ui.py @@ -0,0 +1,1142 @@ +import json +import re +import time +from copy import copy +from datetime import datetime +from functools import partial +import subprocess +from subprocess import Popen, PIPE +from threading import Thread +from collections import namedtuple +from kivy.logger import Logger +import io +import os +import atexit +import yaml +from PIL import Image as PilImage +import pandas as pd +import numpy as np +import plotly.express as px + +from kivy.clock import Clock +from kivy.app import App +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.image import Image +from kivy.core.image import Image as CoreImage +from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \ + ListProperty, BooleanProperty +from kivy.uix.label import Label +from kivy.uix.popup import Popup +from kivy.lang.builder import Builder +from kivy.core.window import Window +from kivy.uix.screenmanager import ScreenManager, Screen +from kivy.uix.scrollview import ScrollView +from kivy.uix.spinner import SpinnerOption, Spinner + +from donkeycar import load_config +from donkeycar.parts.tub_v2 import Tub +from donkeycar.pipeline.augmentations import ImageAugmentation +from donkeycar.pipeline.database import PilotDatabase +from donkeycar.pipeline.types import TubRecord +from donkeycar.utils import get_model_by_type +from donkeycar.pipeline.training import train + + +Builder.load_file(os.path.join(os.path.dirname(__file__), 'ui.kv')) +Window.clearcolor = (0.2, 0.2, 0.2, 1) +LABEL_SPINNER_TEXT = 'Add/remove' + +# Data struct to show tub field in the progress bar, containing the name, +# the name of the maximum value in the config file and if it is centered. +FieldProperty = namedtuple('FieldProperty', + ['field', 'max_value_id', 'centered']) + + +def get_norm_value(value, cfg, field_property, normalised=True): + max_val_key = field_property.max_value_id + max_value = getattr(cfg, max_val_key, 1.0) + out_val = value / max_value if normalised else value * max_value + return out_val + + +def tub_screen(): + return App.get_running_app().tub_screen if App.get_running_app() else None + + +def pilot_screen(): + return App.get_running_app().pilot_screen if App.get_running_app() else None + + +def train_screen(): + return App.get_running_app().train_screen if App.get_running_app() else None + + +def car_screen(): + return App.get_running_app().car_screen if App.get_running_app() else None + + +def recursive_update(target, source): + """ Recursively update dictionary """ + if isinstance(target, dict) and isinstance(source, dict): + for k, v in source.items(): + v_t = target.get(k) + if not recursive_update(v_t, v): + target[k] = v + return True + else: + return False + + +def decompose(field): + """ Function to decompose a string vector field like 'gyroscope_1' into a + tuple ('gyroscope', 1) """ + field_split = field.split('_') + if len(field_split) > 1 and field_split[-1].isdigit(): + return '_'.join(field_split[:-1]), int(field_split[-1]) + return field, None + + +class RcFileHandler: + """ This handles the config file which stores the data, like the field + mapping for displaying of bars and last opened car, tub directory. """ + + # These entries are expected in every tub, so they don't need to be in + # the file + known_entries = [ + FieldProperty('user/angle', '', centered=True), + FieldProperty('user/throttle', '', centered=False), + FieldProperty('pilot/angle', '', centered=True), + FieldProperty('pilot/throttle', '', centered=False), + ] + + def __init__(self, file_path='~/.donkeyrc'): + self.file_path = os.path.expanduser(file_path) + self.data = self.create_data() + recursive_update(self.data, self.read_file()) + self.field_properties = self.create_field_properties() + + def exit_hook(): + self.write_file() + # Automatically save config when program ends + atexit.register(exit_hook) + + def create_field_properties(self): + """ Merges known field properties with the ones from the file """ + field_properties = {entry.field: entry for entry in self.known_entries} + field_list = self.data.get('field_mapping') + if field_list is None: + field_list = {} + for entry in field_list: + assert isinstance(entry, dict), \ + 'Dictionary required in each entry in the field_mapping list' + field_property = FieldProperty(**entry) + field_properties[field_property.field] = field_property + return field_properties + + def create_data(self): + data = dict() + data['user_pilot_map'] = {'user/throttle': 'pilot/throttle', + 'user/angle': 'pilot/angle'} + return data + + def read_file(self): + if os.path.exists(self.file_path): + with open(self.file_path) as f: + data = yaml.load(f, Loader=yaml.FullLoader) + Logger.info(f'Donkeyrc: Donkey file {self.file_path} loaded.') + return data + else: + Logger.warn(f'Donkeyrc: Donkey file {self.file_path} does not ' + f'exist.') + return {} + + def write_file(self): + if os.path.exists(self.file_path): + Logger.info(f'Donkeyrc: Donkey file {self.file_path} updated.') + with open(self.file_path, mode='w') as f: + self.data['time_stamp'] = datetime.now() + data = yaml.dump(self.data, f) + return data + + +rc_handler = RcFileHandler() + + +class MySpinnerOption(SpinnerOption): + """ Customization for Spinner """ + pass + + +class MySpinner(Spinner): + """ Customization of Spinner drop down menu """ + def __init__(self, **kwargs): + super().__init__(option_cls=MySpinnerOption, **kwargs) + + +class FileChooserPopup(Popup): + """ File Chooser popup window""" + load = ObjectProperty() + root_path = StringProperty() + filters = ListProperty() + + +class FileChooserBase: + """ Base class for file chooser widgets""" + file_path = StringProperty("No file chosen") + popup = ObjectProperty(None) + root_path = os.path.expanduser('~') + title = StringProperty(None) + filters = ListProperty() + + def open_popup(self): + self.popup = FileChooserPopup(load=self.load, root_path=self.root_path, + title=self.title, filters=self.filters) + self.popup.open() + + def load(self, selection): + """ Method to load the chosen file into the path and call an action""" + self.file_path = str(selection[0]) + self.popup.dismiss() + self.load_action() + + def load_action(self): + """ Virtual method to run when file_path has been updated """ + pass + + +class ConfigManager(BoxLayout, FileChooserBase): + """ Class to mange loading of the config file from the car directory""" + config = ObjectProperty(None) + file_path = StringProperty(rc_handler.data.get('car_dir', '')) + + def load_action(self): + """ Load the config from the file path""" + if self.file_path: + try: + path = os.path.join(self.file_path, 'config.py') + self.config = load_config(path) + # If load successful, store into app config + rc_handler.data['car_dir'] = self.file_path + except FileNotFoundError: + Logger.error(f'Config: Directory {self.file_path} has no ' + f'config.py') + except Exception as e: + Logger.error(f'Config: {e}') + + +class TubLoader(BoxLayout, FileChooserBase): + """ Class to manage loading or reloading of the Tub from the tub directory. + Loading triggers many actions on other widgets of the app. """ + file_path = StringProperty(rc_handler.data.get('last_tub', '')) + tub = ObjectProperty(None) + len = NumericProperty(1) + records = None + + def load_action(self): + """ Update tub from the file path""" + if self.update_tub(): + # If update successful, store into app config + rc_handler.data['last_tub'] = self.file_path + + def update_tub(self, event=None): + if not self.file_path: + return False + # If config not yet loaded return + cfg = tub_screen().ids.config_manager.config + if not cfg: + return False + # At least check if there is a manifest file in the tub path + if not os.path.exists(os.path.join(self.file_path, 'manifest.json')): + tub_screen().status(f'Path {self.file_path} is not a valid tub.') + return False + try: + self.tub = Tub(self.file_path) + except Exception as e: + tub_screen().status(f'Failed loading tub: {str(e)}') + return False + # Check if filter is set in tub screen + expression = tub_screen().ids.tub_filter.filter_expression + + # Use filter, this defines the function + def select(underlying): + if not expression: + return True + else: + try: + record = TubRecord(cfg, self.tub.base_path, underlying) + res = eval(expression) + return res + except KeyError as err: + Logger.error(f'Filter: {err}') + return True + + self.records = [TubRecord(cfg, self.tub.base_path, record) + for record in self.tub if select(record)] + self.len = len(self.records) + if self.len > 0: + tub_screen().index = 0 + tub_screen().ids.data_plot.update_dataframe_from_tub() + msg = f'Loaded tub {self.file_path} with {self.len} records' + else: + msg = f'No records in tub {self.file_path}' + if expression: + msg += f' using filter {tub_screen().ids.tub_filter.record_filter}' + tub_screen().status(msg) + return True + + +class LabelBar(BoxLayout): + """ Widget that combines a label with a progress bar. This is used to + display the record fields in the data panel.""" + field = StringProperty() + field_property = ObjectProperty() + config = ObjectProperty() + msg = '' + + def update(self, record): + """ This function is called everytime the current record is updated""" + if not record: + return + field, index = decompose(self.field) + if field in record.underlying: + val = record.underlying[field] + if index is not None: + val = val[index] + # Update bar if a field property for this field is known + if self.field_property: + norm_value = get_norm_value(val, self.config, + self.field_property) + new_bar_val = (norm_value + 1) * 50 if \ + self.field_property.centered else norm_value * 100 + self.ids.bar.value = new_bar_val + self.ids.field_label.text = self.field + if isinstance(val, float) or isinstance(val, np.float32): + text = f'{val:+07.3f}' + elif isinstance(val, int): + text = f'{val:10}' + else: + text = str(val) + self.ids.value_label.text = text + else: + Logger.error(f'Record: Bad record {record.underlying["_index"]} - ' + f'missing field {self.field}') + + +class DataPanel(BoxLayout): + """ Data panel widget that contains the label/bar widgets and the drop + down menu to select/deselect fields.""" + record = ObjectProperty() + # dual mode is used in the pilot arena where we only show angle and + # throttle or speed + dual_mode = BooleanProperty(False) + auto_text = StringProperty(LABEL_SPINNER_TEXT) + throttle_field = StringProperty('user/throttle') + link = False + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.labels = {} + self.screen = ObjectProperty() + + def add_remove(self): + """ Method to add or remove a LabelBar. Depending on the value of the + drop down menu the LabelBar is added if it is not present otherwise + removed.""" + field = self.ids.data_spinner.text + if field is LABEL_SPINNER_TEXT: + return + if field in self.labels and not self.dual_mode: + self.remove_widget(self.labels[field]) + del(self.labels[field]) + self.screen.status(f'Removing {field}') + else: + # in dual mode replace the second entry with the new one + if self.dual_mode and len(self.labels) == 2: + k, v = list(self.labels.items())[-1] + self.remove_widget(v) + del(self.labels[k]) + field_property = rc_handler.field_properties.get(decompose(field)[0]) + cfg = tub_screen().ids.config_manager.config + lb = LabelBar(field=field, field_property=field_property, config=cfg) + self.labels[field] = lb + self.add_widget(lb) + lb.update(self.record) + if len(self.labels) == 2: + self.throttle_field = field + self.screen.status(f'Adding {field}') + if self.screen.name == 'tub': + self.screen.ids.data_plot.plot_from_current_bars() + self.ids.data_spinner.text = LABEL_SPINNER_TEXT + self.auto_text = field + + def on_record(self, obj, record): + """ Kivy function that is called every time self.record changes""" + for v in self.labels.values(): + v.update(record) + + def clear(self): + for v in self.labels.values(): + self.remove_widget(v) + self.labels.clear() + + +class FullImage(Image): + """ Widget to display an image that fills the space. """ + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.core_image = None + + def update(self, record): + """ This method is called ever time a record gets updated. """ + try: + img_arr = self.get_image(record) + pil_image = PilImage.fromarray(img_arr) + bytes_io = io.BytesIO() + pil_image.save(bytes_io, format='png') + bytes_io.seek(0) + self.core_image = CoreImage(bytes_io, ext='png') + self.texture = self.core_image.texture + except KeyError as e: + Logger.error('Record: Missing key:', e) + except Exception as e: + Logger.error('Record: Bad record:', e) + + def get_image(self, record): + return record.image(cached=False) + + +class ControlPanel(BoxLayout): + """ Class for control panel navigation. """ + screen = ObjectProperty() + speed = NumericProperty(1.0) + record_display = StringProperty() + clock = None + fwd = None + + def start(self, fwd=True, continuous=False): + """ + Method to cycle through records if either single <,> or continuous + <<, >> buttons are pressed + :param fwd: If we go forward or backward + :param continuous: If we do <<, >> or <, > + :return: None + """ + time.sleep(0.1) + call = partial(self.step, fwd, continuous) + if continuous: + self.fwd = fwd + s = float(self.speed) * tub_screen().ids.config_manager.config.DRIVE_LOOP_HZ + cycle_time = 1.0 / s + else: + cycle_time = 0.08 + self.clock = Clock.schedule_interval(call, cycle_time) + + def step(self, fwd=True, continuous=False, *largs): + """ + Updating a single step and cap/floor the index so we stay w/in the tub. + :param fwd: If we go forward or backward + :param continuous: If we are in continuous mode <<, >> + :param largs: dummy + :return: None + """ + new_index = self.screen.index + (1 if fwd else -1) + if new_index >= tub_screen().ids.tub_loader.len: + new_index = 0 + elif new_index < 0: + new_index = tub_screen().ids.tub_loader.len - 1 + self.screen.index = new_index + msg = f'Donkey {"run" if continuous else "step"} ' \ + f'{"forward" if fwd else "backward"}' + if not continuous: + msg += f' - you can also use {"" if fwd else ""} key' + else: + msg += ' - you can toggle run/stop with ' + self.screen.status(msg) + + def stop(self): + if self.clock: + self.clock.cancel() + self.clock = None + + def restart(self): + if self.clock: + self.stop() + self.start(self.fwd, True) + + def update_speed(self, up=True): + """ Method to update the speed on the controller""" + values = self.ids.control_spinner.values + idx = values.index(self.ids.control_spinner.text) + if up and idx < len(values) - 1: + self.ids.control_spinner.text = values[idx + 1] + elif not up and idx > 0: + self.ids.control_spinner.text = values[idx - 1] + + def set_button_status(self, disabled=True): + """ Method to disable(enable) all buttons. """ + self.ids.run_bwd.disabled = self.ids.run_fwd.disabled = \ + self.ids.step_fwd.disabled = self.ids.step_bwd.disabled = disabled + + def on_keyboard(self, key, scancode): + """ Method to chack with keystroke has ben sent. """ + if key == ' ': + if self.clock and self.clock.is_triggered: + self.stop() + self.set_button_status(disabled=False) + self.screen.status('Donkey stopped') + else: + self.start(continuous=True) + self.set_button_status(disabled=True) + elif scancode == 79: + self.step(fwd=True) + elif scancode == 80: + self.step(fwd=False) + elif scancode == 45: + self.update_speed(up=False) + elif scancode == 46: + self.update_speed(up=True) + + +class PaddedBoxLayout(BoxLayout): + pass + + +class TubEditor(PaddedBoxLayout): + """ Tub editor widget. Contains left/right index interval and the + manipulator buttons for deleting / restoring and reloading """ + lr = ListProperty([0, 0]) + + def set_lr(self, is_l=True): + """ Sets left or right range to the current tub record index """ + if not tub_screen().current_record: + return + self.lr[0 if is_l else 1] = tub_screen().current_record.underlying['_index'] + + def del_lr(self, is_del): + """ Deletes or restores records in chosen range """ + tub = tub_screen().ids.tub_loader.tub + if self.lr[1] >= self.lr[0]: + selected = list(range(*self.lr)) + else: + last_id = tub.manifest.current_index + selected = list(range(self.lr[0], last_id)) + selected += list(range(self.lr[1])) + tub.delete_records(selected) if is_del else tub.restore_records(selected) + + +class TubFilter(PaddedBoxLayout): + """ Tub filter widget. """ + filter_expression = StringProperty(None) + record_filter = StringProperty(rc_handler.data.get('record_filter', '')) + + def update_filter(self): + filter_text = self.ids.record_filter.text + # empty string resets the filter + if filter_text == '': + self.record_filter = '' + self.filter_expression = '' + rc_handler.data['record_filter'] = self.record_filter + tub_screen().status(f'Filter cleared') + return + filter_expression = self.create_filter_string(filter_text) + try: + record = tub_screen().current_record + res = eval(filter_expression) + status = f'Filter result on current record: {res}' + if isinstance(res, bool): + self.record_filter = filter_text + self.filter_expression = filter_expression + rc_handler.data['record_filter'] = self.record_filter + else: + status += ' - non bool expression can\'t be applied' + status += ' - press to see effect' + tub_screen().status(status) + except Exception as e: + tub_screen().status(f'Filter error on current record: {e}') + + @staticmethod + def create_filter_string(filter_text, record_name='record'): + """ Converts text like 'user/angle' into 'record.underlying['user/angle'] + so that it can be used in a filter. Will replace only expressions that + are found in the tub inputs list. + + :param filter_text: input text like 'user/throttle > 0.1' + :param record_name: name of the record in the expression + :return: updated string that has all input fields wrapped + """ + for field in tub_screen().current_record.underlying.keys(): + field_list = filter_text.split(field) + if len(field_list) > 1: + filter_text = f'{record_name}.underlying["{field}"]'\ + .join(field_list) + return filter_text + + +class DataPlot(PaddedBoxLayout): + """ Data plot panel which embeds matplotlib interactive graph""" + df = ObjectProperty(force_dispatch=True, allownone=True) + + def plot_from_current_bars(self, in_app=True): + """ Plotting from current selected bars. The DataFrame for plotting + should contain all bars except for strings fields and all data is + selected if bars are empty. """ + tub = tub_screen().ids.tub_loader.tub + field_map = dict(zip(tub.manifest.inputs, tub.manifest.types)) + # Use selected fields or all fields if nothing is slected + all_cols = tub_screen().ids.data_panel.labels.keys() or self.df.columns + cols = [c for c in all_cols if decompose(c)[0] in field_map + and field_map[decompose(c)[0]] not in ('image_array', 'str')] + + df = self.df[cols] + if df is None: + return + # Don't plot the milliseconds time stamp as this is a too big number + df = df.drop(labels=['_timestamp_ms'], axis=1, errors='ignore') + + if in_app: + tub_screen().ids.graph.df = df + else: + fig = px.line(df, x=df.index, y=df.columns, title=tub.base_path) + fig.update_xaxes(rangeslider=dict(visible=True)) + fig.show() + + def unravel_vectors(self): + """ Unravels vector and list entries in tub which are created + when the DataFrame is created from a list of records""" + manifest = tub_screen().ids.tub_loader.tub.manifest + for k, v in zip(manifest.inputs, manifest.types): + if v == 'vector' or v == 'list': + dim = len(tub_screen().current_record.underlying[k]) + df_keys = [k + f'_{i}' for i in range(dim)] + self.df[df_keys] = pd.DataFrame(self.df[k].tolist(), + index=self.df.index) + self.df.drop(k, axis=1, inplace=True) + + def update_dataframe_from_tub(self): + """ Called from TubManager when a tub is reloaded/recreated. Fills + the DataFrame from records, and updates the dropdown menu in the + data panel.""" + generator = (t.underlying for t in tub_screen().ids.tub_loader.records) + self.df = pd.DataFrame(generator).dropna() + to_drop = {'cam/image_array'} + self.df.drop(labels=to_drop, axis=1, errors='ignore', inplace=True) + self.df.set_index('_index', inplace=True) + self.unravel_vectors() + tub_screen().ids.data_panel.ids.data_spinner.values = self.df.columns + self.plot_from_current_bars() + + +class TabBar(BoxLayout): + manager = ObjectProperty(None) + + def disable_only(self, bar_name): + this_button_name = bar_name + '_btn' + for button_name, button in self.ids.items(): + button.disabled = button_name == this_button_name + + +class TubScreen(Screen): + """ First screen of the app managing the tub data. """ + index = NumericProperty(None, force_dispatch=True) + current_record = ObjectProperty(None) + keys_enabled = BooleanProperty(True) + + def initialise(self, e): + self.ids.config_manager.load_action() + self.ids.tub_loader.update_tub() + + def on_index(self, obj, index): + """ Kivy method that is called if self.index changes""" + self.current_record = self.ids.tub_loader.records[index] + self.ids.slider.value = index + + def on_current_record(self, obj, record): + """ Kivy method that is called if self.current_record changes.""" + self.ids.img.update(record) + i = record.underlying['_index'] + self.ids.control_panel.record_display = f"Record {i:06}" + + def status(self, msg): + self.ids.status.text = msg + + def on_keyboard(self, instance, keycode, scancode, key, modifiers): + if self.keys_enabled: + self.ids.control_panel.on_keyboard(key, scancode) + + +class PilotLoader(BoxLayout, FileChooserBase): + """ Class to mange loading of the config file from the car directory""" + num = StringProperty() + model_type = StringProperty() + pilot = ObjectProperty(None) + filters = ['*.h5', '*.tflite'] + + def load_action(self): + if self.file_path and self.pilot: + try: + self.pilot.load(os.path.join(self.file_path)) + rc_handler.data['pilot_' + self.num] = self.file_path + rc_handler.data['model_type_' + self.num] = self.model_type + except FileNotFoundError: + Logger.error(f'Pilot: Model {self.file_path} not found') + except Exception as e: + Logger.error(f'Pilot: {e}') + + def on_model_type(self, obj, model_type): + """ Kivy method that is called if self.model_type changes. """ + if self.model_type and self.model_type != 'Model type': + cfg = tub_screen().ids.config_manager.config + if cfg: + self.pilot = get_model_by_type(self.model_type, cfg) + self.ids.pilot_button.disabled = False + + def on_num(self, e, num): + """ Kivy method that is called if self.num changes. """ + self.file_path = rc_handler.data.get('pilot_' + self.num, '') + self.model_type = rc_handler.data.get('model_type_' + self.num, '') + + +class OverlayImage(FullImage): + """ Widget to display the image and the user/pilot data for the tub. """ + keras_part = ObjectProperty() + pilot_record = ObjectProperty() + throttle_field = StringProperty('user/throttle') + + def get_image(self, record): + from donkeycar.management.makemovie import MakeMovie + img_arr = copy(super().get_image(record)) + augmentation = pilot_screen().augmentation if pilot_screen().auglist \ + else None + if augmentation: + img_arr = pilot_screen().augmentation.augment(img_arr) + angle = record.underlying['user/angle'] + throttle = get_norm_value(record.underlying[self.throttle_field], + tub_screen().ids.config_manager.config, + rc_handler.field_properties[ + self.throttle_field]) + rgb = (0, 255, 0) + MakeMovie.draw_line_into_image(angle, throttle, False, img_arr, rgb) + if not self.keras_part: + return img_arr + + output = self.keras_part.evaluate(record, augmentation) + rgb = (0, 0, 255) + MakeMovie.draw_line_into_image(output[0], output[1], True, img_arr, rgb) + out_record = copy(record) + out_record.underlying['pilot/angle'] = output[0] + # rename and denormalise the throttle output + pilot_throttle_field \ + = rc_handler.data['user_pilot_map'][self.throttle_field] + out_record.underlying[pilot_throttle_field] \ + = get_norm_value(output[1], tub_screen().ids.config_manager.config, + rc_handler.field_properties[self.throttle_field], + normalised=False) + self.pilot_record = out_record + return img_arr + + +class PilotScreen(Screen): + """ Screen to do the pilot vs pilot comparison .""" + index = NumericProperty(None, force_dispatch=True) + current_record = ObjectProperty(None) + keys_enabled = BooleanProperty(False) + auglist = ListProperty(force_dispatch=True) + augmentation = ObjectProperty() + config = ObjectProperty() + + def on_index(self, obj, index): + """ Kivy method that is called if self.index changes. Here we update + self.current_record and the slider value. """ + if tub_screen().ids.tub_loader.records: + self.current_record = tub_screen().ids.tub_loader.records[index] + self.ids.slider.value = index + + def on_current_record(self, obj, record): + """ Kivy method that is called when self.current_index changes. Here + we update the images and the control panel entry.""" + i = record.underlying['_index'] + self.ids.pilot_control.record_display = f"Record {i:06}" + self.ids.img_1.update(record) + self.ids.img_2.update(record) + + def initialise(self, e): + self.ids.pilot_loader_1.on_model_type(None, None) + self.ids.pilot_loader_1.load_action() + self.ids.pilot_loader_2.on_model_type(None, None) + self.ids.pilot_loader_2.load_action() + mapping = copy(rc_handler.data['user_pilot_map']) + del(mapping['user/angle']) + self.ids.data_in.ids.data_spinner.values = mapping.keys() + self.ids.data_in.ids.data_spinner.text = 'user/angle' + self.ids.data_panel_1.ids.data_spinner.disabled = True + self.ids.data_panel_2.ids.data_spinner.disabled = True + + def map_pilot_field(self, text): + """ Method to return user -> pilot mapped fields except for the + intial vale called Add/remove. """ + if text == LABEL_SPINNER_TEXT: + return text + return rc_handler.data['user_pilot_map'][text] + + def set_brightness(self, val=None): + if self.ids.button_bright.state == 'down': + self.config.AUG_MULTIPLY_RANGE = (val, val) + if self.ids.button_blur.state == 'down': + self.auglist = ['MULTIPLY', 'BLUR'] + else: + self.auglist = ['MULTIPLY'] + + def remove_brightness(self): + self.auglist = ['BLUR'] if self.ids.button_blur.state == 'down' else[] + + def set_blur(self, val=None): + if self.ids.button_blur.state == 'down': + self.config.AUG_BLUR_RANGE = (val, val) + if self.ids.button_bright.state == 'down': + self.auglist = ['MULTIPLY', 'BLUR'] + else: + self.auglist = ['BLUR'] + + def remove_blur(self): + self.auglist = ['MULTIPLY'] if self.ids.button_bright.state == 'down' \ + else [] + + def on_auglist(self, obj, auglist): + self.config.AUGMENTATIONS = self.auglist + self.augmentation = ImageAugmentation(self.config) + self.on_current_record(None, self.current_record) + + def status(self, msg): + self.ids.status.text = msg + + def on_keyboard(self, instance, keycode, scancode, key, modifiers): + if self.keys_enabled: + self.ids.pilot_control.on_keyboard(key, scancode) + + +class ScrollableLabel(ScrollView): + pass + + +class DataFrameLabel(Label): + pass + + +class TransferSelector(BoxLayout, FileChooserBase): + """ Class to select transfer model""" + filters = ['*.h5'] + + +class TrainScreen(Screen): + """ Class showing the training screen. """ + config = ObjectProperty(force_dispatch=True, allownone=True) + database = ObjectProperty() + pilot_df = ObjectProperty(force_dispatch=True) + tub_df = ObjectProperty(force_dispatch=True) + + def train_call(self, model_type, *args): + # remove car directory from path + tub_path = tub_screen().ids.tub_loader.tub.base_path + transfer = self.ids.transfer_spinner.text + if transfer != 'Choose transfer model': + transfer = os.path.join(self.config.MODELS_PATH, transfer + '.h5') + else: + transfer = None + try: + history = train(self.config, tub_paths=tub_path, + model_type=model_type, + transfer=transfer, + comment=self.ids.comment.text) + self.ids.status.text = f'Training completed.' + self.ids.train_button.state = 'normal' + self.ids.transfer_spinner.text = 'Choose transfer model' + self.reload_database() + except Exception as e: + self.ids.status.text = f'Train error {e}' + + def train(self, model_type): + self.config.SHOW_PLOT = False + Thread(target=self.train_call, args=(model_type,)).start() + self.ids.status.text = f'Training started.' + self.ids.comment.text = 'Comment' + + def set_config_attribute(self, input): + try: + val = json.loads(input) + except ValueError: + val = input + + att = self.ids.cfg_spinner.text.split(':')[0] + setattr(self.config, att, val) + self.ids.cfg_spinner.values = self.value_list() + self.ids.status.text = f'Setting {att} to {val} of type ' \ + f'{type(val).__name__}' + + def value_list(self): + if self.config: + return [f'{k}: {v}' for k, v in self.config.__dict__.items()] + else: + return ['select'] + + def on_config(self, obj, config): + if self.config and self.ids: + self.ids.cfg_spinner.values = self.value_list() + self.reload_database() + + def reload_database(self): + if self.config: + self.database = PilotDatabase(self.config) + + def on_database(self, obj, database): + if self.ids.check.state == 'down': + self.pilot_df, self.tub_df = self.database.to_df_tubgrouped() + self.ids.scroll_tubs.text = self.tub_df.to_string() + else: + self.pilot_df = self.database.to_df() + self.tub_df = pd.DataFrame() + self.ids.scroll_tubs.text = '' + + self.pilot_df.drop(columns=['History', 'Config'], errors='ignore', + inplace=True) + text = self.pilot_df.to_string(formatters=self.formatter()) + self.ids.scroll_pilots.text = text + values = ['Choose transfer model'] + if not self.pilot_df.empty: + values += self.pilot_df['Name'].tolist() + self.ids.transfer_spinner.values = values + + @staticmethod + def formatter(): + def time_fmt(t): + fmt = '%Y-%m-%d %H:%M:%S' + return datetime.fromtimestamp(t).strftime(format=fmt) + + def transfer_fmt(model_name): + return model_name.replace('.h5', '') + + return {'Time': time_fmt, 'Transfer': transfer_fmt} + + +class CarScreen(Screen): + """ Screen for interacting with the car. """ + config = ObjectProperty(force_dispatch=True, allownone=True) + files = ListProperty() + car_dir = StringProperty(rc_handler.data.get('robot_car_dir', '~/mycar')) + pull_bar = NumericProperty(0) + push_bar = NumericProperty(0) + event = ObjectProperty(None, allownone=True) + connection = ObjectProperty(None, allownone=True) + pid = NumericProperty(None, allownone=True) + pilots = ListProperty() + is_connected = BooleanProperty(False) + + def initialise(self): + self.event = Clock.schedule_interval(self.connected, 3) + + def list_remote_dir(self, dir): + if self.is_connected: + cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ + f' "ls {dir}"' + listing = os.popen(cmd).read() + adjusted_listing = listing.split('\n')[1:-1] + return adjusted_listing + else: + return [] + + def list_car_dir(self, dir): + self.car_dir = dir + self.files = self.list_remote_dir(dir) + # non-empty director found + if self.files: + rc_handler.data['robot_car_dir'] = dir + + def update_pilots(self): + model_dir = os.path.join(self.car_dir, 'models') + self.pilots = self.list_remote_dir(model_dir) + + def pull(self, tub_dir): + target = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ + f':{os.path.join(self.car_dir, tub_dir)}' + if self.ids.create_dir.state == 'normal': + target += '/' + dest = self.config.DATA_PATH + cmd = ['rsync', '-rv', '--progress', '--partial', target, dest] + Logger.info('car pull: ' + str(cmd)) + proc = Popen(cmd, shell=False, stdout=PIPE, text=True, + encoding='utf-8', universal_newlines=True) + repeats = 100 + call = partial(self.show_progress, proc, repeats, True) + event = Clock.schedule_interval(call, 0.0001) + + def send_pilot(self): + src = self.config.MODELS_PATH + cmd = ['rsync', '-rv', '--progress', '--partial', src, + f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}:' + + f'{self.car_dir}'] + Logger.info('car push: ' + ' '.join(cmd)) + proc = Popen(cmd, shell=False, stdout=PIPE, + encoding='utf-8', universal_newlines=True) + repeats = 1 + call = partial(self.show_progress, proc, repeats, False) + event = Clock.schedule_interval(call, 0.0001) + + def show_progress(self, proc, repeats, is_pull, e): + if proc.poll() is not None: + # call ended this stops the schedule + return False + # find the next repeats lines with update info + count = 0 + while True: + stdout_data = proc.stdout.readline() + if stdout_data: + # find 'to-check=33/4551)' which is end of line + pattern = 'to-check=(.*)\)' + res = re.search(pattern, stdout_data) + if res: + if count < repeats: + count += 1 + else: + remain, total = tuple(res.group(1).split('/')) + bar = 100 * (1. - float(remain) / float(total)) + if is_pull: + self.pull_bar = bar + else: + self.push_bar = bar + return True + else: + # end of stream command completed + if is_pull: + button = self.ids['pull_tub'] + self.pull_bar = 0 + else: + button = self.ids['send_pilots'] + self.push_bar = 0 + self.update_pilots() + button.disabled = False + return False + + def connected(self, event): + if not self.config: + return + if self.connection is None: + if not hasattr(self.config, 'PI_USERNAME') or \ + not hasattr(self.config, 'PI_HOSTNAME'): + self.ids.connected.text = 'Requires PI_USERNAME, PI_HOSTNAME' + return + # run new command to check connection status + cmd = ['ssh', + '-o ConnectTimeout=3', + f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', + 'date'] + Logger.info('car check: ' + ' '.join(cmd)) + self.connection = Popen(cmd, shell=False, stdout=PIPE, text=True, + encoding='utf-8', universal_newlines=True) + else: + # ssh is already running, check where we are + return_val = self.connection.poll() + self.is_connected = False + if return_val is None: + # command still running, do nothing and check next time again + status = 'Awaiting connection...' + self.ids.connected.color = 0.8, 0.8, 0.0, 1 + else: + # command finished, check if successful and reset connection + if return_val == 0: + status = 'Connected' + self.ids.connected.color = 0, 0.9, 0, 1 + self.is_connected = True + else: + status = 'Disconnected' + self.ids.connected.color = 0.9, 0, 0, 1 + self.connection = None + self.ids.connected.text = status + + def drive(self): + model_args = '' + if self.ids.pilot_spinner.text != 'No pilot': + model_path = os.path.join(self.car_dir, "models", + self.ids.pilot_spinner.text) + model_args = f'--type {self.ids.type_spinner.text} ' + \ + f'--model {model_path}' + cmd = ['ssh', + f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', + f'source env/bin/activate; cd {self.car_dir}; ./manage.py ' + f'drive {model_args} 2>&1'] + Logger.info(f'car connect: {cmd}') + proc = Popen(cmd, shell=False, stdout=PIPE, text=True, + encoding='utf-8', universal_newlines=True) + while True: + stdout_data = proc.stdout.readline() + if stdout_data: + # find 'PID: 12345' + pattern = 'PID: .*' + res = re.search(pattern, stdout_data) + if res: + try: + self.pid = int(res.group(0).split('PID: ')[1]) + Logger.info(f'car connect: manage.py drive PID: ' + f'{self.pid}') + except Exception as e: + Logger.error(f'car connect: {e}') + return + Logger.info(f'car connect: {stdout_data}') + else: + return + + def stop(self): + if self.pid: + cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME} '\ + + f'kill {self.pid}' + out = os.popen(cmd).read() + Logger.info(f"car connect: Kill PID {self.pid} + {out}") + self.pid = None + + +class StartScreen(Screen): + img_path = os.path.realpath(os.path.join( + os.path.dirname(__file__), + '../parts/web_controller/templates/static/donkeycar-logo-sideways.png')) + pass + + +class DonkeyApp(App): + start_screen = None + tub_screen = None + train_screen = None + pilot_screen = None + car_screen = None + title = 'Donkey Manager' + + def initialise(self, event): + self.tub_screen.ids.config_manager.load_action() + self.pilot_screen.initialise(event) + self.car_screen.initialise() + # This builds the graph which can only happen after everything else + # has run, therefore delay until the next round. + Clock.schedule_once(self.tub_screen.ids.tub_loader.update_tub) + + def build(self): + self.start_screen = StartScreen(name='donkey') + self.tub_screen = TubScreen(name='tub') + self.train_screen = TrainScreen(name='train') + self.pilot_screen = PilotScreen(name='pilot') + self.car_screen = CarScreen(name='car') + Window.bind(on_keyboard=self.tub_screen.on_keyboard) + Window.bind(on_keyboard=self.pilot_screen.on_keyboard) + Clock.schedule_once(self.initialise) + sm = ScreenManager() + sm.add_widget(self.start_screen) + sm.add_widget(self.tub_screen) + sm.add_widget(self.train_screen) + sm.add_widget(self.pilot_screen) + sm.add_widget(self.car_screen) + return sm + + +def main(): + tub_app = DonkeyApp() + tub_app.run() + + +if __name__ == '__main__': + main() diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index 3135ee120..a9c3ea49e 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -16,9 +16,10 @@ from donkeycar.utils import * +DEG_TO_RAD = math.pi / 180.0 + + class MakeMovie(object): - def __init__(self): - self.deg_to_rad = math.pi / 180.0 def run(self, args, parser): ''' @@ -80,30 +81,30 @@ def run(self, args, parser): clip = mpy.VideoClip(self.make_frame, duration=((num_frames - 1) / self.cfg.DRIVE_LOOP_HZ)) clip.write_videofile(args.out, fps=self.cfg.DRIVE_LOOP_HZ) - def draw_user_input(self, record, img): - ''' - Draw the user input as a green line on the image - ''' - + @staticmethod + def draw_line_into_image(angle, throttle, is_left, img, color): import cv2 - user_angle = float(record["user/angle"]) - user_throttle = float(record["user/throttle"]) - height, width, _ = img.shape - length = height - a1 = user_angle * 45.0 - l1 = user_throttle * length - - mid = width // 2 - 1 + a1 = angle * 45.0 + l1 = throttle * length + mid = width // 2 + (- 1 if is_left else +1) p1 = tuple((mid - 2, height - 1)) - p11 = tuple((int(p1[0] + l1 * math.cos((a1 + 270.0) * self.deg_to_rad)), - int(p1[1] + l1 * math.sin((a1 + 270.0) * self.deg_to_rad)))) + p11 = tuple((int(p1[0] + l1 * math.cos((a1 + 270.0) * DEG_TO_RAD)), + int(p1[1] + l1 * math.sin((a1 + 270.0) * DEG_TO_RAD)))) - # user is green, pilot is blue - cv2.line(img, p1, p11, (0, 255, 0), 2) + cv2.line(img, p1, p11, color, 2) + + def draw_user_input(self, record, img): + """ + Draw the user input as a green line on the image + """ + user_angle = float(record["user/angle"]) + user_throttle = float(record["user/throttle"]) + green = (0, 255, 0) + self.draw_line_into_image(user_angle, user_throttle, False, img, green) def draw_model_prediction(self, img): """ @@ -113,8 +114,6 @@ def draw_model_prediction(self, img): if self.keras_part is None: return - import cv2 - expected = tuple(self.keras_part.get_input_shape()[1:]) actual = img.shape @@ -126,29 +125,19 @@ def draw_model_prediction(self, img): img = grey_img.reshape(grey_img.shape + (1,)) if expected != actual: - print("expected input dim", expected, "didn't match actual dim", actual) + print(f"expected input dim {expected} didn't match actual dim " + f"{actual}") return + blue = (0, 0, 255) pilot_angle, pilot_throttle = self.keras_part.run(img) - height, width, _ = img.shape - - length = height - a2 = pilot_angle * 45.0 - l2 = pilot_throttle * length - - mid = width // 2 - 1 - - p2 = tuple((mid + 2, height - 1)) - p22 = tuple((int(p2[0] + l2 * math.cos((a2 + 270.0) * self.deg_to_rad)), - int(p2[1] + l2 * math.sin((a2 + 270.0) * self.deg_to_rad)))) - - # user is green, pilot is blue - cv2.line(img, p2, p22, (0, 0, 255), 2) + self.draw_line_into_image(pilot_angle, pilot_throttle, True, img, blue) def draw_steering_distribution(self, img): - ''' - query the model for it's prediction, draw the distribution of steering choices - ''' + """ + query the model for it's prediction, draw the distribution of + steering choices + """ from donkeycar.parts.keras import KerasCategorical if self.keras_part is None or type(self.keras_part) is not KerasCategorical: diff --git a/donkeycar/management/tub.py b/donkeycar/management/tub.py index 022d54bb9..7e60ab27f 100644 --- a/donkeycar/management/tub.py +++ b/donkeycar/management/tub.py @@ -113,5 +113,4 @@ def post(self, tub_id): new_indexes.add(frame['_index']) frames_to_delete = [index for index in old_indexes if index not in new_indexes] - for frm in frames_to_delete: - tub.delete_record(frm) + tub.delete_records(frames_to_delete) diff --git a/donkeycar/management/ui.kv b/donkeycar/management/ui.kv new file mode 100644 index 000000000..5960979d4 --- /dev/null +++ b/donkeycar/management/ui.kv @@ -0,0 +1,683 @@ +#: import TsPlot donkeycar.management.graph.TsPlot +#: import get_model_by_type donkeycar.utils.get_model_by_type +#: import platform sys.platform +#: import os os + +#:set common_height 30 if platform != 'darwin' else 60 +#:set layout_pad_x 10 if platform != 'darwin' else 20 +#:set layout_height common_height + layout_pad_x +#:set layout_height_double 2 * common_height + layout_pad_x +#:set layout_pad_xy [layout_pad_x, layout_pad_x // 2] + +: + size_hint_y: None + height: layout_height + padding: layout_pad_xy + spacing: layout_pad_x + + +: + title: "Choose the directory" + size_hint: 1.0, 1.0 + auto_dismiss: False + pos_hint: {'center_x': .5, 'center_y': .5} + + BoxLayout: + orientation: "vertical" + FileChooser: + id: file_chooser + rootpath: root.root_path + dirselect: True + filters: root.filters + FileChooserListLayout + + BoxLayout: + size_hint: (1, 0.1) + pos_hint: {'center_x': .5, 'center_y': .5} + spacing: 20 + Button: + text: "Cancel" + on_release: root.dismiss() + Button: + text: "Load" + on_release: root.load(file_chooser.selection) + disabled: file_chooser.selection==[] + + + text_size: self.size + halign: 'center' + valign: 'middle' + + + title: 'Choose the car directory' + orientation: 'horizontal' + on_config: + app.tub_screen.ids.tub_loader.ids.tub_button.disabled = False + app.tub_screen.ids.tub_loader.root_path = self.file_path + app.pilot_screen.ids.pilot_loader_1.root_path = self.config.MODELS_PATH + app.pilot_screen.ids.pilot_loader_2.root_path = self.config.MODELS_PATH + app.tub_screen.ids.status.text = 'Config loaded from' + self.file_path + Button: + text: 'Load car directory' + size_hint_x: 0.5 + on_press: root.open_popup() + AutoLabel: + id: car_dir + text: root.file_path + + + title: 'Choose the tub directory' + orientation: 'horizontal' + Button: + id: tub_button + text: 'Load tub' + size_hint_x: 0.5 + disabled: True + on_press: root.open_popup() + AutoLabel: + id: tub_dir + text: root.file_path + + +: + orientation: 'horizontal' + spacing: 4 + Label: + id: field_label + text_size: self.size + halign: 'left' + valign: 'middle' + font_size: '14sp' # 14 if platform == 'linux' else 28 + canvas.before: + Color: + rgba: 0.17, 0.18, 0.25, 1 + Rectangle: + pos: self.pos + size: self.size + Label: + id: value_label + text_size: self.size + halign: 'right' + valign: 'middle' + font_size: 14 if platform == 'linux' else 28 + size_hint_x: 0.8 + padding_x: 10 + canvas.before: + Color: + rgba: 0.14, 0.15, 0.22, 1 + Rectangle: + pos: self.pos + size: self.size + ProgressBar: + id: bar + canvas.before: + Color: + rgba: 0.12, 0.13, 0.20, 1 + Rectangle: + pos: self.pos + size: self.size + +: + text_size: self.size + halign: 'center' + valign: 'middle' + height: common_height + + + + orientation: 'vertical' + spacing: 2 + GridLayout: + cols: 2 + Label: + text: 'Record field' + size_hint_y: None + height: common_height + MySpinner: + id: data_spinner + size_hint_y: None + height: common_height + text: root.auto_text if root.link else 'Add/remove' + on_text: root.add_remove() + on_values: root.clear() + + + orientation: 'vertical' + GridLayout: + cols: 2 + Label: + id: record_num + text: root.record_display + MySpinner: + id: control_spinner + pos_hint: {'center': (.5, .5)} + text: '1.00' + values: ['0.25', '0.50', '1.00', '1.50', '2.00', '3.00', '4.00'] + on_text: + root.speed = float(self.text) + app.tub_screen.ids.status.text = f'Setting speed to {self.text} - you can also use the +/- keys.' + root.restart() + Button: + id: step_bwd + text: '<' + on_press: + root.start(fwd=False) + on_release: + root.stop() + Button: + id: step_fwd + text: '>' + on_press: + root.start(fwd=True) + on_release: + root.stop() + Button: + id: run_bwd + text: '<<' + on_press: + root.start(fwd=False, continuous=True) + root.set_button_status(disabled=True) + Button: + id: run_fwd + text: '>>' + on_press: + root.start(fwd=True, continuous=True) + root.set_button_status(disabled=True) + Button: + size_hint_y: 0.3 + text: 'Stop' + on_press: + root.stop() + root.set_button_status(disabled=False) + + + orientation: 'horizontal' + on_lr: + msg = f'Setting range, ' + if root.lr[0] < root.lr[1]: msg += (f'affecting records inside [{root.lr[0]}, {root.lr[1]}) ' + \ + '- you can affect records outside by setting left > right') + else: msg += (f'affecting records outside ({root.lr[1]}, {root.lr[0]}] ' + \ + '- you can affect records inside by setting left < right') + app.tub_screen.ids.status.text = msg + Button: + text: 'Set left' + on_press: root.set_lr(True) + Button: + text: 'Set right' + on_press: root.set_lr(False) + Label: + text: '[' + str(root.lr[0]) + ', ' + str(root.lr[1]) + ')' + # text: f'Index range [{root.lr[0]}, {root.lr[1]})' + Button: + text: 'Delete' + on_press: + root.del_lr(True) + msg = f'Delete records {root.lr} - press to see the ' \ + + f'effect, but you can delete multiple ranges before doing so.' + app.tub_screen.ids.status.text = msg + Button: + text: 'Restore' + on_press: + root.del_lr(False) + msg = f'Restore records {root.lr} - press to see the ' \ + + f'effect, but you can restore multiple ranges before doing so.' + app.tub_screen.ids.status.text = msg + Button: + text: 'Reload Tub' + on_press: + app.tub_screen.ids.tub_loader.update_tub() + +: + id: tub_filter + orientation: 'horizontal' + Button: + text: 'Set filter' + size_hint_x: 0.2 + on_press: root.update_filter() + TextInput: + id: record_filter + text: root.record_filter + multiline: False + on_focus: app.tub_screen.keys_enabled = not app.tub_screen.keys_enabled + +<-FullImage>: + size_hint_x: 1.2 + size_hint_y: 1.2 + canvas: + Color: + rgb: (1, 1, 1) + Rectangle: + texture: self.texture + size: self.width, self.height / 1.2 + pos: self.x, self.y + +: + Button: + text: 'Reload Graph' + on_press: root.plot_from_current_bars() + Button: + text: 'Browser Graph' + on_press: root.plot_from_current_bars(in_app=False) + +: + size_hint_y: None + height: 30 + text_size: self.size + halign: 'left' + valign: 'bottom' + text: 'Donkey ready' + +: + size_hint_y: None + height: common_height + 40 + padding: [20, 20] + spacing: 20 + canvas.before: + Color: + rgba: 0.25, 0.138, 0.0, 1 + Rectangle: + pos: self.pos + size: self.size + Button: + id: tub_btn + text: 'Tub Manager' + on_press: + root.keys_enabled = False + root.manager.current = 'tub' + root.manager.current_screen.keys_enabled = True + root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) + Button: + id: train_btn + text: 'Trainer' + on_press: + root.keys_enabled = False + root.manager.current = 'train' + root.manager.current_screen.keys_enabled = True + root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) + Button: + id: pilot_btn + text: 'Pilot Arena' + on_press: + root.keys_enabled = False + root.manager.current = 'pilot' + if not root.manager.current_screen.index: root.manager.current_screen.index = 0 + root.manager.current_screen.keys_enabled = True + root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) + Button: + id: car_btn + text: 'Car Connector' + on_press: + root.keys_enabled = False + root.manager.current = 'car' + root.manager.current_screen.keys_enabled = True + root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) + root.manager.current_screen.update_pilots() + +: + BoxLayout: + orientation: 'vertical' + TabBar: + id: tab_bar + manager: root.manager + PaddedBoxLayout: + orientation: 'horizontal' + ConfigManager: + id: config_manager + TubLoader: + id: tub_loader + PaddedBoxLayout: + size_hint_y: 1.0 + DataPanel: + id: data_panel + screen: root + record: root.current_record + FullImage: + id: img + ControlPanel + id: control_panel + screen: root + Slider: + id: slider + min: 0 + max: tub_loader.len - 1 + value: 0 + size_hint_y: None + height: common_height + on_value: root.index = int(self.value) + TubEditor + TubFilter: + id: tub_filter + TsPlot: + id: graph + #size_hint_y: 3.0 if platform == 'linux' else None + DataPlot: + id: data_plot + StatusLabel: + id: status + + +: + title: 'Choose the pilot' + orientation: 'horizontal' + model_type: pilot_spinner.text + Button: + id: pilot_button + text: 'Choose pilot' + size_hint_x: 0.5 + disabled: True + on_press: root.open_popup() + MySpinner: + id: pilot_spinner + pos_hint: {'center': (.5, .5)} + size_hint_x: 0.5 + text: 'Model type' + values: ['linear', 'categorical', 'tflite_linear'] + AutoLabel: + id: pilot_file + text: root.file_path + + +: + config: app.tub_screen.ids.config_manager.config + BoxLayout: + orientation: 'vertical' + TabBar: + id: tab_bar + manager: root.manager + PaddedBoxLayout: + PilotLoader: + num: num + id: pilot_loader_1 + num: '1' + data_panel: data_panel_1 + PilotLoader: + id: pilot_loader_2 + num: '2' + data_panel: data_panel_2 + BoxLayout: + padding: layout_pad_xy + spacing: layout_pad_x + OverlayImage: + id: img_1 + keras_part: pilot_loader_1.pilot + throttle_field: data_in.throttle_field + OverlayImage: + id: img_2 + keras_part: pilot_loader_2.pilot + throttle_field: data_in.throttle_field + PaddedBoxLayout: + size_hint_y: 0.5 + DataPanel: + id: data_panel_1 + screen: root + record: img_1.pilot_record + dual_mode: True + link: True + auto_text: root.map_pilot_field(data_in.auto_text) + DataPanel: + id: data_panel_2 + screen: root + record: img_2.pilot_record + dual_mode: True + link: True + auto_text: root.map_pilot_field(data_in.auto_text) + Slider: + id: slider + min: 0 + max: app.tub_screen.ids.tub_loader.len - 1 + value: 0 + size_hint_y: None + height: common_height + on_value: root.index = int(self.value) + + PaddedBoxLayout: + ToggleButton: + id: button_bright + size_hint_x: 0.5 + text: 'Brightness {:4.2f}'.format(slider_bright.value) + on_press: root.set_brightness(slider_bright.value) if self.state == 'down' else root.remove_brightness() + Slider: + id: slider_bright + value: 1 + min: 0 + max: 3 + on_value: root.set_brightness(self.value) + ToggleButton: + id: button_blur + size_hint_x: 0.5 + text: 'Blur {:4.2f}'.format(slider_blur.value) + on_press: root.set_blur(slider_blur.value) if self.state == 'down' else root.remove_blur() + Slider: + id: slider_blur + value: 0 + min: 0 + max: 4 + on_value: root.set_blur(self.value) + PaddedBoxLayout: + size_hint_y: 0.5 + ControlPanel: + id: pilot_control + screen: root + DataPanel: + id: data_in + screen: root + record: root.current_record + dual_mode: True + StatusLabel: + id: status + +: + font_name: 'data/fonts/RobotoMono-Regular.ttf' + font_size: 12 if platform == 'linux' else 24 + size_hint_y: None + height: self.texture_size[1] + text_size: self.width, None + +: + title: 'Choose transfer model' + orientation: 'horizontal' + Button: + id: transfer_button + text: 'Transfer model' + size_hint_x: 0.5 + on_press: root.open_popup() + AutoLabel: + text: root.file_path + +: + config: app.tub_screen.ids.config_manager.config + BoxLayout: + orientation: 'vertical' + TabBar: + id: tab_bar + manager: root.manager + Label: + size_hint_y: None + height: common_height + text: 'Overwrite config' + BoxLayout: + size_hint_y: None + height: layout_height + padding: layout_pad_xy + spacing: layout_pad_x + MySpinner: + id: cfg_spinner + text: 'select' + values: root.value_list() + TextInput: + id: cfg_overwrite + multiline: False + text: 'New value' + on_text_validate: root.set_config_attribute(self.text) + Label: + size_hint_y: None + height: common_height + text: 'Train pilot' + BoxLayout: + size_hint_y: None + height: layout_height + padding: layout_pad_xy + spacing: layout_pad_x + Label: + size_hint_x: 0.5 + text: 'Model type' + MySpinner: + id: train_spinner + size_hint_x: 0.5 + text: 'linear' + values: ['linear', 'categorical', 'tflite_linear'] + TextInput: + id: comment + multiline: False + text: 'Comment' + BoxLayout: + size_hint_y: None + height: layout_height + padding: layout_pad_xy + spacing: layout_pad_x + MySpinner: + id: transfer_spinner + text: 'Choose transfer model' + ToggleButton: + id: train_button + text: 'Training running...' if self.state == 'down' else 'Train' + on_press: root.train(train_spinner.text) + + ScrollableLabel: + DataFrameLabel: + id: scroll_pilots + ScrollableLabel: + size_hint_y: 1.0 if check.state == 'down' else 0.1 + DataFrameLabel: + id: scroll_tubs + BoxLayout: + size_hint_y: None + height: layout_height + padding: layout_pad_xy + spacing: layout_pad_x + Label: + text: 'Group multiple tubs' + ToggleButton: + id: check + on_press: root.on_database(None, None) + text: 'On' if self.state == 'down' else 'Off' + StatusLabel: + id: status + +: + config: app.tub_screen.ids.config_manager.config + BoxLayout: + orientation: 'vertical' + TabBar: + id: tab_bar + manager: root.manager + AutoLabel: + text: 'This screen is experimental and only works on Linux and OSX if ssh is configured to login without password. Please see the docs.' + color: (1, 0.35, 0, 1) + BoxLayout: + size_hint_y: None + height: common_height + Label: + text: 'Connection status' + Label: + id: connected + text: 'Checking...' + Label: + size_hint_y: None + height: common_height + text: 'Transfer tub from car -> pc and pilot from pc -> car' + GridLayout: + cols: 4 + row_default_height: layout_height + row_force_default: True + padding: layout_pad_xy + spacing: layout_pad_x + # first row + Label: + text: 'Car directory (hit return)' + ToggleButton: + id: create_dir + text: 'Create new folder' + ProgressBar: + value: root.pull_bar + ProgressBar: + value: root.push_bar + # second row + TextInput: + id: car_dir + multiline: False + text: root.car_dir + on_text_validate: + root.list_car_dir(self.text) + root.update_pilots() + MySpinner: + id: tub_dir_spinner + text: 'select' + values: root.files + Button: + id: pull_tub + text: 'Pull tub ' + tub_dir_spinner.text + on_press: + self.disabled = True + root.pull(tub_dir_spinner.text) + Button: + id: send_pilots + multiline: False + text: 'Send pilots' + on_press: + self.disabled = True + root.send_pilot() + Label: + size_hint_y: None + height: common_height + text: 'Drive car' + BoxLayout: + size_hint_y: None + height: layout_height_double + padding: layout_pad_xy + spacing: layout_pad_x + MySpinner: + id: type_spinner + text: 'linear' + values: ['linear', 'categorical', 'tflite_linear'] + MySpinner: + id: pilot_spinner + text: 'No pilot' + values: ['No pilot'] + root.pilots + + BoxLayout: + size_hint_y: None + height: layout_height_double + padding: layout_pad_xy + spacing: layout_pad_x + Button: + id: drive_btn + text: 'Drive' + on_press: + root.drive() + self.disabled = True + stop_btn.disabled = False + Button: + id: stop_btn + text: 'Stop' + disabled: True + on_press: + root.stop() + self.disabled = True + drive_btn.disabled = False + + Label: + size_hint_y: 1.0 + text: '' + + +: + BoxLayout: + orientation: 'vertical' + TabBar: + id: tab_bar + manager: root.manager + Image: + source: root.img_path + size: self.texture_size + diff --git a/donkeycar/parts/actuator.py b/donkeycar/parts/actuator.py index e2cbfaca4..144272341 100644 --- a/donkeycar/parts/actuator.py +++ b/donkeycar/parts/actuator.py @@ -49,7 +49,7 @@ class PiGPIO_PWM(): # for PCA9685. # # Install and setup: - # sudo update && sudo apt install pigpio python3-pigpio + # sudo apt update && sudo apt install pigpio python3-pigpio # sudo systemctl start pigpiod # # Note: the range of pulses will differ from those required for PCA9685 diff --git a/donkeycar/parts/camera.py b/donkeycar/parts/camera.py index 1c0363130..8463b72f7 100644 --- a/donkeycar/parts/camera.py +++ b/donkeycar/parts/camera.py @@ -122,7 +122,7 @@ def run_threaded(self): def shutdown(self): # indicate that the thread should be stopped self.on = False - print('stoping Webcam') + print('stopping Webcam') time.sleep(.5) @@ -189,7 +189,7 @@ def run_threaded(self): def shutdown(self): self.running = False - print('stoping CSICamera') + print('stopping CSICamera') time.sleep(.5) del(self.camera) diff --git a/donkeycar/parts/controller.py b/donkeycar/parts/controller.py index 2f0d518d7..11510c772 100644 --- a/donkeycar/parts/controller.py +++ b/donkeycar/parts/controller.py @@ -1,4 +1,3 @@ - import os import array import time @@ -229,6 +228,108 @@ def set_deadzone(self, val): self.dead_zone = val +class RCReceiver: + """ + Class to read PWM from an RC control and convert into a float output number. + Uses pigpio library. The code is essentially a copy of + http://abyz.me.uk/rpi/pigpio/code/read_PWM_py.zip. You will need a voltage + divider from a 5V RC receiver to a 3.3V Pi input pin if the receiver runs + on 5V. If your receiver accepts 3.3V input, then it can be connected + directly to the Pi. + """ + MIN_OUT = -1 + MAX_OUT = 1 + + def __init__(self, gpio, invert=False, jitter=0.025, no_action=None): + """ + :param gpio: gpio pin connected to RC channel + :param invert: invert value of run() within [MIN_OUT,MAX_OUT] + :param jitter: threshold below which no signal is reported + :param no_action: value within [MIN_OUT,MAX_OUT] if no RC signal is + sent. This is usually zero for throttle and steering + being the center values when the controls are not + pressed. + """ + import pigpio + self.pi = pigpio.pi() + self.gpio = gpio + self.high_tick = None + self.period = None + self.high = None + self.min_pwm = 1000 + self.max_pwm = 2000 + self.invert = invert + self.jitter = jitter + if no_action is not None: + self.no_action = no_action + else: + self.no_action = (self.MAX_OUT - self.MIN_OUT) / 2.0 + + self.factor = (self.MAX_OUT - self.MIN_OUT) / (self.max_pwm - self.min_pwm) + self.pi.set_mode(self.gpio, pigpio.INPUT) + self.cb = self.pi.callback(self.gpio, pigpio.EITHER_EDGE, self._cbf) + print('RCReceiver gpio ' + str(gpio) + ' created') + + def _update_param(self, tick): + """ Helper function for callback function _cbf. + :param tick: current tick in mu s + :return: difference in ticks + """ + import pigpio + if self.high_tick is not None: + t = pigpio.tickDiff(self.high_tick, tick) + return t + + def _cbf(self, gpio, level, tick): + """ Callback function for pigpio interrupt gpio. Signature is determined + by pigpiod library. This function is called every time the gpio + changes state as we specified EITHER_EDGE. + :param gpio: gpio to listen for state changes + :param level: rising/falling edge + :param tick: # of mu s since boot, 32 bit int + """ + if level == 1: + self.period = self._update_param(tick) + self.high_tick = tick + elif level == 0: + self.high = self._update_param(tick) + + def pulse_width(self): + """ + :return: the PWM pulse width in microseconds. + """ + if self.high is not None: + return self.high + else: + return 0.0 + + def run(self): + """ + Donkey parts interface, returns pulse mapped into [MIN_OUT,MAX_OUT] or + [MAX_OUT,MIN_OUT] + """ + # signal is a value in [0, (MAX_OUT-MIN_OUT)] + signal = (self.pulse_width() - self.min_pwm) * self.factor + # Assuming non-activity if the pulse is at no_action point + is_action = abs(signal - self.no_action) > self.jitter + # if deemed noise assume no signal + if not is_action: + signal = self.no_action + # convert into min max interval + if self.invert: + signal = -signal + self.MAX_OUT + else: + signal += self.MIN_OUT + return signal, is_action + + def shutdown(self): + """ + Donkey parts interface + """ + import pigpio + self.cb.cancel() + + class JoystickCreator(Joystick): ''' A Helper class to create a new joystick mapping @@ -517,9 +618,14 @@ def __init__(self, *args, **kwargs): super(XboxOneJoystick, self).__init__(*args, **kwargs) self.axis_names = { - 0x00: 'left_stick_horz', - 0x02: 'left_trigger', - 0x05: 'right_trigger' + 0x00 : 'left_stick_horz', + 0x01 : 'left_stick_vert', + 0x05 : 'right_stick_vert', + 0x02 : 'right_stick_horz', + 0x0a : 'left_trigger', + 0x09 : 'right_trigger', + 0x10 : 'dpad_horiz', + 0x11 : 'dpad_vert' } self.button_names = { @@ -527,12 +633,11 @@ def __init__(self, *args, **kwargs): 0x131: 'b_button', 0x133: 'x_button', 0x134: 'y_button', + 0x13b: 'options', 0x136: 'left_shoulder', 0x137: 'right_shoulder', - 0x13b: 'options', } - class LogitechJoystick(Joystick): ''' An interface to a physical Logitech joystick available at /dev/input/js0 @@ -1514,6 +1619,7 @@ def get_js_controller(cfg): ctr.set_deadzone(cfg.JOYSTICK_DEADZONE) return ctr + if __name__ == "__main__": # Testing the XboxOneJoystickController js = XboxOneJoystick('/dev/input/js0') @@ -1521,6 +1627,6 @@ def get_js_controller(cfg): while True: button, button_state, axis, axis_val = js.poll() - if (button is not None or axis is not None): + if button is not None or axis is not None: print(button, button_state, axis, axis_val) time.sleep(0.1) diff --git a/donkeycar/parts/datastore.py b/donkeycar/parts/datastore.py index 9c6a50ec5..5e49c4ed8 100644 --- a/donkeycar/parts/datastore.py +++ b/donkeycar/parts/datastore.py @@ -235,6 +235,10 @@ def put_record(self, data): name = self.make_file_name(key, ext='.png') img.save(os.path.join(self.path, name)) json_data[key]=name + + elif typ == 'nparray': + # convert np array to python so it is jsonable + json_data[key] = val.tolist() else: msg = 'Tub does not know what to do with this type {}'.format(typ) diff --git a/donkeycar/parts/datastore_v2.py b/donkeycar/parts/datastore_v2.py index c851d62cf..959693155 100644 --- a/donkeycar/parts/datastore_v2.py +++ b/donkeycar/parts/datastore_v2.py @@ -10,19 +10,22 @@ class Seekable(object): - ''' - A seekable file reader, writer which deals with newline delimited records. \n - This reader maintains an index of line lengths, so seeking a line is a O(1) operation. - ''' + """ + A seekable file reader, writer which deals with newline delimited + records. \n + This reader maintains an index of line lengths, so seeking a line is a + O(1) operation. + """ def __init__(self, file, read_only=False, line_lengths=list()): self.line_lengths = list() self.cumulative_lengths = list() self.method = 'r' if read_only else 'a+' self.file = open(file, self.method, newline=NEWLINE) + # If file is read only improve performance by memory mapping the file. if self.method == 'r': - # If file is read only improve performance by memory mappping the file. - self.file = mmap.mmap(self.file.fileno(), length=0, access=mmap.ACCESS_READ) + self.file = mmap.mmap(self.file.fileno(), length=0, + access=mmap.ACCESS_READ) self.total_length = 0 if len(line_lengths) <= 0: self._read_contents() @@ -74,7 +77,8 @@ def _line_end_offset(self, line_number): def _offset_until(self, line_index): end_index = line_index - 1 - return self.cumulative_lengths[end_index] if end_index >= 0 and end_index < len(self.cumulative_lengths) else 0 + return self.cumulative_lengths[end_index] \ + if 0 <= end_index < len(self.cumulative_lengths) else 0 def readline(self): contents = self.file.readline() @@ -92,7 +96,8 @@ def seek_end_of_file(self): def truncate_until_end(self, line_number): self.line_lengths = self.line_lengths[:line_number] self.cumulative_lengths = self.cumulative_lengths[:line_number] - self.total_length = self.cumulative_lengths[-1] if len(self.cumulative_lengths) > 0 else 0 + self.total_length = self.cumulative_lengths[-1] \ + if len(self.cumulative_lengths) > 0 else 0 self.seek_end_of_file() self.file.truncate() @@ -140,8 +145,12 @@ class Catalog(object): ''' def __init__(self, path, read_only=False, start_index=0): self.path = Path(os.path.expanduser(path)) - self.manifest = CatalogMetadata(self.path, read_only=read_only, start_index=start_index) - self.seekable = Seekable(self.path.as_posix(), line_lengths=self.manifest.line_lengths(), read_only=read_only) + self.manifest = CatalogMetadata(self.path, + read_only=read_only, + start_index=start_index) + self.seekable = Seekable(self.path.as_posix(), + line_lengths=self.manifest.line_lengths(), + read_only=read_only) def _exit_handler(self): self.close() @@ -164,8 +173,9 @@ class CatalogMetadata(object): ''' def __init__(self, catalog_path, read_only=False, start_index=0): path = Path(catalog_path) - manifest_name = '%s.catalog_manifest' % (path.stem) - self.manifest_path = Path(os.path.join(path.parent.as_posix(), manifest_name)) + manifest_name = f'{path.stem}.catalog_manifest' + self.manifest_path = Path(os.path.join(path.parent.as_posix(), + manifest_name)) self.seekeable = Seekable(self.manifest_path, read_only=read_only) has_contents = False if os.path.exists(self.manifest_path) and self.seekeable.has_content(): @@ -230,6 +240,7 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], self.catalog_paths = list() self.catalog_metadata = dict() self.deleted_indexes = set() + self._updated_session = False has_catalogs = False if self.manifest_path.exists(): @@ -237,24 +248,32 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], if self.seekeable.has_content(): self._read_contents() has_catalogs = len(self.catalog_paths) > 0 + else: created_at = time.time() self.manifest_metadata['created_at'] = created_at if not self.base_path.exists(): self.base_path.mkdir(parents=True, exist_ok=True) - print('Created a new datastore at %s' % (self.base_path.as_posix())) + print(f'Created a new datastore at {self.base_path.as_posix()}') self.seekeable = Seekable(self.manifest_path, read_only=self.read_only) if not has_catalogs: self._write_contents() self._add_catalog() else: - last_known_catalog = os.path.join(self.base_path, self.catalog_paths[-1]) - print('Using catalog %s' % (last_known_catalog)) - self.current_catalog = Catalog(last_known_catalog, read_only=self.read_only, start_index=self.current_index) + last_known_catalog = os.path.join(self.base_path, + self.catalog_paths[-1]) + print(f'Using catalog {last_known_catalog}') + self.current_catalog = Catalog(last_known_catalog, + read_only=self.read_only, + start_index=self.current_index) + # Create a new session_id, which will be added to each record in the + # tub, when Tub.write_record() is called. + self.session_id = self.create_new_session() def write_record(self, record): - new_catalog = self.current_index > 0 and (self.current_index % self.max_len) == 0 + new_catalog = self.current_index > 0 \ + and (self.current_index % self.max_len) == 0 if new_catalog: self._add_catalog() @@ -262,18 +281,34 @@ def write_record(self, record): self.current_index += 1 # Update metadata to keep track of the last index self._update_catalog_metadata(update=True) + # Set session_id update status to True if this method is called at + # least once. Then session id metadata will be updated when the + # session gets closed + if not self._updated_session: + self._updated_session = True + + def delete_records(self, record_indexes): + # Does not actually delete the record, but marks it as deleted. + if isinstance(record_indexes, int): + record_indexes = {record_indexes} + self.deleted_indexes.update(record_indexes) + self._update_catalog_metadata(update=True) - def delete_record(self, record_index): + def restore_records(self, record_indexes): # Does not actually delete the record, but marks it as deleted. - self.deleted_indexes.add(record_index) + if isinstance(record_indexes, int): + record_indexes = {record_indexes} + self.deleted_indexes.difference_update(record_indexes) self._update_catalog_metadata(update=True) def _add_catalog(self): current_length = len(self.catalog_paths) - catalog_name = 'catalog_%s.catalog' % (current_length) + catalog_name = f'catalog_{current_length}.catalog' catalog_path = os.path.join(self.base_path, catalog_name) current_catalog = self.current_catalog - self.current_catalog = Catalog(catalog_path, start_index=self.current_index, read_only=self.read_only) + self.current_catalog = Catalog(catalog_path, + start_index=self.current_index, + read_only=self.read_only) # Store relative paths self.catalog_paths.append(catalog_name) self._update_catalog_metadata(update=True) @@ -318,7 +353,30 @@ def _update_catalog_metadata(self, update=True): self.catalog_metadata = catalog_metadata self.seekeable.writeline(json.dumps(catalog_metadata)) + def create_new_session(self): + """ Creates a new session id and appends it to the metadata.""" + sessions = self.manifest_metadata.get('sessions', {}) + last_id = -1 + if sessions: + last_id = sessions['last_id'] + else: + sessions['all_full_ids'] = [] + this_id = last_id + 1 + date = time.strftime('%y-%m-%d') + this_full_id = date + '_' + str(this_id) + sessions['last_id'] = this_id + sessions['last_full_id'] = this_full_id + sessions['all_full_ids'].append(this_full_id) + self.manifest_metadata['sessions'] = sessions + return this_full_id + def close(self): + """ Closing tub closes open files for catalog, catalog manifest and + manifest.json""" + # If records were received, write updated session_id dictionary into + # the metadata, otherwise keep the session_id information unchanged + if self._updated_session: + self.seekeable.update_line(4, json.dumps(self.manifest_metadata)) self.current_catalog.close() self.seekeable.close() @@ -331,11 +389,11 @@ def __len__(self): class ManifestIterator(object): - ''' + """ An iterator for the Manifest type. \n Returns catalog entries lazily when a consumer calls __next__(). - ''' + """ def __init__(self, manifest): self.manifest = manifest self.has_catalogs = len(self.manifest.catalog_paths) > 0 @@ -344,37 +402,40 @@ def __init__(self, manifest): self.current_catalog = None def __next__(self): - if not self.has_catalogs: - raise StopIteration('No catalogs') - - if self.current_catalog_index >= len(self.manifest.catalog_paths): - raise StopIteration('No more catalogs') - - if self.current_catalog is None: - current_catalog_path = os.path.join(self.manifest.base_path, self.manifest.catalog_paths[self.current_catalog_index]) - self.current_catalog = Catalog(current_catalog_path, read_only=self.manifest.read_only) - self.current_catalog.seekable.seek_line_start(1) - - contents = self.current_catalog.seekable.readline() - - if contents is not None and len(contents) > 0: - # Check for current_index when we are ready to advance the underlying iterator. - current_index = self.current_index - self.current_index += 1 - if current_index in self.manifest.deleted_indexes: - # Skip over index, because it has been marked deleted - return self.__next__() + while True: + if not self.has_catalogs: + raise StopIteration('No catalogs') + + if self.current_catalog_index >= len(self.manifest.catalog_paths): + raise StopIteration('No more catalogs') + + if self.current_catalog is None: + current_catalog_path = os.path.join( + self.manifest.base_path, + self.manifest.catalog_paths[self.current_catalog_index]) + self.current_catalog = Catalog(current_catalog_path, + read_only=self.manifest.read_only) + self.current_catalog.seekable.seek_line_start(1) + + contents = self.current_catalog.seekable.readline() + if contents is not None and len(contents) > 0: + # Check for current_index when we are ready to advance the + # underlying iterator. + current_index = self.current_index + self.current_index += 1 + if current_index in self.manifest.deleted_indexes: + # Skip over index, because it has been marked deleted + continue + else: + try: + record = json.loads(contents) + return record + except Exception: + print(f'Ignoring record at index {current_index}') + continue else: - try: - record = json.loads(contents) - return record - except Exception: - print('Ignoring record at index %s' % (current_index)) - return self.__next__() - else: - self.current_catalog = None - self.current_catalog_index += 1 - return self.__next__() + self.current_catalog = None + self.current_catalog_index += 1 next = __next__ diff --git a/donkeycar/parts/dgym.py b/donkeycar/parts/dgym.py index e651f3ea6..389fad9de 100644 --- a/donkeycar/parts/dgym.py +++ b/donkeycar/parts/dgym.py @@ -9,7 +9,7 @@ def is_exe(fpath): class DonkeyGymEnv(object): - def __init__(self, sim_path, host="127.0.0.1", port=9091, headless=0, env_name="donkey-generated-track-v0", sync="asynchronous", conf={}, delay=0): + def __init__(self, sim_path, host="127.0.0.1", port=9091, headless=0, env_name="donkey-generated-track-v0", sync="asynchronous", conf={}, record_location=False, record_gyroaccel=False, record_velocity=False, delay=0): if sim_path != "remote": if not os.path.exists(sim_path): @@ -26,8 +26,13 @@ def __init__(self, sim_path, host="127.0.0.1", port=9091, headless=0, env_name=" self.frame = self.env.reset() self.action = [0.0, 0.0, 0.0] self.running = True - self.info = { 'pos' : (0., 0., 0.)} + self.info = { 'pos' : (0., 0., 0.), 'speed' : 0, 'cte': 0, 'gyro': (0., 0., 0.), 'accel': (0.,0.,0.), 'vel': (0., 0., 0.)} self.delay = float(delay) + self.record_location = record_location + self.record_gyroaccel = record_gyroaccel + self.record_velocity = record_velocity + + def update(self): while self.running: @@ -42,8 +47,17 @@ def run_threaded(self, steering, throttle, brake=None): if self.delay > 0.0: time.sleep(self.delay / 1000.0) self.action = [steering, throttle, brake] - return self.frame - + + # Output Sim-car position information if configured + outputs = [self.frame] + if self.record_location: outputs += self.info['pos'][0], self.info['pos'][1], self.info['pos'][2], self.info['speed'], self.info['cte'] + if self.record_gyroaccel: outputs += self.info['gyro'][0], self.info['gyro'][1], self.info['gyro'][2], self.info['accel'][0], self.info['accel'][1], self.info['accel'][2] + if self.record_velocity: outputs += self.info['vel'][0], self.info['vel'][1], self.info['vel'][2] + if len(outputs)==1: + return self.frame + else: + return outputs + def shutdown(self): self.running = False time.sleep(0.2) diff --git a/donkeycar/parts/encoder.py b/donkeycar/parts/encoder.py index e98032e12..c1c158750 100644 --- a/donkeycar/parts/encoder.py +++ b/donkeycar/parts/encoder.py @@ -1,19 +1,71 @@ """ -Rotary Encoder +Encoders and odometry """ from datetime import datetime -from donkeycar.parts.teensy import TeensyRCin import re import time + +# The Arduino class is for a quadrature wheel or motor encoder that is being read by an offboard microcontroller +# such as an Arduino or Teensy that is feeding serial data to the RaspberryPy or Nano via USB serial. +# The microcontroller should be flashed with this sketch (use the Arduino IDE to do that): https://github.com/zlite/donkeycar/tree/master/donkeycar/parts/encoder/encoder +# Make sure you check the sketch using the "test_encoder.py code in the Donkeycar tests folder to make sure you've got your +# encoder plugged into the right pins, or edit it to reflect the pins you are using. + +# You will need to calibrate the mm_per_tick line below to reflect your own car. Just measure out a meter and roll your car +# along it. Change the number below until it the distance reads out almost exactly 1.0 + +# This samples the odometer at 10HZ and does a moving average over the past ten readings to derive a velocity + +class ArduinoEncoder(object): + def __init__(self, mm_per_tick=0.0000599, debug=False): + import serial + import serial.tools.list_ports + from donkeycar.parts.pigpio_enc import OdomDist + for item in serial.tools.list_ports.comports(): + print(item) # list all the serial ports + self.ser = serial.Serial('/dev/ttyACM0', 115200, 8, 'N', 1, timeout=0.1) + # initialize the odometer values + self.ser.write(str.encode('r')) # restart the encoder to zero + self.ticks = 0 + self.lasttick = 0 + self.debug = debug + self.on = True + self.mm_per_tick = mm_per_tick + + def update(self): + while self.on: + input = '' + while (self.ser.in_waiting > 0): # read the serial port and see if there's any data there + buffer = self.ser.readline() + input = buffer.decode('ascii') + self.ser.write(str.encode('p')) # queue up another reading by sending the "p" character to the Arduino + if input != '': + temp = input.strip() # remove any whitespace + if (temp.isnumeric()): + self.ticks = int(temp) + self.lasttick = self.ticks + else: self.ticks = self.lasttick + self.speed, self.distance = self.OdomDist(self.ticks, self.mm_per_tick) + + def run_threaded(self): + self.speed + return self.speed + + + def shutdown(self): + # indicate that the thread should be stopped + print('stopping Arduino encoder') + self.on = False + time.sleep(.5) + class AStarSpeed: def __init__(self): + from donkeycar.parts.teensy import TeensyRCin self.speed = 0 self.linaccel = None - self.sensor = TeensyRCin(0) - self.on = True def update(self): @@ -68,10 +120,13 @@ def shutdown(self): class RotaryEncoder(): def __init__(self, mm_per_tick=0.306096, pin=13, poll_delay=0.0166, debug=False): - import RPi.GPIO as GPIO - GPIO.setmode(GPIO.BOARD) - GPIO.setup(pin, GPIO.IN) - GPIO.add_event_detect(pin, GPIO.RISING, callback=self.isr) + import pigpio + self.pi = pigpio.pi() + self.pin = pin + self.pi.set_mode(self.pin, pigpio.INPUT) + self.pi.set_pull_up_down(self.pin, pigpio.PUD_DOWN) + self.cb = self.pi.callback(self.pin, pigpio.FALLING_EDGE, self._cb) + # initialize the odometer values self.m_per_tick = mm_per_tick / 1000.0 @@ -84,10 +139,10 @@ def __init__(self, mm_per_tick=0.306096, pin=13, poll_delay=0.0166, debug=False) self.debug = debug self.top_speed = 0 self.prev_dist = 0. - - def isr(self, channel): + + def _cb(self, pin, level, tick): self.counter += 1 - + def update(self): # keep looping infinitely until the thread is stopped while(self.on): @@ -123,10 +178,9 @@ def update(self): time.sleep(self.poll_delay) - def run_threaded(self): - delta = self.meters - self.prev_dist + def run_threaded(self, throttle): self.prev_dist = self.meters - return self.meters, self.meters_per_second, delta + return self.meters_per_second def shutdown(self): # indicate that the thread should be stopped @@ -134,7 +188,7 @@ def shutdown(self): print('Stopping Rotary Encoder') print("\tDistance Travelled: {} meters".format(round(self.meters, 4))) print("\tTop Speed: {} meters/second".format(round(self.top_speed, 4))) - time.sleep(.5) - - import RPi.GPIO as GPIO - GPIO.cleanup() + if self.cb != None: + self.cb.cancel() + self.cb = None + self.pi.stop() \ No newline at end of file diff --git a/donkeycar/parts/keras.py b/donkeycar/parts/keras.py index d18c71acd..2fe0a238e 100644 --- a/donkeycar/parts/keras.py +++ b/donkeycar/parts/keras.py @@ -12,6 +12,7 @@ import numpy as np from typing import Dict, Any, Tuple, Optional, Union import donkeycar as dk + from donkeycar.utils import normalize_image, linear_bin from donkeycar.pipeline.types import TubRecord @@ -27,7 +28,6 @@ from tensorflow.keras.backend import concatenate from tensorflow.keras.models import Model, Sequential from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint -from tensorflow.keras.optimizers import Optimizer ONE_BYTE_SCALE = 1.0 / 255.0 @@ -46,6 +46,7 @@ def __init__(self) -> None: print(f'Created {self}') def load(self, model_path: str) -> None: + print(f'Loading model {model_path}') self.model = keras.models.load_model(model_path, compile=False) def load_weights(self, model_path: str, by_name: bool = True) -> None: @@ -102,6 +103,21 @@ def inference(self, img_arr: np.ndarray, other_arr: np.ndarray) \ """ pass + def evaluate(self, record: TubRecord, + augmentation: 'ImageAugmentation' = None) \ + -> Tuple[Union[float, np.ndarray], ...]: + # extract model input from record + x0 = self.x_transform(record) + x1 = x0[0] if isinstance(x0, tuple) else x0 + # apply augmentation to training data only + x2 = augmentation.augment(x1) if augmentation else x1 + # normalise image, assume other input data comes already normalised + x3 = normalize_image(x2) + if isinstance(x0, tuple): + return self.inference(x3, *x0[1:]) + else: + return self.inference(x3, None) + def train(self, model_path: str, train_data: 'BatchSequence', @@ -112,7 +128,8 @@ def train(self, epochs: int, verbose: int = 1, min_delta: float = .0005, - patience: int = 5) -> tf.keras.callbacks.History: + patience: int = 5, + show_plot: bool = False) -> tf.keras.callbacks.History: """ trains the model """ @@ -128,7 +145,7 @@ def train(self, save_best_only=True, verbose=verbose)] - history: Dict[str, Any] = model.fit( + history: tf.keras.callbacks.History = model.fit( x=train_data, steps_per_epoch=train_steps, batch_size=batch_size, @@ -140,6 +157,41 @@ def train(self, workers=1, use_multiprocessing=False ) + + if show_plot: + try: + import matplotlib.pyplot as plt + from pathlib import Path + + plt.figure(1) + # Only do accuracy if we have that data + # (e.g. categorical outputs) + if 'angle_out_acc' in history.history: + plt.subplot(121) + + # summarize history for loss + plt.plot(history.history['loss']) + plt.plot(history.history['val_loss']) + plt.title('model loss') + plt.ylabel('loss') + plt.xlabel('epoch') + plt.legend(['train', 'validate'], loc='upper right') + + # summarize history for acc + if 'angle_out_acc' in history.history: + plt.subplot(122) + plt.plot(history.history['angle_out_acc']) + plt.plot(history.history['val_angle_out_acc']) + plt.title('model angle accuracy') + plt.ylabel('acc') + plt.xlabel('epoch') + + plt.savefig(Path(model_path).with_suffix('.png')) + # plt.show() + + except Exception as ex: + print(f"problems with loss graph: {ex}") + return history def _get_train_model(self) -> Model: @@ -161,18 +213,14 @@ def y_translate(self, y: XY) -> Dict[str, Union[float, np.ndarray]]: raise NotImplementedError(f'{self} not ready yet for new training ' f'pipeline') - def output_types(self) -> Dict[str, np.typename]: - raise NotImplementedError(f'{self} not ready yet for new training ' - f'pipeline') - - def output_types(self): + def output_types(self) -> Tuple[Dict[str, np.typename], ...]: """ Used in tf.data, assume all types are doubles""" shapes = self.output_shapes() types = tuple({k: tf.float64 for k in d} for d in shapes) return types - def output_shapes(self) -> Optional[Dict[str, tf.TensorShape]]: - return None + def output_shapes(self) -> Dict[str, tf.TensorShape]: + return {} def __str__(self) -> str: """ For printing model initialisation """ diff --git a/donkeycar/parts/lidar.py b/donkeycar/parts/lidar.py index e3a6adec6..f9b7e2dba 100644 --- a/donkeycar/parts/lidar.py +++ b/donkeycar/parts/lidar.py @@ -2,6 +2,9 @@ Lidar """ +# requies glob to be installed: "pip3 install glob2" +# requires rplidar to be installed: "pip3 install rplidar" + import time import math import pickle @@ -14,17 +17,35 @@ class RPLidar(object): ''' https://github.com/SkoltechRobotics/rplidar ''' - def __init__(self, port='/dev/ttyUSB0'): + def __init__(self, lower_limit = 0, upper_limit = 360, debug=False): from rplidar import RPLidar - self.port = port - self.distances = [] #a list of distance measurements - self.angles = [] # a list of angles corresponding to dist meas above - self.lidar = RPLidar(self.port) - self.lidar.clear_input() - time.sleep(1) - self.on = True - #print(self.lidar.get_info()) - #print(self.lidar.get_health()) + import glob + port_found = False + self.lower_limit = lower_limit + self.upper_limit = upper_limit + temp_list = glob.glob ('/dev/ttyUSB*') + result = [] + for a_port in temp_list: + try: + s = serial.Serial(a_port) + s.close() + result.append(a_port) + port_found = True + except serial.SerialException: + pass + if port_found: + self.port = result[0] + self.distances = [] #a list of distance measurements + self.angles = [] # a list of angles corresponding to dist meas above + self.lidar = RPLidar(self.port, baudrate=115200) + self.lidar.clear_input() + time.sleep(1) + self.on = True + #print(self.lidar.get_info()) + #print(self.lidar.get_health()) + else: + print("No Lidar found") + def update(self): @@ -38,7 +59,19 @@ def update(self): print('serial.serialutil.SerialException from Lidar. common when shutting down.') def run_threaded(self): - return self.distances, self.angles + sorted_distances = [] + if (self.angles != []) and (self.distances != []): + angs = np.copy(self.angles) + dists = np.copy(self.distances) + + filter_angs = angs[(angs > self.lower_limit) & (angs < self.upper_limit)] + filter_dist = dists[(angs > self.lower_limit) & (angs < self.upper_limit)] #sorts distances based on angle values + + angles_ind = np.argsort(filter_angs) # returns the indexes that sorts filter_angs + if angles_ind != []: + sorted_distances = np.argsort(filter_dist) # sorts distances based on angle indexes + return sorted_distances + def shutdown(self): self.on = False @@ -47,6 +80,62 @@ def shutdown(self): self.lidar.stop_motor() self.lidar.disconnect() +class YDLidar(object): + ''' + https://pypi.org/project/PyLidar3/ + ''' + def __init__(self, port='/dev/ttyUSB0'): + import PyLidar3 + self.port = port + self.distances = [] #a list of distance measurements + self.angles = [] # a list of angles corresponding to dist meas above + self.lidar = PyLidar3.YdLidarX4(port) + if(self.lidar.Connect()): + print(self.lidar.GetDeviceInfo()) + self.gen = self.lidar.StartScanning() + else: + print("Error connecting to lidar") + self.on = True + + + def init(self, port='/dev/ttyUSB0'): + import PyLidar3 + print("Starting lidar...") + self.port = port + self.distances = [] #a list of distance measurements + self.angles = [] # a list of angles corresponding to dist meas above + self.lidar = PyLidar3.YdLidarX4(port) + if(self.lidar.Connect()): + print(self.lidar.GetDeviceInfo()) + gen = self.lidar.StartScanning() + return gen + else: + print("Error connecting to lidar") + self.on = True + #print(self.lidar.get_info()) + #print(self.lidar.get_health()) + + def update(self, lidar, debug = False): + while self.on: + try: + self.data = next(lidar) + for angle in range(0,360): + if(self.data[angle]>1000): + self.angles = [angle] + self.distances = [self.data[angle]] + if debug: + return self.distances, self.angles + except serial.serialutil.SerialException: + print('serial.serialutil.SerialException from Lidar. common when shutting down.') + + def run_threaded(self): + return self.distances, self.angles + + def shutdown(self): + self.on = False + time.sleep(2) + self.lidar.StopScanning() + self.lidar.Disconnect() class LidarPlot(object): ''' @@ -178,4 +267,3 @@ def run(self, map_bytes): def shutdown(self): pass - diff --git a/donkeycar/parts/oled.py b/donkeycar/parts/oled.py index 3cbf4fe43..8bb80f3cf 100644 --- a/donkeycar/parts/oled.py +++ b/donkeycar/parts/oled.py @@ -110,7 +110,6 @@ def run_threaded(self, recording, num_records, user_mode): self.recording = 'NO (Records = %s)' % (self.num_records) self.user_mode = 'User Mode (%s)' % (user_mode) - self.update() def update_slots(self): updates = [self.eth0, self.wlan0, self.recording, self.user_mode] @@ -125,7 +124,10 @@ def update_slots(self): self.oled.update() def update(self): - self.update_slots() + self.on = True + # Run threaded loop by itself + while self.on: + self.update_slots() def shutdown(self): self.oled.clear_display() diff --git a/donkeycar/parts/perfmon.py b/donkeycar/parts/perfmon.py new file mode 100644 index 000000000..35b913015 --- /dev/null +++ b/donkeycar/parts/perfmon.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Performance monitor for analyzing real-time CPU/mem/execution frequency + +author: @miro (Meir Tseitlin) 2020 + +Note: +""" +import time +import psutil + + +class PerfMonitor: + + def __init__(self, cfg): + + self.STATS_BUFFER_SIZE = 10 + self._calc_buffer = [cfg.DRIVE_LOOP_HZ for i in range(self.STATS_BUFFER_SIZE)] + self._runs_counter = 0 + self._last_calc_time = time.time() + self._on = True + self._update_metrics() + print("Performance monitor activated.") + + def _update_metrics(self): + self._mem_percent = psutil.virtual_memory().percent + self._cpu_percent = psutil.cpu_percent() + + def update(self): + while self._on: + self._update_metrics() + time.sleep(2) + + def shutdown(self): + # indicate that the thread should be stopped + self._on = False + print('Stopping Perf Monitor') + time.sleep(.2) + + def run_threaded(self): + + # Calc real frequency + curr_time = time.time() + if curr_time - self._last_calc_time > 1: + self._calc_buffer[int(curr_time) % self.STATS_BUFFER_SIZE] = self._runs_counter + self._runs_counter = 0 + self._last_calc_time = curr_time + + self._runs_counter += 1 + + vehicle_frequency = float(sum(self._calc_buffer)) / self.STATS_BUFFER_SIZE + + return self._cpu_percent, self._mem_percent, vehicle_frequency diff --git a/donkeycar/parts/pytorch/ResNet18.py b/donkeycar/parts/pytorch/ResNet18.py index ac32b86ee..4052b34d2 100644 --- a/donkeycar/parts/pytorch/ResNet18.py +++ b/donkeycar/parts/pytorch/ResNet18.py @@ -33,10 +33,8 @@ def __init__(self, input_shape=(128, 3, 224, 224), output_size=(2,)): self.example_input_array = torch.rand(input_shape) # Metrics - self.train_acc = pl.metrics.Accuracy() - self.valid_acc = pl.metrics.Accuracy() - self.train_precision = pl.metrics.Precision() - self.valid_precision = pl.metrics.Precision() + self.train_mse = pl.metrics.MeanSquaredError() + self.valid_mse = pl.metrics.MeanSquaredError() self.model = load_resnet18(num_classes=output_size[0]) @@ -55,15 +53,11 @@ def training_step(self, batch, batch_idx): loss = F.l1_loss(logits, y) self.loss_history.append(loss) - self.log('train_loss', loss) + self.log("train_loss", loss) # Log Metrics - self.train_acc(logits, y) - self.log('train_acc', self.train_acc, on_step=False, on_epoch=True) - - self.train_precision(logits, y) - self.log('train_precision', self.train_precision, - on_step=False, on_epoch=True) + self.train_mse(logits, y) + self.log("train_mse", self.train_mse, on_step=False, on_epoch=True) return loss @@ -72,19 +66,14 @@ def validation_step(self, batch, batch_idx): logits = self.forward(x) loss = F.l1_loss(logits, y) - # Log Metrics - self.log('val_loss', loss) - - self.valid_acc(logits, y) - self.log('valid_acc', self.valid_acc, on_step=False, on_epoch=True) + self.log("val_loss", loss) - self.valid_precision(logits, y) - self.log('valid_precision', self.valid_precision, - on_step=False, on_epoch=True) + # Log Metrics + self.valid_mse(logits, y) + self.log("valid_mse", self.valid_mse, on_step=False, on_epoch=True) def configure_optimizers(self): - optimizer = optim.Adam( - self.model.parameters(), lr=0.0001, weight_decay=0.0005) + optimizer = optim.Adam(self.model.parameters(), lr=0.0001, weight_decay=0.0005) return optimizer def run(self, img_arr: np.ndarray, other_arr: np.ndarray = None): @@ -98,6 +87,7 @@ def run(self, img_arr: np.ndarray, other_arr: np.ndarray = None): :return: tuple of (angle, throttle) """ from PIL import Image + pil_image = Image.fromarray(img_arr) tensor_image = self.inference_transform(pil_image) tensor_image = tensor_image.unsqueeze(0) @@ -107,5 +97,8 @@ def run(self, img_arr: np.ndarray, other_arr: np.ndarray = None): # Resize to (2,) result = result.reshape(-1) + + # Convert from being normalized between [0, 1] to being between [-1, 1] + result = result * 2 - 1 print("ResNet18 result: {}".format(result)) return result diff --git a/donkeycar/parts/pytorch/torch_data.py b/donkeycar/parts/pytorch/torch_data.py index 1c60606b4..09292de82 100644 --- a/donkeycar/parts/pytorch/torch_data.py +++ b/donkeycar/parts/pytorch/torch_data.py @@ -72,7 +72,12 @@ def _create_pipeline(self): def y_transform(record: TubRecord): angle: float = record.underlying['user/angle'] throttle: float = record.underlying['user/throttle'] - return torch.tensor([angle, throttle]) + predictions = torch.tensor([angle, throttle], dtype=torch.float) + + # Normalize to be between [0, 1] + # angle and throttle are originally between [-1, 1] + predictions = (predictions + 1) / 2 + return predictions def x_transform(record: TubRecord): # Loads the result of Image.open() diff --git a/donkeycar/parts/telemetry.py b/donkeycar/parts/telemetry.py index a7a392fea..f6a32aaca 100644 --- a/donkeycar/parts/telemetry.py +++ b/donkeycar/parts/telemetry.py @@ -10,9 +10,15 @@ import queue import time import json +import logging +import numpy as np from logging import StreamHandler from paho.mqtt.client import Client as MQTTClient +logger = logging.getLogger() + +LOG_MQTT_KEY = 'log/default' + class MqttTelemetry(StreamHandler): """ @@ -20,15 +26,15 @@ class MqttTelemetry(StreamHandler): Telemetry reports are timestamped and stored in memory until it is pushed to the server """ - def __init__(self, cfg, default_inputs=None, default_types=None): + def __init__(self, cfg): StreamHandler.__init__(self) self.PUBLISH_PERIOD = cfg.TELEMETRY_PUBLISH_PERIOD self._last_publish = time.time() self._telem_q = queue.Queue() - self._default_inputs = default_inputs or [] - self._default_types = default_types or [] + self._step_inputs = cfg.TELEMETRY_DEFAULT_INPUTS.split(',') + self._step_types = cfg.TELEMETRY_DEFAULT_TYPES.split(',') self._total_updates = 0 self._donkey_name = os.environ.get('DONKEY_NAME', cfg.TELEMETRY_DONKEY_NAME) self._mqtt_broker = os.environ.get('DONKEY_MQTT_BROKER', cfg.TELEMETRY_MQTT_BROKER_HOST) # 'iot.eclipse.org' @@ -38,7 +44,20 @@ def __init__(self, cfg, default_inputs=None, default_types=None): self._mqtt_client.connect(self._mqtt_broker, cfg.TELEMETRY_MQTT_BROKER_PORT) self._mqtt_client.loop_start() self._on = True - print(f"Telemetry MQTT server connected (publishing: {', '.join(self._default_inputs)}") + if cfg.TELEMETRY_LOGGING_ENABLE: + self.setLevel(logging.getLevelName(cfg.TELEMETRY_LOGGING_LEVEL)) + self.setFormatter(logging.Formatter(cfg.TELEMETRY_LOGGING_FORMAT)) + logger.addHandler(self) + + def add_step_inputs(self, inputs, types): + + # Add inputs if supported and not yet registered + for ind in range(0, len(inputs or [])): + if types[ind] in ['float', 'str', 'int'] and inputs[ind] not in self._step_inputs: + self._step_inputs.append(inputs[ind]) + self._step_types.append(types[ind]) + + return self._step_inputs, self._step_types @staticmethod def filter_supported_metrics(inputs, types): @@ -66,10 +85,10 @@ def report(self, metrics): def emit(self, record): """ - FUTURE: Added to support Logging interface (to allow to use Python logging module to log directly to telemetry) + Logging interface (to allow to use Python logging module to log directly to telemetry) """ - # msg = self.format(record) - self.report(record) + msg = {LOG_MQTT_KEY: self.format(record)} + self.report(msg) @property def qsize(self): @@ -85,19 +104,41 @@ def publish(self): if not packet: return - + if self._use_json_format: packet = [{'ts': k, 'values': v} for k, v in packet.items()] payload = json.dumps(packet) - self._mqtt_client.publish(self._topic, payload) - # print(f'Total updates - {self._total_updates} (payload size={len(payload)})') + try: + self._mqtt_client.publish(self._topic, payload) + except Exception as e: + logger.error(f'Error publishing log {self._topic}: {e}') else: - # Publish only the last timestamp + # Publish only the last timestamp for per step metrics last_sample = packet[list(packet)[-1]] for k, v in last_sample.items(): - self._mqtt_client.publish('{}/{}'.format(self._topic, k), v) - # print(f'Total updates - {self._total_updates} (values ={len(last_sample)})') + if k in self._step_inputs: + topic = f'{self._topic}/{k}' + + try: + # Convert unsupported numpy types to python standard + if isinstance(v, np.generic): + v = np.asscalar(v) + + self._mqtt_client.publish(topic, v) + except TypeError: + logger.error(f'Cannot publish topic "{topic}" with value of type {type(v)}') + except Exception as e: + logger.error(f'Error publishing log {topic}: {e}') + + # Publish all logs + for tm, sample in packet.items(): + if LOG_MQTT_KEY in sample: + topic = f'{self._topic}/{LOG_MQTT_KEY}' + try: + self._mqtt_client.publish(topic, sample[LOG_MQTT_KEY]) + except Exception as e: + logger.error(f'Error publishing log {topic}: {e}') self._total_updates += 1 return @@ -107,10 +148,10 @@ def run(self, *args): API function needed to use as a Donkey part. Accepts values, pairs them with their inputs keys and saves them to disk. """ - assert len(self._default_inputs) == len(args) - + assert len(self._step_inputs) == len(args) + # Add to queue - record = dict(zip(self._default_inputs, args)) + record = dict(zip(self._step_inputs, args)) self.report(record) # Periodic publish @@ -123,14 +164,15 @@ def run(self, *args): def run_threaded(self, *args): - assert len(self._default_inputs) == len(args) + assert len(self._step_inputs) == len(args) # Add to queue - record = dict(zip(self._default_inputs, args)) + record = dict(zip(self._step_inputs, args)) self.report(record) return self.qsize def update(self): + logger.info(f"Telemetry MQTT server connected (publishing: { ', '.join(self._step_inputs) })") while self._on: self.publish() time.sleep(self.PUBLISH_PERIOD) @@ -138,5 +180,5 @@ def update(self): def shutdown(self): # indicate that the thread should be stopped self._on = False - print('Stopping MQTT Telemetry') - time.sleep(.2) \ No newline at end of file + logger.debug('Stopping MQTT Telemetry') + time.sleep(.2) diff --git a/donkeycar/parts/tflite.py b/donkeycar/parts/tflite.py index 052fe29f7..186a9cbde 100755 --- a/donkeycar/parts/tflite.py +++ b/donkeycar/parts/tflite.py @@ -1,7 +1,9 @@ import os import tensorflow as tf +import numpy as np +from typing import Dict, Union -from donkeycar.parts.keras import KerasPilot +from donkeycar.parts.keras import KerasPilot, KerasLinear, XY def keras_model_to_tflite(in_filename, out_filename, data_gen=None): @@ -43,6 +45,7 @@ def __init__(self): def load(self, model_path): assert os.path.splitext(model_path)[1] == '.tflite', \ 'TFlitePilot should load only .tflite files' + print(f'Loading model {model_path}') # Load TFLite model and allocate tensors. self.interpreter = tf.lite.Interpreter(model_path=model_path) self.interpreter.allocate_tensors() @@ -55,7 +58,7 @@ def load(self, model_path): self.input_shape = self.input_details[0]['shape'] def inference(self, img_arr, other_arr): - input_data = img_arr.reshape(self.input_shape) + input_data = np.float32(img_arr.reshape(self.input_shape)) self.interpreter.set_tensor(self.input_details[0]['index'], input_data) self.interpreter.invoke() @@ -66,9 +69,9 @@ def inference(self, img_arr, other_arr): output_data = self.interpreter.get_tensor(tensor['index']) outputs.append(output_data[0][0]) - steering = outputs[0] + steering = float(outputs[0]) if len(outputs) > 1: - throttle = outputs[1] + throttle = float(outputs[1]) return steering, throttle @@ -76,3 +79,6 @@ def get_input_shape(self): assert self.input_shape is not None, "Need to load model first" return self.input_shape + def y_translate(self, y: XY) -> Dict[str, Union[float, np.ndarray]]: + """ For now only support keras linear""" + return KerasLinear.y_translate(self, y) diff --git a/donkeycar/parts/tub_v2.py b/donkeycar/parts/tub_v2.py index e34f91d16..17cf8ed02 100644 --- a/donkeycar/parts/tub_v2.py +++ b/donkeycar/parts/tub_v2.py @@ -1,6 +1,8 @@ import atexit import os import time +from datetime import datetime +import json import numpy as np from PIL import Image @@ -50,6 +52,8 @@ def write_record(self, record=None): contents[key] = int(value) elif input_type == 'boolean': contents[key] = bool(value) + elif input_type == 'nparray': + contents[key] = value.tolist() elif input_type == 'list' or input_type == 'vector': contents[key] = list(value) elif input_type == 'image_array': @@ -63,20 +67,20 @@ def write_record(self, record=None): # Private properties contents['_timestamp_ms'] = int(round(time.time() * 1000)) contents['_index'] = self.manifest.current_index + contents['_session_id'] = self.manifest.session_id self.manifest.write_record(contents) - def delete_record(self, record_index): - self.manifest.delete_record(record_index) + def delete_records(self, record_indexes): + self.manifest.delete_records(record_indexes) def delete_last_n_records(self, n): last_index = self.manifest.current_index - first_index = last_index - n - for index in range(first_index, last_index): - if index < 0: - continue - else: - self.manifest.delete_record(index) + first_index = max(last_index - n, 0) + self.manifest.delete_records(range(first_index, last_index)) + + def restore_records(self, record_indexes): + self.manifest.restore_records(record_indexes) def close(self): self.manifest.close() @@ -106,14 +110,10 @@ class TubWriter(object): def __init__(self, base_path, inputs=[], types=[], metadata=[], max_catalog_len=1000): self.tub = Tub(base_path, inputs, types, metadata, max_catalog_len) - def shutdown_hook(): - self.close() - - # Register hook - atexit.register(shutdown_hook) def run(self, *args): - assert len(self.tub.inputs) == len(args) + assert len(self.tub.inputs) == len(args), \ + f'Expected {len(self.tub.inputs)} inputs but received {len(args)}' record = dict(zip(self.tub.inputs, args)) self.tub.write_record(record) return self.tub.manifest.current_index @@ -122,4 +122,44 @@ def __iter__(self): return self.tub.__iter__() def close(self): - self.tub.manifest.close() + self.tub.close() + + def shutdown(self): + self.close() + + + + +class TubWiper: + """ + Donkey part which deletes a bunch of records from the end of tub. + This allows to remove bad data already during recording. As this gets called + in the vehicle loop the deletion runs only once in each continuous + activation. A new execution requires to release of the input trigger. The + action could result in a multiple number of executions otherwise. + """ + def __init__(self, tub, num_records=20): + """ + :param tub: tub to operate on + :param num_records: number or records to delete + """ + self._tub = tub + self._num_records = num_records + self._active_loop = False + + def run(self, is_delete): + """ + Method in the vehicle loop. Delete records when trigger switches from + False to True only. + :param is_delete: if deletion has been triggered by the caller + """ + # only run if input is true and debounced + if is_delete: + if not self._active_loop: + # action command + self._tub.delete_last_n_records(self._num_records) + # increase the loop counter + self._active_loop = True + else: + # trigger released, reset active loop + self._active_loop = False \ No newline at end of file diff --git a/donkeycar/parts/web_controller/templates/vehicle.html b/donkeycar/parts/web_controller/templates/vehicle.html index 8848cdb10..e63258710 100755 --- a/donkeycar/parts/web_controller/templates/vehicle.html +++ b/donkeycar/parts/web_controller/templates/vehicle.html @@ -52,6 +52,7 @@
+
@@ -115,6 +116,9 @@
+
+

Caution: If a Physical Joystick is Used, It overides the Web Controller.

+
diff --git a/donkeycar/pipeline/augmentations.py b/donkeycar/pipeline/augmentations.py index f0b934037..289b4bf12 100644 --- a/donkeycar/pipeline/augmentations.py +++ b/donkeycar/pipeline/augmentations.py @@ -1,9 +1,13 @@ import cv2 import numpy as np +import logging import imgaug.augmenters as iaa from donkeycar.config import Config +logger = logging.getLogger() + + class Augmentations(object): """ Some ready to use image augumentations. @@ -89,10 +93,12 @@ def create(cls, aug_type: str, config: Config) -> iaa.meta.Augmenter: elif aug_type == 'MULTIPLY': interval = getattr(config, 'AUG_MULTIPLY_RANGE', (0.5, 1.5)) + logger.info(f'Creating augmentation {aug_type} {interval}') return iaa.Multiply(interval) elif aug_type == 'BLUR': interval = getattr(config, 'AUG_BLUR_RANGE', (0.0, 3.0)) + logger.info(f'Creating augmentation {aug_type} {interval}') return iaa.GaussianBlur(sigma=interval) def augment(self, img_arr): diff --git a/donkeycar/pipeline/database.py b/donkeycar/pipeline/database.py new file mode 100644 index 000000000..d6502dbee --- /dev/null +++ b/donkeycar/pipeline/database.py @@ -0,0 +1,83 @@ +import json +import os +import time +from typing import Dict, List, Tuple +import pandas as pd + +from donkeycar.config import Config + + +FILE = 'database.json' + + +class PilotDatabase: + def __init__(self, cfg: Config) -> None: + self.cfg = cfg + self.path = os.path.join(cfg.MODELS_PATH, FILE) + self.entries = self.read() + + def read(self) -> List[Dict]: + if os.path.exists(self.path): + with open(self.path, "r") as read_file: + data = json.load(read_file) + return data + else: + return [] + + def generate_model_name(self) -> Tuple[str, int]: + if self.entries: + df = self.to_df() + # otherwise this will be a numpy int + last_num = int(df.index.max()) + this_num = last_num + 1 + else: + this_num = 0 + date = time.strftime('%y-%m-%d') + name = 'pilot_' + date + '_' + str(this_num) + return name, this_num + + def to_df(self) -> pd.DataFrame: + if self.entries: + df = pd.DataFrame.from_records(self.entries) + df.set_index('Number', inplace=True) + return df + else: + return pd.DataFrame() + + def write(self): + try: + with open(self.path, "w") as data_file: + json.dump(self.entries, data_file) + except Exception as e: + print(e) + + def add_entry(self, entry: Dict): + self.entries.append(entry) + + def to_df_tubgrouped(self): + def sorted_string(comma_separated_string): + """ Return sorted list of comma separated string list""" + return ','.join(sorted(comma_separated_string.split(','))) + + df_pilots = self.to_df() + if df_pilots.empty: + return pd.DataFrame(), pd.DataFrame() + tubs = df_pilots.Tubs + multi_tubs = [tub for tub in tubs if ',' in tub] + # We might still have 'duplicates in here as 'tub_1,tub2' and 'tub_2, + # tub_1' would be two different entries. Hence we need to compress these + multi_tub_set = set([sorted_string(tub) for tub in multi_tubs]) + # Because set is only using unique entries we can now map each list to a + # group and give it a name + d = dict(zip(multi_tub_set, + ['tub_group_' + str(i) for i in range(len(multi_tubs))])) + new_tubs = [d[sorted_string(tub)] if tub in multi_tubs + else tub for tub in df_pilots['Tubs']] + df_pilots['Tubs'] = new_tubs + df_pilots.sort_index(inplace=True) + # pandas explode normalises multiplicity of arrays as entries in data + # frame + df_tubs = pd.DataFrame( + zip(d.values(), [k.split(',') for k in d.keys()]), + columns=['TubGroup', 'Tubs']).explode('Tubs') + return df_pilots, df_tubs diff --git a/donkeycar/pipeline/training.py b/donkeycar/pipeline/training.py index 8b3a27d87..56bd640fe 100644 --- a/donkeycar/pipeline/training.py +++ b/donkeycar/pipeline/training.py @@ -1,10 +1,12 @@ import math import os -from typing import List, Dict, Union +from time import time +from typing import List, Dict, Union, Tuple from donkeycar.config import Config from donkeycar.parts.keras import KerasPilot from donkeycar.parts.tflite import keras_model_to_tflite +from donkeycar.pipeline.database import PilotDatabase from donkeycar.pipeline.sequence import TubRecord, TubSequence, TfmIterator from donkeycar.pipeline.types import TubDataset from donkeycar.pipeline.augmentations import ImageAugmentation @@ -76,38 +78,57 @@ def create_tf_data(self) -> tf.data.Dataset: return dataset.repeat().batch(self.batch_size) -def train(cfg: Config, tub_paths: str, model: str, model_type: str) \ +def get_model_train_details(cfg: Config, database: PilotDatabase, + model: str = None, model_type: str = None) \ + -> Tuple[str, int, str, bool]: + if not model_type: + model_type = cfg.DEFAULT_MODEL_TYPE + train_type = model_type + is_tflite = False + if 'tflite_' in train_type: + train_type = train_type.replace('tflite_', '') + is_tflite = True + model_num = 0 + if not model: + model_name, model_num = database.generate_model_name() + else: + model_name, model_ext = os.path.splitext(model) + is_tflite = model_ext == '.tflite' + + return model_name, model_num, train_type, is_tflite + + +def train(cfg: Config, tub_paths: str, model: str = None, + model_type: str = None, transfer: str = None, comment: str = None) \ -> tf.keras.callbacks.History: """ Train the model """ - model_name, model_ext = os.path.splitext(model) - is_tflite = model_ext == '.tflite' - if is_tflite: - model = f'{model_name}.h5' - - if not model_type: - model_type = cfg.DEFAULT_MODEL_TYPE - - tubs = tub_paths.split(',') - all_tub_paths = [os.path.expanduser(tub) for tub in tubs] - output_path = os.path.expanduser(model) - train_type = 'linear' if 'linear' in model_type else model_type + database = PilotDatabase(cfg) + model_name, model_num, train_type, is_tflite = \ + get_model_train_details(cfg, database, model, model_type) + output_path = os.path.join(cfg.MODELS_PATH, model_name + '.h5') kl = get_model_by_type(train_type, cfg) + if transfer: + kl.load(transfer) if cfg.PRINT_MODEL_SUMMARY: print(kl.model.summary()) + tubs = tub_paths.split(',') + all_tub_paths = [os.path.expanduser(tub) for tub in tubs] dataset = TubDataset(cfg, all_tub_paths) training_records, validation_records = dataset.train_test_split() - print('Records # Training %s' % len(training_records)) - print('Records # Validation %s' % len(validation_records)) + print(f'Records # Training {len(training_records)}') + print(f'Records # Validation {len(validation_records)}') training_pipe = BatchSequence(kl, cfg, training_records, is_train=True) validation_pipe = BatchSequence(kl, cfg, validation_records, is_train=False) - dataset_train = training_pipe.create_tf_data() - dataset_validate = validation_pipe.create_tf_data() + dataset_train = training_pipe.create_tf_data().prefetch( + tf.data.experimental.AUTOTUNE) + dataset_validate = validation_pipe.create_tf_data().prefetch( + tf.data.experimental.AUTOTUNE) train_size = len(training_pipe) val_size = len(validation_pipe) @@ -123,10 +144,25 @@ def train(cfg: Config, tub_paths: str, model: str, model_type: str) \ epochs=cfg.MAX_EPOCHS, verbose=cfg.VERBOSE_TRAIN, min_delta=cfg.MIN_DELTA, - patience=cfg.EARLY_STOP_PATIENCE) + patience=cfg.EARLY_STOP_PATIENCE, + show_plot=cfg.SHOW_PLOT) if is_tflite: tf_lite_model_path = f'{os.path.splitext(output_path)[0]}.tflite' keras_model_to_tflite(output_path, tf_lite_model_path) + database_entry = { + 'Number': model_num, + 'Name': model_name, + 'Type': str(kl), + 'Tubs': tub_paths, + 'Time': time(), + 'History': history.history, + 'Transfer': os.path.basename(transfer) if transfer else None, + 'Comment': comment, + 'Config': str(cfg) + } + database.add_entry(database_entry) + database.write() + return history diff --git a/donkeycar/pipeline/types.py b/donkeycar/pipeline/types.py index a9dca3cea..7971fb8da 100644 --- a/donkeycar/pipeline/types.py +++ b/donkeycar/pipeline/types.py @@ -4,7 +4,7 @@ import numpy as np from donkeycar.config import Config from donkeycar.parts.tub_v2 import Tub -from donkeycar.utils import load_image, load_pil_image, normalize_image, train_test_split +from donkeycar.utils import load_image, load_pil_image, train_test_split from typing_extensions import TypedDict X = TypeVar('X', covariant=True) @@ -60,14 +60,15 @@ def image(self, cached=True, as_nparray=True) -> np.ndarray: else: _image = self._image return _image + def __repr__(self) -> str: return repr(self.underlying) class TubDataset(object): - ''' + """ Loads the dataset, and creates a train/test split. - ''' + """ def __init__(self, config: Config, tub_paths: List[str], shuffle: bool = True) -> None: @@ -77,15 +78,20 @@ def __init__(self, config: Config, tub_paths: List[str], self.tubs: List[Tub] = [Tub(tub_path, read_only=True) for tub_path in self.tub_paths] self.records: List[TubRecord] = list() + self.train_filter = getattr(config, 'TRAIN_FILTER', None) def train_test_split(self) -> Tuple[List[TubRecord], List[TubRecord]]: - print(f'Loading tubs from paths {self.tub_paths}') + msg = f'Loading tubs from paths {self.tub_paths}' + f' with filter ' \ + f'{self.train_filter}' if self.train_filter else '' + print(msg) self.records.clear() for tub in self.tubs: for underlying in tub: - record = TubRecord(self.config, tub.base_path, - underlying=underlying) - self.records.append(record) + record = TubRecord(self.config, tub.base_path, underlying) + if not self.train_filter or self.train_filter(record): + self.records.append(record) return train_test_split(self.records, shuffle=self.shuffle, test_size=(1. - self.config.TRAIN_TEST_SPLIT)) + + diff --git a/donkeycar/templates/basic.py b/donkeycar/templates/basic.py index 0ae575331..8d5daf3e0 100755 --- a/donkeycar/templates/basic.py +++ b/donkeycar/templates/basic.py @@ -4,19 +4,25 @@ Usage: manage.py drive [--model=] [--type=(linear|categorical|tflite_linear)] + manage.py calibrate Options: -h --help Show this screen. """ from docopt import docopt +import logging +import os import donkeycar as dk -from donkeycar.parts.tub_v2 import TubWriter +from donkeycar.parts.tub_v2 import TubWriter, TubWiper from donkeycar.parts.datastore import TubHandler -from donkeycar.parts.controller import LocalWebController +from donkeycar.parts.controller import LocalWebController, RCReceiver from donkeycar.parts.actuator import PCA9685, PWMSteering, PWMThrottle +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + class DriveMode: """ Helper class to dispatch between ai and user driving""" @@ -36,7 +42,7 @@ def run(self, mode, user_angle, user_throttle, pilot_angle, pilot_throttle): class PilotCondition: - """ Helper class to determine how is in charge of driving""" + """ Helper class to determine who is in charge of driving""" def run(self, mode): return mode != 'user' @@ -46,20 +52,22 @@ def drive(cfg, model_path=None, model_type=None): Construct a minimal robotic vehicle from many parts. Here, we use a single camera, web or joystick controller, autopilot and tubwriter. - Each part runs as a job in the Vehicle loop, calling either it's run or + Each part runs as a job in the Vehicle loop, calling either its run or run_threaded method depending on the constructor flag `threaded`. All parts are updated one after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. Parts may have named outputs and inputs. The framework handles passing named outputs to parts requesting the same named input. """ - + logger.info(f'PID: {os.getpid()}') car = dk.vehicle.Vehicle() # add camera inputs = [] if cfg.DONKEY_GYM: from donkeycar.parts.dgym import DonkeyGymEnv - cam = DonkeyGymEnv(cfg.DONKEY_SIM_PATH, host=cfg.SIM_HOST, env_name=cfg.DONKEY_GYM_ENV_NAME, conf=cfg.GYM_CONF, delay=cfg.SIM_ARTIFICIAL_LATENCY) + cam = DonkeyGymEnv(cfg.DONKEY_SIM_PATH, host=cfg.SIM_HOST, + env_name=cfg.DONKEY_GYM_ENV_NAME, conf=cfg.GYM_CONF, + delay=cfg.SIM_ARTIFICIAL_LATENCY) inputs = ['angle', 'throttle', 'brake'] elif cfg.CAMERA_TYPE == "PICAM": from donkeycar.parts.camera import PiCamera @@ -96,22 +104,38 @@ def drive(cfg, model_path=None, model_type=None): car.add(cam, inputs=inputs, outputs=['cam/image_array'], threaded=True) # add controller - if cfg.USE_JOYSTICK_AS_DEFAULT: - from donkeycar.parts.controller import get_js_controller - ctr = get_js_controller(cfg) - if cfg.USE_NETWORKED_JS: - from donkeycar.parts.controller import JoyStickSub - netwkJs = JoyStickSub(cfg.NETWORK_JS_SERVER_IP) - car.add(netwkJs, threaded=True) - ctr.js = netwkJs - else: + if cfg.USE_RC: + rc_steering = RCReceiver(cfg.STEERING_RC_GPIO, invert=True) + rc_throttle = RCReceiver(cfg.THROTTLE_RC_GPIO) + rc_wiper = RCReceiver(cfg.DATA_WIPER_RC_GPIO, jitter=0.05, no_action=0) + car.add(rc_steering, outputs=['user/angle', 'user/angle_on']) + car.add(rc_throttle, outputs=['user/throttle', 'user/throttle_on']) + car.add(rc_wiper, outputs=['user/wiper', 'user/wiper_on']) ctr = LocalWebController(port=cfg.WEB_CONTROL_PORT, mode=cfg.WEB_INIT_MODE) + # web controller sets user mode, its angle, throttle are not used. + car.add(ctr, inputs=['cam/image_array'], + outputs=['webcontroller/angle', 'webcontroller/throttle', + 'user/mode', 'recording'], + threaded=True) - car.add(ctr, - inputs=['cam/image_array'], - outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], - threaded=True) + else: + if cfg.USE_JOYSTICK_AS_DEFAULT: + from donkeycar.parts.controller import get_js_controller + ctr = get_js_controller(cfg) + if cfg.USE_NETWORKED_JS: + from donkeycar.parts.controller import JoyStickSub + netwkJs = JoyStickSub(cfg.NETWORK_JS_SERVER_IP) + car.add(netwkJs, threaded=True) + ctr.js = netwkJs + else: + ctr = LocalWebController(port=cfg.WEB_CONTROL_PORT, + mode=cfg.WEB_INIT_MODE) + car.add(ctr, + inputs=['cam/image_array'], + outputs=['user/angle', 'user/throttle', 'user/mode', + 'recording'], + threaded=True) # pilot condition to determine if user or ai are driving car.add(PilotCondition(), inputs=['user/mode'], outputs=['run_pilot']) @@ -136,18 +160,20 @@ def drive(cfg, model_path=None, model_type=None): if cfg.DONKEY_GYM or cfg.DRIVE_TRAIN_TYPE == "MOCK": pass else: - steering_controller = PCA9685(cfg.STEERING_CHANNEL, cfg.PCA9685_I2C_ADDR, - busnum=cfg.PCA9685_I2C_BUSNUM) + steering_controller = PCA9685(cfg.STEERING_CHANNEL, + cfg.PCA9685_I2C_ADDR, + busnum=cfg.PCA9685_I2C_BUSNUM) steering = PWMSteering(controller=steering_controller, - left_pulse=cfg.STEERING_LEFT_PWM, - right_pulse=cfg.STEERING_RIGHT_PWM) + left_pulse=cfg.STEERING_LEFT_PWM, + right_pulse=cfg.STEERING_RIGHT_PWM) - throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL, cfg.PCA9685_I2C_ADDR, - busnum=cfg.PCA9685_I2C_BUSNUM) + throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL, + cfg.PCA9685_I2C_ADDR, + busnum=cfg.PCA9685_I2C_BUSNUM) throttle = PWMThrottle(controller=throttle_controller, - max_pulse=cfg.THROTTLE_FORWARD_PWM, - zero_pulse=cfg.THROTTLE_STOPPED_PWM, - min_pulse=cfg.THROTTLE_REVERSE_PWM) + max_pulse=cfg.THROTTLE_FORWARD_PWM, + zero_pulse=cfg.THROTTLE_STOPPED_PWM, + min_pulse=cfg.THROTTLE_REVERSE_PWM) car.add(steering, inputs=['angle']) car.add(throttle, inputs=['throttle']) @@ -155,17 +181,58 @@ def drive(cfg, model_path=None, model_type=None): # add tub to save data inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode'] types = ['image_array', 'float', 'float', 'str'] + # do we want to store new records into own dir or append to existing tub_path = TubHandler(path=cfg.DATA_PATH).create_tub_path() if \ cfg.AUTO_CREATE_NEW_TUB else cfg.DATA_PATH tub_writer = TubWriter(base_path=tub_path, inputs=inputs, types=types) car.add(tub_writer, inputs=inputs, outputs=["tub/num_records"], run_condition='recording') + if not model_path and cfg.USE_RC: + tub_wiper = TubWiper(tub_writer.tub, num_records=cfg.DRIVE_LOOP_HZ) + car.add(tub_wiper, inputs=['user/wiper_on']) # start the car car.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS) +def calibrate(cfg): + """ + Construct an auxiliary robotic vehicle from only the RC controllers and + prints their values. The RC remote usually has a tuning pot for the throttle + and steering channel. In this loop we run the controllers and simply print + their values in order to allow centering the RC pwm signals. If there is a + third channel on the remote we can use it for wiping bad data while + recording, so we print its values here, too. + """ + donkey_car = dk.vehicle.Vehicle() + + # create the RC receiver + rc_steering = RCReceiver(cfg.STEERING_RC_GPIO, invert=True) + rc_throttle = RCReceiver(cfg.THROTTLE_RC_GPIO) + rc_wiper = RCReceiver(cfg.DATA_WIPER_RC_GPIO, jitter=0.05, no_action=0) + donkey_car.add(rc_steering, outputs=['user/angle', 'user/steering_on']) + donkey_car.add(rc_throttle, outputs=['user/throttle', 'user/throttle_on']) + donkey_car.add(rc_wiper, outputs=['user/wiper', 'user/wiper_on']) + + # create plotter part for printing into the shell + class Plotter: + def run(self, angle, steering_on, throttle, throttle_on, wiper, wiper_on): + print('angle=%+5.4f, steering_on=%1d, throttle=%+5.4f, ' + 'throttle_on=%1d wiper=%+5.4f, wiper_on=%1d' % + (angle, steering_on, throttle, throttle_on, wiper, wiper_on)) + + # add plotter part + donkey_car.add(Plotter(), inputs=['user/angle', 'user/steering_on', + 'user/throttle', 'user/throttle_on', + 'user/wiper', 'user/wiper_on']) + # run the vehicle at 5Hz to keep network traffic down + donkey_car.start(rate_hz=10, max_loop_count=cfg.MAX_LOOPS) + + if __name__ == '__main__': args = docopt(__doc__) cfg = dk.load_config() - drive(cfg, model_path=args['--model'], model_type=args['--type']) + if args['drive']: + drive(cfg, model_path=args['--model'], model_type=args['--type']) + elif args['calibrate']: + calibrate(cfg) diff --git a/donkeycar/templates/cfg_basic.py b/donkeycar/templates/cfg_basic.py index c66be226c..9f296826b 100755 --- a/donkeycar/templates/cfg_basic.py +++ b/donkeycar/templates/cfg_basic.py @@ -4,7 +4,7 @@ This file is read by your car application's manage.py script to change the car performance. -EXMAPLE +EXAMPLE ----------- import dk cfg = dk.load_config(config_path='~/mycar/config.py') @@ -23,7 +23,6 @@ #VEHICLE DRIVE_LOOP_HZ = 20 MAX_LOOPS = 100000 -DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM|PIGPIO_PWM|MM1|MOCK #CAMERA CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST) @@ -33,6 +32,8 @@ CAMERA_FRAMERATE = DRIVE_LOOP_HZ CAMERA_VFLIP = False CAMERA_HFLIP = False +# For CSIC camera - If the camera is mounted in a rotated position, changing the below parameter will correct the output frame orientation +CSIC_CAM_GSTREAMER_FLIP_PARM = 0 # (0 => none , 4 => Flip horizontally, 6 => Flip vertically) #9865, over rides only if needed, ie. TX2.. PCA9685_I2C_ADDR = 0x40 @@ -49,7 +50,30 @@ THROTTLE_STOPPED_PWM = 370 THROTTLE_REVERSE_PWM = 220 +# DRIVETRAIN +DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM|PIGPIO_PWM|MM1|MOCK + +# #LIDAR +USE_LIDAR = False +LIDAR_TYPE = 'RP' #(RP|YD) +LIDAR_LOWER_LIMIT = 44 # angles that will be recorded. Use this to block out obstructed areas on your car, or looking backwards. Note that for the RP A1M8 Lidar, "0" is in the direction of the motor +LIDAR_UPPER_LIMIT = 136 + +# #RC CONTROL +USE_RC = False +STEERING_RC_GPIO = 26 +THROTTLE_RC_GPIO = 20 +DATA_WIPER_RC_GPIO = 19 + + +#LOGGING +HAVE_CONSOLE_LOGGING = True +LOGGING_LEVEL = 'INFO' # (Python logging level) 'NOTSET' / 'DEBUG' / 'INFO' / 'WARNING' / 'ERROR' / 'FATAL' / 'CRITICAL' +LOGGING_FORMAT = '%(message)s' # (Python logging format - https://docs.python.org/3/library/logging.html#formatter-objects + + #TRAINING +DEFAULT_AI_FRAMEWORK = 'tensorflow' # The default AI framework to use. Choose from (tensorflow|pytorch) DEFAULT_MODEL_TYPE = 'linear' #(linear|categorical|rnn|imu|behavior|3d|localizer|latent) BATCH_SIZE = 128 TRAIN_TEST_SPLIT = 0.8 @@ -106,6 +130,9 @@ WEB_CONTROL_PORT = int(os.getenv("WEB_CONTROL_PORT", 8887)) # which port to listen on when making a web controller WEB_INIT_MODE = "user" # which control mode to start in. one of user|local_angle|local. Setting local will start in ai mode. +#DRIVING +AI_THROTTLE_MULT = 1.0 # this multiplier will scale every throttle value for all output from NN models + #DonkeyGym #Only on Ubuntu linux, you can use the simulator as a virtual donkey and @@ -115,7 +142,7 @@ #then extract that and modify DONKEY_SIM_PATH. DONKEY_GYM = False DONKEY_SIM_PATH = "path to sim" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" when racing on virtual-race-league use "remote", or user "remote" when you want to start the sim manually first. -DONKEY_GYM_ENV_NAME = "donkey-mountain-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") +DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") GYM_CONF = { "body_style" : "donkey", "body_rgb" : (128, 128, 128), "car_name" : "car", "font_size" : 100} # body style(donkey|bare|car01) body rgb 0-255 GYM_CONF["racer_name"] = "Your Name" GYM_CONF["country"] = "Place" diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index 5b13358ad..ef0761c1b 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -4,7 +4,7 @@ This file is read by your car application's manage.py script to change the car performance. -EXMAPLE +EXAMPLE ----------- import dk cfg = dk.load_config(config_path='~/mycar/config.py') @@ -23,7 +23,6 @@ #VEHICLE DRIVE_LOOP_HZ = 20 # the vehicle loop will pause if faster than this speed. MAX_LOOPS = None # the vehicle loop can abort after this many iterations, when given a positive integer. -DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM|PIGPIO_PWM|MM1|MOCK #CAMERA CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST) @@ -53,7 +52,7 @@ #DC_TWO_WHEEL uses HBridge pwm to control two drive motors, one on the left, and one on the right. #SERVO_HBRIDGE_PWM use ServoBlaster to output pwm control from the PiZero directly to control steering, and HBridge for a drive motor. #PIGPIO_PWM uses Raspberrys internal PWM -DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM|PIGPIO_PWM|MM1|MOCK +DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|DC_TWO_WHEEL_L298N|SERVO_HBRIDGE_PWM|PIGPIO_PWM|MM1|MOCK #STEERING STEERING_CHANNEL = 1 #channel on the 9685 pwm board 0-15 @@ -91,9 +90,40 @@ HBRIDGE_PIN_RIGHT_BWD = 13 +#ODOMETRY +HAVE_ODOM = False # Do you have an odometer/encoder +ENCODER_TYPE = 'GPIO' # What kind of encoder? GPIO|Arduino|Astar +MM_PER_TICK = 12.7625 # How much travel with a single tick, in mm. Roll you car a meter and divide total ticks measured by 1,000 +ODOM_PIN = 13 # if using GPIO, which GPIO board mode pin to use as input +ODOM_DEBUG = False # Write out values on vel and distance as it runs + +# #LIDAR +USE_LIDAR = False +LIDAR_TYPE = 'RP' #(RP|YD) +LIDAR_LOWER_LIMIT = 90 # angles that will be recorded. Use this to block out obstructed areas on your car, or looking backwards. Note that for the RP A1M8 Lidar, "0" is in the direction of the motor +LIDAR_UPPER_LIMIT = 270 + + +# #RC CONTROL +USE_RC = False +STEERING_RC_GPIO = 26 +THROTTLE_RC_GPIO = 20 +DATA_WIPER_RC_GPIO = 19 + +#DC_TWO_WHEEL_L298N - with two wheels as drive, left and right. +#these GPIO pinouts are only used for the DRIVE_TRAIN_TYPE=DC_TWO_WHEEL_L298N +HBRIDGE_L298N_PIN_LEFT_FWD = 16 +HBRIDGE_L298N_PIN_LEFT_BWD = 18 +HBRIDGE_L298N_PIN_LEFT_EN = 22 + +HBRIDGE_L298N_PIN_RIGHT_FWD = 15 +HBRIDGE_L298N_PIN_RIGHT_BWD = 13 +HBRIDGE_L298N_PIN_RIGHT_EN = 11 + + #TRAINING # The default AI framework to use. Choose from (tensorflow|pytorch) -DEFAULT_AI_FRAMEWORK='tensorflow' +DEFAULT_AI_FRAMEWORK = 'tensorflow' #The DEFAULT_MODEL_TYPE will choose which model will be created at training time. This chooses #between different neural network designs. You can override this setting by passing the command @@ -140,7 +170,7 @@ WEB_INIT_MODE = "user" # which control mode to start in. one of user|local_angle|local. Setting local will start in ai mode. #JOYSTICK -USE_JOYSTICK_AS_DEFAULT = True #when starting the manage.py, when True, will not require a --js option to use the joystick +USE_JOYSTICK_AS_DEFAULT = False #when starting the manage.py, when True, will not require a --js option to use the joystick JOYSTICK_MAX_THROTTLE = 0.5 #this scalar is multiplied with the -1 to 1 throttle value to limit the maximum throttle. This can help if you drop the controller or just don't need the full speed available. JOYSTICK_STEERING_SCALE = 1.0 #some people want a steering that is less sensitve. This scalar is multiplied with the steering -1 to 1. It can be negative to reverse dir. AUTO_RECORD_ON_THROTTLE = True #if true, we will record whenever throttle is not zero. if false, you must manually toggle recording with some other trigger. Usually circle button on joystick. @@ -148,7 +178,7 @@ USE_NETWORKED_JS = False #should we listen for remote joystick control over the network? NETWORK_JS_SERVER_IP = None #when listening for network joystick control, which ip is serving this information JOYSTICK_DEADZONE = 0.01 # when non zero, this is the smallest throttle before recording triggered. -JOYSTICK_THROTTLE_DIR = 1.0 # use -1.0 to flip forward/backward, use 1.0 to use joystick's natural forward/backward +JOYSTICK_THROTTLE_DIR = -1.0 # use -1.0 to flip forward/backward, use 1.0 to use joystick's natural forward/backward USE_FPV = False # send camera data to FPV webserver JOYSTICK_DEVICE_FILE = "/dev/input/js0" # this is the unix file use to access the joystick. @@ -184,14 +214,27 @@ # eg.'/dev/tty.usbmodemXXXXXX' and replace the port accordingly MM1_SERIAL_PORT = '/dev/ttyS0' # Serial Port for reading and sending MM1 data. +#LOGGING +HAVE_CONSOLE_LOGGING = True +LOGGING_LEVEL = 'INFO' # (Python logging level) 'NOTSET' / 'DEBUG' / 'INFO' / 'WARNING' / 'ERROR' / 'FATAL' / 'CRITICAL' +LOGGING_FORMAT = '%(message)s' # (Python logging format - https://docs.python.org/3/library/logging.html#formatter-objects + #TELEMETRY -TELEMETRY_DONKEY_NAME = 'my_robot1234' -TELEMETRY_PUBLISH_PERIOD = 1 HAVE_MQTT_TELEMETRY = False +TELEMETRY_DONKEY_NAME = 'my_robot1234' TELEMETRY_MQTT_TOPIC_TEMPLATE = 'donkey/%s/telemetry' -TELEMETRY_MQTT_JSON_ENABLE = True -TELEMETRY_MQTT_BROKER_HOST = 'broker.emqx.io' +TELEMETRY_MQTT_JSON_ENABLE = False +TELEMETRY_MQTT_BROKER_HOST = 'broker.hivemq.com' TELEMETRY_MQTT_BROKER_PORT = 1883 +TELEMETRY_PUBLISH_PERIOD = 1 +TELEMETRY_LOGGING_ENABLE = True +TELEMETRY_LOGGING_LEVEL = 'INFO' # (Python logging level) 'NOTSET' / 'DEBUG' / 'INFO' / 'WARNING' / 'ERROR' / 'FATAL' / 'CRITICAL' +TELEMETRY_LOGGING_FORMAT = '%(message)s' # (Python logging format - https://docs.python.org/3/library/logging.html#formatter-objects +TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle,recording' +TELEMETRY_DEFAULT_TYPES = 'float,float' + +# PERF MONITOR +HAVE_PERFMON = False #RECORD OPTIONS RECORD_DURING_AI = False #normally we do not record during ai mode. Set this to true to get image and steering records for your Ai. Be careful not to use them to train. @@ -256,7 +299,7 @@ #then extract that and modify DONKEY_SIM_PATH. DONKEY_GYM = False DONKEY_SIM_PATH = "path to sim" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" when racing on virtual-race-league use "remote", or user "remote" when you want to start the sim manually first. -DONKEY_GYM_ENV_NAME = "donkey-mountain-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") +DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") GYM_CONF = { "body_style" : "donkey", "body_rgb" : (128, 128, 128), "car_name" : "car", "font_size" : 100} # body style(donkey|bare|car01) body rgb 0-255 GYM_CONF["racer_name"] = "Your Name" GYM_CONF["country"] = "Place" @@ -265,6 +308,11 @@ SIM_HOST = "127.0.0.1" # when racing on virtual-race-league use host "trainmydonkey.com" SIM_ARTIFICIAL_LATENCY = 0 # this is the millisecond latency in controls. Can use useful in emulating the delay when useing a remote server. values of 100 to 400 probably reasonable. +# Save info from Simulator (pln) +SIM_RECORD_LOCATION = False +SIM_RECORD_GYROACCEL= False +SIM_RECORD_VELOCITY = False + #publish camera over network #This is used to create a tcp service to pushlish the camera feed PUB_CAMERA_IMAGES = False diff --git a/donkeycar/templates/cfg_cv_control.py b/donkeycar/templates/cfg_cv_control.py index c22fb9455..fa2284a2b 100755 --- a/donkeycar/templates/cfg_cv_control.py +++ b/donkeycar/templates/cfg_cv_control.py @@ -53,7 +53,7 @@ #then extract that and modify DONKEY_SIM_PATH. DONKEY_GYM = False DONKEY_SIM_PATH = "remote" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" when racing on virtual-race-league use "remote", or user "remote" when you want to start the sim manually first. -DONKEY_GYM_ENV_NAME = "donkey-mountain-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") +DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") GYM_CONF = { "body_style" : "donkey", "body_rgb" : (128, 128, 128), "car_name" : "car", "font_size" : 100} # body style(donkey|bare|car01) body rgb 0-255 GYM_CONF["racer_name"] = "Your Name" GYM_CONF["country"] = "Place" diff --git a/donkeycar/templates/cfg_path_follow.py b/donkeycar/templates/cfg_path_follow.py index b8bebff04..1cc7e9b04 100644 --- a/donkeycar/templates/cfg_path_follow.py +++ b/donkeycar/templates/cfg_path_follow.py @@ -8,81 +8,87 @@ # The update operation will not touch this file. # """ -# import os +import os # # #PATHS -# CAR_PATH = PACKAGE_PATH = os.path.dirname(os.path.realpath(__file__)) -# DATA_PATH = os.path.join(CAR_PATH, 'data') -# MODELS_PATH = os.path.join(CAR_PATH, 'models') +CAR_PATH = PACKAGE_PATH = os.path.dirname(os.path.realpath(__file__)) +DATA_PATH = os.path.join(CAR_PATH, 'data') +MODELS_PATH = os.path.join(CAR_PATH, 'models') # # #VEHICLE -# DRIVE_LOOP_HZ = 20 # the vehicle loop will pause if faster than this speed. -# MAX_LOOPS = None # the vehicle loop can abort after this many iterations, when given a positive integer. -# +DRIVE_LOOP_HZ = 20 # the vehicle loop will pause if faster than this speed. +MAX_LOOPS = None # the vehicle loop can abort after this many iterations, when given a positive integer. # + +#WEB CONTROL +WEB_CONTROL_PORT = int(os.getenv("WEB_CONTROL_PORT", 8887)) # which port to listen on when making a web controller +WEB_INIT_MODE = "user" # which control mode to start in. one of user|local_angle|local. Setting local will start in ai mode. + + # #9865, over rides only if needed, ie. TX2.. -# PCA9685_I2C_ADDR = 0x40 #I2C address, use i2cdetect to validate this number -# PCA9685_I2C_BUSNUM = None #None will auto detect, which is fine on the pi. But other platforms should specify the bus num. +PCA9685_I2C_ADDR = 0x40 #I2C address, use i2cdetect to validate this number +PCA9685_I2C_BUSNUM = None #None will auto detect, which is fine on the pi. But other platforms should specify the bus num. # # #DRIVETRAIN # #These options specify which chasis and motor setup you are using. Most are using SERVO_ESC. # #DC_STEER_THROTTLE uses HBridge pwm to control one steering dc motor, and one drive wheel motor # #DC_TWO_WHEEL uses HBridge pwm to control two drive motors, one on the left, and one on the right. # #SERVO_HBRIDGE_PWM use ServoBlaster to output pwm control from the PiZero directly to control steering, and HBridge for a drive motor. -# DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM +DRIVE_TRAIN_TYPE = "SERVO_ESC" # SERVO_ESC|DC_STEER_THROTTLE|DC_TWO_WHEEL|SERVO_HBRIDGE_PWM # # #STEERING -# STEERING_CHANNEL = 1 #channel on the 9685 pwm board 0-15 +STEERING_CHANNEL = 1 #channel on the 9685 pwm board 0-15 STEERING_LEFT_PWM = 460 #pwm value for full left steering STEERING_RIGHT_PWM = 340 #pwm value for full right steering # # #THROTTLE -# THROTTLE_CHANNEL = 0 #channel on the 9685 pwm board 0-15 -# THROTTLE_FORWARD_PWM = 400 #pwm value for auto mode throttle -# THROTTLE_STOPPED_PWM = 370 #pwm value for no movement -# THROTTLE_REVERSE_PWM = 220 #pwm value for max reverse throttle +THROTTLE_CHANNEL = 0 #channel on the 9685 pwm board 0-15 +THROTTLE_FORWARD_PWM = 400 #pwm value for auto mode throttle +THROTTLE_STOPPED_PWM = 370 #pwm value for no movement +THROTTLE_REVERSE_PWM = 220 #pwm value for max reverse throttle # # #DC_STEER_THROTTLE with one motor as steering, one as drive # #these GPIO pinouts are only used for the DRIVE_TRAIN_TYPE=DC_STEER_THROTTLE -# HBRIDGE_PIN_LEFT = 18 -# HBRIDGE_PIN_RIGHT = 16 -# HBRIDGE_PIN_FWD = 15 -# HBRIDGE_PIN_BWD = 13 +HBRIDGE_PIN_LEFT = 18 +HBRIDGE_PIN_RIGHT = 16 +HBRIDGE_PIN_FWD = 15 +HBRIDGE_PIN_BWD = 13 # # #DC_TWO_WHEEL - with two wheels as drive, left and right. # #these GPIO pinouts are only used for the DRIVE_TRAIN_TYPE=DC_TWO_WHEEL -# HBRIDGE_PIN_LEFT_FWD = 18 -# HBRIDGE_PIN_LEFT_BWD = 16 -# HBRIDGE_PIN_RIGHT_FWD = 15 -# HBRIDGE_PIN_RIGHT_BWD = 13 +HBRIDGE_PIN_LEFT_FWD = 18 +HBRIDGE_PIN_LEFT_BWD = 16 +HBRIDGE_PIN_RIGHT_FWD = 15 +HBRIDGE_PIN_RIGHT_BWD = 13 # # # # #JOYSTICK -# JOYSTICK_MAX_THROTTLE = 0.5 #this scalar is multiplied with the -1 to 1 throttle value to limit the maximum throttle. This can help if you drop the controller or just don't need the full speed available. -# JOYSTICK_STEERING_SCALE = 1.0 #some people want a steering that is less sensitve. This scalar is multiplied with the steering -1 to 1. It can be negative to reverse dir. -# AUTO_RECORD_ON_THROTTLE = True #if true, we will record whenever throttle is not zero. if false, you must manually toggle recording with some other trigger. Usually circle button on joystick. +USE_JOYSTICK_AS_DEFAULT = False #when starting the manage.py, when True, will not require a --js option to use the joystick +JOYSTICK_MAX_THROTTLE = 0.5 #this scalar is multiplied with the -1 to 1 throttle value to limit the maximum throttle. This can help if you drop the controller or just don't need the full speed available. +JOYSTICK_STEERING_SCALE = 1.0 #some people want a steering that is less sensitve. This scalar is multiplied with the steering -1 to 1. It can be negative to reverse dir. +AUTO_RECORD_ON_THROTTLE = True #if true, we will record whenever throttle is not zero. if false, you must manually toggle recording with some other trigger. Usually circle button on joystick. CONTROLLER_TYPE='ps4' #(ps3|ps4|xbox|nimbus|wiiu|F710|rc3) -# USE_NETWORKED_JS = False #should we listen for remote joystick control over the network? -# NETWORK_JS_SERVER_IP = "192.168.0.1"#when listening for network joystick control, which ip is serving this information -# JOYSTICK_DEADZONE = 0.0 # when non zero, this is the smallest throttle before recording triggered. -# JOYSTICK_THROTTLE_DIR = -1.0 # use -1.0 to flip forward/backward, use 1.0 to use joystick's natural forward/backward -# JOYSTICK_DEVICE_FILE = "/dev/input/js0" # this is the unix file use to access the joystick. +USE_NETWORKED_JS = False #should we listen for remote joystick control over the network? +NETWORK_JS_SERVER_IP = "192.168.0.1"#when listening for network joystick control, which ip is serving this information +JOYSTICK_DEADZONE = 0.0 # when non zero, this is the smallest throttle before recording triggered. +JOYSTICK_THROTTLE_DIR = -1.0 # use -1.0 to flip forward/backward, use 1.0 to use joystick's natural forward/backward +JOYSTICK_DEVICE_FILE = "/dev/input/js0" # this is the unix file use to access the joystick. USE_FPV = False # send camera data to FPV webserver # # # #SOMBRERO -# HAVE_SOMBRERO = False #set to true when using the sombrero hat from the Donkeycar store. This will enable pwm on the hat. +HAVE_SOMBRERO = False #set to true when using the sombrero hat from the Donkeycar store. This will enable pwm on the hat. # # -# #Path following -# PATH_FILENAME = "donkey_path.pkl" # the path will be saved to this filename -# PATH_SCALE = 10.0 # the path display will be scaled by this factor in the web page -# PATH_OFFSET = (255, 255) # 255, 255 is the center of the map. This offset controls where the origin is displayed. +#Path following +PATH_FILENAME = "donkey_path.pkl" # the path will be saved to this filename +PATH_SCALE = 10.0 # the path display will be scaled by this factor in the web page +PATH_OFFSET = (255, 255) # 255, 255 is the center of the map. This offset controls where the origin is displayed. PATH_MIN_DIST = 0.2 # after travelling this distance (m), save a path point PID_P = -0.5 # proportional mult for PID path follower -# PID_I = 0.000 # integral mult for PID path follower +PID_I = 0.000 # integral mult for PID path follower PID_D = -0.3 # differential mult for PID path follower PID_THROTTLE = 0.30 # constant throttle value during path following @@ -95,13 +101,13 @@ # # #Odometry -# HAVE_ODOM = False # Do you have an odometer? Uses pigpio -# MM_PER_TICK = 12.7625 # How much travel with a single tick, in mm -# ODOM_PIN = 4 # Which GPIO board mode pin to use as input -# ODOM_DEBUG = False # Write out values on vel and distance as it runs +HAVE_ODOM = False # Do you have an odometer? Uses pigpio +MM_PER_TICK = 12.7625 # How much travel with a single tick, in mm +ODOM_PIN = 4 # Which GPIO board mode pin to use as input +ODOM_DEBUG = False # Write out values on vel and distance as it runs # # #Intel T265 -# WHEEL_ODOM_CALIB = "calibration_odometry.json" +WHEEL_ODOM_CALIB = "calibration_odometry.json" # # #DonkeyGym # #Only on Ubuntu linux, you can use the simulator as a virtual donkey and @@ -109,6 +115,6 @@ # #This enables that, and sets the path to the simualator and the environment. # #You will want to download the simulator binary from: https://github.com/tawnkramer/donkey_gym/releases/download/v18.9/DonkeySimLinux.zip # #then extract that and modify DONKEY_SIM_PATH. -# DONKEY_GYM = False -# DONKEY_SIM_PATH = "path to sim" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" -# DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") +DONKEY_GYM = False +DONKEY_SIM_PATH = "path to sim" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" +DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") diff --git a/donkeycar/templates/complete.py b/donkeycar/templates/complete.py index 7b18da1bb..bbb77c9da 100644 --- a/donkeycar/templates/complete.py +++ b/donkeycar/templates/complete.py @@ -16,12 +16,10 @@ """ import os import time - +import logging from docopt import docopt -import numpy as np import donkeycar as dk - from donkeycar.parts.transform import TriggeredCallback, DelayedTrigger from donkeycar.parts.tub_v2 import TubWriter from donkeycar.parts.datastore import TubHandler @@ -32,18 +30,22 @@ from donkeycar.parts.launch import AiLaunch from donkeycar.utils import * - -def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type='single', meta=[]): - ''' - Construct a working robotic vehicle from many parts. - Each part runs as a job in the Vehicle loop, calling either - it's run or run_threaded method depending on the constructor flag `threaded`. - All parts are updated one after another at the framerate given in - cfg.DRIVE_LOOP_HZ assuming each part finishes processing in a timely manner. - Parts may have named outputs and inputs. The framework handles passing named outputs - to parts requesting the same named input. - ''' - +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + + +def drive(cfg, model_path=None, use_joystick=False, model_type=None, + camera_type='single', meta=[]): + """ + Construct a working robotic vehicle from many parts. Each part runs as a + job in the Vehicle loop, calling either it's run or run_threaded method + depending on the constructor flag `threaded`. All parts are updated one + after another at the framerate given in cfg.DRIVE_LOOP_HZ assuming each + part finishes processing in a timely manner. Parts may have named outputs + and inputs. The framework handles passing named outputs to parts + requesting the same named input. + """ + logger.info(f'PID: {os.getpid()}') if cfg.DONKEY_GYM: #the simulator will use cuda and then we usually run out of resources #if we also try to use cuda. so disable for donkey_gym. @@ -60,7 +62,30 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type #Initialize car V = dk.vehicle.Vehicle() - print("cfg.CAMERA_TYPE", cfg.CAMERA_TYPE) + #Initialize logging before anything else to allow console logging + if cfg.HAVE_CONSOLE_LOGGING: + logger.setLevel(logging.getLevelName(cfg.LOGGING_LEVEL)) + ch = logging.StreamHandler() + ch.setFormatter(logging.Formatter(cfg.LOGGING_FORMAT)) + logger.addHandler(ch) + + if cfg.HAVE_MQTT_TELEMETRY: + from donkeycar.parts.telemetry import MqttTelemetry + tel = MqttTelemetry(cfg) + + if cfg.HAVE_ODOM: + if cfg.ENCODER_TYPE == "GPIO": + from donkeycar.parts.encoder import RotaryEncoder + enc = RotaryEncoder(mm_per_tick=0.306096, pin = cfg.ODOM_PIN, debug = cfg.ODOM_DEBUG) + V.add(enc, inputs=['throttle'], outputs=['enc/speed'], threaded=True) + elif cfg.ENCODER_TYPE == "arduino": + from donkeycar.parts.encoder import ArduinoEncoder + enc = ArduinoEncoder() + V.add(enc, outputs=['enc/speed'], threaded=True) + else: + print("No supported encoder found") + + logger.info("cfg.CAMERA_TYPE %s"%cfg.CAMERA_TYPE) if camera_type == "stereo": if cfg.CAMERA_TYPE == "WEBCAM": @@ -102,12 +127,14 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type from donkeycar.parts.dgym import DonkeyGymEnv inputs = [] + outputs = ['cam/image_array'] threaded = True if cfg.DONKEY_GYM: from donkeycar.parts.dgym import DonkeyGymEnv - cam = DonkeyGymEnv(cfg.DONKEY_SIM_PATH, host=cfg.SIM_HOST, env_name=cfg.DONKEY_GYM_ENV_NAME, conf=cfg.GYM_CONF, delay=cfg.SIM_ARTIFICIAL_LATENCY) + #rbx + cam = DonkeyGymEnv(cfg.DONKEY_SIM_PATH, host=cfg.SIM_HOST, env_name=cfg.DONKEY_GYM_ENV_NAME, conf=cfg.GYM_CONF, record_location=cfg.SIM_RECORD_LOCATION, record_gyroaccel=cfg.SIM_RECORD_GYROACCEL, record_velocity=cfg.SIM_RECORD_VELOCITY, delay=cfg.SIM_ARTIFICIAL_LATENCY) threaded = True - inputs = ['angle', 'throttle'] + inputs = ['angle', 'throttle'] elif cfg.CAMERA_TYPE == "PICAM": from donkeycar.parts.camera import PiCamera cam = PiCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, framerate=cfg.CAMERA_FRAMERATE, vflip=cfg.CAMERA_VFLIP, hflip=cfg.CAMERA_HFLIP) @@ -135,8 +162,36 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type else: raise(Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE)) - V.add(cam, inputs=inputs, outputs=['cam/image_array'], threaded=threaded) - + # add lidar + if cfg.USE_LIDAR: + from donkeycar.parts.lidar import RPLidar + if cfg.LIDAR_TYPE == 'RP': + print("adding RP lidar part") + lidar = RPLidar(lower_limit = cfg.LIDAR_LOWER_LIMIT, upper_limit = cfg.LIDAR_UPPER_LIMIT) + V.add(lidar, inputs=[],outputs=['lidar/dist_array'], threaded=True) + if cfg.LIDAR_TYPE == 'YD': + print("YD Lidar not yet supported") + + # Donkey gym part will output position information if it is configured + if cfg.DONKEY_GYM: + if cfg.SIM_RECORD_LOCATION: + outputs += ['pos/pos_x', 'pos/pos_y', 'pos/pos_z', 'pos/speed', 'pos/cte'] + if cfg.SIM_RECORD_GYROACCEL: + outputs += ['gyro/gyro_x', 'gyro/gyro_y', 'gyro/gyro_z', 'accel/accel_x', 'accel/accel_y', 'accel/accel_z'] + if cfg.SIM_RECORD_VELOCITY: + outputs += ['vel/vel_x', 'vel/vel_y', 'vel/vel_z'] + + V.add(cam, inputs=inputs, outputs=outputs, threaded=threaded) + + #This web controller will create a web server that is capable + #of managing steering, throttle, and modes, and more. + ctr = LocalWebController(port=cfg.WEB_CONTROL_PORT, mode=cfg.WEB_INIT_MODE) + + V.add(ctr, + inputs=['cam/image_array', 'tub/num_records'], + outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], + threaded=True) + if use_joystick or cfg.USE_JOYSTICK_AS_DEFAULT: #modify max_throttle closer to 1.0 to have more power #modify steering_scale lower than 1.0 to have less responsive steering @@ -170,16 +225,6 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, camera_type outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], threaded=True) - else: - #This web controller will create a web server that is capable - #of managing steering, throttle, and modes, and more. - ctr = LocalWebController(port=cfg.WEB_CONTROL_PORT, mode=cfg.WEB_INIT_MODE) - - V.add(ctr, - inputs=['cam/image_array', 'tub/num_records'], - outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], - threaded=True) - #this throttle filter will allow one tap back for esc reverse th_filter = ThrottleFilter() V.add(th_filter, inputs=['user/throttle'], outputs=['user/throttle']) @@ -315,12 +360,20 @@ def show_record_acount_status(): inputs = ['cam/image_array', "behavior/one_hot_state_array"] #IMU + elif cfg.USE_LIDAR: + inputs = ['cam/image_array', 'lidar/dist_array'] + + elif cfg.HAVE_ODOM: + inputs = ['cam/image_array', 'enc/speed'] + elif model_type == "imu": assert(cfg.HAVE_IMU) #Run the pilot if the mode is not user. inputs=['cam/image_array', 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'] + elif cfg.USE_LIDAR: + inputs = ['cam/image_array', 'lidar/dist_array'] else: inputs=['cam/image_array'] @@ -508,6 +561,21 @@ def run(self, mode, recording): V.add(left_motor, inputs=['left_motor_speed']) V.add(right_motor, inputs=['right_motor_speed']) + elif cfg.DRIVE_TRAIN_TYPE == "DC_TWO_WHEEL_L298N": + from donkeycar.parts.actuator import TwoWheelSteeringThrottle, L298N_HBridge_DC_Motor + + left_motor = L298N_HBridge_DC_Motor(cfg.HBRIDGE_L298N_PIN_LEFT_FWD, cfg.HBRIDGE_L298N_PIN_LEFT_BWD, cfg.HBRIDGE_L298N_PIN_LEFT_EN) + right_motor = L298N_HBridge_DC_Motor(cfg.HBRIDGE_L298N_PIN_RIGHT_FWD, cfg.HBRIDGE_L298N_PIN_RIGHT_BWD, cfg.HBRIDGE_L298N_PIN_RIGHT_EN) + two_wheel_control = TwoWheelSteeringThrottle() + + V.add(two_wheel_control, + inputs=['throttle', 'angle'], + outputs=['left_motor_speed', 'right_motor_speed']) + + V.add(left_motor, inputs=['left_motor_speed']) + V.add(right_motor, inputs=['right_motor_speed']) + + elif cfg.DRIVE_TRAIN_TYPE == "SERVO_HBRIDGE_PWM": from donkeycar.parts.actuator import ServoBlaster, PWMSteering steering_controller = ServoBlaster(cfg.STEERING_CHANNEL) #really pin @@ -553,13 +621,20 @@ def run(self, mode, recording): #add tub to save data - inputs=['cam/image_array', - 'user/angle', 'user/throttle', - 'user/mode'] + if cfg.USE_LIDAR: + inputs = ['cam/image_array', 'lidar/dist_array', 'user/angle', 'user/throttle', 'user/mode'] + types = ['image_array', 'nparray','float', 'float', 'str'] + else: + inputs=['cam/image_array','user/angle', 'user/throttle', 'user/mode'] + types=['image_array','float', 'float','str'] - types=['image_array', - 'float', 'float', - 'str'] + if cfg.USE_LIDAR: + inputs += ['lidar/dist_array'] + types += ['nparray'] + + if cfg.HAVE_ODOM: + inputs += ['enc/speed'] + types += ['float'] if cfg.TRAIN_BEHAVIORS: inputs += ['behavior/state', 'behavior/label', "behavior/one_hot_state_array"] @@ -576,10 +651,30 @@ def run(self, mode, recording): types +=['float', 'float', 'float', 'float', 'float', 'float'] + # rbx + if cfg.DONKEY_GYM: + if cfg.SIM_RECORD_LOCATION: + inputs += ['pos/pos_x', 'pos/pos_y', 'pos/pos_z', 'pos/speed', 'pos/cte'] + types += ['float', 'float', 'float', 'float', 'float'] + if cfg.SIM_RECORD_GYROACCEL: + inputs += ['gyro/gyro_x', 'gyro/gyro_y', 'gyro/gyro_z', 'accel/accel_x', 'accel/accel_y', 'accel/accel_z'] + types += ['float', 'float', 'float', 'float', 'float', 'float'] + if cfg.SIM_RECORD_VELOCITY: + inputs += ['vel/vel_x', 'vel/vel_y', 'vel/vel_z'] + types += ['float', 'float', 'float'] + if cfg.RECORD_DURING_AI: inputs += ['pilot/angle', 'pilot/throttle'] types += ['float', 'float'] + if cfg.HAVE_PERFMON: + from donkeycar.parts.perfmon import PerfMonitor + mon = PerfMonitor(cfg) + perfmon_outputs = ['perf/cpu', 'perf/mem', 'perf/freq'] + inputs += perfmon_outputs + types += ['float', 'float', 'float'] + V.add(mon, inputs=[], outputs=perfmon_outputs, threaded=True) + # do we want to store new records into own dir or append to existing tub_path = TubHandler(path=cfg.DATA_PATH).create_tub_path() if \ cfg.AUTO_CREATE_NEW_TUB else cfg.DATA_PATH @@ -588,10 +683,8 @@ def run(self, mode, recording): # Telemetry (we add the same metrics added to the TubHandler if cfg.HAVE_MQTT_TELEMETRY: - from donkeycar.parts.telemetry import MqttTelemetry - published_inputs, published_types = MqttTelemetry.filter_supported_metrics(inputs, types) - tel = MqttTelemetry(cfg, default_inputs=published_inputs, default_types=published_types) - V.add(tel, inputs=published_inputs, outputs=["tub/queue_size"], threaded=False) + telem_inputs, _ = tel.add_step_inputs(inputs, types) + V.add(tel, inputs=telem_inputs, outputs=["tub/queue_size"], threaded=True) if cfg.PUB_CAMERA_IMAGES: from donkeycar.parts.network import TCPServeValue @@ -626,4 +719,3 @@ def run(self, mode, recording): meta=args['--meta']) elif args['train']: print('Use python train.py instead.\n') - diff --git a/donkeycar/templates/path_follow.py b/donkeycar/templates/path_follow.py index 48fcf7148..b6e0ec835 100644 --- a/donkeycar/templates/path_follow.py +++ b/donkeycar/templates/path_follow.py @@ -25,7 +25,7 @@ import pigpio import donkeycar as dk -from donkeycar.parts.controller import WebFpv, get_js_controller +from donkeycar.parts.controller import WebFpv, get_js_controller, LocalWebController from donkeycar.parts.actuator import PCA9685, PWMSteering, PWMThrottle from donkeycar.parts.path import Path, PathPlot, CTE, PID_Pilot, PlotCircle, PImage, OriginOffset from donkeycar.parts.transform import PIDController @@ -53,10 +53,10 @@ def drive(cfg): ctr = get_js_controller(cfg) - V.add(ctr, - inputs=['null'], - outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], - threaded=True) + V.add(ctr, + inputs=['cam/image_array'], + outputs=['user/angle', 'user/throttle', 'user/mode', 'recording'], + threaded=True) if cfg.HAVE_ODOM: pi = pigpio.pi() @@ -197,19 +197,15 @@ def inc_pid_d(): ctr.set_button_down_trigger("L2", dec_pid_d) ctr.set_button_down_trigger("R2", inc_pid_d) - # Plot a circle on the map where the car is located + # #This web controller will create a web server. We aren't using any controls, just for visualization. + web_ctr = LocalWebController(port=cfg.WEB_CONTROL_PORT, + mode=cfg.WEB_INIT_MODE) - carcolor = 'green' - - loc_plot = PlotCircle(scale=cfg.PATH_SCALE, offset=cfg.PATH_OFFSET, color = carcolor) - V.add(loc_plot, inputs=['map/image', 'pos/x', 'pos/y'], outputs=['map/image']) - - #This web controller will create a web server. We aren't using any controls, just for visualization. - web_ctr = WebFpv() V.add(web_ctr, - inputs=['map/image'], - threaded=True) + inputs=['map/image'], + outputs=['web/angle', 'web/throttle', 'web/mode', 'web/recording'], + threaded=True) #Choose what inputs should change the car. @@ -280,6 +276,9 @@ def run(self, mode, print("follow the path using 'select' to change to ai drive mode.") print("You can also press the Square button to reset the origin") print("###############################################################################") + carcolor = 'green' + loc_plot = PlotCircle(scale=cfg.PATH_SCALE, offset=cfg.PATH_OFFSET, color = carcolor) + V.add(loc_plot, inputs=['map/image', 'pos/x', 'pos/y'], outputs=['map/image']) V.start(rate_hz=cfg.DRIVE_LOOP_HZ, max_loop_count=cfg.MAX_LOOPS) @@ -297,4 +296,4 @@ def run(self, mode, if args['drive']: - drive(cfg) + drive(cfg) \ No newline at end of file diff --git a/donkeycar/templates/train.py b/donkeycar/templates/train.py index 729d99a8d..4517c35ff 100644 --- a/donkeycar/templates/train.py +++ b/donkeycar/templates/train.py @@ -4,7 +4,9 @@ Basic usage should feel familiar: train.py --tubs data/ --model models/mypilot.h5 Usage: - train.py [--tubs=tubs] (--model=) [--type=(linear|inferred|tensorrt_linear|tflite_linear)] + train.py [--tubs=tubs] (--model=) + [--type=(linear|inferred|tensorrt_linear|tflite_linear)] + [--comment=] Options: -h --help Show this screen. @@ -21,7 +23,8 @@ def main(): tubs = args['--tubs'] model = args['--model'] model_type = args['--type'] - train(cfg, tubs, model, model_type) + comment = args['--comment'] + train(cfg, tubs, model, model_type, comment) if __name__ == "__main__": diff --git a/donkeycar/tests/test_datastore_v2.py b/donkeycar/tests/test_datastore_v2.py index 1edad5ac6..a4eaadfc2 100644 --- a/donkeycar/tests/test_datastore_v2.py +++ b/donkeycar/tests/test_datastore_v2.py @@ -35,15 +35,37 @@ def test_deletion(self): manifest.write_record(self._newRecord()) for i in range(deleted): - manifest.delete_record(i) + manifest.delete_records(i) read_records = 0 for entry in manifest: - print('Entry %s' % (entry)) + print(f'Entry {entry}') read_records += 1 self.assertEqual((count - deleted), read_records) + def test_delete_and_restore_by_set(self): + manifest = Manifest(self._path, max_len=2) + count = 10 + deleted = range(3, 7) + for i in range(count): + manifest.write_record(self._newRecord()) + + manifest.delete_records(deleted) + read_records = 0 + for entry in manifest: + print(f'Entry {entry}') + read_records += 1 + + self.assertEqual(count - len(deleted), read_records) + + manifest.restore_records(deleted) + read_records = 0 + for entry in manifest: + print(f'Entry {entry}') + read_records += 1 + + self.assertEqual(count, read_records) def test_memory_mapped_read(self): manifest = Manifest(self._path, max_len=2) @@ -60,7 +82,6 @@ def test_memory_mapped_read(self): self.assertEqual(10, read_records) - def tearDown(self): shutil.rmtree(self._path) @@ -68,5 +89,6 @@ def _newRecord(self): record = {'at' : time.time()} return record + if __name__ == '__main__': unittest.main() diff --git a/donkeycar/tests/test_lidar.py b/donkeycar/tests/test_lidar.py new file mode 100644 index 000000000..37622a080 --- /dev/null +++ b/donkeycar/tests/test_lidar.py @@ -0,0 +1,97 @@ +import pytest + + +def has_lidar(): + """ Determine if test platform (nano, pi) has lidar""" + # for now just return false, better to use an environ + return False + + +@pytest.mark.skipif(not has_lidar(), reason='Need lidar installed') +def test_rp_lidar(): + from rplidar import RPLidar + import serial + import glob + + temp_list = glob.glob('/dev/ttyUSB*') + result = [] + for a_port in temp_list: + try: + s = serial.Serial(a_port) + s.close() + result.append(a_port) + except serial.SerialException: + pass + print("available ports", result) + lidar = RPLidar(result[0], baudrate=115200) + + info = lidar.get_info() + print(info) + + health = lidar.get_health() + print(health) + + for i, scan in enumerate(lidar.iter_scans()): + print(f'{i}: Got {len(scan)} measurements') + if i > 10: + break + + lidar.stop() + lidar.stop_motor() + lidar.disconnect() + + +@pytest.mark.skipif(not has_lidar(), reason='Need lidar installed') +def test_py_lidar3(): + import PyLidar3 + import serial + import glob + import time # Time module + temp_list = glob.glob ('/dev/ttyUSB*') + result = [] + for a_port in temp_list: + try: + s = serial.Serial(a_port) + s.close() + result.append(a_port) + except serial.SerialException: + pass + print("available ports", result) + + port = result[0] # linux + # PyLidar3.your_version_of_lidar(port,chunk_size) + ydlidar = PyLidar3.YdLidarX4(port) + if ydlidar.Connect(): + print(ydlidar.GetDeviceInfo()) + gen = ydlidar.StartScanning() + t = time.time() # start time + while (time.time() - t) < 30: # scan for 30 seconds + print(next(gen)) + time.sleep(0.5) + ydlidar.StopScanning() + ydlidar.Disconnect() + else: + print("Error connecting to device") + + +@pytest.mark.skipif(not has_lidar(), reason='Need lidar installed') +def test_simple_express_scan(): + from pyrplidar import PyRPlidar + import time + lidar = PyRPlidar() + lidar.connect(port="/dev/ttyUSB0", baudrate=115200, timeout=3) + # Linux : "/dev/ttyUSB0" + # MacOS : "/dev/cu.SLAB_USBtoUART" + # Windows : "COM5" + + lidar.set_motor_pwm(500) + time.sleep(2) + scan_generator = lidar.start_scan_express(4) + + for count, scan in enumerate(scan_generator()): + print(count, scan) + if count == 20: break + + lidar.stop() + lidar.set_motor_pwm(0) + lidar.disconnect() diff --git a/donkeycar/tests/test_telemetry.py b/donkeycar/tests/test_telemetry.py index b522f9580..872cf8620 100644 --- a/donkeycar/tests/test_telemetry.py +++ b/donkeycar/tests/test_telemetry.py @@ -6,10 +6,15 @@ import donkeycar.templates.cfg_complete as cfg from donkeycar.parts.telemetry import MqttTelemetry import pytest +from random import randint def test_mqtt_telemetry(): + cfg.TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle' + cfg.TELEMETRY_DONKEY_NAME = 'test{}'.format(randint(0, 1000)) + cfg.TELEMETRY_MQTT_JSON_ENABLE = True + # Create receiver sub = Client(clean_session=True) @@ -21,14 +26,15 @@ def test_mqtt_telemetry(): sub.on_message = on_message_mock sub.connect(cfg.TELEMETRY_MQTT_BROKER_HOST) sub.loop_start() - name = "donkey/%s/telemetry" % cfg.TELEMETRY_DONKEY_NAME + name = "donkey/%s/#" % cfg.TELEMETRY_DONKEY_NAME sub.subscribe(name) - t = MqttTelemetry(cfg, default_inputs=['angle'], default_types=['float']) + t = MqttTelemetry(cfg) + t.add_step_inputs(inputs=['my/voltage'], types=['float']) t.publish() - timestamp = t.report({'speed': 16, 'voltage': 12}) - t.run(33.3) + timestamp = t.report({'my/speed': 16, 'my/voltage': 12}) + t.run(33.3, 22.2, 11.1) assert t.qsize == 2 time.sleep(1.5) @@ -38,5 +44,6 @@ def test_mqtt_telemetry(): time.sleep(0.5) - res = str.encode('[{"ts": %s, "values": {"speed": 16, "voltage": 12, "angle": 33.3}}]' % timestamp) + res = str.encode('[{"ts": %s, "values": {"my/speed": 16, "my/voltage": 11.1, "pilot/angle": 33.3, ' + '"pilot/throttle": 22.2}}]' % timestamp) assert on_message_mock.call_args_list[0][0][2].payload == res diff --git a/donkeycar/tests/test_train.py b/donkeycar/tests/test_train.py index cf840f658..f3db4bdc8 100644 --- a/donkeycar/tests/test_train.py +++ b/donkeycar/tests/test_train.py @@ -3,17 +3,18 @@ import os import numpy as np from collections import defaultdict, namedtuple +from typing import Callable from donkeycar.pipeline.training import train, BatchSequence from donkeycar.config import Config -from donkeycar.pipeline.types import TubDataset +from donkeycar.pipeline.types import TubDataset, TubRecord from donkeycar.utils import get_model_by_type, normalize_image Data = namedtuple('Data', ['type', 'name', 'convergence', 'pretrained']) @pytest.fixture -def config() -> Config: +def config(car_dir) -> Config: """ Config for the test with relevant parameters""" cfg = Config() cfg.BATCH_SIZE = 64 @@ -23,10 +24,13 @@ def config() -> Config: cfg.IMAGE_DEPTH = 3 cfg.PRINT_MODEL_SUMMARY = True cfg.EARLY_STOP_PATIENCE = 1000 - cfg.MAX_EPOCHS = 20 + cfg.MAX_EPOCHS = 5 cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8 cfg.VERBOSE_TRAIN = True cfg.MIN_DELTA = 0.0005 + cfg.MODELS_PATH = os.path.join(car_dir, 'models') + cfg.DATA_PATH = os.path.join(car_dir, 'tub') + cfg.SHOW_PLOT = False return cfg @@ -44,8 +48,8 @@ def car_dir(tmpdir_factory): # define the test data d1 = Data(type='linear', name='lin1', convergence=0.6, pretrained=None) -d2 = Data(type='categorical', name='cat1', convergence=0.85, pretrained=None) -d3 = Data(type='inferred', name='inf1', convergence=0.6, pretrained=None) +d2 = Data(type='categorical', name='cat1', convergence=0.9, pretrained=None) +d3 = Data(type='inferred', name='inf1', convergence=0.9, pretrained=None) d4 = Data(type='latent', name='lat1', convergence=0.5, pretrained=None) d5 = Data(type='latent', name='lat2', convergence=0.5, pretrained='lat1') test_data = [d1, d2, d3] @@ -54,43 +58,50 @@ def car_dir(tmpdir_factory): @pytest.mark.skipif("GITHUB_ACTIONS" in os.environ, reason='Suppress training test in CI') @pytest.mark.parametrize('data', test_data) -def test_train(config: Config, car_dir: str, data: Data) -> None: +def test_train(config: Config, data: Data) -> None: """ Testing convergence of the linear an categorical models - :param config: donkey config - :param car_dir: car directory (this is a temp dir) :param data: test case data :return: None """ def pilot_path(name): pilot_name = f'pilot_{name}.h5' - return os.path.join(car_dir, 'models', pilot_name) + return os.path.join(config.MODELS_PATH, pilot_name) if data.pretrained: config.LATENT_TRAINED = pilot_path(data.pretrained) - tub_dir = os.path.join(car_dir, 'tub') + tub_dir = config.DATA_PATH history = train(config, tub_dir, pilot_path(data.name), data.type) loss = history.history['loss'] # check loss is converging assert loss[-1] < loss[0] * data.convergence +filters = [lambda r: r.underlying['user/throttle'] > 0.5, + lambda r: r.underlying['user/angle'] < 0, + lambda r: r.underlying['user/throttle'] < 0.6 and + r.underlying['user/angle'] > -0.5] + + @pytest.mark.parametrize('model_type', ['linear', 'categorical', 'inferred']) -def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ - -> None: +@pytest.mark.parametrize('train_filter', filters) +def test_training_pipeline(config: Config, model_type: str, + train_filter: Callable[[TubRecord], bool]) -> None: """ Testing consistency of the model interfaces and data used in training pipeline. :param config: donkey config :param model_type: test specification of model type - :param tub_dir: tub directory (car_dir/tub) + :param train_filter: filter for records :return: None """ + config.TRAIN_FILTER = train_filter kl = get_model_by_type(model_type, config) - tub_dir = os.path.join(car_dir, 'tub') + tub_dir = config.DATA_PATH # don't shuffle so we can identify data for testing + config.TRAIN_FILTER = train_filter dataset = TubDataset(config, [tub_dir], shuffle=False) training_records, validation_records = dataset.train_test_split() seq = BatchSequence(kl, config, training_records, True) diff --git a/donkeycar/tests/test_tub_v2.py b/donkeycar/tests/test_tub_v2.py index e693633d3..2450090a9 100644 --- a/donkeycar/tests/test_tub_v2.py +++ b/donkeycar/tests/test_tub_v2.py @@ -24,7 +24,7 @@ def test_basic_tub_operations(self): self.tub.write_record(record) for index in delete_indexes: - self.tub.delete_record(index) + self.tub.delete_records(index) count = 0 for record in self.tub: diff --git a/donkeycar/tests/test_tubwriter.py b/donkeycar/tests/test_tubwriter.py new file mode 100644 index 000000000..924ee48d9 --- /dev/null +++ b/donkeycar/tests/test_tubwriter.py @@ -0,0 +1,43 @@ +import shutil +import tempfile +import unittest +from random import randint + +from donkeycar.parts.tub_v2 import Tub, TubWriter + + +class TestTub(unittest.TestCase): + def setUp(self): + self._path = tempfile.mkdtemp() + + def test_tubwriter_sessions(self): + # run tubwriter multiple times on the same tub directory + write_counts = [] + for _ in range(5): + tub_writer = TubWriter(self._path, inputs=['input'], types=['int']) + write_count = randint(1, 10) + for i in range(write_count): + tub_writer.run(i) + tub_writer.close() + write_counts.append(write_count) + + # Check we have good session id for all new records: + id = 0 + total = 0 + for record in tub_writer.tub: + print(f'Record: {record}') + session_number = int(record['_session_id'].split('_')[1]) + self.assertEqual(session_number, id , + 'Session id not correctly generated') + total += 1 + if total == write_counts[0]: + total = 0 + id += 1 + write_counts.pop(0) + + def tearDown(self): + shutil.rmtree(self._path) + + +if __name__ == '__main__': + unittest.main() diff --git a/donkeycar/vehicle.py b/donkeycar/vehicle.py index ab504500b..b54cc4104 100644 --- a/donkeycar/vehicle.py +++ b/donkeycar/vehicle.py @@ -8,11 +8,14 @@ import time import numpy as np +import logging from threading import Thread from .memory import Memory from prettytable import PrettyTable import traceback +logger = logging.getLogger(__name__) + class PartProfiler: def __init__(self): @@ -34,7 +37,7 @@ def on_part_finished(self, p): self.records[p]['times'][-1] = delta def report(self): - print("Part Profile Summary: (times in ms)") + logger.info("Part Profile Summary: (times in ms)") pt = PrettyTable() field_names = ["part", "max", "min", "avg"] pctile = [50, 90, 99, 99.9] @@ -52,7 +55,7 @@ def report(self): "%.2f" % (sum(arr) / len(arr) * 1000)] row += ["%.2f" % (np.percentile(arr, p) * 1000) for p in pctile] pt.add_row(row) - print(pt) + logger.info('\n' + str(pt)) class Vehicle: @@ -89,7 +92,7 @@ def add(self, part, inputs=[], outputs=[], assert type(threaded) is bool, "threaded is not a boolean: %r" % threaded p = part - print('Adding part {}.'.format(p.__class__.__name__)) + logger.info('Adding part {}.'.format(p.__class__.__name__)) entry = {} entry['part'] = p entry['inputs'] = inputs @@ -141,7 +144,7 @@ def start(self, rate_hz=10, max_loop_count=None, verbose=False): entry.get('thread').start() # wait until the parts warm up. - print('Starting vehicle at {} Hz'.format(rate_hz)) + logger.info('Starting vehicle at {} Hz'.format(rate_hz)) loop_count = 0 while self.on: @@ -160,7 +163,7 @@ def start(self, rate_hz=10, max_loop_count=None, verbose=False): else: # print a message when could not maintain loop rate. if verbose: - print('WARN::Vehicle: jitter violation in vehicle loop ' + logger.info('WARN::Vehicle: jitter violation in vehicle loop ' 'with {0:4.0f}ms'.format(abs(1000 * sleep_time))) if verbose and loop_count % 200 == 0: @@ -205,7 +208,7 @@ def update_parts(self): self.profiler.on_part_finished(p) def stop(self): - print('Shutting down vehicle and its parts...') + logger.info('Shutting down vehicle and its parts...') for entry in self.parts: try: entry['part'].shutdown() @@ -213,6 +216,6 @@ def stop(self): # usually from missing shutdown method, which should be optional pass except Exception as e: - print(e) + logger.error(e) self.profiler.report() diff --git a/install/envs/mac.yml b/install/envs/mac.yml index ea41bc99a..d542d72b8 100644 --- a/install/envs/mac.yml +++ b/install/envs/mac.yml @@ -31,6 +31,9 @@ dependencies: - torchaudio - pytorch-lightning - numpy + - psutil + - kivy=2.0.0 + - plotly - pip: - tensorflow==2.2.0 - git+https://github.com/autorope/keras-vis.git diff --git a/install/envs/ubuntu.yml b/install/envs/ubuntu.yml index 20603688f..eccc6e18c 100644 --- a/install/envs/ubuntu.yml +++ b/install/envs/ubuntu.yml @@ -31,6 +31,9 @@ dependencies: - torchaudio - pytorch-lightning - numpy + - psutil + - kivy=2.0.0 + - plotly - tensorflow==2.2.0 - pip: - git+https://github.com/autorope/keras-vis.git diff --git a/install/envs/windows.yml b/install/envs/windows.yml index e030b5664..bf1cb522c 100644 --- a/install/envs/windows.yml +++ b/install/envs/windows.yml @@ -31,6 +31,8 @@ dependencies: - torchaudio - pytorch-lightning - numpy + - kivy=2.0.0 + - plotly - psutil - pip: - git+https://github.com/autorope/keras-vis.git diff --git a/scripts/convert_to_tub_v2.py b/scripts/convert_to_tub_v2.py index b57c137fc..1b20382e1 100755 --- a/scripts/convert_to_tub_v2.py +++ b/scripts/convert_to_tub_v2.py @@ -4,7 +4,7 @@ convert_to_tub_v2.py --tub= --output= Note: - This script converts the old datastore format, to the new datastore format. + This script converts the old datastore format to the new datastore format. ''' import json @@ -16,28 +16,30 @@ from PIL import Image from progress.bar import IncrementalBar -import donkeycar as dk from donkeycar.parts.datastore import Tub as LegacyTub from donkeycar.parts.tub_v2 import Tub def convert_to_tub_v2(paths, output_path): - empty_record = {'__empty__': True} + """ + Convert from old tubs to new one + :param paths: legacy tub paths + :param output_path: new tub output path + :return: None + """ + empty_record = {'__empty__': True} if type(paths) is str: paths = [paths] legacy_tubs = [LegacyTub(path) for path in paths] - output_tub = None print(f'Total number of tubs: {len(legacy_tubs)}') for legacy_tub in legacy_tubs: - if not output_tub: - # add input and type for empty records recording - inputs = legacy_tub.inputs + ['__empty__'] - types = legacy_tub.types + ['boolean'] - output_tub = Tub(output_path, inputs, types, - list(legacy_tub.meta.items())) - + # add input and type for empty records recording + inputs = legacy_tub.inputs + ['__empty__'] + types = legacy_tub.types + ['boolean'] + output_tub = Tub(output_path, inputs, types, + list(legacy_tub.meta.items())) record_paths = legacy_tub.gather_records() bar = IncrementalBar('Converting', max=len(record_paths)) previous_index = None @@ -54,22 +56,26 @@ def convert_to_tub_v2(paths, output_path): if not previous_index or current_index == previous_index + 1: output_tub.write_record(record) previous_index = current_index - # otherwise fill the gap with dummy records + # otherwise fill the gap with empty records else: # Skipping over previous record here because it has # already been written. previous_index += 1 # Adding empty record nodes, and marking them deleted # until the next valid record. + delete_list = [] while previous_index < current_index: idx = output_tub.manifest.current_index output_tub.write_record(empty_record) - output_tub.delete_record(idx) + delete_list.append(idx) previous_index += 1 + output_tub.delete_records(delete_list) bar.next() except Exception as exception: print(f'Ignoring record path {record_path}\n', exception) traceback.print_exc() + # writing session id into manifest metadata + output_tub.close() if __name__ == '__main__': diff --git a/scripts/freeze_model.py b/scripts/freeze_model.py index 35b317f15..410a4b276 100755 --- a/scripts/freeze_model.py +++ b/scripts/freeze_model.py @@ -19,13 +19,14 @@ output_path = Path(output) output_meta = Path('%s/%s.metadata' % (output_path.parent.as_posix(), output_path.stem)) +tf.compat.v1.disable_eager_execution() # Reset session tf.keras.backend.clear_session() tf.keras.backend.set_learning_phase(0) -model = tf.keras.models.load_model(in_model, compile=False) -session = tf.keras.backend.get_session() +model = tf.compat.v1.keras.models.load_model(in_model, compile=False) +session = tf.compat.v1.keras.backend.get_session() input_names = sorted([layer.op.name for layer in model.inputs]) output_names = sorted([layer.op.name for layer in model.outputs]) diff --git a/scripts/multi_train.py b/scripts/multi_train.py index 4a57e12b3..3c4f68bed 100755 --- a/scripts/multi_train.py +++ b/scripts/multi_train.py @@ -24,7 +24,7 @@ outfile.write('WEB_CONTROL_PORT = 888%d\n' % i) outfile.write('WEB_INIT_MODE = "local"\n') outfile.write('DONKEY_GYM = True\n') - outfile.write('DONKEY_GYM_ENV_NAME = "donkey-mountain-track-v0"\n') + outfile.write('DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0"\n') outfile.write('DONKEY_SIM_PATH = "remote"\n') outfile.write('SIM_HOST = "%s"\n' % host) iStyle = random.randint(0, len(body_styles) - 1) diff --git a/setup.py b/setup.py index 79e545862..3eee96b75 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ def package_files(directory, strip_leading): long_description = fh.read() setup(name='donkeycar', - version='4.1.0', + version='4.2.0', long_description=long_description, description='Self driving library for python.', url='https://github.com/autorope/donkeycar', @@ -48,7 +48,8 @@ def package_files(directory, strip_leading): "simple_pid", 'progress', 'typing_extensions', - 'pyfiglet' + 'pyfiglet', + 'psutil' ], extras_require={ 'pi': [ @@ -66,6 +67,7 @@ def package_files(directory, strip_leading): 'pc': [ 'matplotlib', 'imgaug', + 'kivy' ], 'dev': [ 'pytest', @@ -74,7 +76,7 @@ def package_files(directory, strip_leading): 'mypy' ], 'ci': ['codecov'], - 'tf': ['tensorflow>=2.2.0'], + 'tf': ['tensorflow==2.2.0'], 'torch': [ 'pytorch>=1.7.1', 'torchvision',