From 0182bb89c331d76476031a4e920f9737b7dcead6 Mon Sep 17 00:00:00 2001 From: DocGarbanzo <47540921+DocGarbanzo@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:48:47 +0000 Subject: [PATCH] Release 5.0 and installation with pip (#1153) * Created setup.cfg and pyproject.toml for DC install. * Remove obsolete conda yml files. * Avoid moviepy import error and fix ui behavior of merging deleted indexes which was missing its implementation in the datastore. * Adding Pi user and hostname, as this is required by the ui. * Add missing opencv for RPi * include scripts folder in pypi package * Make torch import local in test * Fixed tubwriter test after updating of session_id to be a tuple * Froze nano versions for numpy * Added target macos for install on apple arm in setup.cfg * Set version to right version for main branch commit. * Remove obsolete argument in tub --- .github/workflows/python-package-conda.yml | 12 +- MANIFEST.in | 2 +- donkeycar/__init__.py | 3 +- donkeycar/management/makemovie.py | 73 ++++++++++-- donkeycar/parts/datastore_v2.py | 70 ++++++++--- donkeycar/parts/tub_v2.py | 3 +- donkeycar/templates/cfg_complete.py | 4 + donkeycar/tests/test_torch.py | 22 ++-- install/README.md | 10 -- install/envs/jetson.yml | 41 ------- install/envs/jetson46.yml | 34 ------ install/envs/mac.yml | 47 -------- install/envs/sagemaker.yml | 16 --- install/envs/ubuntu.yml | 46 -------- install/envs/windows.yml | 45 ------- install/nano/install-jp44.sh | 60 ---------- install/nano/requirements.txt | 28 ----- install/pi/install.sh | 46 -------- install/pi/opencv.sh | 19 --- install/pi/prepare.sh | 40 ------- pyproject.toml | 6 + scripts/profile.py | 2 +- setup.cfg | 102 +++++++++++++++- setup.py | 131 --------------------- 24 files changed, 246 insertions(+), 616 deletions(-) delete mode 100644 install/README.md delete mode 100644 install/envs/jetson.yml delete mode 100644 install/envs/jetson46.yml delete mode 100644 install/envs/mac.yml delete mode 100644 install/envs/sagemaker.yml delete mode 100644 install/envs/ubuntu.yml delete mode 100644 install/envs/windows.yml delete mode 100644 install/nano/install-jp44.sh delete mode 100644 install/nano/requirements.txt delete mode 100644 install/pi/install.sh delete mode 100644 install/pi/opencv.sh delete mode 100644 install/pi/prepare.sh create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index c168fab54..6b0311720 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,11 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - include: - - os: macos-latest - ENV_FILE: install/envs/mac.yml - - os: ubuntu-latest - ENV_FILE: install/envs/ubuntu.yml + os: ["macos-latest", "ubuntu-latest"] fail-fast: false defaults: run: @@ -28,16 +24,16 @@ jobs: python-version: 3.9 mamba-version: "*" activate-environment: donkey - environment-file: ${{matrix.ENV_FILE}} auto-activate-base: false channels: default, conda-forge, pytorch channel-priority: true - name: Conda info and list run: | - echo Environment file ${{matrix.ENV_FILE}} conda info conda list - name: Install donkey - run: pip install -e .[pc] + run: | + pip install -e .[pc,dev] + pip list - name: Run tests run: pytest diff --git a/MANIFEST.in b/MANIFEST.in index 2d41b7d62..b9013ea14 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include donkeycar/templates/* -include VERSION +include scripts recursive-include donkeycar/parts/web_controller/templates/ * diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 6338ce4dc..69d83cf52 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -2,9 +2,8 @@ import sys from pyfiglet import Figlet import logging -from pkg_resources import get_distribution -__version__ = get_distribution('donkeycar').version +__version__ = '5.1.dev0' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index d17e5f34b..df6106f55 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -1,13 +1,12 @@ -import moviepy.editor as mpy +import tempfile + from tensorflow.python.keras import activations from tensorflow.python.keras import backend as K +from tensorflow.python.keras.models import load_model import tensorflow as tf import cv2 from matplotlib import cm -try: - from vis.utils import utils -except: - raise Exception("Please install keras-vis: pip install git+https://github.com/autorope/keras-vis.git") + import donkeycar as dk from donkeycar.parts.tub_v2 import Tub @@ -17,6 +16,52 @@ DEG_TO_RAD = math.pi / 180.0 +def apply_modifications(model, custom_objects=None): + """Applies modifications to the model layers to create a new Graph. For + example, simply changing `model.layers[idx].activation = new activation` + does not change the graph. The entire graph needs to be updated with + modified inbound and outbound tensors because of change in layer building + function. + + Args: + model: The `keras.models.Model` instance. + + Returns: + The modified model with changes applied. Does not mutate the original + `model`. + """ + # The strategy is to save the modified model and load it back. This is + # done because setting the activation in a Keras layer doesnt actually + # change the graph. We have to iterate the entire graph and change the + # layer inbound and outbound nodes with modified tensors. This is doubly + # complicated in Keras 2.x since multiple inbound and outbound nodes are + # allowed with the Graph API. + model_path = os.path.join(tempfile.gettempdir(), + next(tempfile._get_candidate_names()) + '.h5') + try: + model.save(model_path) + return load_model(model_path, custom_objects=custom_objects) + finally: + os.remove(model_path) + + +def normalize(array, min_value=0., max_value=1.): + """Normalizes the numpy array to (min_value, max_value) + + Args: + array: The numpy array + min_value: The min value in normalized array (Default value = 0) + max_value: The max value in normalized array (Default value = 1) + + Returns: + The array normalized to range between (min_value, max_value) + """ + arr_min = np.min(array) + arr_max = np.max(array) + normalized = (array - arr_min) / (arr_max - arr_min + K.epsilon()) + return (max_value - min_value) * normalized + min_value + + class MakeMovie(object): def run(self, args, parser): @@ -24,6 +69,11 @@ def run(self, args, parser): Load the images from a tub and create a movie from them. Movie ''' + try: + import moviepy.editor as mpy + except ImportError as e: + logger.error(f'Please install moviepy first: {e}') + return if args.tub is None: print("ERR>> --tub argument missing.") @@ -40,15 +90,18 @@ def run(self, args, parser): if args.type is None and args.model is not None: args.type = self.cfg.DEFAULT_MODEL_TYPE - print("Model type not provided. Using default model type from config file") + print("Model type not provided. Using default model type from " + "config file") if args.salient: if args.model is None: - print("ERR>> salient visualization requires a model. Pass with the --model arg.") + print("ERR>> salient visualization requires a model. Pass " + "with the --model arg.") parser.print_help() if args.type not in ['linear', 'categorical']: - print("Model type {} is not supported. Only linear or categorical is supported for salient visualization".format(args.type)) + print(f"Model type {args.type} is not supported. Only linear " + f"or categorical is supported for salient visualization") parser.print_help() return @@ -181,7 +234,7 @@ def init_salient(self, model): for li in layer_idx: model.layers[li].activation = activations.linear # build salient model and optimizer - sal_model = utils.apply_modifications(model) + sal_model = apply_modifications(model) self.sal_model = sal_model return True @@ -210,7 +263,7 @@ def compute_visualisation_mask(self, img): channel_idx = 1 if K.image_data_format() == 'channels_first' else -1 grads = np.sum(grads, axis=channel_idx) - res = utils.normalize(grads)[0] + res = normalize(grads)[0] return res def draw_salient(self, img): diff --git a/donkeycar/parts/datastore_v2.py b/donkeycar/parts/datastore_v2.py index ac48bf503..c1ae56120 100644 --- a/donkeycar/parts/datastore_v2.py +++ b/donkeycar/parts/datastore_v2.py @@ -1,3 +1,4 @@ +import atexit import json import mmap import os @@ -7,7 +8,6 @@ logger = logging.getLogger(__name__) - NEWLINE = '\n' NEWLINE_STRIP = '\r\n' @@ -244,6 +244,7 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], self.catalog_metadata = dict() self.deleted_indexes = set() self._updated_session = False + self._is_closed = False has_catalogs = False if self.manifest_path.exists(): @@ -258,10 +259,12 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], self.manifest_metadata['created_at'] = created_at if not self.base_path.exists(): self.base_path.mkdir(parents=True, exist_ok=True) - logger.info(f'Created a new datastore at' + logger.info(f'Creating a new datastore at' f' {self.base_path.as_posix()}') self.seekeable = Seekable(self.manifest_path, read_only=self.read_only) + logger.info(f'Creating a new manifest at ' + f'{self.manifest_path.as_posix()}') if not has_catalogs: self._write_contents() @@ -275,7 +278,14 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], start_index=self.current_index) # Create a new session_id, which will be added to each record in the # tub, when Tub.write_record() is called. - self.session_id = self.create_new_session() + self.session_id = self.create_new_session_id() + + def exit_hook(): + if not self._is_closed: + logger.error(f"Unexpected closing manifest {self.base_path}") + self.close() + # Automatically save config when program ends + atexit.register(exit_hook) def write_record(self, record): new_catalog = self.current_index > 0 \ @@ -300,8 +310,8 @@ def delete_records(self, record_indexes): self.deleted_indexes.update(record_indexes) self._update_catalog_metadata(update=True) if record_indexes: - logger.info(f'Deleted records {min(record_indexes)} - ' - f'{max(record_indexes)}') + logger.info(f'Deleting {len(record_indexes)} records: ' + f'{min(record_indexes)} - {max(record_indexes)}') def restore_records(self, record_indexes): # Does not actually delete the record, but marks it as deleted. @@ -339,8 +349,19 @@ def _read_metadata(self, metadata=[]): def _read_contents(self): self.seekeable.seek_line_start(1) - self.inputs = json.loads(self.seekeable.readline()) - self.types = json.loads(self.seekeable.readline()) + manifest_inputs = json.loads(self.seekeable.readline()) + manifest_types = json.loads(self.seekeable.readline()) + if not self.inputs and not self.types: + self.inputs = manifest_inputs + self.types = manifest_types + else: + assert self.inputs == manifest_inputs \ + and self.types == manifest_types, \ + f'Trying to create a tub with different inputs/types than ' \ + f'the stored tub. This is only allowed when new tub ' \ + f'specifies no inputs. New inputs: {self.inputs} vs ' \ + f'stored inputs: {manifest_inputs}, new types {self.types}'\ + f' vs stored types: {manifest_types}' self.metadata = json.loads(self.seekeable.readline()) self.manifest_metadata = json.loads(self.seekeable.readline()) # Catalog metadata @@ -370,22 +391,29 @@ def _update_catalog_metadata(self, update=True): self.catalog_metadata = catalog_metadata self.seekeable.writeline(json.dumps(catalog_metadata)) - def create_new_session(self): + def _update_session_info(self): """ Creates a new session id and appends it to the metadata.""" sessions = self.manifest_metadata.get('sessions', {}) - last_id = -1 - if sessions: - last_id = sessions['last_id'] - else: + if not sessions: sessions['all_full_ids'] = [] - this_id = last_id + 1 - date = time.strftime('%y-%m-%d') - this_full_id = date + '_' + str(this_id) + this_id, this_full_id = self.session_id sessions['last_id'] = this_id sessions['last_full_id'] = this_full_id sessions['all_full_ids'].append(this_full_id) self.manifest_metadata['sessions'] = sessions - return this_full_id + + def create_new_session_id(self): + """ Creates a new session id and appends it to the metadata.""" + sessions = self.manifest_metadata.get('sessions', {}) + new_id = sessions['last_id'] + 1 if sessions else 0 + new_full_id = f"{time.strftime('%y-%m-%d')}_{new_id}" + return new_id, new_full_id + + def add_deleted_indexes(self, indexes): + if isinstance(indexes, int): + indexes = {indexes} + self.deleted_indexes.update(indexes) + self._update_catalog_metadata(update=True) def close(self): """ Closing tub closes open files for catalog, catalog manifest and @@ -393,9 +421,17 @@ def close(self): # If records were received, write updated session_id dictionary into # the metadata, otherwise keep the session_id information unchanged if self._updated_session: - self.seekeable.update_line(4, json.dumps(self.manifest_metadata)) + logger.info(f'Saving new session {self.session_id[1]}') + self._update_session_info() + self.write_metadata() self.current_catalog.close() self.seekeable.close() + self._is_closed = True + logger.info(f'Closing manifest {self.base_path}') + + def write_metadata(self): + self.seekeable.update_line(3, json.dumps(self.metadata)) + self.seekeable.update_line(4, json.dumps(self.manifest_metadata)) def __iter__(self): return ManifestIterator(self) diff --git a/donkeycar/parts/tub_v2.py b/donkeycar/parts/tub_v2.py index 1a804379b..d33b353ed 100644 --- a/donkeycar/parts/tub_v2.py +++ b/donkeycar/parts/tub_v2.py @@ -78,8 +78,7 @@ def write_record(self, record=None): # Private properties contents['_timestamp_ms'] = int(round(time.time() * 1000)) contents['_index'] = self.manifest.current_index - contents['_session_id'] = self.manifest.session_id - + contents['_session_id'] = self.manifest.session_id[1] self.manifest.write_record(contents) def delete_records(self, record_indexes): diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index ac790f202..464e8837f 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -760,3 +760,7 @@ # FPS counter SHOW_FPS = False FPS_DEBUG_INTERVAL = 10 # the interval in seconds for printing the frequency info into the shell + +# PI connection +PI_USERNAME = "pi" +PI_HOSTNAME = "donkeypi.local" diff --git a/donkeycar/tests/test_torch.py b/donkeycar/tests/test_torch.py index 5f9cb38ad..3137e50dd 100644 --- a/donkeycar/tests/test_torch.py +++ b/donkeycar/tests/test_torch.py @@ -2,14 +2,7 @@ import tarfile import os import platform -from collections import defaultdict, namedtuple - -import torch -import pytorch_lightning as pl -from donkeycar.parts.pytorch.torch_train import train -from donkeycar.parts.pytorch.torch_data import TorchTubDataModule -from donkeycar.parts.pytorch.torch_utils import get_model_by_type - +from collections import namedtuple from donkeycar.config import Config Data = namedtuple('Data', ['type', 'name', 'convergence', 'pretrained']) @@ -68,6 +61,8 @@ def test_train(config: Config, car_dir: str, data: Data) -> None: :param data: test case data :return: None """ + from donkeycar.parts.pytorch.torch_train import train + def pilot_path(name): pilot_name = f'pilot_{name}.ckpt' return os.path.join(car_dir, 'models', pilot_name) @@ -95,6 +90,11 @@ def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ :param tub_dir: tub directory (car_dir/tub) :return: None """ + import torch + import pytorch_lightning as pl + from donkeycar.parts.pytorch.torch_data import TorchTubDataModule + from donkeycar.parts.pytorch.torch_utils import get_model_by_type + model = get_model_by_type( model_type, config, checkpoint_path=None) @@ -120,11 +120,11 @@ def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ x, y = batch # In order to use a model pre-trained on ImageNet, the image # will be re-sized to 3x224x224 regardless of what the user chooses. - assert(x.shape == (config.BATCH_SIZE, 3, 224, 224)) - assert(y.shape == (config.BATCH_SIZE, 2)) + assert x.shape == (config.BATCH_SIZE, 3, 224, 224), "shape mismatch" + assert y.shape == (config.BATCH_SIZE, 2), "shape mismatch" break # Check inference val_x, val_y = next(iter(data_module.val_dataloader())) output = model(val_x) - assert(output.shape == (config.BATCH_SIZE, 2)) + assert output.shape == (config.BATCH_SIZE, 2), "shape mismatch" diff --git a/install/README.md b/install/README.md deleted file mode 100644 index 6952850c3..000000000 --- a/install/README.md +++ /dev/null @@ -1,10 +0,0 @@ -### Install on raspberry pi -The easiest way to get donkey running on a pi is with -a prebuilt disk image. To create your own disk -image you can use the scripts in /pi. - - -### Install on other systems -Create a conda environment using the env files - - diff --git a/install/envs/jetson.yml b/install/envs/jetson.yml deleted file mode 100644 index c27b4db2d..000000000 --- a/install/envs/jetson.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - fastai - -dependencies: - - python=3.8 - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - numpy - - psutil - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - - pip: - - --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v502 - - tensorflow==2.9.1+nv22.09 - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - kivy diff --git a/install/envs/jetson46.yml b/install/envs/jetson46.yml deleted file mode 100644 index 1a1740ea9..000000000 --- a/install/envs/jetson46.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - fastai - -dependencies: - - python=3.9 - - h5py - - pillow - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - numpy - - psutil - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - diff --git a/install/envs/mac.yml b/install/envs/mac.yml deleted file mode 100644 index be4f9844b..000000000 --- a/install/envs/mac.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - numpy - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning>=1.9,<2.0 - - psutil - - kivy=2.1 - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - tensorflow==2.9 - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - moviepy - diff --git a/install/envs/sagemaker.yml b/install/envs/sagemaker.yml deleted file mode 100644 index f1b3d82c0..000000000 --- a/install/envs/sagemaker.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: donkey -dependencies: - -- h5py -- pillow -- tensorflow-gpu -- opencv -- matplotlib -- tornado -- docopt -- pandas - -- pip: - - moviepy - - paho-mqtt - - PrettyTable diff --git a/install/envs/ubuntu.yml b/install/envs/ubuntu.yml deleted file mode 100644 index a43798092..000000000 --- a/install/envs/ubuntu.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - numpy - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning>=1.9,<2.0 - - psutil - - plotly - - pyyaml - - tensorflow=2.9 - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - kivy==2.1 - - moviepy diff --git a/install/envs/windows.yml b/install/envs/windows.yml deleted file mode 100644 index 4a0d89d2a..000000000 --- a/install/envs/windows.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning - - numpy - - kivy=2.1 - - plotly - - pyyaml - - psutil - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - git+https://github.com/autorope/keras-vis.git - - simple-pid diff --git a/install/nano/install-jp44.sh b/install/nano/install-jp44.sh deleted file mode 100644 index 1cb8f3cec..000000000 --- a/install/nano/install-jp44.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -e - -password='jetson' -# Get the full dir name of this script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# Keep updating the existing sudo time stamp -sudo -v -while true; do sudo -n true; sleep 120; kill -0 "$$" || exit; done 2>/dev/null & - -######################################## -# Install DonkeyCar -######################################## -sudo apt-get update -y -sudo apt-get upgrade -y -sudo apt-get install -y libhdf5-serial-dev hdf5-tools libhdf5-dev zlib1g-dev zip libjpeg8-dev liblapack-dev libblas-dev gfortran -sudo apt-get install -y python3-dev python3-pip -sudo apt-get install -y libxslt1-dev libxml2-dev libffi-dev libcurl4-openssl-dev libssl-dev libpng-dev libopenblas-dev -sudo apt-get install -y git -sudo apt-get install -y openmpi-doc openmpi-bin libopenmpi-dev libopenblas-dev - -# Install Tensorflow as system package -sudo -H pip3 install -r requirements.txt -#sudo -H pip3 install --pre --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v44 'tensorflow>2' -sudo -H pip3 install --pre --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v44 tensorflow==2.2.0+nv20.6 - -######################################## -# Install PyTorch v1.7 - torchvision v0.8.1 -# pytorch 1.6.0-rc2 -# https://forums.developer.nvidia.com/t/pytorch-for-jetson-nano-version-1-5-0-now-available/72048/392 -wget https://nvidia.box.com/shared/static/wa34qwrwtk9njtyarwt5nvo6imenfy26.whl -O torch-1.7.0-cp36-cp36m-linux_aarch64.whl -sudo -H pip3 install ./torch-1.7.0-cp36-cp36m-linux_aarch64.whl - -# Install PyTorch Vision -sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libavcodec-dev libavformat-dev libswscale-dev -mkdir -p ~/projects; cd ~/projects -git clone --branch v0.8.1 https://github.com/pytorch/vision torchvision -cd torchvision -export BUILD_VERSION=0.8.1 -sudo python3 setup.py install - - -# Create virtual enviroment -pip3 install virtualenv -python3 -m virtualenv -p python3 ~/.virtualenv/donkeycar --system-site-packages -echo "source ~/.virtualenv/donkeycar/bin/activate" >> ~/.bashrc -# "source ~/.virtualenv/donkeycar/bin/activate" in the shell script -. ~/.virtualenv/donkeycar/bin/activate - - -# Install DonkeyCar as user package -cd ~/projects -git clone https://github.com/autorope/donkeycar -cd donkeycar -git checkout dev -pip install -e .[nano] - -# https://github.com/keras-team/keras-tuner/issues/317 -echo "export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1" >> ~/.bashrc -#export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1 \ No newline at end of file diff --git a/install/nano/requirements.txt b/install/nano/requirements.txt deleted file mode 100644 index daa2dc778..000000000 --- a/install/nano/requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -pip -testresources -setuptools -futures -protobuf -pybind11 -cython -numpy -future -mock -h5py==2.10.0 -keras_preprocessing -keras_applications -gast -grpcio -absl-py -py-cpuinfo -psutil -portpicker -six -requests -astor -termcolor -wrapt -google-pasta -scipy -pandas -gdown \ No newline at end of file diff --git a/install/pi/install.sh b/install/pi/install.sh deleted file mode 100644 index bdf69f237..000000000 --- a/install/pi/install.sh +++ /dev/null @@ -1,46 +0,0 @@ -# Script to install everything needed for donkeycar except the donkeycar library - -#standard updates (5 min) -sudo apt update -y -sudo apt upgrade -y -sudo rpi-update -y - -#helpful libraries (2 min) -sudo apt install build-essential python3-dev python3-distlib python3-setuptools python3-pip python3-wheel -y - -sudo apt-get install git cmake pkg-config -y -sudo apt-get install libjpeg-dev libtiff5-dev libjasper-dev libpng12-dev -y -sudo apt-get install libavcodec-dev libavformat-dev libswscale-dev libv4l-dev -y -sudo apt-get install libxvidcore-dev libx264-dev -y -sudo apt-get install libatlas-base-dev gfortran -y - -sudo apt install libzmq-dev -y -sudo apt install xsel xclip -y -sudo apt install python3-h5py -y - -#install numpy and pandas (3 min) -sudo apt install libxml2-dev python3-lxml -y -sudo apt install libxslt-dev -y - -#remove python2 (1 min) -sudo apt-get remove python2.7 -y -sudo apt-get autoremove -y - -#install redis-server (1 min) -sudo apt install redis-server -y - -#create a python virtualenv (2 min) -sudo apt install virtualenv -y -virtualenv ~/env --system-site-packages --python python3 -echo '#start env' >> ~/.bashrc -echo 'source ~/env/bin/activate' >> ~/.bashrc -source ~/env/bin/activate - - -#make sure the virtual environment is active -source ~/env/bin/activate - -# install pandas and numpy and Adafruit CircuitPython -pip install pandas #also installs numpy - -pip install tensorflow==1.9 \ No newline at end of file diff --git a/install/pi/opencv.sh b/install/pi/opencv.sh deleted file mode 100644 index e6008a72c..000000000 --- a/install/pi/opencv.sh +++ /dev/null @@ -1,19 +0,0 @@ -#install opencv (1 hour) -#instructions from:https://raspberrypi.stackexchange.com/questions/69169/how-to-install-opencv-on-raspberry-pi-3-in-raspbian-jessie - -# NOTE: this gets the dev version. Use tags to get specific version -git clone --branch 3.4.1 --depth 1 https://github.com/opencv/opencv.git -git clone --branch 3.4.1 --depth 1 https://github.com/opencv/opencv_contrib.git - -cd ~/opencv -mkdir build -cd build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D INSTALL_C_EXAMPLES=OFF \ - -D INSTALL_PYTHON_EXAMPLES=OFF \ - -D OPENCV_EXTRA_MODULES_PATH=~/opencv_contrib/modules \ - -D BUILD_EXAMPLES=OFF .. -make -j4 -sudo make install -sudo ldconfig \ No newline at end of file diff --git a/install/pi/prepare.sh b/install/pi/prepare.sh deleted file mode 100644 index b1698f14f..000000000 --- a/install/pi/prepare.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -#Script to setup pi disk after base rpi image. - -MEDIA=/media/$USER -BOOT=$MEDIA/boot -ROOTFS=$MEDIA/rootfs - -sudo touch $BOOT/ssh - -#enable camera -echo "start_x=1" | sudo tee -a $BOOT/config.txt -echo "gpu_mem=128" | sudo tee -a $BOOT/config.txt - -#enable i2c -echo "i2c-dev" | sudo tee -a $ROOTFS/etc/modules - -#Change hostname -hostn=$(cat $ROOTFS/etc/hostname) -echo "Existing hostname is $hostn" -echo "Enter new hostname: " -read newhost - -#change hostname in /etc/hosts & /etc/hostname -sudo sed -i "s/$hostn/$newhost/g" $ROOTFS/etc/hosts -sudo sed -i "s/$hostn/$newhost/g" $ROOTFS/etc/hostname -echo "Your new hostname is $newhost" - -# setup default wifi config -sudo truncate -s 0 $ROOTFS/etc/wpa_supplicant/wpa_supplicant.conf -cat <<'EOF' | sudo tee $ROOTFS/etc/wpa_supplicant/wpa_supplicant.conf -country=US -ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev -update_config=1 - -network={ - ssid="network_name" - psk="password" -} -EOF \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..145c91223 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + + + diff --git a/scripts/profile.py b/scripts/profile.py index c6d48f3b7..cee6ca696 100755 --- a/scripts/profile.py +++ b/scripts/profile.py @@ -42,4 +42,4 @@ def profile(model_path, model_type): if __name__ == '__main__': args = docopt(__doc__) - profile(model_path = args['--model'], model_type = args['--type']) + profile(model_path=args['--model'], model_type=args['--type']) diff --git a/setup.cfg b/setup.cfg index b88034e41..294f98074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,102 @@ [metadata] -description-file = README.md +name = donkeycar_pypi +version = attr: donkeycar.__version__ +author = Will Roscoe, Adam Conway, Tawn Kramer +url = https://github.com/autorope/donkeycar +description = Self driving library for python. +long_description = file: README.md +long_description_content_type = text/markdown +keywords = selfdriving cars donkeycar diyrobocars +license = MIT +classifiers = + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + Development Status :: 4 - Beta + # Indicate who your project is intended for + Intended Audience :: Developers + Topic :: Scientific/Engineering :: Artificial Intelligence + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +packages = find_namespace: +zip_safe = True +include_package_data = True +python_requires = >=3.8,<4 +install_requires = + numpy + pillow + docopt + tornado + requests + PrettyTable + paho-mqtt + simple_pid + progress + typing_extensions + pyfiglet + psutil + pynmea2 + pyserial + utm + pandas + pyyaml + +[options.extras_require] +pi = + picamera2 + Adafruit_PCA9685 + adafruit-circuitpython-ssd1306 + adafruit-circuitpython-rplidar + RPi.GPIO + tensorflow-aarch64==2.9.3 + opencv-contrib-python + +nano = + Adafruit_PCA9685 + adafruit-circuitpython-ssd1306 + adafruit-circuitpython-rplidar + Jetson.GPIO + numpy==1.23 + matplotlib==3.7 + kivy==2.1 + plotly + pandas==2.0 + +pc = + tensorflow==2.9 + matplotlib + kivy==2.1 + pandas + plotly + albumentations + +macos = + tensorflow-macos==2.9 + matplotlib + kivy==2.1 + pandas + plotly + albumentations + +dev = + pytest + pytest-cov + responses + mypy + +torch = + pytorch + torchvision==0.12 + torchaudio + fastai + + +[options.package_data] +* = *.html, *.ini, *.txt, *.kv + +[options.entry_points] +console_scripts = + donkey = donkeycar.management.base:execute_from_command_line diff --git a/setup.py b/setup.py deleted file mode 100644 index ca9a2b639..000000000 --- a/setup.py +++ /dev/null @@ -1,131 +0,0 @@ -import os - -from setuptools import find_packages, setup - - -# include the non python files -def package_files(directory, strip_leading): - paths = [] - for (path, directories, filenames) in os.walk(directory): - for filename in filenames: - package_file = os.path.join(path, filename) - paths.append(package_file[len(strip_leading):]) - return paths - - -car_templates = ['templates/*'] -web_controller_html = package_files('donkeycar/parts/controllers/templates', - 'donkeycar/') - -extra_files = car_templates + web_controller_html -print('extra_files', extra_files) - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup(name='donkeycar', - version="5.0.dev3", - long_description=long_description, - description='Self driving library for python.', - url='https://github.com/autorope/donkeycar', - author='Will Roscoe, Adam Conway, Tawn Kramer', - author_email='wroscoe@gmail.com, adam@casaconway.com, tawnkramer@gmail.com', - license='MIT', - entry_points={ - 'console_scripts': [ - 'donkey=donkeycar.management.base:execute_from_command_line', - ], - }, - install_requires=[ - 'numpy', - 'pillow', - 'docopt', - 'tornado', - 'requests', - 'h5py', - 'PrettyTable', - 'paho-mqtt', - "simple_pid", - 'progress', - 'typing_extensions', - 'pyfiglet', - 'psutil', - "pynmea2", - 'pyserial', - "utm", - 'pandas', - 'pyyaml', - ], - extras_require={ - # if installing into a conda (i.e. miniforge) env on Pi we have to - # run 'sudo apt-get install libcap-dev' first. - 'pi': [ - 'picamera2', - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'RPi.GPIO', - 'tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl' - ], - 'nano45': [ - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'Jetson.GPIO', - 'matplotlib', - ], - 'nano': [ - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'Jetson.GPIO', - 'matplotlib', - 'kivy-jetson', - 'plotly' - ], - 'pc': [ - 'matplotlib', - 'kivy', - 'pandas', - 'plotly', - 'albumentations' - ], - 'dev': [ - 'pytest', - 'pytest-cov', - 'responses', - 'mypy' - ], - 'ci': ['codecov'], - 'tf': ['tensorflow==2.9'], - 'torch': [ - 'pytorch', - 'torchvision==0.12', - 'torchaudio', - 'fastai' - ], - 'mm1': ['pyserial'] - }, - package_data={ - 'donkeycar': extra_files, - }, - include_package_data=True, - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 4 - Beta', - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - keywords='selfdriving cars donkeycar diyrobocars', - packages=find_packages(exclude=(['tests', 'docs', 'site', 'env'])), - )