diff --git a/.dockerignore b/.dockerignore index f9520e3c6..51907712d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,4 +10,8 @@ saved/ */saved/ dataset/ config/ -tmp/ +*.github/ +*tmp/ +*.tox/ +*docs/ +.firebase/ diff --git a/.jenkins/README.md b/.jenkins/README.md deleted file mode 100644 index 5c96df1e9..000000000 --- a/.jenkins/README.md +++ /dev/null @@ -1,32 +0,0 @@ -## Jenkins Server -Here are Jenkins servers for CI -* With GPU : https://jenkins.blue-oil.org -* With FPGA : https://jenkins.leapmind.local:8080 (only for local access) - -## Auto Test with PR -This test will be run automatically by PR. - -The triggers to run tests are below. -* Create PR -* Push commit to branch related PR -* Comment in PR - * Run all tests - * Comment `run test` - * Run the specified test individually - * blueoil test : `run blueoil test` - * lmnet test : `run lmnet test` - * dlk test : `run dlk test` - - -## Test Jobs -### [blueoil] jenkins test -* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_main/configure -* Script run by Jenkins : `./.jenkins/test_blueoil.sh` - -### [lmnet] jenkins test -* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_lmnet/configure -* Script run by Jenkins : `./.jenkins/test_lmnet.sh` - -### [dlk] jenkins test -* Cofiguration URL : http://jenkins.leapmind.local:8080/job/blueoil_dlk_test/configure -* Script run by Jenkins : `./.jenkins/test_dlk.sh` diff --git a/.jenkins/test_blueoil.sh b/.jenkins/test_blueoil.sh deleted file mode 100755 index 9bec4f6a9..000000000 --- a/.jenkins/test_blueoil.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -xe - -# select GPU:0 -export CUDA_VISIBLE_DEVICES=0 - -echo "# build docker container" -./docker_build.sh - -echo "# run test" -./blueoil_test.sh \ No newline at end of file diff --git a/.jenkins/test_dlk.sh b/.jenkins/test_dlk.sh deleted file mode 100755 index fbf342eb2..000000000 --- a/.jenkins/test_dlk.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -xe - -echo "# build docker container" -./docker_build.sh - -echo "# run test" -DOCKER_OPTION="-e PYTHONPATH=python/dlk -v /root/.ssh:/root/.ssh --net=host" -docker run --rm -t ${DOCKER_OPTION} $(id -un)_blueoil:local_build /bin/bash -c \ - "apt-get update && apt-get install -y iputils-ping && cd dlk && python setup.py test" - -echo "# check PEP8" -docker run --rm -t $(id -un)_blueoil:local_build /bin/bash -c \ - "cd dlk && pycodestyle --ignore=W --max-line-length=120 --exclude='*static/pb*','*docs/*','*.eggs*','*tvm/*','*tests/*','backends/*' ." diff --git a/.jenkins/test_lmnet.sh b/.jenkins/test_lmnet.sh deleted file mode 100755 index e345b15bd..000000000 --- a/.jenkins/test_lmnet.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -xe - -# select GPU:1 -export CUDA_VISIBLE_DEVICES=1 - -cd lmnet - -echo "# build docker container" -docker-compose build --pull --no-cache tensorflow > /dev/null - -echo "# build coco PythonAPI" -docker-compose run --rm tensorflow bash -c "cd third_party/coco/PythonAPI && make -j8" - -echo "# run test with python3.5" -docker-compose run --rm tensorflow tox -e py35 diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..8e9b2775c --- /dev/null +++ b/Makefile @@ -0,0 +1,41 @@ +IMAGE_NAME:=blueoil_$$(id -un) +BUILD_VERSION:=$(shell git describe --tags --always --dirty --match="v*" 2> /dev/null || cat $(CURDIR/.version 2> /dev/null || echo v0)) +DOCKER_OPT:=--runtime=nvidia + +default: build + +.PHONY: deps +deps: + # Update dependencies + git submodule update --init --recursive + +.PHONY: build +build: deps + # Build docker image + docker build -t $(IMAGE_NAME):$(BUILD_VERSION) --build-arg python_version="3.6.3" -f docker/Dockerfile . + +.PHONY: test +test: build + # Run Blueoil test + CUDA_VISIBLE_DEVICES=$(CUDA_VISIBLE_DEVICES) bash ./blueoil_test.sh + +.PHONY: test-lmnet +test-lmnet: build + # Run lmnet test with Python3.6 (only available on Jenkins) + docker run $(DOCKER_OPT) -v /storage/dataset:/storage/dataset -v lmnet_saved:/storage/lmnet/saved -e CUDA_VISIBLE_DEVICES=$(CUDA_VISIBLE_DEVICES) -e DATA_DIR=/storage/dataset --rm $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "cd lmnet; tox -e py36" + +.PHONY: test-dlk +test-dlk: build + # Run dlk test (only available on Jenkins) + docker run --rm -t -v /root/.ssh:/root/.ssh --net=host $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "apt-get update && apt-get install -y iputils-ping && cd dlk && python setup.py test" + +.PHONY: pep8-dlk +pep8-dlk: + # Check dlk PEP8 + docker run --rm -t $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "cd dlk && pycodestyle --ignore=W --max-line-length=120 --exclude='*static/pb*','*docs/*','*.eggs*','*tvm/*','*tests/*','backends/*' ." + +.PHONY: clean +clean: + # Clean created files + docker rmi $(IMAGE_NAME):$(BUILD_VERSION) + rm -rf tmp/* diff --git a/README.md b/README.md index deff8a012..d2c9897db 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Please see the detail in the nvidia-docker's [prerequisites](https://github.com/ ## Set up There are some submodules in this repositry, so you should run `git submodule update --init --recursive` after cloning or `git clone --recursive [this repository]`. ``` -./docker_build.sh +make build ``` Note: The private repository submodules are set to connect by ssh, if you want to use HTTPS, you should edit URLs in `.gitmodules` and run `git submodule sync` before `git submodule update --init --recursive` command. ([see how to edit](https://stackoverflow.com/a/30885128)) @@ -72,4 +72,4 @@ Arguments: ``` ### Tests for CI -* [Jenkins settings for CI](./.jenkins/README.md) +* [Jenkins settings for CI](./tests/README.md) diff --git a/blueoil.sh b/blueoil.sh index e238c45a4..7395719a4 100755 --- a/blueoil.sh +++ b/blueoil.sh @@ -74,7 +74,9 @@ NAME=$0 # Name of the script BASE_DIR=$(dirname $0) ABS_BASE_DIR=$(get_abs_path ${BASE_DIR}) # Docker image of blueoil -DOCKER_IMAGE=$(id -un)_blueoil:local_build +IMAGE_NAME=blueoil_$(id -un) +BUILD_VERSION=$(git describe --tags --always --dirty --match="v*" 2> /dev/null || cat $(CURDIR/.version 2> /dev/null || echo v0)) +DOCKER_IMAGE=$IMAGE_NAME:$BUILD_VERSION # Argument of path for docker needs to be absolute path. GUEST_HOME_DIR="/home/blueoil" GUEST_CONFIG_DIR="${GUEST_HOME_DIR}/config" diff --git a/docker/Dockerfile b/docker/Dockerfile index 4cb634979..d988518a6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,7 @@ COPY lmnet/third_party third_party # https://github.com/cocodataset/cocoapi/blob/440d145a30b410a2a6032827c568cff5dc1d2abf/PythonAPI/setup.py#L2 RUN cd third_party/coco/PythonAPI && pip install -e . -# For development +# For development RUN apt-get update && apt-get install -y \ x11-apps \ imagemagick \ @@ -132,5 +132,4 @@ RUN chmod 777 /home/blueoil/dlk # Copy output template COPY output_template /home/blueoil/output_template -# Set __init__.py for python modules -RUN touch lmnet/lmnet/__init__.py +ENV PYTHONPATH $PYTHONPATH:/home/blueoil:/home/blueoil/lmnet:/home/blueoil/dlk/python/dlk diff --git a/docker_build.sh b/docker_build.sh deleted file mode 100755 index 0f92238ae..000000000 --- a/docker_build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -if [ -z "$(ls -A lmnet/third_party/coco)" ]; then - echo "seems submodule is not initialized correctly. Please execute following command, before re-run this script." - echo "git submodule update --init --recursive" - exit 1 -fi - -docker build -t $(id -un)_blueoil:local_build --build-arg python_version="3.6.3" -f docker/Dockerfile . - diff --git a/docs/install/install.md b/docs/install/install.md index aa1428da3..b8cc9dbdd 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -32,7 +32,7 @@ You need the following devices. --- ## Install / Uninstall Blueoil -This section describes how to setup an environment to train a neural network and convert to FPGA ready format by Blueoil. +This section describes how to setup an environment to train a neural network and convert to FPGA ready format by Blueoil. ### Install (On server) @@ -46,14 +46,14 @@ Clone the Blueoil repository with recursive option. You should run below to update some submodules if cloned without recursive option. - $ git submodule update --init --recursive + $ make deps #### Build the docker image including Blueoil libraries $ cd blueoil - $ ./docker_build.sh + $ make build -After the build has succeeded, you can use the docker image named `[User ID]_blueoil:local_build` +After the build has succeeded, you can use the docker image named `blueoil_[User ID]:[TAG based on commit tag]` ### Uninstall (On server) @@ -77,7 +77,7 @@ Please set the empty microSD card (8GB+) into your PC and write the downloaded i $ cat de10nano_ubuntu.img.gz | gunzip | sudo dd of=/dev/mmcblk0 bs=4M Caution & Note: -- In Linux +- In Linux - Please unmount the microSD with umount command before writing - /dev/xxx might have a different name, so please confirm the name in the target microSD - In macOS @@ -142,7 +142,7 @@ Please unset the microSD from your host system after dd && sync operation has be ### Update some files for Blueoil (On your PC) -We need to update some files on the microSD. +We need to update some files on the microSD. To update it, we need to perform some copy operations. REQUIRED_FILES are shown in the following list. These files are necessary in later step. @@ -206,7 +206,7 @@ Third, update kernel modules with: ## Install required packages -Login to FPGA board and update required packages. +Login to FPGA board and update required packages. $ apt-get update $ apt-get install python-dev python-setuptools python-pip unzip @@ -215,4 +215,4 @@ Login to FPGA board and update required packages. ## Other Information -- Network of the FPGA board is initially set up as DHCP network. +- Network of the FPGA board is initially set up as DHCP network. diff --git a/lmnet/.gitignore b/lmnet/.gitignore index 69c652b90..022541779 100644 --- a/lmnet/.gitignore +++ b/lmnet/.gitignore @@ -1,32 +1,18 @@ # osx .DS_Store - # python *.pyc __pycache__/ - # pyenv .python-version - # project dataset/ tmp/ saved/ - -# personal -rsync.sh -rsync.*.sh -tags - -# ansible -ansible/servers.retry -ansible/dataset_master.retry -ansible/env - # doc _build/ _static/ diff --git a/lmnet/lmnet/__init__.py b/lmnet/lmnet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..3cd32e2cb --- /dev/null +++ b/tests/README.md @@ -0,0 +1,35 @@ +# CI Servers for Blueoil +We have several CI Servers +* With GPU/OpenDatasets : https://jenkins.blue-oil.org +* With FPGA device : https://jenkins.leapmind.local:8080 (only for LeapMind local access for now) + +## Automated Test with PR +The tests run automatically by making PR. + +The triggers to run tests are below. +* Create PR +* Push commit to branch related PR +* Comment in PR + * Run all tests + * Comment `run test` + * Run the specified test individually + * blueoil test : `run blueoil test` + * lmnet test : `run lmnet test` + * dlk test : `run dlk test` + + +## Test Jobs +All `make` tasks should be made in repository root dir. + +### [blueoil] test blueoil entire workflow with GPU +* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_main/configure +* Script run by Jenkins : `make test` + +### [lmnet] test lmnet training part with GPU +* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_lmnet/configure +* Script run by Jenkins : `make test-lmnet` + +### [dlk] test compiling and inference part on FPGA device +* Cofiguration URL : http://jenkins.leapmind.local:8080/job/blueoil_dlk_test/configure +* Now this test is available in LeapMind internal only because of the device limitation +* Script run by Jenkins : `make test-dlk`