Create Makefile and unify test commands in it (blue-oil#194)

* Remove confusing config
* Create Makefile to make initializing easier
* Wrap all test tasks by Makefile
* Fix unreadable connfigs around docker

.dockerignore

@@ -10,4 +10,8 @@ saved/
 */saved/
 dataset/
 config/
-tmp/
+*.github/
+*tmp/
+*.tox/
+*docs/
+.firebase/

Makefile

@@ -0,0 +1,41 @@
+IMAGE_NAME:=blueoil_$$(id -un)
+BUILD_VERSION:=$(shell git describe --tags --always --dirty --match="v*" 2> /dev/null || cat $(CURDIR/.version 2> /dev/null || echo v0))
+DOCKER_OPT:=--runtime=nvidia
+
+default: build
+
+.PHONY: deps
+deps:
+	# Update dependencies
+	git submodule update --init --recursive
+
+.PHONY: build
+build: deps
+	# Build docker image
+	docker build -t $(IMAGE_NAME):$(BUILD_VERSION) --build-arg python_version="3.6.3" -f docker/Dockerfile .
+
+.PHONY: test
+test: build
+	# Run Blueoil test
+	CUDA_VISIBLE_DEVICES=$(CUDA_VISIBLE_DEVICES) bash ./blueoil_test.sh
+
+.PHONY: test-lmnet
+test-lmnet: build
+	# Run lmnet test with Python3.6 (only available on Jenkins)
+	docker run $(DOCKER_OPT) -v /storage/dataset:/storage/dataset -v lmnet_saved:/storage/lmnet/saved -e CUDA_VISIBLE_DEVICES=$(CUDA_VISIBLE_DEVICES) -e DATA_DIR=/storage/dataset --rm $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "cd lmnet; tox -e py36"
+
+.PHONY: test-dlk
+test-dlk: build
+	# Run dlk test (only available on Jenkins)
+	docker run --rm -t -v /root/.ssh:/root/.ssh --net=host $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "apt-get update && apt-get install -y iputils-ping && cd dlk && python setup.py test"
+
+.PHONY: pep8-dlk
+pep8-dlk:
+	# Check dlk PEP8
+	docker run --rm -t $(IMAGE_NAME):$(BUILD_VERSION) /bin/bash -c "cd dlk && pycodestyle --ignore=W --max-line-length=120 --exclude='*static/pb*','*docs/*','*.eggs*','*tvm/*','*tests/*','backends/*' ."
+
+.PHONY: clean
+clean:
+	# Clean created files
+	docker rmi  $(IMAGE_NAME):$(BUILD_VERSION)
+	rm -rf tmp/*

README.md

@@ -38,7 +38,7 @@ Please see the detail in the nvidia-docker's [prerequisites](https://github.com/
 ## Set up
 There are some submodules in this repositry, so you should run `git submodule update --init --recursive` after cloning or `git clone --recursive [this repository]`.
 ```
-./docker_build.sh
+make build
 ```
 Note: The private repository submodules are set to connect by ssh, if you want to use HTTPS, you should edit URLs in `.gitmodules` and run `git submodule sync` before `git submodule update --init --recursive` command. ([see how to edit](https://stackoverflow.com/a/30885128))
 
@@ -72,4 +72,4 @@ Arguments:
 ```
 
 ### Tests for CI
-* [Jenkins settings for CI](./.jenkins/README.md)
+* [Jenkins settings for CI](./tests/README.md)

blueoil.sh

@@ -74,7 +74,9 @@ NAME=$0 # Name of the script
 BASE_DIR=$(dirname $0)
 ABS_BASE_DIR=$(get_abs_path ${BASE_DIR})
 # Docker image of blueoil
-DOCKER_IMAGE=$(id -un)_blueoil:local_build
+IMAGE_NAME=blueoil_$(id -un)
+BUILD_VERSION=$(git describe --tags --always --dirty --match="v*" 2> /dev/null || cat $(CURDIR/.version 2> /dev/null || echo v0))
+DOCKER_IMAGE=$IMAGE_NAME:$BUILD_VERSION
 # Argument of path for docker needs to be absolute path.
 GUEST_HOME_DIR="/home/blueoil"
 GUEST_CONFIG_DIR="${GUEST_HOME_DIR}/config"

docker/Dockerfile

@@ -104,7 +104,7 @@ COPY lmnet/third_party third_party
 # https://github.com/cocodataset/cocoapi/blob/440d145a30b410a2a6032827c568cff5dc1d2abf/PythonAPI/setup.py#L2
 RUN cd third_party/coco/PythonAPI && pip install -e .
 
-# For development 
+# For development
 RUN apt-get update && apt-get install -y \
     x11-apps \
     imagemagick \
@@ -132,5 +132,4 @@ RUN chmod 777 /home/blueoil/dlk
 # Copy output template
 COPY output_template /home/blueoil/output_template
 
-# Set __init__.py for python modules
-RUN touch lmnet/lmnet/__init__.py
+ENV PYTHONPATH $PYTHONPATH:/home/blueoil:/home/blueoil/lmnet:/home/blueoil/dlk/python/dlk

docs/install/install.md

@@ -32,7 +32,7 @@ You need the following devices.
 ---
 
 ## Install / Uninstall Blueoil
-This section describes how to setup an environment to train a neural network and convert to FPGA ready format by Blueoil. 
+This section describes how to setup an environment to train a neural network and convert to FPGA ready format by Blueoil.
 
 ### Install (On server)
 
@@ -46,14 +46,14 @@ Clone the Blueoil repository with recursive option.
 
 You should run below to update some submodules if cloned without recursive option.
 
-    $ git submodule update --init --recursive
+    $ make deps
 
 #### Build the docker image including Blueoil libraries
 
     $ cd blueoil
-    $ ./docker_build.sh
+    $ make build
 
-After the build has succeeded, you can use the docker image named `[User ID]_blueoil:local_build`
+After the build has succeeded, you can use the docker image named `blueoil_[User ID]:[TAG based on commit tag]`
 
 ### Uninstall (On server)
 
@@ -77,7 +77,7 @@ Please set the empty microSD card (8GB+) into your PC and write the downloaded i
     $ cat de10nano_ubuntu.img.gz | gunzip | sudo dd of=/dev/mmcblk0 bs=4M
 
 Caution & Note:
-- In Linux 
+- In Linux
     - Please unmount the microSD with umount command before writing
     - /dev/xxx might have a different name, so please confirm the name in the target microSD
 - In macOS
@@ -142,7 +142,7 @@ Please unset the microSD from your host system after dd && sync operation has be
 
 ### Update some files for Blueoil (On your PC)
 
-We need to update some files on the microSD. 
+We need to update some files on the microSD.
 To update it, we need to perform some copy operations.
 
 REQUIRED_FILES are shown in the following list. These files are necessary in later step.
@@ -206,7 +206,7 @@ Third, update kernel modules with:
 
 ## Install required packages
 
-Login to FPGA board and update required packages. 
+Login to FPGA board and update required packages.
 
     $ apt-get update
     $ apt-get install python-dev python-setuptools python-pip unzip
@@ -215,4 +215,4 @@ Login to FPGA board and update required packages.
 
 ## Other Information
 
-- Network of the FPGA board is initially set up as DHCP network. 
+- Network of the FPGA board is initially set up as DHCP network.

lmnet/.gitignore

@@ -1,32 +1,18 @@
 # osx
 .DS_Store
 
-
 # python
 *.pyc
 __pycache__/
 
-
 # pyenv
 .python-version
 
-
 # project
 dataset/
 tmp/
 saved/
 
-
-# personal
-rsync.sh
-rsync.*.sh
-tags
-
-# ansible
-ansible/servers.retry
-ansible/dataset_master.retry
-ansible/env
-
 # doc
 _build/
 _static/

tests/README.md

@@ -0,0 +1,35 @@
+# CI Servers for Blueoil
+We have several CI Servers
+* With GPU/OpenDatasets : https://jenkins.blue-oil.org
+* With FPGA device : https://jenkins.leapmind.local:8080 (only for LeapMind local access for now)
+
+## Automated Test with PR
+The tests run automatically by making PR.
+
+The triggers to run tests are below.
+* Create PR
+* Push commit to branch related PR
+* Comment in PR
+    * Run all tests
+        * Comment `run test`
+    * Run the specified test individually
+        * blueoil test : `run blueoil test`
+        * lmnet test : `run lmnet test`
+        * dlk test : `run dlk test`
+
+
+## Test Jobs
+All `make` tasks should be made in repository root dir.
+
+### [blueoil] test blueoil entire workflow with GPU
+* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_main/configure
+* Script run by Jenkins : `make test`
+
+### [lmnet] test lmnet training part with GPU
+* Cofiguration URL : https://jenkins.blue-oil.org/job/blueoil_lmnet/configure
+* Script run by Jenkins : `make test-lmnet`
+
+### [dlk] test compiling and inference part on FPGA device
+* Cofiguration URL : http://jenkins.leapmind.local:8080/job/blueoil_dlk_test/configure
+* Now this test is available in LeapMind internal only because of the device limitation
+* Script run by Jenkins : `make test-dlk`