Merge pull request #1357 from IndustryEssentials/prerelease

[YMIR] Release-2.0
IndustryEssentials · Nov 8, 2022 · b6f948a · b6f948a
2 parents c8335e8 + f47d73e
commit b6f948a
Show file tree

Hide file tree

Showing 745 changed files with 52,862 additions and 24,390 deletions.
diff --git a/.env b/.env
@@ -61,3 +61,21 @@ ANONYMOUS_UUID=
 
 # nvidia for gpu, runc for cpu.
 SERVER_RUNTIME=nvidia
+
+# Dispatch training task to OPENPAI.
+OPENPAI_HOST=
+OPENPAI_TOKEN=
+OPENPAI_STORAGE=
+OPENPAI_USER=
+OPENPAI_CLUSTER=
+OPENPAI_GPUTYPE=
+
+# viewer
+MONGODB_HOST_PORT=27019
+MONGODB_USE_CACHE=True
+
+# model deployment module env
+# if you don't want to install model deployment module, set DEPLOY_MODULE_HOST_PORT to empty
+DEPLOY_MODULE_HOST_PORT=18801
+DEPLOY_MODULE_URL=${DEPLOY_MODULE_HOST_PORT}
+DEPLOY_MODULE_MYSQL_ROOT_PASSWORD=deploy_db_passwd
diff --git a/.github/workflows/backend-pr.yml b/.github/workflows/backend-pr.yml
@@ -1,12 +1,13 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-name: unittest
+name: backend-unittest
 
 on:
   pull_request:
     branches:
       - master
       - dev
+      - prerelease
 
 env:
   PYTHONPATH: ./ymir/backend
@@ -24,8 +25,17 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        pip install tox
+        time pip install tox
       working-directory: ymir/backend
+    - name: Cache Tox Environment
+      uses: actions/cache@v3
+      with:
+        path: |
+          ymir/backend/.tox
+          ymir/backend/.mypy_cache
+        # requirements.txt and requirements-dev.txt have versioning info that would
+        # impact the tox environment.
+        key: tox-${{ hashFiles('ymir/backend/requirements.txt', 'ymir/backend/requirements-dev.txt') }}
     - name: Test with tox
       run: |
         git config --global user.name 'CI-test'

diff --git a/.github/workflows/command-pr.yml b/.github/workflows/command-pr.yml
@@ -1,13 +1,14 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: uniitest
+name: command-uniitest
 
 on:
   pull_request:
     branches:
       - master
       - dev
+      - prerelease
 
 jobs:
   command-ci:
@@ -20,22 +21,29 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
+    - name: Cache Python Environment
+      uses: actions/cache@v3
+      with:
+        path: |
+          ymir/command/.mypy_cache
+          ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('ymir/command/requirements.txt', 'ymir/command/requirements-ci.txt') }}
     - name: Install dependencies
       run: |
-        pip install -r requirements.txt
-        pip install -r requirements-ci.txt
+        time pip install -r requirements.txt
+        time pip install -r requirements-ci.txt
       working-directory: ymir/command
     - name: mypy
       run: |
-        mypy mir
+        time mypy mir
       working-directory: ymir/command
     - name: flake8
       run: |
-        flake8 mir
+        time flake8 mir
       working-directory: ymir/command
     - name: Test with pytest
       run: |
         git config --global user.name 'CI-TEST'
         git config --global user.email 'ci-test@ymir-team'
-        python tests/__main__.py
+        time python tests/__main__.py
       working-directory: ymir/command
diff --git a/.github/workflows/executor-framework-pr.yml → .github/workflows/exc-sdk-pr.yml b/.github/workflows/executor-framework-pr.yml → .github/workflows/exc-sdk-pr.yml
@@ -8,11 +8,12 @@ on:
     branches:
       - master
       - dev
+      - prerelease
     paths:
       - 'docker_executor/sample_executor/**'
 
 jobs:
-  executor-framework-ci:
+  exc-sdk-ci:
 
     runs-on: ubuntu-latest
 
@@ -24,17 +25,17 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        pip install -r executor/requirements.txt
-        pip install -r executor/requirements-ci.txt
+        pip install -r ymir_exc/requirements.txt
+        pip install -r ymir_exc/requirements-ci.txt
       working-directory: docker_executor/sample_executor
     - name: mypy
       run: |
         mypy .
-      working-directory: docker_executor/sample_executor/executor
+      working-directory: docker_executor/sample_executor/ymir_exc
     - name: flake8
       run: |
         flake8 .
-      working-directory: docker_executor/sample_executor/executor
+      working-directory: docker_executor/sample_executor/ymir_exc
     - name: Test with pytest
       run: |
         python tests/__main__.py

diff --git a/.github/workflows/hel-pr.yml b/.github/workflows/hel-pr.yml
@@ -0,0 +1,48 @@
+name: hel-unittest
+
+on:
+  pull_request:
+    branches:
+      - master
+      - dev
+      - prerelease
+
+jobs:
+  hel-ci:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.18
+
+    - name: Cache Go Environment
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: Verify dependencies
+      working-directory: ymir/backend/src/ymir_hel
+      run: |
+        time go mod verify
+        time go install honnef.co/go/tools/cmd/staticcheck@latest
+
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v3
+      with:
+        version: v1.48
+        working-directory: ymir/backend/src/ymir_hel
+
+    - name: Lint and test code
+      working-directory: ymir/backend/src/ymir_hel
+      run: |
+        git config --global user.name 'CI-TEST'
+        git config --global user.email 'ci-test@ymir-team'
+        bash run_tests.sh
diff --git a/.github/workflows/web-pr.yml b/.github/workflows/web-pr.yml
@@ -5,7 +5,11 @@ name: Node.js CI
 
 on:
   pull_request:
-    branches: [ master, dev ]
+    branches:
+      - master
+      - dev
+      - prerelease
+
 
 jobs:
   web-ci:
@@ -14,14 +18,15 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Use Node.js 16.x
-      uses: actions/setup-node@v2
+    - name: Use Node.js 16.10
+      uses: actions/setup-node@v3
       with:
-        node-version: 16.x
+        # node-version: 16.12 or above has issue at caching.
+        node-version: 16.10
         cache: 'npm'
         cache-dependency-path: ymir/web/package-lock.json
     - name: Test with npm
       run: |
-        npm install
-        npm run test:coverage
+        time npm ci
+        time CI=true npm run test:coverage
       working-directory: ymir/web
diff --git a/.gitignore b/.gitignore
@@ -47,6 +47,9 @@ coverage.xml
 .hypothesis/
 venv/
 .python-version
+Library/
+.mypy_cache/
+.gitconfig
 
 # Translations
 *.mo
@@ -81,7 +84,11 @@ output/
 mysql/
 redis/
 ymir-data/
-ymir-workplace
+ymir-workplace*
 .mir_lock
 .local
 .pre-commit-config.yaml
+ymir/backend/alembic
+ymir/backend/static
+
+*/updater/app/mir/protos/*.pyi