Merge pull request #681 from IndustryEssentials/dev

README.md

@@ -226,6 +226,11 @@ Execute the start command after the modification: `bash ymir.sh start`.
 
 4. After the service successfully started, YMIR will be available at [http://localhost:12001/](http://localhost:12001/). If you need to **stop the service**, run the command: `bash ymir.sh stop`
 
+5. The default initial user is super administrator, you can check account and password through the .env file under the project path and modify it before deployment. It is recommended to change the password through the user management interface after the service deployment is completed.
+<div align="left">
+  <img src="https://github.com/IndustryEssentials/ymir-images/blob/main/doc_images/first_admin.png" width="600"/>
+  <div>&nbsp;</div>
+
 ## 2.3. Installation of **Label Studio** (optional)
 
 **Label Sudio** is also an external labeling system supported by YMIR and can be installed as an alternative labeling tool.
@@ -337,9 +342,9 @@ Users can download the example **Sample.zip** for reference as follows:
 
 1. Download the open-source dataset VOC2012 ([Click to download VOC2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar)) and unzip it. Change the folder name as required, and then compressing them separately into zip packages that meet the import requirements.
 
-2. Place dataset VOC2012 under ymir-workplace/importing_pic.
+2. Place dataset VOC2012 under `ymir-workplace/ymir-sharing`.
 
-3. Select 'path import' and enter the absolute path address of the dataset in the server: /data/sharing/voc2012, as shown in the figure below:
+3. Select 'path import' and enter the absolute path of the dataset in the server: `/ymir-sharing/voc2012`, as shown in the figure below:
 
 ![path import](https://github.com/IndustryEssentials/ymir-images/blob/main/doc_images/path%20import.jpg)
 

README_zh-CN.md

@@ -224,6 +224,11 @@ LABEL_TOOL_HOST_PORT=set_your_label_tool_HOST_PORT
 
 4. 服务启动成功后，默认配置端口为12001，可以直接访问 [http://localhost:12001/](http://localhost:12001/)  显示登录界面即安装成功。如果需要**停止服务**，运行命令为：`bash ymir.sh stop`
 
+5. 默认初始用户权限为超级管理员，可以通过项目路径下.env文件查看账号密码，部署前可自行设置修改。建议在服务部署完成后，通过用户管理界面修改密码。
+<div align="left">
+  <img src="https://github.com/IndustryEssentials/ymir-images/blob/main/doc_images/first_admin.png" width="600"/>
+  <div>&nbsp;</div>
+
 ## 2.3. 安装配置LabelStudio （可选）
 
 label studio同时也是YMIR所支持的外接标注系统，可以作为备选标注工具安装。
@@ -320,9 +325,9 @@ LABEL_TOOL_TOKEN="Token token_value"
 
 1.通过在网络中下载开源数据集VOC2012([点击下载VOC2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar))，解压缩后按要求修改文件夹名称，再分别压缩为符合导入要求的zip包；
 
-2.把VOC2012放到ymir-workplace/importing_pic下面；
+2.把VOC2012放到 `ymir-workplace/ymir-sharing` 下面；
 
-3.选择路径导入，填上路径地址/data/sharing/voc2012_train。
+3.选择路径导入，填上路径地址`/ymir-sharing/voc2012`。
 
 完成初始数据集的导入后，点击【迭代数据准备】，完成对应的数据集和挖掘策略设置。其中训练集已设置为创建项目时默认的系统训练集，不可变更。
 

docker-compose.yml

@@ -12,19 +12,17 @@ services:
       - MODELS_PATH=${YMIR_PATH}/ymir-models
       - ASSETS_PATH=${YMIR_PATH}/ymir-assets
       - CONTROLLER_LOG_PATH=${YMIR_PATH}/ymir-data/logs
-      - DATA_SOURCE=/data/sharing
       # viz
       - VIZ_REDIS_URI=redis://:@viz-redis
       # app
       - DATABASE_URI=mysql+pymysql://${MYSQL_INITIAL_USER}:${MYSQL_INITIAL_PASSWORD}@db/ymir
       - GRPC_CHANNEL=127.0.0.1:50066
       - VIZ_HOST=127.0.0.1:9099
-      - SHARED_DATA_DIR=/data/sharing
+      - SHARED_DATA_DIR=/ymir-sharing
       - NGINX_DATA_PATH=/ymir-storage
       # arq
       - CHECK_INTERVAL_IN_SECONDS=30
       # monitor
-      - POSTMAN_URL=http://127.0.0.1:8090
       - MONITOR_URL=http://127.0.0.1:9098
       - APP_API_HOST=127.0.0.1:80
     volumes:
@@ -40,7 +38,7 @@ services:
       - /lib64/libltdl.so.7:/lib64/libltdl.so.7
       # app
       - ${YMIR_PATH}/ymir-storage:/ymir-storage
-      - ${YMIR_PATH}/importing_pic:/data/sharing
+      - ${YMIR_PATH}/ymir-sharing:/ymir-sharing
       # tensorboard
       - "${TENSORBOARD_ROOT}:${TENSORBOARD_ROOT}"
     depends_on:

docker_executor/public_index.md

@@ -2,6 +2,6 @@
 
 |docker_name|functions|contributor|organization|description|
 |--|--|--|--|--|
-|industryessentials/executor-det-yolov4-training:release-0.5.0|training|alfrat|-|yolov4 detection model training|
-|industryessentials/executor-det-yolov4-mining:release-0.5.0|mining inference|alfrat|-|yolov4 detection model mining & inference|
+|industryessentials/executor-det-yolov4-training:release-1.1.0|training|alfrat|-|yolov4 detection model training|
+|industryessentials/executor-det-yolov4-mining:release-1.1.0|mining inference|alfrat|-|yolov4 detection model mining & inference|
 

ymir.sh

@@ -149,11 +149,16 @@ docker-compose up -d
 }
 
 print_help() {
-    printf '\nUsage: \n  sh ymir.sh start/stop.\n'
+    printf '\nUsage: \n  bash ymir.sh start/stop.\n'
 }
 
 # main
 main() {
+    if [ "$EUID" -eq 0 ]
+        then echo "Error: using sudo, this will cause permission issue."
+        exit
+    fi
+
     if [[ $# -eq 0 ]]; then
         print_help
     else

ymir/Dockerfile.backend

@@ -14,6 +14,8 @@ RUN apt-get update \
   && ln -s /usr/bin/python3 python \
   && pip3 install -i ${PIP_SOURCE} --upgrade pip
 
+RUN git config --global core.fileMode false
+
 RUN pip install --no-cache-dir "uvicorn[standard]" gunicorn -i ${PIP_SOURCE}
 
 RUN mkdir -p /data/sharing/
@@ -23,7 +25,7 @@ COPY ./backend/requirements.txt ./
 RUN pip3 install -r requirements.txt -i ${PIP_SOURCE}
 
 COPY ./command /command
-RUN pip3 install -U /command
+RUN pip3 install -U /command -i ${PIP_SOURCE}
 
 COPY ./backend/src /app
 WORKDIR /app

ymir/backend/.bumpversion.cfg

@@ -1,3 +1,3 @@
 [bumpversion]
-current_version = 1.0.0
+current_version = 1.1.0
 commit = False

ymir/backend/deploy/supervisor/conf.d/ymir_service.ini

@@ -20,7 +20,7 @@ startsecs=50
 
 
 [program:viz_service]
-command=/bin/bash -c "cd ymir_viz&&gunicorn -k gevent -c gunicorn_conf.py wsgi:connexion_app"
+command=/bin/bash -c "cd ymir_viz && gunicorn -k gevent -c gunicorn_conf.py wsgi:connexion_app"
 numprocs=1
 autostart=true
 autorestart=true
@@ -83,7 +83,7 @@ startsecs=50
 
 
 [program:app]
-command=/bin/bash -c "cd ymir_app&&sh prestart.sh&&sh start.sh"
+command=/bin/bash -c "cd ymir_app && sh prestart.sh && sh start.sh"
 numprocs=1
 autostart=true
 autorestart=true
@@ -103,48 +103,6 @@ stopasgroup=true
 startsecs=50
 
 
-[program:postman_app]
-command=/bin/bash -c "cd ymir_postman && uvicorn pm_app_server:app --port 8090 --host 0.0.0.0 --log-config uvicorn_log_config.json"
-numprocs=1
-autostart=true
-autorestart=true
-redirect_stderr=true          ; redirect proc stderr to stdout (default false)
-stdout_logfile=/app_logs/ymir_postman_app.log        ; stdout log path, NONE for none; default AUTO
-stdout_logfile_maxbytes=50MB   ; max # logfile bytes b4 rotation (default 50MB)
-stdout_logfile_backups=10     ; # of stdout logfile backups (0 means none, default 10)stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-stdout_events_enabled=false   ; emit events on stdout writes (default false)
-stdout_syslog=false           ; send stdout to syslog with process name (default false)
-stderr_logfile=/app_logs/ymir_postman_app.log        ; stderr log path, NONE for none; default AUTO
-stderr_logfile_maxbytes=50MB   ; max # logfile bytes b4 rotation (default 50MB)
-stderr_logfile_backups=10     ; # of stderr logfile backups (0 means none, default 10)
-stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-stderr_events_enabled=false   ; emit events on stderr writes (default false)
-stderr_syslog=false           ; send stderr to syslog with process name (default false)
-stopasgroup=true
-startsecs=50
-
-
-[program:postman]
-command=/bin/bash -c "cd ymir_postman && python pm_server.py"
-numprocs=1
-autostart=true
-autorestart=true
-redirect_stderr=true          ; redirect proc stderr to stdout (default false)
-stdout_logfile=/app_logs/ymir_postman.log        ; stdout log path, NONE for none; default AUTO
-stdout_logfile_maxbytes=50MB   ; max # logfile bytes b4 rotation (default 50MB)
-stdout_logfile_backups=10     ; # of stdout logfile backups (0 means none, default 10)stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-stdout_events_enabled=false   ; emit events on stdout writes (default false)
-stdout_syslog=false           ; send stdout to syslog with process name (default false)
-stderr_logfile=/app_logs/ymir_postman.log        ; stderr log path, NONE for none; default AUTO
-stderr_logfile_maxbytes=50MB   ; max # logfile bytes b4 rotation (default 50MB)
-stderr_logfile_backups=10     ; # of stderr logfile backups (0 means none, default 10)
-stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
-stderr_events_enabled=false   ; emit events on stderr writes (default false)
-stderr_syslog=false           ; send stderr to syslog with process name (default false)
-stopasgroup=true
-startsecs=50
-
-
 [program:monitor_crontab]
 command=python ymir_monitor/monitor/utils/crontab_job.py
 numprocs=1

ymir/backend/src/common/common_utils/percent_log_util.py

@@ -57,7 +57,7 @@ def write_percent_log(log_file: str,
             raise RuntimeError("Invalid log_file")
         content_list: List[str] = [tid, f"{datetime.now().timestamp():.6f}", str(percent), str(state.value)]
         if error_code and error_message:
-            content_list.extend([str(error_code), error_message])
+            content_list.extend([str(int(error_code)), error_message])
         content = '\t'.join(content_list)
         if msg:
             content = '\n'.join([content, msg])

ymir/backend/src/common/id_definition/error_codes.py

@@ -17,13 +17,16 @@ class CTLResponseCode(IntEnum):
     INVOKER_INIT_ERROR = 130601
     INVOKER_LABEL_TASK_UNKNOWN_ERROR = 130602
     INVOKER_LABEL_TASK_NETWORK_ERROR = 130603
+    INVOKER_HTTP_ERROR = 130604
+    INVOKER_UNKNOWN_ERROR = 130605
 
 
 @unique
 class VizErrorCode(IntEnum):
     GENERAL_ERROR = 140400
     BRANCH_NOT_EXISTS = 140401
     MODEL_NOT_EXISTS = 140402
+    DATASET_EVALUATION_NOT_EXISTS = 140403
     INTERNAL_ERROR = 140500
 
 
@@ -33,6 +36,7 @@ class MonitorErrorCode(IntEnum):
     DUPLICATE_TASK_ID = 150401
     PERCENT_LOG_FILE_ERROR = 150402
     PERCENT_LOG_WEIGHT_ERROR = 150403
+    PERCENT_LOG_PARSE_ERROR = 150404
     INTERNAL_ERROR = 150500
 
 
@@ -48,6 +52,7 @@ class APIErrorCode(IntEnum):
     FAILED_TO_DOWNLOAD = 110108
     INVALID_CONFIGURATION = 110109
     INVALID_SCOPE = 110110
+    FAILED_TO_PROCESS_PROTECTED_RESOURCES = 110111
 
     USER_NOT_FOUND = 110201
     USER_DUPLICATED_NAME = 110202
@@ -63,6 +68,7 @@ class APIErrorCode(IntEnum):
     DATASET_NOT_ACCESSIBLE = 110403
     DATASET_FAILED_TO_CREATE = 110404
     DATASET_PROTECTED_TO_DELETE = 110405
+    DATASETS_NOT_IN_SAME_GROUP = 110406
 
     ASSET_NOT_FOUND = 110501
 
@@ -114,3 +120,8 @@ class APIErrorCode(IntEnum):
     ITERATION_COULD_NOT_UPDATE_STAGE = 111704
 
     FAILED_TO_IMPORT_MODEL = 111801
+
+    REFUSE_TO_PROCESS_MIXED_OPERATIONS = 111901
+    FAILED_TO_EVALUATE = 111902
+    DATASET_EVALUATION_NOT_FOUND = 111903
+    MISSING_OPERATIONS = 111904

ymir/backend/src/common/proto/backend.proto

@@ -29,7 +29,7 @@ enum TaskType {
     TaskTypeCopyModel = 14;
     TaskTypeDatasetInfer = 15;
 
-    reserved 12;
+    reserved 12, 16;
 };
 
 enum LabelFormat {
@@ -71,6 +71,9 @@ enum RequestType {
     CMD_PULL_IMAGE = 16;
     CMD_GPU_INFO_GET = 17;
     CMD_SAMPLING = 18;
+    CMD_EVALUATE = 19;
+    CMD_REPO_CHECK = 20;
+    CMD_REPO_CLEAR = 21;
 
     // Sandbox path operation
     USER_LIST = 101;
@@ -120,6 +123,7 @@ message GeneralReq {
     }
     string task_parameters = 24;
     LabelCollection label_collection = 25;
+    EvaluateConfig evaluate_config = 26;
     ReqCreateTask req_create_task = 1001;
 
     reserved 17, 1002;
@@ -134,6 +138,7 @@ message GeneralResp {
     map<int32 , string> docker_image_config = 7;
     int32 available_gpu_counts = 8;
     LabelCollection label_collection = 9;
+    bool ops_ret = 10;
     RespCMDInference detection = 1001;
 
     reserved 5, 1000;
@@ -272,6 +277,14 @@ message Label {
     string update_time = 5;  // RFC 3339 date strings
 }
 
+message EvaluateConfig {
+    // confidence threshold, 0 to 1
+    float conf_thr = 1;
+    // from:to:step, to value is excluded (same as python range)
+    string iou_thrs_interval = 2;
+    // need pr curve in evaluation result, default is false
+    bool need_pr_curve = 3;
+}
 
 service mir_controller_service {
     /*