From 243455a422fcf226df7f53a5738804d1a5a016bd Mon Sep 17 00:00:00 2001 From: Lin Han Date: Fri, 27 Jan 2023 10:02:37 -0500 Subject: [PATCH 01/17] feat(ci): test on more py versions --- .github/workflows/cypress.yml | 4 ++-- .github/workflows/update_qr.yml | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/update_qr.yml diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 3bb7c95..d9baf4a 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -22,10 +22,10 @@ jobs: matrix: node-version: ["19.x"] - # python-version: ["3.11", "3.10", "3.9", "3.8", "3.7"] + python-version: ["3.11", "3.10", "3.9", "3.8", "3.7"] os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.11"] + # python-version: ["3.11"] # os: [ubuntu-latest] fail-fast: false diff --git a/.github/workflows/update_qr.yml b/.github/workflows/update_qr.yml deleted file mode 100644 index dac477f..0000000 --- a/.github/workflows/update_qr.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Update group qr -on: - schedule: - - cron: "0 12 * * 0" - - -jobs: - fail: - runs-on: ubuntu-latest - name: Update group QR code in readme ! - steps: - - name: Fail - run: Do it! From d45b0ce6e7f34ac2df5960c8fa8d4d40caff8740 Mon Sep 17 00:00:00 2001 From: Action Bot Date: Fri, 27 Jan 2023 15:03:53 +0000 Subject: [PATCH 02/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index af0b7dd..238d6e8 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.6 +1.0.7 From 5bbca73b7f99100e1128b76d976fc95f4d789561 Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 04:45:30 -0500 Subject: [PATCH 03/17] fix(doc): badge --- .github/workflows/build.yml | 2 +- doc/CN/README.md | 4 +--- paddlelabel/task/classification.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74d6dff..2b37181 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build PaddleLabel package +name: Build & E2E Tests on: push: diff --git a/doc/CN/README.md b/doc/CN/README.md index 1bfed8b..0195eb4 100644 --- a/doc/CN/README.md +++ b/doc/CN/README.md @@ -8,8 +8,6 @@ 飞桨智能标注，让标注快人一步 - -

@@ -19,7 +17,7 @@ - +

diff --git a/paddlelabel/task/classification.py b/paddlelabel/task/classification.py index e43e723..f7c9379 100644 --- a/paddlelabel/task/classification.py +++ b/paddlelabel/task/classification.py @@ -154,3 +154,16 @@ def multi_class_exporter(self, export_dir): # 4. export split self.export_split(osp.join(export_dir), tasks, new_paths) + + +class ProjectSubtypeSelector: + def add_q( + self, + question: str, + required: bool, + type: str, + choices: list[str], + tips: str, + depends_on: tuple[str, str], + ): + pass From 836f89b3227dd745fc647bc175c196bdf4ef9199 Mon Sep 17 00:00:00 2001 From: Action Bot Date: Sat, 28 Jan 2023 09:47:08 +0000 Subject: [PATCH 04/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index 238d6e8..b0f3d96 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.7 +1.0.8 From 7834b9a6a896fbcce2f4cfef7a0f62a7c3204ed2 Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 04:54:10 -0500 Subject: [PATCH 05/17] chore(version) --- .github/workflows/cypress.yml | 8 ++++++-- paddlelabel/version | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index d9baf4a..f685213 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -42,6 +42,7 @@ jobs: channels: conda-forge - name: Download built paddlelabel package + if: github.event_name == 'workflow_call' uses: actions/download-artifact@v3 with: name: PaddleLabel_built_package @@ -63,8 +64,11 @@ jobs: echo $bgid # download latest build - # curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip - # unzip PaddleLabel_built_package.zip + if [ ${{ github.event_name }} -eq workflow_call ] + then + curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip + unzip PaddleLabel_built_package.zip + fi # install latest build echo ======================================================== diff --git a/paddlelabel/version b/paddlelabel/version index b0f3d96..7dea76e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.8 +1.0.1 From e21c632ce13ec93673ecb08cf851244a6d5c288a Mon Sep 17 00:00:00 2001 From: Action Bot Date: Sat, 28 Jan 2023 10:24:26 +0000 Subject: [PATCH 06/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index 7dea76e..6d7de6e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.1 +1.0.2 From 8af6e05aa8e223bf42386d2b58b7298bde461c7a Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 05:39:16 -0500 Subject: [PATCH 07/17] fix(ci) --- .github/workflows/cypress.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index f685213..135ae5c 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -41,6 +41,8 @@ jobs: cache-env: true channels: conda-forge + - run: echo ${{ github.event_name }} + - name: Download built paddlelabel package if: github.event_name == 'workflow_call' uses: actions/download-artifact@v3 @@ -64,7 +66,7 @@ jobs: echo $bgid # download latest build - if [ ${{ github.event_name }} -eq workflow_call ] + if [ ${{ github.event_name }} -ne workflow_call ] then curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip unzip PaddleLabel_built_package.zip From e68cd92ef9d330af8eae9757210fa0cfbfdcaa2f Mon Sep 17 00:00:00 2001 From: Action Bot Date: Sat, 28 Jan 2023 10:40:49 +0000 Subject: [PATCH 08/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index 6d7de6e..21e8796 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.2 +1.0.3 From dfd0307c3bc5f3eb842c572b8487e16263c27dcc Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 05:45:14 -0500 Subject: [PATCH 09/17] fix(ci) --- .github/workflows/cypress.yml | 4 ++-- paddlelabel/version | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 135ae5c..d11f209 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -44,7 +44,7 @@ jobs: - run: echo ${{ github.event_name }} - name: Download built paddlelabel package - if: github.event_name == 'workflow_call' + if: github.event_name == 'push' uses: actions/download-artifact@v3 with: name: PaddleLabel_built_package @@ -66,7 +66,7 @@ jobs: echo $bgid # download latest build - if [ ${{ github.event_name }} -ne workflow_call ] + if [[ ${{ github.event_name }} != "workflow_call" ]] then curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip unzip PaddleLabel_built_package.zip diff --git a/paddlelabel/version b/paddlelabel/version index 21e8796..7dea76e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.3 +1.0.1 From 0f3ec5ac78687461175285fe89f23caaf0e29b7f Mon Sep 17 00:00:00 2001 From: Action Bot Date: Sat, 28 Jan 2023 10:46:40 +0000 Subject: [PATCH 10/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index 7dea76e..6d7de6e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.1 +1.0.2 From 729d131136b5499af82e788d53833342f779f93d Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 05:54:47 -0500 Subject: [PATCH 11/17] fix(ci) --- .github/workflows/build.yml | 22 +- .github/workflows/cypress.yml | 4 + paddlelabel/openapi.yml | 513 +++++++++++++++++----------------- paddlelabel/version | 2 +- 4 files changed, 275 insertions(+), 266 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b37181..8196044 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,13 +93,6 @@ jobs: version: $(cat paddlelabel/version) " >> $GITHUB_STEP_SUMMARY - - name: Bump version - id: bump_version - if: github.event_name == 'push' - run: | - python tool/bumpversion.py - git config --global user.email "bot@github.com" && git config --global user.name "Action Bot" - git add paddlelabel/version; git commit -m "bump version"; git push - name: Save built package uses: actions/upload-artifact@v3 @@ -112,3 +105,18 @@ jobs: needs: build # uses: PaddleCV-SIG/PaddleLabel/.github/workflows/cypress.yml@develop uses: ./.github/workflows/cypress.yml + + bump_version: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + needs: cypress_e2e_test + steps: + - name: Bump version + id: bump_version + if: github.event_name == 'push' + run: | + python tool/bumpversion.py + git config --global user.email "bot@github.com" && git config --global user.name "Action Bot" + git add paddlelabel/version; git commit -m "bump version"; git push diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index d11f209..c2aad92 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -72,9 +72,13 @@ jobs: unzip PaddleLabel_built_package.zip fi + pwd + ls + # install latest build echo ======================================================== pip uninstall paddlelabel -y + pip uninstall paddlelabel -y pip install paddlelabel-*-py3-none-any.whl # wait for bg job diff --git a/paddlelabel/openapi.yml b/paddlelabel/openapi.yml index 30a4a2a..6a973a9 100644 --- a/paddlelabel/openapi.yml +++ b/paddlelabel/openapi.yml @@ -1,34 +1,32 @@ openapi: 3.0.0 info: title: PaddleLabel API Specs - version: 1.0.0 - description: Web backend APIs for PP-Label + version: 1.0.2 + description: Web backend APIs for PaddleLabel contact: - name: PaddleLabel Team - url: "https://github.com/PaddleCV-SIG/PaddleLabel/issues" + name: PaddleLabel Developers + url: 'https://github.com/PaddleCV-SIG/PaddleLabel/issues' email: me@linhan.email servers: - url: /api description: Same origion - - url: "http://localhost:17995/api" - description: Local server paths: - "/projects": + /projects: parameters: - - $ref: "#/components/parameters/request_id" + - $ref: '#/components/parameters/request_id' get: tags: - Project - summary: "Read all projects, sort by last modify date" + summary: 'Read all projects, sort by last modify date' responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Project" + $ref: '#/components/schemas/Project' operationId: getAll parameters: - schema: @@ -39,38 +37,38 @@ paths: tags: - Project summary: Create a new project - description: "" + description: '' requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/Project" + $ref: '#/components/schemas/Project' responses: - "201": + '201': description: success content: application/json: schema: - $ref: "#/components/schemas/Project" + $ref: '#/components/schemas/Project' operationId: create - "/projects/{project_id}": + '/projects/{project_id}': parameters: - - $ref: "#/components/parameters/project_id" - - $ref: "#/components/parameters/request_id" + - $ref: '#/components/parameters/project_id' + - $ref: '#/components/parameters/request_id' get: tags: - Project summary: Get info of a specific project - description: "" + description: '' responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Project" - "404": + $ref: '#/components/schemas/Project' + '404': description: Project not fond operationId: get delete: @@ -79,9 +77,9 @@ paths: tags: - Project responses: - "200": + '200': description: OK - "404": + '404': description: No project with such project id operationId: remove put: @@ -94,18 +92,18 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Project" + $ref: '#/components/schemas/Project' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Project" - "404": - description: "Project with project id not fond, or project dont have requested property" + $ref: '#/components/schemas/Project' + '404': + description: 'Project with project id not fond, or project dont have requested property' operationId: update - "/projects/{project_id}/tasks": + '/projects/{project_id}/tasks': parameters: - name: project_id in: path @@ -118,28 +116,28 @@ paths: tags: - Project responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Task" + $ref: '#/components/schemas/Task' operationId: getTasks - description: "" + description: '' parameters: - schema: type: string in: query name: order_by put: - summary: "" + summary: '' tags: - Project operationId: setAll responses: - "200": + '200': description: OK requestBody: content: @@ -149,100 +147,100 @@ paths: properties: data_predicted: type: boolean - "/projects/{project_id}/labels": + '/projects/{project_id}/labels': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' get: summary: Get all labels under a project tags: - Project responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' operationId: getLabels - description: "" + description: '' post: - summary: "Set all labels under a project, will delete previous labels" + summary: 'Set all labels under a project, will delete previous labels' tags: - Project responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' operationId: setLabels - description: "" + description: '' requestBody: content: application/json: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' delete: summary: Delete all labels under a project tags: - Project responses: - "200": + '200': description: success operationId: removeLabels - description: "" - "/projects/{project_id}/annotations": + description: '' + '/projects/{project_id}/annotations': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' get: summary: Get all annotations under a project tags: - Project responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' operationId: getAnnotations - description: "" - "/projects/{project_id}/tags": + description: '' + '/projects/{project_id}/tags': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' get: summary: Get all tags under a project tags: - Project responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' operationId: getTags - description: "" - "/projects/{project_id}/progress": + description: '' + '/projects/{project_id}/progress': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' get: summary: Get project progress tags: - Project responses: - "200": + '200': description: OK content: application/json: @@ -254,16 +252,16 @@ paths: total: type: integer operationId: getProgress - description: "" - "/projects/{project_id}/split": + description: '' + '/projects/{project_id}/split': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' post: - summary: "Split this project's data into train, validation and test dataset." + summary: 'Split this project''s data into train, validation and test dataset.' tags: - Project responses: - "200": + '200': description: success content: application/json: @@ -277,7 +275,7 @@ paths: test: type: integer operationId: splitDataset - description: "" + description: '' requestBody: content: application/json: @@ -290,10 +288,10 @@ paths: type: integer test: type: integer - description: "" - "/projects/{project_id}/export": + description: '' + '/projects/{project_id}/export': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' post: summary: Export dataset to specified directory tags: @@ -313,20 +311,20 @@ paths: required: - export_dir responses: - "200": + '200': description: success operationId: exportDataset - description: "" - "/projects/{project_id}/import": + description: '' + '/projects/{project_id}/import': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' post: - summary: "" + summary: '' operationId: importDataset tags: - Project responses: - "200": + '200': description: OK requestBody: content: @@ -339,10 +337,10 @@ paths: import_format: type: string nullable: true - description: "" - "/projects/{project_id}/predict": + description: '' + '/projects/{project_id}/predict': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' post: summary: Run prediction on all data in the dataset tags: @@ -369,20 +367,20 @@ paths: - ml_backend_url - model responses: - "200": + '200': description: success operationId: predict - description: "" - "/projects/{project_id}/toEasydata": + description: '' + '/projects/{project_id}/toEasydata': parameters: - - $ref: "#/components/parameters/project_id" + - $ref: '#/components/parameters/project_id' post: - summary: "" + summary: '' tags: - Project operationId: toEasydata responses: - "200": + '200': description: OK requestBody: content: @@ -397,20 +395,20 @@ paths: required: - access_token - dataset_id - "/labels": + /labels: get: tags: - Label - summary: "Get all labels, sort by last modify" + summary: 'Get all labels, sort by last modify' responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' operationId: getAll post: tags: @@ -423,39 +421,39 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' responses: - "201": + '201': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' parameters: - - $ref: "#/components/parameters/request_id" + - $ref: '#/components/parameters/request_id' - schema: type: boolean in: header name: remove_duplicate_by_name description: 是否根据标签名去重，标签名如果存在不会创建 operationId: create - "/labels/{label_id}": + '/labels/{label_id}': parameters: - - $ref: "#/components/parameters/label_id" + - $ref: '#/components/parameters/label_id' get: tags: - Label summary: Get info about a specific label responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Label" - "404": + $ref: '#/components/schemas/Label' + '404': description: Label not fond operationId: get delete: @@ -464,9 +462,9 @@ paths: tags: - Label responses: - "200": + '200': description: OK - "404": + '404': description: The label specified is not found operationId: remove put: @@ -479,16 +477,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Label" - "404": - description: "Label with specified label id not fond, or project dont have requested property" + $ref: '#/components/schemas/Label' + '404': + description: 'Label with specified label id not fond, or project dont have requested property' operationId: update /tasks: get: @@ -496,14 +494,14 @@ paths: tags: - Task responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Task" + $ref: '#/components/schemas/Task' operationId: getAll parameters: - schema: @@ -515,16 +513,16 @@ paths: - Task summary: Create a new task responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Task" + $ref: '#/components/schemas/Task' operationId: create - "/tasks/{task_id}": + '/tasks/{task_id}': parameters: - - $ref: "#/components/parameters/task_id" + - $ref: '#/components/parameters/task_id' get: tags: - Task @@ -537,13 +535,13 @@ paths: type: integer required: true responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Task" - "404": + $ref: '#/components/schemas/Task' + '404': description: Task not fond operationId: get delete: @@ -559,9 +557,9 @@ paths: type: integer required: true responses: - "200": + '200': description: OK - "404": + '404': description: The task specified is not found operationId: remove put: @@ -581,51 +579,51 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Task" + $ref: '#/components/schemas/Task' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Task" - "404": - description: "Task with task id not fond, or task dont have requested property" + $ref: '#/components/schemas/Task' + '404': + description: 'Task with task id not fond, or task dont have requested property' operationId: update - "/tasks/{task_id}/tags": + '/tasks/{task_id}/tags': parameters: - - $ref: "#/components/parameters/task_id" + - $ref: '#/components/parameters/task_id' get: tags: - Task summary: Get all tags of the task responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' operationId: getTags - description: "" + description: '' post: tags: - Task summary: Add a new tag to the task responses: - "201": + '201': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' parameters: - - $ref: "#/components/parameters/request_id" - description: "Add a tag to a task, the tag has to exist." + - $ref: '#/components/parameters/request_id' + description: 'Add a tag to a task, the tag has to exist.' requestBody: content: application/json: @@ -636,54 +634,54 @@ paths: tag_id: type: integer operationId: addTag - "/tasks/{task_id}/datas": + '/tasks/{task_id}/datas': parameters: - - $ref: "#/components/parameters/task_id" + - $ref: '#/components/parameters/task_id' get: tags: - Task summary: Get all datas of a task responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Data" + $ref: '#/components/schemas/Data' operationId: getDatas - description: "" - "/tasks/{task_id}/annotations": + description: '' + '/tasks/{task_id}/annotations': parameters: - - $ref: "#/components/parameters/task_id" + - $ref: '#/components/parameters/task_id' get: tags: - Task summary: Get all annotations of a task responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' operationId: getAnnotations - description: "" + description: '' /datas/: get: tags: - Data - summary: "Get all data, sort by last modified" + summary: 'Get all data, sort by last modified' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Data" + $ref: '#/components/schemas/Data' operationId: getAll post: tags: @@ -694,41 +692,41 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Data" + $ref: '#/components/schemas/Data' responses: - "201": + '201': description: success content: application/json: schema: - $ref: "#/components/schemas/Data" + $ref: '#/components/schemas/Data' operationId: create - "/datas/{data_id}/": + '/datas/{data_id}/': parameters: - - $ref: "#/components/parameters/data_id" + - $ref: '#/components/parameters/data_id' get: tags: - Data summary: Get info of a specific data record responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Data" - "404": + $ref: '#/components/schemas/Data' + '404': description: Data record not fond operationId: get delete: summary: Delete a data record - description: "Delete a data record, file on file system will not be deleted" + description: 'Delete a data record, file on file system will not be deleted' tags: - Data responses: - "200": + '200': description: OK - "404": + '404': description: The data record specified is not found operationId: remove put: @@ -741,18 +739,18 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Data" + $ref: '#/components/schemas/Data' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Data" - "404": - description: "Data record with data id not fond, or data dont have requested property" + $ref: '#/components/schemas/Data' + '404': + description: 'Data record with data id not fond, or data dont have requested property' operationId: update - "/datas/{data_id}/image": + '/datas/{data_id}/image': parameters: - name: data_id in: path @@ -765,7 +763,7 @@ paths: tags: - Data responses: - "200": + '200': description: success operationId: getImage parameters: @@ -773,7 +771,7 @@ paths: type: string in: query name: sault - "/datas/{data_id}/mask": + '/datas/{data_id}/mask': parameters: - schema: type: string @@ -785,10 +783,10 @@ paths: tags: - Data responses: - "200": + '200': description: OK operationId: getMask - "/datas/{data_id}/annotations": + '/datas/{data_id}/annotations': parameters: - name: data_id in: path @@ -801,14 +799,14 @@ paths: tags: - Data responses: - "200": + '200': description: OK content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' operationId: getAnnotations delete: summary: Delete all annotations of a data record @@ -816,7 +814,7 @@ paths: - Data operationId: deleteAnnotations responses: - "200": + '200': description: OK post: summary: Set the annotations of a data record @@ -824,35 +822,35 @@ paths: - Data operationId: setAnnotations responses: - "200": + '200': description: OK content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' requestBody: content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' /annotations/: get: tags: - Annotation - summary: "Get all annotations, sort by last modified" + summary: 'Get all annotations, sort by last modified' responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' operationId: getAll post: tags: @@ -865,25 +863,25 @@ paths: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' responses: - "201": + '201': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' parameters: - - $ref: "#/components/parameters/request_id" + - $ref: '#/components/parameters/request_id' - schema: type: boolean in: header name: deduplicate description: 是否进行去重操作 operationId: create - "/annotations/{annotation_id}": + '/annotations/{annotation_id}': get: tags: - Annotation @@ -896,13 +894,13 @@ paths: type: integer required: true responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Annotation" - "404": + $ref: '#/components/schemas/Annotation' + '404': description: Annotation not fond operationId: get delete: @@ -918,9 +916,9 @@ paths: type: integer required: true responses: - "200": + '200': description: OK - "404": + '404': description: The annotation record specified is not found operationId: remove put: @@ -940,16 +938,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Annotation" - "404": - description: "Annotation record with data id not fond, or data dont have requested property" + $ref: '#/components/schemas/Annotation' + '404': + description: 'Annotation record with data id not fond, or data dont have requested property' operationId: update parameters: - name: annotation_id @@ -962,16 +960,16 @@ paths: get: tags: - Tag - summary: "Get all tags, sort by last modify date" + summary: 'Get all tags, sort by last modify date' responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' operationId: getAll post: tags: @@ -982,33 +980,33 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' responses: - "201": + '201': description: success content: application/json: schema: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' parameters: - - $ref: "#/components/parameters/request_id" - description: "" + - $ref: '#/components/parameters/request_id' + description: '' operationId: create - "/tags/{tag_id}": + '/tags/{tag_id}': parameters: - - $ref: "#/components/parameters/tag_id" + - $ref: '#/components/parameters/tag_id' get: tags: - Tag summary: Get info of a specific tag responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/Tag" - "404": + $ref: '#/components/schemas/Tag' + '404': description: Tag not fond operationId: get delete: @@ -1017,9 +1015,9 @@ paths: tags: - Tag responses: - "200": + '200': description: OK - "404": + '404': description: The tag specified is not found operationId: remove put: @@ -1032,16 +1030,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/Tag" + $ref: '#/components/schemas/Tag' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/Tag" - "404": - description: "Tag with tag id not fond, or tag dont have requested property" + $ref: '#/components/schemas/Tag' + '404': + description: 'Tag with tag id not fond, or tag dont have requested property' operationId: update /users: get: @@ -1049,45 +1047,45 @@ paths: tags: - User responses: - "200": + '200': description: success content: application/json: schema: type: array items: - $ref: "#/components/schemas/User" + $ref: '#/components/schemas/User' operationId: getAll post: tags: - User summary: Add a new user responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/User" - description: "" + $ref: '#/components/schemas/User' + description: '' operationId: create - "/users/{uuid}": + '/users/{uuid}': parameters: - - $ref: "#/components/parameters/uuid" + - $ref: '#/components/parameters/uuid' get: tags: - User summary: Get info of a specific user responses: - "200": + '200': description: OK content: application/json: schema: - $ref: "#/components/schemas/User" - "404": + $ref: '#/components/schemas/User' + '404': description: User not fond - description: "" + description: '' operationId: get delete: summary: Delete a user @@ -1095,9 +1093,9 @@ paths: tags: - User responses: - "200": + '200': description: OK - "404": + '404': description: The task specified is not found operationId: remove put: @@ -1110,16 +1108,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/User" + $ref: '#/components/schemas/User' responses: - "200": + '200': description: success content: application/json: schema: - $ref: "#/components/schemas/User" - "404": - description: "Use with user id not fond, or user dont have requested property" + $ref: '#/components/schemas/User' + '404': + description: 'Use with user id not fond, or user dont have requested property' operationId: update /users/login: post: @@ -1128,7 +1126,7 @@ paths: summary: Login and get JWT operationId: paddlelabel.api.controller.user.login responses: - "200": + '200': description: OK requestBody: content: @@ -1145,12 +1143,12 @@ paths: - password /rpc/folders: post: - summary: "" + summary: '' tags: - rpc operationId: getFolders responses: - "200": + '200': description: OK requestBody: content: @@ -1162,12 +1160,12 @@ paths: type: string /rpc/seg/polygon2points: post: - summary: "" + summary: '' tags: - rpc operationId: polygon2points responses: - "200": + '200': description: OK content: application/json: @@ -1183,12 +1181,12 @@ paths: type: string /rpc/seg/points2polygon: post: - summary: "" + summary: '' operationId: points2polygon tags: - rpc responses: - "200": + '200': description: OK content: application/json: @@ -1204,7 +1202,7 @@ paths: properties: points: type: string - description: "" + description: '' /version: get: summary: Get backend version @@ -1212,7 +1210,7 @@ paths: - manage operationId: getVersion responses: - "200": + '200': description: OK content: application/json: @@ -1233,7 +1231,7 @@ paths: task_category_id: type: integer responses: - "200": + '200': description: OK content: application/json: @@ -1248,7 +1246,7 @@ paths: tags: - sample responses: - "200": + '200': description: OK content: application/json: @@ -1270,7 +1268,7 @@ paths: tags: - sample responses: - "200": + '200': description: OK operationId: getFile parameters: @@ -1279,7 +1277,7 @@ paths: in: query name: path parameters: [] - "/debug/printid/{debug_id}": + '/debug/printid/{debug_id}': parameters: - schema: type: string @@ -1291,17 +1289,17 @@ paths: tags: - rpc responses: - "200": + '200': description: OK operationId: printDebugId /rpc/cache: post: - summary: "" + summary: '' tags: - rpc operationId: createCache responses: - "200": + '200': description: OK content: application/json: @@ -1318,7 +1316,7 @@ paths: properties: content: type: string - "/rpc/cache/{cache_id}": + '/rpc/cache/{cache_id}': parameters: - schema: type: string @@ -1330,7 +1328,7 @@ paths: tags: - rpc responses: - "200": + '200': description: OK content: application/json: @@ -1352,7 +1350,7 @@ components: type: object x-examples: {} additionalProperties: false - title: "" + title: '' description: 项目基本信息和设置 properties: project_id: @@ -1374,7 +1372,7 @@ components: example: This project is very cool task_category_id: type: integer - description: "Top level annotation task category, see TODO for int <-> category map" + description: 'Top level annotation task category, see TODO for int <-> category map' task_category: type: string readOnly: true @@ -1388,10 +1386,10 @@ components: labels: type: array items: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' label_format: type: string - description: "eg: single_class/multi_class for classification" + description: 'eg: single_class/multi_class for classification' created: type: string description: Project creation timestamp in UTC @@ -1412,8 +1410,8 @@ components: properties: mlBackendUrl: type: string - description: "ML后端地址，一般是 http://localhost:1234" - example: "http://localhost:1234" + description: 'ML后端地址，一般是 http://localhost:1234' + example: 'http://localhost:1234' perviousModel: type: string deprecated: true @@ -1454,7 +1452,6 @@ components: type: string lang: type: string - Task: title: Task type: object @@ -1473,9 +1470,9 @@ components: annotations: type: array items: - $ref: "#/components/schemas/Annotation" + $ref: '#/components/schemas/Annotation' project: - $ref: "#/components/schemas/Project" + $ref: '#/components/schemas/Project' set: type: integer modified: @@ -1500,7 +1497,7 @@ components: size: type: string task: - $ref: "#/components/schemas/Task" + $ref: '#/components/schemas/Task' created: type: string readOnly: true @@ -1514,7 +1511,7 @@ components: Annotation: type: object title: Annotation - description: "" + description: '' additionalProperties: false properties: annotation_id: @@ -1527,7 +1524,7 @@ components: label_id: type: integer label: - $ref: "#/components/schemas/Label" + $ref: '#/components/schemas/Label' project_id: type: integer deprecated: true @@ -1553,7 +1550,7 @@ components: Label: title: Label type: object - description: "" + description: '' properties: label_id: type: integer @@ -1604,7 +1601,7 @@ components: User: title: User type: object - description: "" + description: '' properties: user_id: type: integer @@ -1626,7 +1623,7 @@ components: schema: type: integer maxLength: 30 - description: "Assign a unique random string each request, backend will reject requests with same request_id within several seconds. Prevent critical operations (mostly post) from executing multiple times" + description: 'Assign a unique random string each request, backend will reject requests with same request_id within several seconds. Prevent critical operations (mostly post) from executing multiple times' project_id: name: project_id in: path diff --git a/paddlelabel/version b/paddlelabel/version index 6d7de6e..7dea76e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.2 +1.0.1 From ac530caf51fa2a1a3643e319f522175c66fda57f Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 06:03:00 -0500 Subject: [PATCH 12/17] fix(ci) --- .github/workflows/cypress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index c2aad92..4ddea1a 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -66,7 +66,7 @@ jobs: echo $bgid # download latest build - if [[ ${{ github.event_name }} != "workflow_call" ]] + if [[ ${{ github.event_name }} != "push" ]] then curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip unzip PaddleLabel_built_package.zip From 7fb0a54d0d59e46a549bc966aa99c6e660dd4ea1 Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 07:23:24 -0500 Subject: [PATCH 13/17] fix(ci): build --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8196044..14725dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,7 +93,6 @@ jobs: version: $(cat paddlelabel/version) " >> $GITHUB_STEP_SUMMARY - - name: Save built package uses: actions/upload-artifact@v3 with: @@ -113,6 +112,9 @@ jobs: python-version: ["3.11"] needs: cypress_e2e_test steps: + - name: Checkout backend code + uses: actions/checkout@v3 + - name: Bump version id: bump_version if: github.event_name == 'push' From 911ae17da110458d144d1ab8818c045e4cf4a05e Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 07:43:29 -0500 Subject: [PATCH 14/17] feat(ci): retry cypress --- .github/workflows/cypress.yml | 44 ++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 4ddea1a..504b7de 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -57,10 +57,7 @@ jobs: cd ~/测试路径/3rd_party/ curl -LO https://github.com/linhandev/static/releases/download/PaddleLabel%E7%9B%B8%E5%85%B3/paddlelabel_3rd_party_tests.zip echo ---------------------------------- - # ls unzip -q paddlelabel_3rd_party_tests.zip - # pwd - # ls } & bgid=$! echo $bgid @@ -68,7 +65,10 @@ jobs: # download latest build if [[ ${{ github.event_name }} != "push" ]] then + echo "here" curl -LO https://nightly.link/PaddleCV-SIG/PaddleLabel/workflows/build/develop/PaddleLabel_built_package.zip + pwd + ls unzip PaddleLabel_built_package.zip fi @@ -124,19 +124,35 @@ jobs: # ls echo "tests=$(python cypress/order_tests.py)" >> $GITHUB_OUTPUT - - name: Run tests - uses: cypress-io/github-action@v5 + - uses: Wandalen/wretry.action@master id: test with: - working-directory: ./cypress - browser: chrome - config: baseUrl=http://localhost:1111 - env: os=${{ matrix.os }} - spec: ${{ steps.test_order.outputs.tests }} - wait-on: "http://localhost:1111" - wait-on-timeout: 120 - start-windows: yarn run test:win - start: yarn run test:unix + action: cypress-io/github-action@v5 + with: | + working-directory: ./cypress + browser: chrome + config: baseUrl=http://localhost:1111 + env: os=${{ matrix.os }} + spec: ${{ steps.test_order.outputs.tests }} + wait-on: "http://localhost:1111" + wait-on-timeout: 120 + start-windows: yarn run test:win + start: yarn run test:unix + attempt_limit: 2 + + # - name: Run tests + # uses: cypress-io/github-action@v5 + # id: test + # with: + # working-directory: ./cypress + # browser: chrome + # config: baseUrl=http://localhost:1111 + # env: os=${{ matrix.os }} + # spec: ${{ steps.test_order.outputs.tests }} + # wait-on: "http://localhost:1111" + # wait-on-timeout: 120 + # start-windows: yarn run test:win + # start: yarn run test:unix - name: Backend logs, cypress screenshots if: failure() From 48281724107bf73a047956985d8ffc7d668cfead Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 07:51:28 -0500 Subject: [PATCH 15/17] fix(ci) --- .github/workflows/cypress.yml | 54 +++++++++++++++++------------------ paddlelabel/openapi.yml | 27 ++++++++++++++++++ 2 files changed, 54 insertions(+), 27 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 504b7de..e9154fa 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -124,35 +124,35 @@ jobs: # ls echo "tests=$(python cypress/order_tests.py)" >> $GITHUB_OUTPUT - - uses: Wandalen/wretry.action@master - id: test - with: - action: cypress-io/github-action@v5 - with: | - working-directory: ./cypress - browser: chrome - config: baseUrl=http://localhost:1111 - env: os=${{ matrix.os }} - spec: ${{ steps.test_order.outputs.tests }} - wait-on: "http://localhost:1111" - wait-on-timeout: 120 - start-windows: yarn run test:win - start: yarn run test:unix - attempt_limit: 2 - - # - name: Run tests - # uses: cypress-io/github-action@v5 + # - uses: Wandalen/wretry.action@master # id: test # with: - # working-directory: ./cypress - # browser: chrome - # config: baseUrl=http://localhost:1111 - # env: os=${{ matrix.os }} - # spec: ${{ steps.test_order.outputs.tests }} - # wait-on: "http://localhost:1111" - # wait-on-timeout: 120 - # start-windows: yarn run test:win - # start: yarn run test:unix + # action: cypress-io/github-action@v5 + # with: | + # working-directory: ./cypress + # browser: chrome + # config: baseUrl=http://localhost:1111 + # env: os=${{ matrix.os }} + # spec: ${{ steps.test_order.outputs.tests }} + # wait-on: "http://localhost:1111" + # wait-on-timeout: 120 + # start-windows: yarn run test:win + # start: yarn run test:unix + # attempt_limit: 2 + + - name: Run tests + uses: cypress-io/github-action@v5 + id: test + with: + working-directory: ./cypress + browser: chrome + config: baseUrl=http://localhost:1111 + env: os=${{ matrix.os }} + spec: ${{ steps.test_order.outputs.tests }} + wait-on: "http://localhost:1111" + wait-on-timeout: 120 + start-windows: yarn run test:win + start: yarn run test:unix - name: Backend logs, cypress screenshots if: failure() diff --git a/paddlelabel/openapi.yml b/paddlelabel/openapi.yml index 6a973a9..e016a42 100644 --- a/paddlelabel/openapi.yml +++ b/paddlelabel/openapi.yml @@ -7,6 +7,9 @@ info: name: PaddleLabel Developers url: 'https://github.com/PaddleCV-SIG/PaddleLabel/issues' email: me@linhan.email + license: + name: Apache License 2.0 + url: 'http://www.apache.org/licenses/LICENSE-2.0' servers: - url: /api description: Same origion @@ -52,6 +55,30 @@ paths: schema: $ref: '#/components/schemas/Project' operationId: create + /projects/import_options: + parameters: + - name: request_id + in: header + required: false + schema: + type: integer + maxLength: 30 + description: 'Assign a unique random string each request, backend will reject requests with same request_id within several seconds. Prevent critical operations (mostly post) from executing multiple times' + get: + tags: + - Project + summary: 'Read all projects, sort by last modify date' + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + operationId: getImportOptions + parameters: [] '/projects/{project_id}': parameters: - $ref: '#/components/parameters/project_id' From 39f5e2de52b6ec3334af4786a215b6c8dc4fcf6b Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sat, 28 Jan 2023 10:06:47 -0500 Subject: [PATCH 16/17] feat(thunder): api testing --- paddlelabel/api/controller/data.py | 3 +- paddlelabel/api/controller/project.py | 77 +++---- paddlelabel/openapi.yml | 7 +- paddlelabel/task/__init__.py | 5 +- paddlelabel/task/classification.py | 31 ++- ...gmentation.py => instance_segmentation.py} | 184 ---------------- ...cr.py => optical_character_recognition.py} | 0 paddlelabel/task/semantic_segmentation.py | 197 ++++++++++++++++++ paddlelabel/util.py | 1 + thunder-tests/thunderActivity.json | 1 + thunder-tests/thunderCollection.json | 9 + thunder-tests/thunderEnvironment.json | 16 ++ thunder-tests/thunderclient.json | 22 ++ 13 files changed, 326 insertions(+), 227 deletions(-) rename paddlelabel/task/{segmentation.py => instance_segmentation.py} (72%) rename paddlelabel/task/{ocr.py => optical_character_recognition.py} (100%) create mode 100644 paddlelabel/task/semantic_segmentation.py create mode 100644 thunder-tests/thunderActivity.json create mode 100644 thunder-tests/thunderCollection.json create mode 100644 thunder-tests/thunderEnvironment.json create mode 100644 thunder-tests/thunderclient.json diff --git a/paddlelabel/api/controller/data.py b/paddlelabel/api/controller/data.py index b66c676..b925345 100644 --- a/paddlelabel/api/controller/data.py +++ b/paddlelabel/api/controller/data.py @@ -8,12 +8,11 @@ import flask import tempfile -from paddlelabel.config import db from .base import crud from ..model import Data, Project, Task from ..schema import DataSchema from paddlelabel.api.util import abort -from paddlelabel.task.segmentation import draw_mask +from paddlelabel.task.instance_segmentation import draw_mask get_all, get, post, put, delete = crud(Data, DataSchema) diff --git a/paddlelabel/api/controller/project.py b/paddlelabel/api/controller/project.py index a14dd47..a5bac2d 100644 --- a/paddlelabel/api/controller/project.py +++ b/paddlelabel/api/controller/project.py @@ -14,7 +14,7 @@ import connexion from paddlelabel.config import db -from paddlelabel.api.model import Project, Task, TaskCategory, Annotation, Label +from paddlelabel.api.model import Project, Task, TaskCategory, Annotation, Label, TaskCategory from paddlelabel.api.schema import ProjectSchema from paddlelabel.api.controller.base import crud from paddlelabel.api.util import abort @@ -199,6 +199,42 @@ def pre_put(project, body, se): return project, body +# TODO: move to label controller +def create_label(project, label_name): + color = rand_hex_color([l.color for l in project.labels]) + ids = [l.id for l in project.labels] + ids.append(0) + label = Label( + id=max(ids) + 1, + project_id=project.project_id, + name=label_name, + color=color, + ) + project.labels.append(label) + db.session.commit() + return label + + +def post_delete(project, se): + warning_path = Path(project.data_dir) / "paddlelabel.warning" + if warning_path.exists(): + warning_path.unlink() + + +get_all, get, post, put, delete = crud( + Project, + ProjectSchema, + triggers=[pre_add, post_add, pre_put, post_delete], +) + + +def to_easydata(project_id): + _, project = Project._exists(project_id) + task_category = TaskCategory._get(task_category_id=project.task_category_id) + handler = eval(task_category.handler)(project) + handler.to_easydata(project_id=project_id, **{k: connexion.request.json[k] for k in ["access_token", "dataset_id"]}) + + def split_dataset(project_id): Project._exists(project_id) split = connexion.request.json @@ -239,22 +275,6 @@ def split_dataset(project_id): }, 200 -# TODO: move to label controller -def create_label(project, label_name): - color = rand_hex_color([l.color for l in project.labels]) - ids = [l.id for l in project.labels] - ids.append(0) - label = Label( - id=max(ids) + 1, - project_id=project.project_id, - name=label_name, - color=color, - ) - project.labels.append(label) - db.session.commit() - return label - - def predict(project_id): _, project = Project._exists(project_id) @@ -305,21 +325,8 @@ def predict(project_id): return "finished" -def to_easydata(project_id): - _, project = Project._exists(project_id) - task_category = TaskCategory._get(task_category_id=project.task_category_id) - handler = eval(task_category.handler)(project) - handler.to_easydata(project_id=project_id, **{k: connexion.request.json[k] for k in ["access_token", "dataset_id"]}) - - -def post_delete(project, se): - warning_path = osp.join(project.data_dir, "paddlelabel.warning") - if osp.exists(warning_path): - os.remove(warning_path) - - -get_all, get, post, put, delete = crud( - Project, - ProjectSchema, - triggers=[pre_add, post_add, pre_put, post_delete], -) +def import_options(project_type): + all_catgs = TaskCategory._get(many=True) + assert project_type in [c.name for c in all_catgs], f"Project type specified {project_type} isn't supported" + selector = eval(f"paddlelabel.task.{project_type}.ProjectSubtypeSelector")() + print(selector.questions) diff --git a/paddlelabel/openapi.yml b/paddlelabel/openapi.yml index e016a42..f2c6692 100644 --- a/paddlelabel/openapi.yml +++ b/paddlelabel/openapi.yml @@ -55,7 +55,7 @@ paths: schema: $ref: '#/components/schemas/Project' operationId: create - /projects/import_options: + '/projects/importOptions/{project_type}': parameters: - name: request_id in: header @@ -64,6 +64,11 @@ paths: type: integer maxLength: 30 description: 'Assign a unique random string each request, backend will reject requests with same request_id within several seconds. Prevent critical operations (mostly post) from executing multiple times' + - schema: + type: string + name: project_type + in: path + required: true get: tags: - Project diff --git a/paddlelabel/task/__init__.py b/paddlelabel/task/__init__.py index b89d8bd..7823543 100644 --- a/paddlelabel/task/__init__.py +++ b/paddlelabel/task/__init__.py @@ -2,5 +2,6 @@ from .base import BaseTask from .classification import Classification from .detection import Detection -from .segmentation import InstanceSegmentation, SemanticSegmentation -from .ocr import OpticalCharacterRecognition +from .semantic_segmentation import SemanticSegmentation +from .instance_segmentation import InstanceSegmentation +from .optical_character_recognition import OpticalCharacterRecognition diff --git a/paddlelabel/task/classification.py b/paddlelabel/task/classification.py index f7c9379..648c48f 100644 --- a/paddlelabel/task/classification.py +++ b/paddlelabel/task/classification.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import annotations +from functools import partial from pathlib import Path import os.path as osp # TODO: remove this dep @@ -157,13 +158,37 @@ def multi_class_exporter(self, export_dir): class ProjectSubtypeSelector: + def __init__(self): + self.import_questions = [] + self.export_questions = [] + iq = partial(self.add_q, self.import_questions) + eq = partial(self.add_q, self.export_questions) + iq( + "clasSubCags", + True, + "choice", + ["singleClass", "multiClass"], + None, + None, + ) + def add_q( self, + question_set: list, question: str, required: bool, type: str, choices: list[str], - tips: str, - depends_on: tuple[str, str], + tips: str | None, + show_after: tuple[str, str] | None, ): - pass + question_set.append( + { + "question": question, + "required": required, + "type": type, + "choices": choices, + "tips": tips, + # "show_after" + } + ) diff --git a/paddlelabel/task/segmentation.py b/paddlelabel/task/instance_segmentation.py similarity index 72% rename from paddlelabel/task/segmentation.py rename to paddlelabel/task/instance_segmentation.py index 0d9e28f..8ba2a38 100644 --- a/paddlelabel/task/segmentation.py +++ b/paddlelabel/task/instance_segmentation.py @@ -118,72 +118,6 @@ def draw_mask(data, mask_type="grayscale"): return catg_mask -def parse_semantic_mask(annotation_path, labels, image_path=None): - # ann = cv2.imread(annotation_path, cv2.IMREAD_UNCHANGED) - ann_img = Image.open(annotation_path) - ann = np.array(ann_img.convert(mode=ann_img.mode)) # size is hwc - print(ann.shape) - - if image_path is not None: - # img = cv2.imread(annotation_path, cv2.IMREAD_UNCHANGED) - img = Image.open(annotation_path) - if img.size[::-1] != ann.shape[:2]: - raise RuntimeError( - f"Image ({img.size[::-1]}) and annotation ({ann.shape[:2]}) has different shapes, please check image {image_path} and annotation {annotation_path}", - ) - frontend_id = 1 - anns = [] - if len(ann.shape) == 3: - # ann = cv2.cvtColor(ann, cv2.COLOR_BGR2RGB) - ann_gray = np.zeros(ann.shape[:2], dtype="uint8") - for label in labels: - color = hex_to_rgb(label.color) - label_mask = np.all(ann == color, axis=2) - ann_gray[label_mask == 1] = label.id - ann[label_mask == 1] = 0 - if ann.sum() != 0: - ann = ann.reshape((-1, ann.shape[-1])) - raise RuntimeError( - f"Pseudo color mask {annotation_path} contains color that's not specified in labels {np.unique(ann, axis=0)[1:].tolist()} . Maybe you didn't include a background class in the first line of labels.txt or didn't specify label color?" - ) - ann = ann_gray - - for label in labels: - label_mask = deepcopy(ann) - label_mask[label_mask != label.id] = 0 - label_mask[label_mask != 0] = 255 - - if label_mask.sum() == 0: - continue - - ann[ann == label.id] = 0 - (cc_num, cc_mask, values, centroid) = cv2.connectedComponentsWithStats(label_mask, connectivity=8) - for cc_id in range(1, cc_num): - h, w = np.where(cc_mask == cc_id) - result = ",".join([f"{w},{h}" for h, w in zip(h, w)]) - # result = f"{1},{frontend_id}," + result - # TODO: patch. points type will be set by ann.type - result = f"{0},{0}," + result - anns.append( - { - "label_name": label.name, - "result": result, - "type": "brush", - "frontend_id": label.id, - } - ) - frontend_id += 1 - - if ann.sum() != 0: - msg = f"Mask {annotation_path} contains unspecified labels {np.unique(ann)[1:].tolist()} . Maybe you didn't include a background class in the first line of labels.txt or didn't specify label id?" - abort(msg, 404) - - s = (1,) + tuple(ann.shape[:2]) - s = [str(s) for s in s] - size = ",".join(s) - return size, anns - - def parse_instance_mask(annotation_path, labels, image_path=None): mask = tif.imread(annotation_path) if image_path is not None: @@ -533,121 +467,3 @@ def eiseg_importer( self.add_task([{"path": str(data_path), "size": size}], [anns]) json_paths.remove(json_path) self.commit() - - -class SemanticSegmentation(InstanceSegmentation): - def __init__(self, project, data_dir=None, is_export=False): - super().__init__(project, data_dir=data_dir, is_export=is_export) - self.importers = { - "mask": self.mask_importer, - "coco": self.coco_importer, - "eiseg": self.eiseg_importer, - } - self.exporters = { - "mask": self.mask_exporter, - "coco": self.coco_exporter, - } - self.default_exporter = self.mask_exporter - - def mask_importer( - self, - data_dir=None, - filters={"exclude_prefix": ["."], "include_postfix": image_extensions}, - ): - - # 1. set params - project = self.project - - base_dir = project.data_dir if data_dir is None else data_dir - - data_dir = osp.join(base_dir, "JPEGImages") - ann_dirs = [ - Path(base_dir) / "Annotations", - Path(base_dir) / "label", # EISeg - ] - - background_line = self.import_labels(ignore_first=True) - other_settings = project._get_other_settings() - other_settings["background_line"] = background_line - project.other_settings = json.dumps(other_settings) - - ann_dict = {} - for ann_dir in ann_dirs: - paths = listdir(ann_dir, filters) - ann_dict.update({osp.basename(p).split(".")[0]: ann_dir / p for p in paths}) - if ann_dir.name == "label": - ann_dict.update( - { - osp.basename(p).split(".")[0][: -len("_pseudo")]: ann_dir / p - for p in paths - if "_pseudo" in Path(p).name - } - ) # NOTE: EISeg pseudo color label export - - # 2. import records - data_paths = listdir(data_dir, filters) - if len(data_paths) == 0: - raise RuntimeError("No image found. Did you put images under JPEGImages folder?") - - for data_path in data_paths: - id = osp.basename(data_path).split(".")[0] - data_path = osp.join(data_dir, data_path) - if id in ann_dict.keys(): - ann_path = osp.join(ann_dir, ann_dict[id]) - size, anns = parse_semantic_mask(ann_path, project.labels, data_path) - else: - anns = [] - size, _, _ = getSize(Path(data_path)) - - self.add_task([{"path": data_path, "size": size}], [anns]) - self.commit() - - def mask_exporter(self, export_dir: str, seg_mask_type: str = "grayscale"): - """Export semantic segmentation dataset in mask format - - Args: - export_dir (str): The folder to export to. - seg_mask_type (str): grayscale|pseudo - """ - - # 1. set params - project = self.project - # other_settings = project._get_other_settings() - # mask_type = other_settings.get("segMaskType", "grayscale") - - export_data_dir = osp.join(export_dir, "JPEGImages") - export_label_dir = osp.join(export_dir, "Annotations") - create_dir(export_data_dir) - create_dir(export_label_dir) - - tasks = Task._get(project_id=project.project_id, many=True) - export_data_paths = [] - export_label_paths = [] - - for task in tasks: - data = task.datas[0] - data_path = osp.join(project.data_dir, data.path) - - export_data_path = osp.join("JPEGImages", osp.basename(data.path)) - - # TODO: strip ext - export_label_path = osp.join(export_label_dir, osp.basename(data_path).split(".")[0] + ".png") - - copy(data_path, export_data_dir) - - mask = draw_mask(data, mask_type=seg_mask_type) - mask_img = Image.fromarray(mask.astype("uint8"), "L") - mask_img.save(export_label_path) - - export_data_paths.append([export_data_path]) - export_label_paths.append([export_label_path]) - - self.export_split( - Path(export_dir), - tasks, - export_data_paths, - with_labels=False, - annotation_ext=".png", - ) - bg = project._get_other_settings().get("background_line", "background") - self.export_labels(osp.join(export_dir, "labels.txt"), bg) diff --git a/paddlelabel/task/ocr.py b/paddlelabel/task/optical_character_recognition.py similarity index 100% rename from paddlelabel/task/ocr.py rename to paddlelabel/task/optical_character_recognition.py diff --git a/paddlelabel/task/semantic_segmentation.py b/paddlelabel/task/semantic_segmentation.py new file mode 100644 index 0000000..b30486e --- /dev/null +++ b/paddlelabel/task/semantic_segmentation.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations +import os.path as osp +import json + +from copy import deepcopy +from PIL import Image +from pathlib import Path +import numpy as np +import cv2 + +from paddlelabel.io.image import getSize +from paddlelabel.task.instance_segmentation import InstanceSegmentation, draw_mask +from paddlelabel.task.util import create_dir, listdir, image_extensions, copy +from paddlelabel.task.util.color import hex_to_rgb +from paddlelabel.api.model import Task + + +def parse_semantic_mask(annotation_path, labels, image_path=None): + ann_img = Image.open(annotation_path) + ann = np.array(ann_img.convert(mode=ann_img.mode)) # size is hwc + + if image_path is not None: + img = Image.open(annotation_path) + if img.size[::-1] != ann.shape[:2]: + raise RuntimeError( + f"Image ({img.size[::-1]}) and annotation ({ann.shape[:2]}) has different shapes, please check image {image_path} and annotation {annotation_path}", + ) + frontend_id = 1 + anns = [] + if len(ann.shape) == 3: + # ann = cv2.cvtColor(ann, cv2.COLOR_BGR2RGB) + ann_gray = np.zeros(ann.shape[:2], dtype="uint8") + for label in labels: + color = hex_to_rgb(label.color) + label_mask = np.all(ann == color, axis=2) + ann_gray[label_mask == 1] = label.id + ann[label_mask == 1] = 0 + if ann.sum() != 0: + ann = ann.reshape((-1, ann.shape[-1])) + raise RuntimeError( + f"Pseudo color mask {annotation_path} contains color that's not specified in labels {np.unique(ann, axis=0)[1:].tolist()} . Maybe you didn't include a background class in the first line of labels.txt or didn't specify label color?" + ) + ann = ann_gray + + for label in labels: + label_mask = deepcopy(ann) + label_mask[label_mask != label.id] = 0 + label_mask[label_mask != 0] = 255 + + if label_mask.sum() == 0: + continue + + ann[ann == label.id] = 0 + (cc_num, cc_mask, values, centroid) = cv2.connectedComponentsWithStats(label_mask, connectivity=8) + for cc_id in range(1, cc_num): + h, w = np.where(cc_mask == cc_id) + result = ",".join([f"{w},{h}" for h, w in zip(h, w)]) + # result = f"{1},{frontend_id}," + result + # TODO: patch. points type will be set by ann.type + result = f"{0},{0}," + result + anns.append( + { + "label_name": label.name, + "result": result, + "type": "brush", + "frontend_id": label.id, + } + ) + frontend_id += 1 + + if ann.sum() != 0: + msg = f"Mask {annotation_path} contains unspecified labels {np.unique(ann)[1:].tolist()} . Maybe you didn't include a background class in the first line of labels.txt or didn't specify label id?" + abort(msg, 404) + + s = (1,) + tuple(ann.shape[:2]) + s = [str(s) for s in s] + size = ",".join(s) + return size, anns + + +class SemanticSegmentation(InstanceSegmentation): + def __init__(self, project, data_dir=None, is_export=False): + super().__init__(project, data_dir=data_dir, is_export=is_export) + self.importers = { + "mask": self.mask_importer, + "coco": self.coco_importer, + "eiseg": self.eiseg_importer, + } + self.exporters = { + "mask": self.mask_exporter, + "coco": self.coco_exporter, + } + self.default_exporter = self.mask_exporter + + def mask_importer( + self, + data_dir=None, + filters={"exclude_prefix": ["."], "include_postfix": image_extensions}, + ): + + # 1. set params + project = self.project + + base_dir = project.data_dir if data_dir is None else data_dir + + data_dir = osp.join(base_dir, "JPEGImages") + ann_dirs = [ + Path(base_dir) / "Annotations", + Path(base_dir) / "label", # EISeg + ] + + background_line = self.import_labels(ignore_first=True) + other_settings = project._get_other_settings() + other_settings["background_line"] = background_line + project.other_settings = json.dumps(other_settings) + + ann_dict = {} + for ann_dir in ann_dirs: + paths = listdir(ann_dir, filters) + ann_dict.update({osp.basename(p).split(".")[0]: ann_dir / p for p in paths}) + if ann_dir.name == "label": + ann_dict.update( + { + osp.basename(p).split(".")[0][: -len("_pseudo")]: ann_dir / p + for p in paths + if "_pseudo" in Path(p).name + } + ) # NOTE: EISeg pseudo color label export + + # 2. import records + data_paths = listdir(data_dir, filters) + if len(data_paths) == 0: + raise RuntimeError("No image found. Did you put images under JPEGImages folder?") + + for data_path in data_paths: + id = osp.basename(data_path).split(".")[0] + data_path = osp.join(data_dir, data_path) + if id in ann_dict.keys(): + ann_path = osp.join(ann_dir, ann_dict[id]) + size, anns = parse_semantic_mask(ann_path, project.labels, data_path) + else: + anns = [] + size, _, _ = getSize(Path(data_path)) + + self.add_task([{"path": data_path, "size": size}], [anns]) + self.commit() + + def mask_exporter(self, export_dir: str, seg_mask_type: str = "grayscale"): + """Export semantic segmentation dataset in mask format + + Args: + export_dir (str): The folder to export to. + seg_mask_type (str): grayscale|pseudo + """ + + # 1. set params + project = self.project + # other_settings = project._get_other_settings() + # mask_type = other_settings.get("segMaskType", "grayscale") + + export_data_dir = osp.join(export_dir, "JPEGImages") + export_label_dir = osp.join(export_dir, "Annotations") + create_dir(export_data_dir) + create_dir(export_label_dir) + + tasks = Task._get(project_id=project.project_id, many=True) + export_data_paths = [] + export_label_paths = [] + + for task in tasks: + data = task.datas[0] + data_path = osp.join(project.data_dir, data.path) + + export_data_path = osp.join("JPEGImages", osp.basename(data.path)) + + # TODO: strip ext + export_label_path = osp.join(export_label_dir, osp.basename(data_path).split(".")[0] + ".png") + + copy(data_path, export_data_dir) + + mask = draw_mask(data, mask_type=seg_mask_type) + mask_img = Image.fromarray(mask.astype("uint8"), "L") + mask_img.save(export_label_path) + + export_data_paths.append([export_data_path]) + export_label_paths.append([export_label_path]) + + self.export_split( + Path(export_dir), + tasks, + export_data_paths, + with_labels=False, + annotation_ext=".png", + ) + bg = project._get_other_settings().get("background_line", "background") + self.export_labels(osp.join(export_dir, "labels.txt"), bg) diff --git a/paddlelabel/util.py b/paddlelabel/util.py index d3e65fd..742a964 100644 --- a/paddlelabel/util.py +++ b/paddlelabel/util.py @@ -92,6 +92,7 @@ def resolve_operation_id(self, operation): # TODO: auto resolve /collection/{item}/collection special = { + "/projects/importOptions/{project_type} getImportOptions": "paddlelabel.api.controller.project.import_options", "/projects/{project_id}/tasks getTasks": "paddlelabel.api.controller.task.get_by_project", "/projects/{project_id}/tasks setAll": "paddlelabel.api.controller.task.set_all_by_project", "/projects/{project_id}/labels getLabels": "paddlelabel.api.controller.label.get_by_project", diff --git a/thunder-tests/thunderActivity.json b/thunder-tests/thunderActivity.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/thunder-tests/thunderActivity.json @@ -0,0 +1 @@ +[] diff --git a/thunder-tests/thunderCollection.json b/thunder-tests/thunderCollection.json new file mode 100644 index 0000000..f897108 --- /dev/null +++ b/thunder-tests/thunderCollection.json @@ -0,0 +1,9 @@ +[ + { + "_id": "f8fe5699-a721-439d-be48-4d18b9cd9672", + "colName": "project", + "created": "2023-01-28T14:41:00.442Z", + "folders": [], + "sortNum": 10000 + } +] diff --git a/thunder-tests/thunderEnvironment.json b/thunder-tests/thunderEnvironment.json new file mode 100644 index 0000000..060d818 --- /dev/null +++ b/thunder-tests/thunderEnvironment.json @@ -0,0 +1,16 @@ +[ + { + "_id": "d7b67eda-f60d-4050-905a-e0f1612c44b0", + "created": "2023-01-28T14:40:04.587Z", + "data": [ + { + "name": "host", + "value": "http://localhost:17995/api" + } + ], + "default": true, + "modified": "2023-01-28T14:40:04.587Z", + "name": "PaddleLabel", + "sortNum": 10000 + } +] diff --git a/thunder-tests/thunderclient.json b/thunder-tests/thunderclient.json new file mode 100644 index 0000000..38563b2 --- /dev/null +++ b/thunder-tests/thunderclient.json @@ -0,0 +1,22 @@ +[ + { + "_id": "771fb38b-ad2d-4d33-ae4e-fffd90edbaaa", + "colId": "f8fe5699-a721-439d-be48-4d18b9cd9672", + "containerId": "", + "created": "2023-01-28T14:41:11.511Z", + "headers": [], + "method": "GET", + "modified": "2023-01-28T14:42:38.399Z", + "name": "import_settings", + "params": [ + { + "isPath": true, + "name": "project_type", + "value": "classification" + } + ], + "sortNum": 10000, + "tests": [], + "url": "{{host}}/projects/importOptions/{project_type}" + } +] From 5c6889eb57988b95b25721ff33b9802f4dc1edea Mon Sep 17 00:00:00 2001 From: Action Bot Date: Sat, 28 Jan 2023 15:29:15 +0000 Subject: [PATCH 17/17] bump version --- paddlelabel/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlelabel/version b/paddlelabel/version index 7dea76e..6d7de6e 100644 --- a/paddlelabel/version +++ b/paddlelabel/version @@ -1 +1 @@ -1.0.1 +1.0.2