daily_run_test #276
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: daily_run_test | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '56 16 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONDA_ENV: opencompass_regression | |
| PIP_CACHE_PATH: /cpfs01/user/qa-llm-cicd/.cache/pip | |
| USERSPACE_PREFIX: /cpfs01/user/qa-llm-cicd | |
| HF_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/model_weights/hf_hub | |
| DATEASET_CACHE_PATH: /cpfs01/shared/public/public_hdd/llmeval/llm-evaluation-datasets | |
| HF_DATASETS_OFFLINE: 1 | |
| TRANSFORMERS_OFFLINE: 1 | |
| HF_HUB_OFFLINE: 1 | |
| jobs: | |
| daily_run_test: | |
| runs-on: self-hosted | |
| environment: 'prod' | |
| timeout-minutes: 240 #4hours | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v2 | |
| - name: Prepare - create conda env and install torch | |
| run: | | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda create -y --name ${{env.CONDA_ENV}} python=3.10 | |
| conda activate ${{env.CONDA_ENV}} | |
| pip install /cpfs01/user/qa-llm-cicd/packages/lmdeploy-0.5.0+cu118-cp310-cp310-manylinux2014_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} | |
| pip install /cpfs01/user/qa-llm-cicd/packages/vllm-0.5.2+cu118-cp310-cp310-manylinux1_x86_64.whl --cache-dir ${{env.PIP_CACHE_PATH}} | |
| FLASH_ATTENTION_FORCE_BUILD=TRUE pip install /cpfs01/user/qa-llm-cicd/packages/flash_attn-2.5.8+cu118torch2.2cxx11abiFALSE-cp310-cp310-linux_x86_64.whl | |
| pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --cache-dir ${{env.PIP_CACHE_PATH}} --index-url https://download.pytorch.org/whl/cu118 | |
| pip install xformers==0.0.25.post1 --cache-dir ${{env.PIP_CACHE_PATH}} | |
| conda info --envs | |
| - name: Prepare - Pip install code | |
| run: | | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| pip install -e . --cache-dir ${{env.PIP_CACHE_PATH}} | |
| pip install human_eval transformers protobuf pytest --cache-dir ${{env.PIP_CACHE_PATH}} | |
| conda info --envs | |
| - name: Prepare - prepare data and hf model | |
| run: | | |
| ln -s ${{env.DATEASET_CACHE_PATH}} data | |
| rm -rf ~/.cache/huggingface/hub -f && mkdir ~/.cache -p && mkdir ~/.cache/huggingface -p | |
| ln -s ${{env.HF_CACHE_PATH}} ~/.cache/huggingface/hub | |
| - name: Run chat model test | |
| run: | | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| sed -i 's/judgemodel/'$(tail -n 1 /cpfs01/shared/public/llmeval/share_info/compassjuder_ip.txt)'/g' .github/scripts/eval_regression_chat.py | |
| python3 run.py .github/scripts/eval_regression_chat.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/chat/*/summary regression_result_daily | |
| python -m pytest -m chat -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Run base model test | |
| run: | | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| python3 run.py .github/scripts/eval_regression_base.py --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/base/*/summary regression_result_daily | |
| python -m pytest -m base -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Run command testcase | |
| run: | | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| conda info --envs | |
| export from_tf=TRUE | |
| python tools/list_configs.py internlm2_5 mmlu | |
| python run.py --models hf_internlm2_5_7b --datasets race_ppl --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd1/*/summary regression_result_daily | |
| python -m pytest -m case1 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --models hf_internlm2_5_7b_chat --datasets race_gen -a lmdeploy --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd2/*/summary regression_result_daily | |
| python -m pytest -m case2 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --datasets race_ppl --hf-type base --hf-path internlm/internlm2_5-7b --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd3/*/summary regression_result_daily | |
| python -m pytest -m case3 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| python run.py --datasets race_gen --hf-type chat --hf-path internlm/internlm2_5-7b-chat --work-dir /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4 --reuse | |
| rm regression_result_daily -f && ln -s /cpfs01/user/qa-llm-cicd/report/${{ github.run_id }}/cmd4/*/summary regression_result_daily | |
| python -m pytest -m case4 -s -v --color=yes .github/scripts/oc_score_assert.py | |
| - name: Remove Conda Env | |
| if: always() | |
| run: | | |
| rm -rf regression_result_daily | |
| . /cpfs01/shared/public/qa-llm-cicd/miniconda3/bin/activate | |
| conda env remove -y --name ${{env.CONDA_ENV}} | |
| conda info --envs | |
| notify_to_feishu: | |
| if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }} | |
| needs: [daily_run_test] | |
| environment: 'prod' | |
| timeout-minutes: 5 | |
| runs-on: self-hosted | |
| steps: | |
| - name: notify | |
| run: | | |
| curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Opencompass- Daily test failed","content":[[{"tag":"text","text":"branch: ${{github.ref_name}}, run action: ${{github.workflow}} failed. "},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}' ${{ secrets.WEBHOOK_URL }} |