Skip to content

Commit

Permalink
ENG-1293: Running tests with --e2e-results, and saving results on f…
Browse files Browse the repository at this point in the history
…ailure, to assist in troubleshooting
  • Loading branch information
AsadHasan-Rasa committed Sep 13, 2024
1 parent 5512be9 commit 6397722
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 14 deletions.
59 changes: 53 additions & 6 deletions .github/workflows/continous-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -174,14 +174,25 @@ jobs:
run: |
make run-duckling
- name: Make test results directory
run: |
mkdir tests
- name: Run e2e passing tests
env:
OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
RASA_PRO_BETA_INTENTLESS: true
run: |
make test-passing
make test-passing
- name: Save test-passing results
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-passing-results
path: tests/

- name: Run e2e flaky tests
if: always()
Expand All @@ -191,7 +202,7 @@ jobs:
RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
RASA_PRO_BETA_INTENTLESS: true
run: |
make test-flaky || true
make test-flaky || true
- name: Run e2e failing tests
if: always()
Expand All @@ -203,6 +214,13 @@ jobs:
run: |
make test-failing | grep '0 passed'
- name: Save test-failing results
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-failing-results
path: tests/*passed.yml

- name: Stop Duckling server
run: |
make stop-duckling
Expand Down Expand Up @@ -282,14 +300,25 @@ jobs:
run: |
make run-duckling
- name: Make test results directory
run: |
mkdir tests
- name: Run e2e passing tests with assertions
env:
OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
RASA_PRO_BETA_E2E_ASSERTIONS: true
run: |
make test-passing-assertions
make test-passing-assertions
- name: Save test-passing-assertions results
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-passing-assertions-results
path: tests/

- name: Run e2e flaky tests with assertions
if: always()
Expand All @@ -299,7 +328,7 @@ jobs:
RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
RASA_PRO_BETA_E2E_ASSERTIONS: true
run: |
make test-flaky-assertions || true
make test-flaky-assertions || true
- name: Run e2e failing tests with assertions
if: always()
Expand All @@ -309,7 +338,14 @@ jobs:
RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
RASA_PRO_BETA_E2E_ASSERTIONS: true
run: |
make test-failing-assertions | grep '0 passed'
make test-failing-assertions | grep '0 passed'
- name: Save test-failing-assertions results
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-failing-assertions-results
path: tests/*passed.yml

- name: Stop Duckling server
run: |
Expand Down Expand Up @@ -381,6 +417,10 @@ jobs:
run: |
make run-duckling
- name: Make test results directory
run: |
mkdir tests
- name: Run e2e passing tests with stub custom actions
env:
OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
Expand All @@ -389,7 +429,14 @@ jobs:
RASA_PRO_BETA_E2E_ASSERTIONS: true
RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
run: |
make test-passing-stub-custom-actions
make test-passing-stub-custom-actions
- name: Save test-stub-custom-actions-passing results
if: failure()
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: test-stub-custom-actions-passing-results
path: tests/

- name: Stop Duckling server
run: |
Expand Down
16 changes: 8 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -75,16 +75,16 @@ actions:
poetry run rasa run actions

test-passing: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests/passing
poetry run rasa test e2e e2e_tests/passing --e2e-results

test-flaky: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests/flaky
poetry run rasa test e2e e2e_tests/flaky --e2e-results

test-failing: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests/failing
poetry run rasa test e2e e2e_tests/failing --e2e-results

test-multistep: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests/multistep
poetry run rasa test e2e e2e_tests/multistep --e2e-results

test-one: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e $(target) --debug
Expand All @@ -93,13 +93,13 @@ stop-duckling:
docker stop duckling_container

test-passing-assertions: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests_with_assertions/passing
poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results

test-flaky-assertions: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests_with_assertions/flaky
poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results

test-failing-assertions: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests_with_assertions/failing
poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results

make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing
poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results

0 comments on commit 6397722

Please sign in to comment.