diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 2bf17b3..892472a 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -174,6 +174,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -181,7 +185,14 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_INTENTLESS: true run: | - make test-passing + make test-passing + + - name: Save test-passing results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-passing-results + path: tests/ - name: Run e2e flaky tests if: always() @@ -194,6 +205,7 @@ jobs: make test-flaky || true - name: Run e2e failing tests + id: run-e2e-failing-tests if: always() env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -203,6 +215,13 @@ jobs: run: | make test-failing | grep '0 passed' + - name: Save test-failing results + if: failure() && steps.run-e2e-failing-tests.outcome == 'failure' + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-failing-results + path: tests/*passed.yml + - name: Stop Duckling server run: | make stop-duckling @@ -282,6 +301,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests with assertions env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -289,7 +312,14 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_E2E_ASSERTIONS: true run: | - make test-passing-assertions + make test-passing-assertions + + - name: Save test-passing-assertions results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-passing-assertions-results + path: tests/ - name: Run e2e flaky tests with assertions if: always() @@ -302,6 +332,7 @@ jobs: make test-flaky-assertions || true - name: Run e2e failing tests with assertions + id: run-e2e-failing-tests-with-assertions if: always() env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -311,6 +342,13 @@ jobs: run: | make test-failing-assertions | grep '0 passed' + - name: Save test-failing-assertions results + if: failure() && steps.run-e2e-failing-tests-with-assertions.outcome == 'failure' + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-failing-assertions-results + path: tests/*passed.yml + - name: Stop Duckling server run: | make stop-duckling @@ -381,6 +419,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests with stub custom actions env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -389,7 +431,14 @@ jobs: RASA_PRO_BETA_E2E_ASSERTIONS: true RASA_PRO_BETA_STUB_CUSTOM_ACTION: true run: | - make test-passing-stub-custom-actions + make test-passing-stub-custom-actions + + - name: Save test-stub-custom-actions-passing results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-stub-custom-actions-passing-results + path: tests/ - name: Stop Duckling server run: | diff --git a/Makefile b/Makefile index 856e69c..838b7fa 100644 --- a/Makefile +++ b/Makefile @@ -75,16 +75,16 @@ actions: poetry run rasa run actions test-passing: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/passing + poetry run rasa test e2e e2e_tests/passing --e2e-results test-flaky: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/flaky + poetry run rasa test e2e e2e_tests/flaky --e2e-results test-failing: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/failing + poetry run rasa test e2e e2e_tests/failing --e2e-results test-multistep: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/multistep + poetry run rasa test e2e e2e_tests/multistep --e2e-results test-one: .EXPORT_ALL_VARIABLES poetry run rasa test e2e $(target) --debug @@ -93,13 +93,13 @@ stop-duckling: docker stop duckling_container test-passing-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/passing + poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results test-flaky-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/flaky + poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results test-failing-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/failing + poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing + poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results diff --git a/README.md b/README.md index cd23348..af01693 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This demo showcases a chatbot built with Rasa's LLM-native approach: [CALM](http > CALM's current stage of development. > [!NOTE] -> This demo bot is currently compatible with `3.9.3`. +> This demo bot is currently compatible with `3.10.1`. ## Terms of Use @@ -244,8 +244,8 @@ and run end-to-end tests. > please refer our documentation [here](https://rasa.com/docs/rasa-pro/installation/python/installation). > [!NOTE] -> If you want to check out the state of the demo bot compatible with Rasa 3.8.8, please check out the branch -> [3.8.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.8.x). +> If you want to check out the state of the demo bot compatible with Rasa 3.9, please check out the branch +> [3.9.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.9.x). Prerequisites: - rasa pro license @@ -370,7 +370,7 @@ rasa inspect --debug rasa shell --debug ``` -### Running e2e test +### Running e2e tests The demo bot comes with a set of e2e tests, categorized into two primary groups: **failing**, and **passing**. These tests are organized not per individual flow but @@ -444,4 +444,39 @@ make rasa-test-multistep or ```commandline run rasa test e2e e2e_tests/multistep -``` \ No newline at end of file +``` + +#### E2E tests with assertions +To enable the feature, please set the environment variable `RASA_PRO_BETA_E2E_ASSERTIONS` to true in your testing environment. + +`export RASA_PRO_BETA_E2E_ASSERTIONS=true` + +To run **all the tests**: + +```commandline +rasa test e2e e2e_tests_with_assertions +``` + +------ + +To run **passing/failing/flaky** tests: +```commandline +rasa test e2e e2e_tests_with_assertions/passing +``` +```commandline +rasa test e2e e2e_tests_with_assertions/failing +``` +```commandline +rasa test e2e e2e_tests_with_assertions/flaky +``` + +------ + +To run a **single test** , provide the path to a +target test: + +```commandline +rasa test e2e e2e_tests_with_assertions/tests/path/to/a/target/test.yml +``` + +------ diff --git a/actions/action_increase_clarification_count.py b/actions/action_increase_clarification_count.py new file mode 100644 index 0000000..27609ed --- /dev/null +++ b/actions/action_increase_clarification_count.py @@ -0,0 +1,23 @@ +from rasa_sdk.events import SlotSet +from rasa_sdk import Action, Tracker +from rasa_sdk.executor import CollectingDispatcher + + +class ActionIncreaseClarificationCount(Action): + """Action which clarifies which flow to start.""" + + def name(self) -> str: + """Return the flow name.""" + return "action_increase_clarification_count" + + def run( + self, + dispatcher: CollectingDispatcher, + tracker: Tracker, + domain: dict + ) -> list: + attempts = tracker.get_slot("clarification_count") + if not attempts: + attempts = 0 + + return [SlotSet("clarification_count", attempts + 1)] diff --git a/actions/authenticate_user.py b/actions/authenticate_user.py index 6dbbeee..2279f31 100644 --- a/actions/authenticate_user.py +++ b/actions/authenticate_user.py @@ -13,13 +13,16 @@ def run(self, dispatcher: CollectingDispatcher, tracker: Tracker, # Retrieve the user credentials from slots user_name = tracker.get_slot("user_name") user_password = tracker.get_slot("user_password") + attempts = tracker.get_slot("login_failed_attempts") + if not attempts: + attempts = 0 # Dummy authentication. # Placeholder for user authentication, in real scenarios the username and # password should be checked against a database. - authenticated = True + authenticated = not (user_name == "John" and user_password == "1234") if authenticated: return [SlotSet("is_user_logged_in", True)] else: - return [SlotSet("is_user_logged_in", False)] \ No newline at end of file + return [SlotSet("is_user_logged_in", False), SlotSet("login_failed_attempts", attempts + 1)] diff --git a/config/config.yml b/config/config.yml index 24cb2be..23226d3 100644 --- a/config/config.yml +++ b/config/config.yml @@ -29,7 +29,7 @@ pipeline: - name: SingleStepLLMCommandGenerator llm: model: gpt-4 - request_timeout: 7 + timeout: 7 temperature: 0.0 top_p: 0.0 diff --git a/config/multistep-config.yml b/config/multistep-config.yml index 0be1701..71d5c74 100644 --- a/config/multistep-config.yml +++ b/config/multistep-config.yml @@ -28,8 +28,8 @@ pipeline: - name: NLUCommandAdapter - name: MultiStepLLMCommandGenerator llm: - model_name: gpt-3.5-turbo-0125 - request_timeout: 7 + model: gpt-3.5-turbo-0125 + timeout: 7 temperature: 0.0 top_p: 0.0 diff --git a/config/qdrant-config.yml b/config/qdrant-config.yml index 1fa7985..c041b94 100644 --- a/config/qdrant-config.yml +++ b/config/qdrant-config.yml @@ -28,8 +28,8 @@ pipeline: - name: NLUCommandAdapter - name: SingleStepLLMCommandGenerator llm: - model_name: gpt-4 - request_timeout: 7 + model: gpt-4 + timeout: 7 temperature: 0.0 top_p: 0.0 @@ -44,8 +44,8 @@ policies: vector_store: type: "addons.qdrant.Qdrant_Store" embeddings: - type: "huggingface" - model_name: "BAAI/bge-small-en-v1.5" + provider: "huggingface" + model: "BAAI/bge-small-en-v1.5" model_kwargs: device: 'cpu' encode_kwargs: diff --git a/data/flows/authenticate_user.yml b/data/flows/authenticate_user.yml index 40fd5ed..8665258 100644 --- a/data/flows/authenticate_user.yml +++ b/data/flows/authenticate_user.yml @@ -15,9 +15,16 @@ flows: description: "The password of the user." - action: action_authenticate_user next: + - if: slots.login_failed_attempts >= 3 + then: + - action: utter_authentication_failed_multiple_times + next: END - if: not slots.is_user_logged_in then: - action: utter_authentication_failed + - set_slots: + - user_name: null + - user_password: null next: ask_user_credentials - else: - action: utter_authentication_successful diff --git a/data/flows/check_portfolio.yml b/data/flows/check_portfolio.yml index 8bee3d8..85c469d 100644 --- a/data/flows/check_portfolio.yml +++ b/data/flows/check_portfolio.yml @@ -3,7 +3,13 @@ flows: description: "Check the user's investment portfolio, including stocks, bonds, and mutual funds." steps: - call: authenticate_user - - collect: portfolio_type + next: + - if: slots.login_failed_attempts >= 3 + then: + - link: pattern_human_handoff + - else: collect_portfolio_type + - id: collect_portfolio_type + collect: portfolio_type description: "The type of portfolio, for example: stocks, bonds or mutual_funds." - action: action_check_portfolio_exists next: diff --git a/data/flows/patterns.yml b/data/flows/patterns.yml index a164c4c..4ed6009 100644 --- a/data/flows/patterns.yml +++ b/data/flows/patterns.yml @@ -51,4 +51,23 @@ flows: name: pattern completed steps: - action: utter_can_do_something_else - - action: action_reset_routing \ No newline at end of file + - action: action_reset_routing + + pattern_clarification: + description: Conversation repair flow for handling ambiguous requests that could match multiple flows + name: pattern clarification + steps: + - action: action_clarify_flows + next: + - if: context.names contains "add a contact" + then: + - link: add_contact + - else: + - action: action_increase_clarification_count + next: + - if: slots.clarification_count > 2 + then: + - link: pattern_human_handoff + - else: clarify_options + - id: clarify_options + action: utter_clarification_options_rasa diff --git a/domain/flows/authenticate_user.yml b/domain/flows/authenticate_user.yml index 3ac177b..460f91d 100644 --- a/domain/flows/authenticate_user.yml +++ b/domain/flows/authenticate_user.yml @@ -14,6 +14,12 @@ slots: type: text mappings: - type: from_llm + login_failed_attempts: + type: float + initial_value: 0.0 + mappings: + - type: custom + action: action_authenticate_user responses: utter_authentication_failed: @@ -24,6 +30,8 @@ responses: - text: Please enter your user name. utter_ask_user_password: - text: Please enter your password. + utter_authentication_failed_multiple_times: + - text: Authentication failed again. Stop authentication process. actions: - action_authenticate_user diff --git a/domain/flows/patterns.yml b/domain/flows/patterns.yml index de62b6d..bf9ae33 100644 --- a/domain/flows/patterns.yml +++ b/domain/flows/patterns.yml @@ -1,10 +1,20 @@ version: "3.1" +actions: + - action_increase_clarification_count + slots: confirm_slot_correction: type: bool mappings: - type: from_llm + clarification_count: + type: float + initial_value: 0.0 + mappings: + - type: custom + action: action_increase_clarification_count + responses: utter_ask_confirm_slot_correction: @@ -21,3 +31,7 @@ responses: - text: Ok, I did not correct the previous input. metadata: rephrase: True + utter_human_handoff_not_available: + - text: I understand you want to be connected to a human agent, but that's something I cannot help you with at the moment. Is there something else I can help you with? + metadata: + rephrase: False diff --git a/e2e_tests_with_assertions/passing/disambiguation/user_sends_short_verb_only_message.yml b/e2e_tests_with_assertions/passing/disambiguation/user_sends_short_verb_only_message.yml index 3fc3dc3..6e480a8 100644 --- a/e2e_tests_with_assertions/passing/disambiguation/user_sends_short_verb_only_message.yml +++ b/e2e_tests_with_assertions/passing/disambiguation/user_sends_short_verb_only_message.yml @@ -12,9 +12,5 @@ test_cases: - pattern_clarification_contains: - 'add a card' - 'add a contact' - - bot_uttered: - utter_name: utter_clarification_options_rasa - - user: contact - assertions: - bot_uttered: utter_name: utter_ask_add_contact_handle diff --git a/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_2_clarifications.yml b/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_2_clarifications.yml new file mode 100644 index 0000000..bf3db35 --- /dev/null +++ b/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_2_clarifications.yml @@ -0,0 +1,46 @@ +fixtures: + - route_to_calm: + - route_session_to_calm: True + +metadata: +- duplicate_message_1: + turn_idx: 1 +- duplicate_message_2: + turn_idx: 2 +- duplicate_message_3: + turn_idx: 3 + +test_cases: + - test_case: user_is_referred_to_human_after_2_clarifications + fixtures: + - route_to_calm + steps: + - user: cash + metadata: duplicate_message_1 + assertions: + - pattern_clarification_contains: + - 'transfer money' + - 'check your balance' + - slot_was_set: + - name: clarification_count + value: 1 + - bot_uttered: + utter_name: utter_clarification_options_rasa + - user: cash + metadata: duplicate_message_2 + assertions: + - pattern_clarification_contains: + - 'transfer money' + - 'check your balance' + - slot_was_set: + - name: clarification_count + value: 2 + - bot_uttered: + utter_name: utter_clarification_options_rasa + - user: cash + metadata: duplicate_message_3 + assertions: + - slot_was_set: + - name: clarification_count + value: 3 + - flow_started: pattern_human_handoff diff --git a/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_3_portfolio_check_auth_fails.yml b/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_3_portfolio_check_auth_fails.yml new file mode 100644 index 0000000..fd3826b --- /dev/null +++ b/e2e_tests_with_assertions/passing/flow_guards/user_is_referred_to_human_after_3_portfolio_check_auth_fails.yml @@ -0,0 +1,63 @@ +fixtures: + - route_to_calm: + - route_session_to_calm: True + +metadata: +- duplicate_message_1: + turn_idx: 1 +- duplicate_message_2: + turn_idx: 2 +- duplicate_message_3: + turn_idx: 3 + +test_cases: + - test_case: user_is_referred_to_human_after_3_portfolio_check_auth_fails + fixtures: + - route_to_calm + steps: + - user: I want to check my portfolio + assertions: + - bot_uttered: + utter_name: utter_ask_user_name + - user: John + metadata: duplicate_message_1 + assertions: + - bot_uttered: + utter_name: utter_ask_user_password + - user: "1234" + metadata: duplicate_message_1 + assertions: + - bot_uttered: + utter_name: utter_authentication_failed + - slot_was_set: + - name: login_failed_attempts + value: 1 + - bot_uttered: + utter_name: utter_ask_user_name + - user: John + metadata: duplicate_message_2 + assertions: + - bot_uttered: + utter_name: utter_ask_user_password + - user: "1234" + metadata: duplicate_message_2 + assertions: + - bot_uttered: + utter_name: utter_authentication_failed + - slot_was_set: + - name: login_failed_attempts + value: 2 + - bot_uttered: + utter_name: utter_ask_user_name + - user: John + metadata: duplicate_message_3 + assertions: + - bot_uttered: + utter_name: utter_ask_user_password + - user: "1234" + metadata: duplicate_message_3 + assertions: + - slot_was_set: + - name: login_failed_attempts + value: 3 + - flow_started: pattern_human_handoff diff --git a/e2e_tests_with_assertions/passing/negations/users_says_they_dont_want_the_former_option.yml b/e2e_tests_with_assertions/passing/negations/users_says_they_dont_want_the_former_option.yml index b9775a9..529da8f 100644 --- a/e2e_tests_with_assertions/passing/negations/users_says_they_dont_want_the_former_option.yml +++ b/e2e_tests_with_assertions/passing/negations/users_says_they_dont_want_the_former_option.yml @@ -12,8 +12,6 @@ test_cases: - pattern_clarification_contains: - 'add a card' - 'add a contact' - - bot_uttered: - utter_name: utter_clarification_options_rasa - user: not the former assertions: - bot_uttered: diff --git a/poetry.lock b/poetry.lock index 5c3690c..f2f1816 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3072,6 +3072,7 @@ description = "Clang Python Bindings, mirrored from the official LLVM repo: http optional = false python-versions = "*" files = [ + {file = "libclang-18.1.1-1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a"}, {file = "libclang-18.1.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5"}, {file = "libclang-18.1.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:83ce5045d101b669ac38e6da8e58765f12da2d3aafb3b9b98d88b286a60964d8"}, {file = "libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b"}, @@ -5444,13 +5445,13 @@ fire = "*" [[package]] name = "rasa-pro" -version = "3.10.0" +version = "3.10.1" description = "State-of-the-art open-core Conversational AI framework for Enterprises that natively leverages generative AI for effortless assistant development." optional = false python-versions = ">=3.8.1,<3.11" files = [ - {file = "rasa_pro-3.10.0-py3-none-any.whl", hash = "sha256:206262ce346392609643af9a17b2fdd259206483a0847ec4b4f38fb89e8eb4ea"}, - {file = "rasa_pro-3.10.0.tar.gz", hash = "sha256:0b1ead8911212de002c5e87f7897b974cd2bef7896d1da5ddddf28db6635a47a"}, + {file = "rasa_pro-3.10.1-py3-none-any.whl", hash = "sha256:408fd74f8dfca4417976377ce3c8d5b8d685b7e44c6f0e2fed8d6d022b49213a"}, + {file = "rasa_pro-3.10.1.tar.gz", hash = "sha256:a60b5ae6d8bd3e5b687b8951079baff9d5b6f2391ca207282257e272f0f1c332"}, ] [package.dependencies] @@ -8092,4 +8093,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.11" -content-hash = "eadc7fd58b54b314489527296f6cd42a39b598c5cf19f22ffa7beba153c1caa2" +content-hash = "06b45d688de072d4d8d9618d6efc05297a7bcfbb45c9099c6290d61d9f7cd861" diff --git a/pyproject.toml b/pyproject.toml index 077618a..22ccec0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ priority = "supplemental" python = ">=3.8.1,<3.11" [tool.poetry.dependencies.rasa-pro] -version = "3.10.0" +version = "3.10.1" allow-prereleases = true extras = ["mlflow"]