ENG-1293: Update rasa-calm-demo to version 3.10.1 (#57)

* link to add_contact in pattern_clarification * link from check_portfolio to pattern_human_handoff * link from pattern_clarification to pattern_human_handoff * do not rephrase utter_human_handoff_not_available * ENG-1293: Added pattern_human_handoff flow * ENG-1293: Running tests with `--e2e-results`, and saving results on failure, to assist in troubleshooting * ENG-1293: Update `rasa-calm-demo` to version 3.10.1 * ENG-1293: Fix 2 failing tests Remove `utter_clarification_options_rasa` assertion in the following two failing tests, as now this step won't be reached: 1. e2e_tests_with_assertions/passing/disambiguation/user_sends_short_verb_only_message.yml::user sends short verb only message 2. e2e_tests_with_assertions/passing/negations/users_says_they_dont_want_the_former_option.yml::user says they don't want the former option * ENG-1293: fix mistake, had removed assertion from a wrong similar sounding test, corrected it now * ENG-1293: fix/update test `user_sends_short_verb_only_message`, based on recent pattern-clarification changes * ENG-1293: properly add `pattern_human_handoff` to default `pattern_clarification` * ENG-1293: Add tests to verify that link to `pattern_human_handoff` is triggered in relevant conditions On these conditions: 1. When `pattern_clarification` is triggered twice 2. Or when login fails during check-portfolio at least 3 times --------- Co-authored-by: Tanja Bunk <[email protected]>
RasaHQ · Sep 18, 2024 · 1e38976 · 1e38976
1 parent 48e4580
commit 1e38976
Show file tree

Hide file tree

Showing 19 changed files with 306 additions and 38 deletions.
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -174,14 +174,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
-        make test-passing
+        make test-passing 
+
+    - name: Save test-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-results
+        path: tests/
 
     - name: Run e2e flaky tests
       if: always()
@@ -194,6 +205,7 @@ jobs:
         make test-flaky || true
 
     - name: Run e2e failing tests
+      id: run-e2e-failing-tests 
       if: always()
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -203,6 +215,13 @@ jobs:
       run: |
         make test-failing | grep '0 passed'
 
+    - name: Save test-failing results
+      if: failure() && steps.run-e2e-failing-tests.outcome == 'failure'
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-results
+        path: tests/*passed.yml
+
     - name: Stop Duckling server
       run: |
           make stop-duckling
@@ -282,14 +301,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with assertions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_E2E_ASSERTIONS: true
       run: |
-        make test-passing-assertions
+        make test-passing-assertions 
+
+    - name: Save test-passing-assertions results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-assertions-results
+        path: tests/
 
     - name: Run e2e flaky tests with assertions
       if: always()
@@ -302,6 +332,7 @@ jobs:
         make test-flaky-assertions || true
 
     - name: Run e2e failing tests with assertions
+      id: run-e2e-failing-tests-with-assertions
       if: always()
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -311,6 +342,13 @@ jobs:
       run: |
         make test-failing-assertions | grep '0 passed'
 
+    - name: Save test-failing-assertions results
+      if: failure() && steps.run-e2e-failing-tests-with-assertions.outcome == 'failure'
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-assertions-results
+        path: tests/*passed.yml
+
     - name: Stop Duckling server
       run: |
           make stop-duckling
@@ -381,6 +419,10 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with stub custom actions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -389,7 +431,14 @@ jobs:
         RASA_PRO_BETA_E2E_ASSERTIONS: true
         RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
       run: |
-        make test-passing-stub-custom-actions
+        make test-passing-stub-custom-actions 
+
+    - name: Save test-stub-custom-actions-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-stub-custom-actions-passing-results
+        path: tests/
 
     - name: Stop Duckling server
       run: |

diff --git a/Makefile b/Makefile
@@ -75,16 +75,16 @@ actions:
 	poetry run rasa run actions
 
 test-passing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/passing
+	poetry run rasa test e2e e2e_tests/passing --e2e-results
 
 test-flaky: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/flaky
+	poetry run rasa test e2e e2e_tests/flaky --e2e-results
 
 test-failing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/failing
+	poetry run rasa test e2e e2e_tests/failing --e2e-results
 
 test-multistep: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/multistep
+	poetry run rasa test e2e e2e_tests/multistep --e2e-results
 
 test-one: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e $(target) --debug
@@ -93,13 +93,13 @@ stop-duckling:
 	docker stop duckling_container
 
 test-passing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/passing
+	poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results
 
 test-flaky-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/flaky
+	poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results
 
 test-failing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/failing
+	poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results
 
 make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing
+	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results
diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@ This demo showcases a chatbot built with Rasa's LLM-native approach: [CALM](http
 > CALM's current stage of development.
 
 > [!NOTE]
-> This demo bot is currently compatible with `3.9.3`.
+> This demo bot is currently compatible with `3.10.1`.
 
 ## Terms of Use
 
@@ -244,8 +244,8 @@ and run end-to-end tests.
 > please refer our documentation [here](https://rasa.com/docs/rasa-pro/installation/python/installation).
 
 > [!NOTE]
-> If you want to check out the state of the demo bot compatible with Rasa 3.8.8, please check out the branch
-> [3.8.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.8.x).
+> If you want to check out the state of the demo bot compatible with Rasa 3.9, please check out the branch
+> [3.9.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.9.x).
 
 Prerequisites:
 - rasa pro license
@@ -370,7 +370,7 @@ rasa inspect --debug
 rasa shell --debug
 ```
 
-### Running e2e test
+### Running e2e tests
 
 The demo bot comes with a set of e2e tests, categorized into two primary groups: 
 **failing**, and **passing**. These tests are organized not per individual flow but 
@@ -444,4 +444,39 @@ make rasa-test-multistep
 or
 ```commandline
 run rasa test e2e e2e_tests/multistep
-```
+```
+
+#### E2E tests with assertions
+To enable the feature, please set the environment variable `RASA_PRO_BETA_E2E_ASSERTIONS` to true in your testing environment.
+
+`export RASA_PRO_BETA_E2E_ASSERTIONS=true`
+
+To run **all the tests**:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions
+```
+
+------
+
+To run **passing/failing/flaky** tests:
+```commandline
+rasa test e2e e2e_tests_with_assertions/passing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/failing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/flaky
+```
+
+------
+
+To run a **single test** , provide the path to a 
+target test:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions/tests/path/to/a/target/test.yml
+```
+
+------
diff --git a/actions/action_increase_clarification_count.py b/actions/action_increase_clarification_count.py
@@ -0,0 +1,23 @@
+from rasa_sdk.events import SlotSet
+from rasa_sdk import Action, Tracker
+from rasa_sdk.executor import CollectingDispatcher
+
+
+class ActionIncreaseClarificationCount(Action):
+    """Action which clarifies which flow to start."""
+
+    def name(self) -> str:
+        """Return the flow name."""
+        return "action_increase_clarification_count"
+
+    def run(
+        self,
+        dispatcher: CollectingDispatcher,
+        tracker: Tracker,
+        domain: dict
+    ) -> list:
+        attempts = tracker.get_slot("clarification_count")
+        if not attempts:
+            attempts = 0
+
+        return [SlotSet("clarification_count", attempts + 1)]
diff --git a/actions/authenticate_user.py b/actions/authenticate_user.py
@@ -13,13 +13,16 @@ def run(self, dispatcher: CollectingDispatcher, tracker: Tracker,
         # Retrieve the user credentials from slots
         user_name = tracker.get_slot("user_name")
         user_password = tracker.get_slot("user_password")
+        attempts = tracker.get_slot("login_failed_attempts")
+        if not attempts:
+            attempts = 0
 
         # Dummy authentication.
         # Placeholder for user authentication, in real scenarios the username and
         # password should be checked against a database.
-        authenticated = True
+        authenticated = not (user_name == "John" and user_password == "1234")
 
         if authenticated:
             return [SlotSet("is_user_logged_in", True)]
         else:
-            return [SlotSet("is_user_logged_in", False)]
+            return [SlotSet("is_user_logged_in", False), SlotSet("login_failed_attempts", attempts + 1)]
diff --git a/config/config.yml b/config/config.yml
@@ -29,7 +29,7 @@ pipeline:
 - name: SingleStepLLMCommandGenerator
   llm:
     model: gpt-4
-    request_timeout: 7
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 

diff --git a/config/multistep-config.yml b/config/multistep-config.yml
@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: MultiStepLLMCommandGenerator
   llm:
-    model_name: gpt-3.5-turbo-0125
-    request_timeout: 7
+    model: gpt-3.5-turbo-0125
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 

diff --git a/config/qdrant-config.yml b/config/qdrant-config.yml
@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: SingleStepLLMCommandGenerator
   llm:
-    model_name: gpt-4
-    request_timeout: 7
+    model: gpt-4
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 
@@ -44,8 +44,8 @@ policies:
   vector_store:
     type: "addons.qdrant.Qdrant_Store"
   embeddings:
-    type: "huggingface"
-    model_name: "BAAI/bge-small-en-v1.5"
+    provider: "huggingface"
+    model: "BAAI/bge-small-en-v1.5"
     model_kwargs:
       device: 'cpu'
     encode_kwargs:

diff --git a/data/flows/authenticate_user.yml b/data/flows/authenticate_user.yml
@@ -15,9 +15,16 @@ flows:
         description: "The password of the user."
       - action: action_authenticate_user
         next:
+        - if: slots.login_failed_attempts >= 3
+          then:
+            - action: utter_authentication_failed_multiple_times
+              next: END
         - if: not slots.is_user_logged_in
           then:
             - action: utter_authentication_failed
+            - set_slots:
+                - user_name: null
+                - user_password: null
               next: ask_user_credentials
         - else:
             - action: utter_authentication_successful

diff --git a/data/flows/check_portfolio.yml b/data/flows/check_portfolio.yml
@@ -3,7 +3,13 @@ flows:
     description: "Check the user's investment portfolio, including stocks, bonds, and mutual funds."
     steps:
       - call: authenticate_user
-      - collect: portfolio_type
+        next:
+          - if: slots.login_failed_attempts >= 3
+            then:
+              - link: pattern_human_handoff
+          - else: collect_portfolio_type
+      - id: collect_portfolio_type
+        collect: portfolio_type
         description: "The type of portfolio, for example: stocks, bonds or mutual_funds."
       - action: action_check_portfolio_exists
         next:

diff --git a/data/flows/patterns.yml b/data/flows/patterns.yml
@@ -51,4 +51,23 @@ flows:
     name: pattern completed
     steps:
       - action: utter_can_do_something_else
-      - action: action_reset_routing
+      - action: action_reset_routing
+
+  pattern_clarification:
+    description: Conversation repair flow for handling ambiguous requests that could match multiple flows
+    name: pattern clarification
+    steps:
+      - action: action_clarify_flows
+        next:
+          - if: context.names contains "add a contact"
+            then:
+              - link: add_contact
+          - else: 
+            - action: action_increase_clarification_count
+              next:
+                - if: slots.clarification_count > 2
+                  then:
+                    - link: pattern_human_handoff
+                - else: clarify_options
+      - id: clarify_options
+        action: utter_clarification_options_rasa