ENG-1293: Update rasa-calm-demo to version 3.10.1

.github/workflows/continous-integration.yml

@@ -174,14 +174,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
-        make test-passing
+        make test-passing 
+
+    - name: Save test-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-results
+        path: tests/
 
     - name: Run e2e flaky tests
       if: always()
@@ -194,6 +205,7 @@ jobs:
         make test-flaky || true
 
     - name: Run e2e failing tests
+      id: run-e2e-failing-tests 
       if: always()
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -203,6 +215,13 @@ jobs:
       run: |
         make test-failing | grep '0 passed'
 
+    - name: Save test-failing results
+      if: failure() && steps.run-e2e-failing-tests.outcome == 'failure'
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-results
+        path: tests/*passed.yml
+
     - name: Stop Duckling server
       run: |
           make stop-duckling
@@ -282,14 +301,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with assertions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_E2E_ASSERTIONS: true
       run: |
-        make test-passing-assertions
+        make test-passing-assertions 
+
+    - name: Save test-passing-assertions results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-assertions-results
+        path: tests/
 
     - name: Run e2e flaky tests with assertions
       if: always()
@@ -302,6 +332,7 @@ jobs:
         make test-flaky-assertions || true
 
     - name: Run e2e failing tests with assertions
+      id: run-e2e-failing-tests-with-assertions
       if: always()
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -311,6 +342,13 @@ jobs:
       run: |
         make test-failing-assertions | grep '0 passed'
 
+    - name: Save test-failing-assertions results
+      if: failure() && steps.run-e2e-failing-tests-with-assertions.outcome == 'failure'
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-assertions-results
+        path: tests/*passed.yml
+
     - name: Stop Duckling server
       run: |
           make stop-duckling
@@ -381,6 +419,10 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with stub custom actions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -389,7 +431,14 @@ jobs:
         RASA_PRO_BETA_E2E_ASSERTIONS: true
         RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
       run: |
-        make test-passing-stub-custom-actions
+        make test-passing-stub-custom-actions 
+
+    - name: Save test-stub-custom-actions-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-stub-custom-actions-passing-results
+        path: tests/
 
     - name: Stop Duckling server
       run: |

Makefile

@@ -75,16 +75,16 @@ actions:
 	poetry run rasa run actions
 
 test-passing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/passing
+	poetry run rasa test e2e e2e_tests/passing --e2e-results
 
 test-flaky: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/flaky
+	poetry run rasa test e2e e2e_tests/flaky --e2e-results
 
 test-failing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/failing
+	poetry run rasa test e2e e2e_tests/failing --e2e-results
 
 test-multistep: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/multistep
+	poetry run rasa test e2e e2e_tests/multistep --e2e-results
 
 test-one: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e $(target) --debug
@@ -93,13 +93,13 @@ stop-duckling:
 	docker stop duckling_container
 
 test-passing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/passing
+	poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results
 
 test-flaky-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/flaky
+	poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results
 
 test-failing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/failing
+	poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results
 
 make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing
+	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results

README.md

@@ -9,7 +9,7 @@ This demo showcases a chatbot built with Rasa's LLM-native approach: [CALM](http
 > CALM's current stage of development.
 
 > [!NOTE]
-> This demo bot is currently compatible with `3.9.3`.
+> This demo bot is currently compatible with `3.10.1`.
 
 ## Terms of Use
 
@@ -244,8 +244,8 @@ and run end-to-end tests.
 > please refer our documentation [here](https://rasa.com/docs/rasa-pro/installation/python/installation).
 
 > [!NOTE]
-> If you want to check out the state of the demo bot compatible with Rasa 3.8.8, please check out the branch
-> [3.8.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.8.x).
+> If you want to check out the state of the demo bot compatible with Rasa 3.9, please check out the branch
+> [3.9.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.9.x).
 
 Prerequisites:
 - rasa pro license
@@ -370,7 +370,7 @@ rasa inspect --debug
 rasa shell --debug
 ```
 
-### Running e2e test
+### Running e2e tests
 
 The demo bot comes with a set of e2e tests, categorized into two primary groups: 
 **failing**, and **passing**. These tests are organized not per individual flow but 
@@ -444,4 +444,39 @@ make rasa-test-multistep
 or
 ```commandline
 run rasa test e2e e2e_tests/multistep
-```
+```
+
+#### E2E tests with assertions
+To enable the feature, please set the environment variable `RASA_PRO_BETA_E2E_ASSERTIONS` to true in your testing environment.
+
+`export RASA_PRO_BETA_E2E_ASSERTIONS=true`
+
+To run **all the tests**:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions
+```
+
+------
+
+To run **passing/failing/flaky** tests:
+```commandline
+rasa test e2e e2e_tests_with_assertions/passing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/failing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/flaky
+```
+
+------
+
+To run a **single test** , provide the path to a 
+target test:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions/tests/path/to/a/target/test.yml
+```
+
+------

actions/action_increase_clarification_count.py

@@ -0,0 +1,23 @@
+from rasa_sdk.events import SlotSet
+from rasa_sdk import Action, Tracker
+from rasa_sdk.executor import CollectingDispatcher
+
+
+class ActionIncreaseClarificationCount(Action):
+    """Action which clarifies which flow to start."""
+
+    def name(self) -> str:
+        """Return the flow name."""
+        return "action_increase_clarification_count"
+
+    def run(
+        self,
+        dispatcher: CollectingDispatcher,
+        tracker: Tracker,
+        domain: dict
+    ) -> list:
+        attempts = tracker.get_slot("clarification_count")
+        if not attempts:
+            attempts = 0
+
+        return [SlotSet("clarification_count", attempts + 1)]

actions/authenticate_user.py

@@ -13,13 +13,16 @@ def run(self, dispatcher: CollectingDispatcher, tracker: Tracker,
         # Retrieve the user credentials from slots
         user_name = tracker.get_slot("user_name")
         user_password = tracker.get_slot("user_password")
+        attempts = tracker.get_slot("login_failed_attempts")
+        if not attempts:
+            attempts = 0
 
         # Dummy authentication.
         # Placeholder for user authentication, in real scenarios the username and
         # password should be checked against a database.
-        authenticated = True
+        authenticated = not (user_name == "John" and user_password == "1234")
 
         if authenticated:
             return [SlotSet("is_user_logged_in", True)]
         else:
-            return [SlotSet("is_user_logged_in", False)]
+            return [SlotSet("is_user_logged_in", False), SlotSet("login_failed_attempts", attempts + 1)]

config/config.yml

@@ -29,7 +29,7 @@ pipeline:
 - name: SingleStepLLMCommandGenerator
   llm:
     model: gpt-4
-    request_timeout: 7
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 

config/multistep-config.yml

@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: MultiStepLLMCommandGenerator
   llm:
-    model_name: gpt-3.5-turbo-0125
-    request_timeout: 7
+    model: gpt-3.5-turbo-0125
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 

config/qdrant-config.yml

@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: SingleStepLLMCommandGenerator
   llm:
-    model_name: gpt-4
-    request_timeout: 7
+    model: gpt-4
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 
@@ -44,8 +44,8 @@ policies:
   vector_store:
     type: "addons.qdrant.Qdrant_Store"
   embeddings:
-    type: "huggingface"
-    model_name: "BAAI/bge-small-en-v1.5"
+    provider: "huggingface"
+    model: "BAAI/bge-small-en-v1.5"
     model_kwargs:
       device: 'cpu'
     encode_kwargs:

data/flows/authenticate_user.yml

@@ -15,9 +15,16 @@ flows:
         description: "The password of the user."
       - action: action_authenticate_user
         next:
+        - if: slots.login_failed_attempts >= 3
+          then:
+            - action: utter_authentication_failed_multiple_times
+              next: END
         - if: not slots.is_user_logged_in
           then:
             - action: utter_authentication_failed
+            - set_slots:
+                - user_name: null
+                - user_password: null
               next: ask_user_credentials
         - else:
             - action: utter_authentication_successful

data/flows/check_portfolio.yml

@@ -3,7 +3,13 @@ flows:
     description: "Check the user's investment portfolio, including stocks, bonds, and mutual funds."
     steps:
       - call: authenticate_user
-      - collect: portfolio_type
+        next:
+          - if: slots.login_failed_attempts >= 3
+            then:
+              - link: pattern_human_handoff
+          - else: collect_portfolio_type
+      - id: collect_portfolio_type
+        collect: portfolio_type
         description: "The type of portfolio, for example: stocks, bonds or mutual_funds."
       - action: action_check_portfolio_exists
         next:

data/flows/patterns.yml

@@ -51,4 +51,23 @@ flows:
     name: pattern completed
     steps:
       - action: utter_can_do_something_else
-      - action: action_reset_routing
+      - action: action_reset_routing
+
+  pattern_clarification:
+    description: Conversation repair flow for handling ambiguous requests that could match multiple flows
+    name: pattern clarification
+    steps:
+      - action: action_clarify_flows
+        next:
+          - if: context.names contains "add a contact"
+            then:
+              - link: add_contact
+          - else: 
+            - action: action_increase_clarification_count
+              next:
+                - if: slots.clarification_count > 2
+                  then:
+                    - link: pattern_human_handoff
+                - else: clarify_options
+      - id: clarify_options
+        action: utter_clarification_options_rasa