[WIP/DRAFT] ENG-1293: Update rasa calm demo to 3.10.0

.github/workflows/continous-integration.yml

@@ -100,7 +100,7 @@ jobs:
           path: models/${{steps.upload_model.outputs.model}}.tar.gz
 
   run_e2e_tests:
-    name: Run e2e Tests
+    name: Run e2e tests
     runs-on: ubuntu-22.04
     needs: [train-model]
 
@@ -206,3 +206,192 @@ jobs:
     - name: Stop Duckling server
       run: |
           make stop-duckling
+
+  run_e2e_tests_with_assertions:
+    name: Run e2e tests with assertions
+    runs-on: ubuntu-22.04
+    needs: [train-model]
+
+    steps:
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+    - name: Setup Python
+      uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Install poetry 🦄
+      uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
+      with:
+        poetry-version: ${{ env.POETRY_VERSION }}
+
+    - name: Load Poetry Cached Libraries ⬇
+      id: cache-poetry
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .venv
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Create virtual environment
+      if: steps.cache-poetry.outputs.cache-hit != 'true'
+      run: python -m venv create .venv
+
+    - name: Set up virtual environment
+      run: poetry config virtualenvs.in-project true
+
+      # Authenticate with gcloud for release registry (where Rasa is published)
+    - id: "auth-release"
+      name: Authenticate with gcloud for release registry 🎫
+      uses: "google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d"
+      with:
+        token_format: 'access_token'
+        credentials_json: "${{ secrets.RASA_RELEASES_READ }}"
+
+    - name: Configure OAuth token for poetry
+      run: |
+        poetry config http-basic.rasa-plus oauth2accesstoken $(gcloud auth print-access-token)
+
+    - name: Install Dependencies 📦
+      run: |
+        make install
+
+    - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+      with:
+        name: trained-model
+        path: models/
+
+    - name: Init LLM Cache
+      id: cache-llm
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .rasa
+        key: rasa-llm-cache
+
+    - name: Run action server
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_INTENTLESS: true
+      run: |
+        make actions &
+
+    - name: Run duckling server
+      run: |
+        make run-duckling
+
+    - name: Run e2e passing tests with assertions
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_E2E_ASSERTIONS: true
+      run: |
+        make test-passing-assertions
+
+    - name: Run e2e flaky tests with assertions
+      if: always()
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_E2E_ASSERTIONS: true
+      run: |
+        make test-flaky-assertions || true
+
+    - name: Run e2e failing tests with assertions
+      if: always()
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_E2E_ASSERTIONS: true
+      run: |
+        make test-failing-assertions | grep '0 passed'
+
+    - name: Stop Duckling server
+      run: |
+          make stop-duckling
+
+  run_e2e_tests_with_stub_custom_actions:
+    name: Run e2e tests with stub custom actions
+    runs-on: ubuntu-22.04
+    needs: [train-model]
+
+    steps:
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+
+    - name: Setup Python
+      uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Install poetry 🦄
+      uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
+      with:
+        poetry-version: ${{ env.POETRY_VERSION }}
+
+    - name: Load Poetry Cached Libraries ⬇
+      id: cache-poetry
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .venv
+        key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: ${{ runner.os }}-poetry-${{ env.DEFAULT_PYTHON_VERSION }}
+
+    - name: Create virtual environment
+      if: steps.cache-poetry.outputs.cache-hit != 'true'
+      run: python -m venv create .venv
+
+    - name: Set up virtual environment
+      run: poetry config virtualenvs.in-project true
+
+      # Authenticate with gcloud for release registry (where Rasa is published)
+    - id: "auth-release"
+      name: Authenticate with gcloud for release registry 🎫
+      uses: "google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d"
+      with:
+        token_format: 'access_token'
+        credentials_json: "${{ secrets.RASA_RELEASES_READ }}"
+
+    - name: Configure OAuth token for poetry
+      run: |
+        poetry config http-basic.rasa-plus oauth2accesstoken $(gcloud auth print-access-token)
+
+    - name: Install Dependencies 📦
+      run: |
+        make install
+
+    - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+      with:
+        name: trained-model
+        path: models/
+
+    - name: Init LLM Cache
+      id: cache-llm
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
+      with:
+        path: .rasa
+        key: rasa-llm-cache
+
+    - name: Run duckling server
+      run: |
+        make run-duckling
+
+    - name: Run e2e passing tests with stub custom actions
+      env:
+        OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
+        RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
+        RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
+        RASA_PRO_BETA_E2E_ASSERTIONS: true
+        RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
+      run: |
+        make test-passing-stub-custom-actions
+
+    - name: Stop Duckling server
+      run: |
+        make stop-duckling
+

.gitignore

@@ -142,3 +142,6 @@ models/
 prompts/
 tests/
 qdrant_storage/
+
+# mlflow
+mlruns/

Makefile

@@ -91,3 +91,15 @@ test-one: .EXPORT_ALL_VARIABLES
 
 stop-duckling:
 	docker stop duckling_container
+
+test-passing-assertions: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests_with_assertions/passing
+
+test-flaky-assertions: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests_with_assertions/flaky
+
+test-failing-assertions: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests_with_assertions/failing
+
+make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
+	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing

README.md

@@ -9,7 +9,7 @@ This demo showcases a chatbot built with Rasa's LLM-native approach: [CALM](http
 > CALM's current stage of development.
 
 > [!NOTE]
-> This demo bot is currently compatible with `3.9.3`.
+> This demo bot is currently compatible with `3.10.0`.
 
 ## Terms of Use
 
@@ -244,8 +244,8 @@ and run end-to-end tests.
 > please refer our documentation [here](https://rasa.com/docs/rasa-pro/installation/python/installation).
 
 > [!NOTE]
-> If you want to check out the state of the demo bot compatible with Rasa 3.8.8, please check out the branch
-> [3.8.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.8.x).
+> If you want to check out the state of the demo bot compatible with Rasa 3.9, please check out the branch
+> [3.9.x](https://github.com/RasaHQ/rasa-calm-demo/tree/3.9.x).
 
 Prerequisites:
 - rasa pro license
@@ -370,7 +370,7 @@ rasa inspect --debug
 rasa shell --debug
 ```
 
-### Running e2e test
+### Running e2e tests
 
 The demo bot comes with a set of e2e tests, categorized into two primary groups: 
 **failing**, and **passing**. These tests are organized not per individual flow but 
@@ -444,4 +444,39 @@ make rasa-test-multistep
 or
 ```commandline
 run rasa test e2e e2e_tests/multistep
-```
+```
+
+#### E2E tests with assertions
+To enable the feature, please set the environment variable `RASA_PRO_BETA_E2E_ASSERTIONS` to true in your testing environment.
+
+`export RASA_PRO_BETA_E2E_ASSERTIONS=true`
+
+To run **all the tests**:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions
+```
+
+------
+
+To run **passing/failing/flaky** tests:
+```commandline
+rasa test e2e e2e_tests_with_assertions/passing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/failing
+```
+```commandline
+rasa test e2e e2e_tests_with_assertions/flaky
+```
+
+------
+
+To run a **single test** , provide the path to a 
+target test:
+
+```commandline
+rasa test e2e e2e_tests_with_assertions/tests/path/to/a/target/test.yml
+```
+
+------

actions/ask_for_slot_action.py

@@ -47,7 +47,7 @@ def run(
 
         if len(restaurant_names) > 0:
             dispatcher.utter_message(
-                text="Do you know which restaurant you would like me to reverse a table at?",
+                text="Do you know which restaurant you would like me to reserve a table at?",
                 buttons=[
                     {"title": r, "payload": f'/inform{{"restaurant_name":"{r}"}}'}
                     for r in restaurant_names

actions/execute_transfer.py

@@ -1,3 +1,4 @@
+import re
 from typing import Any, Dict
 from datetime import datetime
 from rasa_sdk import Action, Tracker
@@ -6,6 +7,9 @@
 from actions.db import get_account, write_account, add_transaction, Transaction
 
 
+AMOUNT_OF_MONEY_REGEX = re.compile(r"\d*[.,]*\d+")
+
+
 class ExecuteTransfer(Action):
 
     def name(self) -> str:
@@ -25,7 +29,7 @@ def run(self, dispatcher: CollectingDispatcher,
         if recipient == "Jack":
             return [SlotSet("transfer_money_transfer_successful", False)]
 
-        amount_of_money_value = float(amount_of_money.replace("$", "").replace("USD", "").strip())
+        amount_of_money_value = float(re.findall(AMOUNT_OF_MONEY_REGEX, amount_of_money)[0])
         account.funds -= amount_of_money_value
         new_transaction = \
             Transaction(datetime=datetime.now().isoformat(), recipient=recipient,

actions/setup_recurrent_payment.py

@@ -20,7 +20,8 @@ def parse_datetime(text: str) -> Optional[datetime]:
     if isinstance(parsed_value, dict):
         parsed_value = parsed_value["from"]
 
-    return datetime.fromisoformat(parsed_value)
+    result = datetime.fromisoformat(parsed_value)
+    return result.replace(tzinfo=None)
 
 
 class ValidatePaymentStartDate(Action):
@@ -43,7 +44,7 @@ def run(
             dispatcher.utter_message(response="utter_invalid_date")
             return [SlotSet("recurrent_payment_start_date", None)]
 
-        return [SlotSet("recurrent_payment_start_date", start_date.isoformat())]
+        return [SlotSet("recurrent_payment_start_date", start_date.strftime("%Y-%m-%d"))]
 
 
 class ValidatePaymentEndDate(Action):
@@ -66,11 +67,11 @@ def run(
             return [SlotSet("recurrent_payment_end_date", None)]
 
         start_date = tracker.get_slot("recurrent_payment_start_date")
-        if start_date is not None and end_date < datetime.fromisoformat(start_date):
+        if start_date is not None and end_date < datetime.strptime(start_date, "%Y-%m-%d"):
             dispatcher.utter_message(response="utter_invalid_date")
             return [SlotSet("recurrent_payment_end_date", None)]
 
-        return [SlotSet("recurrent_payment_end_date", end_date.isoformat())]
+        return [SlotSet("recurrent_payment_end_date", end_date.strftime("%Y-%m-%d"))]
 
 
 class ExecutePayment(Action):

config/config.yml

@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: SingleStepLLMCommandGenerator
   llm:
-    model_name: gpt-4
-    request_timeout: 7
+    model: gpt-4
+    timeout: 7
     temperature: 0.0
     top_p: 0.0
 

config/multistep-config.yml

@@ -28,8 +28,8 @@ pipeline:
 - name: NLUCommandAdapter
 - name: MultiStepLLMCommandGenerator
   llm:
-    model_name: gpt-3.5-turbo-0125
-    request_timeout: 7
+    model: gpt-3.5-turbo-0125
+    timeout: 7
     temperature: 0.0
     top_p: 0.0