Langsmith-graph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2522ce05-2181-4de1-bc4f-8e92ed703072",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tavily-python 0.3.1 requires tiktoken==0.5.2, but you have tiktoken 0.6.0 which is incompatible.\n"
     ]
    }
   ],
   "source": [
    "# %pip install --upgrade --quiet  langchain langsmith langchainhub --quiet\n",
    "# %pip install --upgrade --quiet  langchain-openai tiktoken pandas duckduckgo-search --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0aaee236-0c7a-43d3-bbab-3c5eac106d7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                                  Version\n",
      "---------------------------------------- ---------------\n",
      "aiohttp                                  3.9.3\n",
      "aiosignal                                1.3.1\n",
      "annotated-types                          0.6.0\n",
      "anyio                                    3.5.0\n",
      "argon2-cffi                              21.3.0\n",
      "argon2-cffi-bindings                     21.2.0\n",
      "arxiv                                    2.1.0\n",
      "asgiref                                  3.7.2\n",
      "asttokens                                2.0.5\n",
      "async-lru                                2.0.4\n",
      "async-timeout                            4.0.3\n",
      "attrs                                    23.1.0\n",
      "autogenstudio                            0.0.31a0\n",
      "Babel                                    2.11.0\n",
      "backcall                                 0.2.0\n",
      "backoff                                  2.2.1\n",
      "bcrypt                                   4.1.2\n",
      "beautifulsoup4                           4.12.2\n",
      "bleach                                   4.1.0\n",
      "Brotli                                   1.0.9\n",
      "cachetools                               5.3.2\n",
      "certifi                                  2023.11.17\n",
      "cffi                                     1.16.0\n",
      "chardet                                  5.2.0\n",
      "charset-normalizer                       2.0.4\n",
      "chess                                    1.10.0\n",
      "chroma-hnswlib                           0.7.3\n",
      "chromadb                                 0.4.21\n",
      "click                                    8.1.7\n",
      "colorama                                 0.4.6\n",
      "coloredlogs                              15.0.1\n",
      "comm                                     0.1.2\n",
      "contourpy                                1.2.0\n",
      "cryptography                             41.0.7\n",
      "curl_cffi                                0.6.0b9\n",
      "cycler                                   0.12.1\n",
      "dataclasses-json                         0.6.4\n",
      "debugpy                                  1.6.7\n",
      "decorator                                5.1.1\n",
      "defusedxml                               0.7.1\n",
      "Deprecated                               1.2.14\n",
      "diskcache                                5.6.3\n",
      "distro                                   1.9.0\n",
      "docker                                   7.0.0\n",
      "duckduckgo_search                        4.4.3\n",
      "exceptiongroup                           1.0.4\n",
      "executing                                0.8.3\n",
      "faiss-cpu                                1.7.4\n",
      "fastapi                                  0.108.0\n",
      "fastjsonschema                           2.16.2\n",
      "feedparser                               6.0.10\n",
      "filelock                                 3.13.1\n",
      "FLAML                                    2.1.1\n",
      "flatbuffers                              23.5.26\n",
      "fonttools                                4.47.2\n",
      "frozenlist                               1.4.1\n",
      "fsspec                                   2023.12.2\n",
      "google-auth                              2.25.2\n",
      "googleapis-common-protos                 1.62.0\n",
      "greenlet                                 3.0.3\n",
      "grpcio                                   1.60.0\n",
      "h11                                      0.14.0\n",
      "httpcore                                 1.0.2\n",
      "httptools                                0.6.1\n",
      "httpx                                    0.26.0\n",
      "huggingface-hub                          0.20.1\n",
      "humanfriendly                            10.0\n",
      "idna                                     3.4\n",
      "importlib-metadata                       6.11.0\n",
      "importlib-resources                      6.1.1\n",
      "ipykernel                                6.25.0\n",
      "ipython                                  8.15.0\n",
      "ipywidgets                               8.0.4\n",
      "jedi                                     0.18.1\n",
      "Jinja2                                   3.1.2\n",
      "joblib                                   1.3.2\n",
      "json5                                    0.9.6\n",
      "jsonpatch                                1.33\n",
      "jsonpointer                              2.4\n",
      "jsonschema                               4.19.2\n",
      "jsonschema-specifications                2023.7.1\n",
      "jupyter                                  1.0.0\n",
      "jupyter_client                           8.6.0\n",
      "jupyter-console                          6.6.3\n",
      "jupyter_core                             5.5.0\n",
      "jupyter-events                           0.8.0\n",
      "jupyter-lsp                              2.2.0\n",
      "jupyter_server                           2.10.0\n",
      "jupyter_server_terminals                 0.4.4\n",
      "jupyterlab                               4.0.8\n",
      "jupyterlab-pygments                      0.1.2\n",
      "jupyterlab_server                        2.25.1\n",
      "jupyterlab-widgets                       3.0.9\n",
      "kiwisolver                               1.4.5\n",
      "kubernetes                               28.1.0\n",
      "langchain                                0.1.8\n",
      "langchain-community                      0.0.21\n",
      "langchain-core                           0.1.25\n",
      "langchain-openai                         0.0.6\n",
      "langchainhub                             0.1.14\n",
      "langsmith                                0.1.5\n",
      "lightgbm                                 4.2.0\n",
      "lxml                                     5.1.0\n",
      "MarkupSafe                               2.1.3\n",
      "marshmallow                              3.20.2\n",
      "matplotlib                               3.8.1\n",
      "matplotlib-inline                        0.1.6\n",
      "mistune                                  2.0.4\n",
      "mmh3                                     4.0.1\n",
      "monotonic                                1.6\n",
      "mpmath                                   1.3.0\n",
      "multidict                                6.0.5\n",
      "mypy-extensions                          1.0.0\n",
      "nbclient                                 0.8.0\n",
      "nbconvert                                7.10.0\n",
      "nbformat                                 5.9.2\n",
      "nest-asyncio                             1.5.8\n",
      "networkx                                 3.2.1\n",
      "nltk                                     3.8.1\n",
      "notebook                                 7.0.6\n",
      "notebook_shim                            0.2.3\n",
      "numpy                                    1.26.2\n",
      "oauthlib                                 3.2.2\n",
      "onnxruntime                              1.16.3\n",
      "openai                                   1.12.0\n",
      "opentelemetry-api                        1.22.0\n",
      "opentelemetry-exporter-otlp-proto-common 1.22.0\n",
      "opentelemetry-exporter-otlp-proto-grpc   1.22.0\n",
      "opentelemetry-instrumentation            0.43b0\n",
      "opentelemetry-instrumentation-asgi       0.43b0\n",
      "opentelemetry-instrumentation-fastapi    0.43b0\n",
      "opentelemetry-proto                      1.22.0\n",
      "opentelemetry-sdk                        1.22.0\n",
      "opentelemetry-semantic-conventions       0.43b0\n",
      "opentelemetry-util-http                  0.43b0\n",
      "overrides                                7.4.0\n",
      "packaging                                23.2\n",
      "pandas                                   2.2.0\n",
      "pandocfilters                            1.5.0\n",
      "parso                                    0.8.3\n",
      "pickleshare                              0.7.5\n",
      "pillow                                   10.2.0\n",
      "pip                                      23.3.1\n",
      "platformdirs                             3.10.0\n",
      "ply                                      3.11\n",
      "posthog                                  3.1.0\n",
      "prometheus-client                        0.14.1\n",
      "prompt-toolkit                           3.0.43\n",
      "protobuf                                 4.25.1\n",
      "psutil                                   5.9.0\n",
      "pulsar-client                            3.3.0\n",
      "pure-eval                                0.2.2\n",
      "pyasn1                                   0.5.1\n",
      "pyasn1-modules                           0.3.0\n",
      "pyautogen                                0.2.5\n",
      "pycparser                                2.21\n",
      "pydantic                                 2.5.3\n",
      "pydantic_core                            2.14.6\n",
      "Pygments                                 2.15.1\n",
      "pyOpenSSL                                23.2.0\n",
      "pyparsing                                3.1.1\n",
      "pypdf                                    3.17.4\n",
      "PyPika                                   0.48.9\n",
      "PyQt5                                    5.15.10\n",
      "PyQt5-sip                                12.13.0\n",
      "pyreadline3                              3.4.1\n",
      "PySocks                                  1.7.1\n",
      "python-dateutil                          2.8.2\n",
      "python-dotenv                            1.0.0\n",
      "python-json-logger                       2.0.7\n",
      "pytz                                     2023.3.post1\n",
      "pywin32                                  305.1\n",
      "pywinpty                                 2.0.10\n",
      "PyYAML                                   6.0.1\n",
      "pyzmq                                    25.1.0\n",
      "qtconsole                                5.5.0\n",
      "QtPy                                     2.4.1\n",
      "referencing                              0.30.2\n",
      "regex                                    2023.12.25\n",
      "requests                                 2.31.0\n",
      "requests-oauthlib                        1.3.1\n",
      "rfc3339-validator                        0.1.4\n",
      "rfc3986-validator                        0.1.1\n",
      "rpds-py                                  0.10.6\n",
      "rsa                                      4.9\n",
      "safetensors                              0.4.1\n",
      "scikit-learn                             1.3.2\n",
      "scipy                                    1.11.4\n",
      "Send2Trash                               1.8.2\n",
      "sentence-transformers                    2.2.2\n",
      "sentencepiece                            0.1.99\n",
      "setuptools                               68.2.2\n",
      "sgmllib3k                                1.0.0\n",
      "sip                                      6.7.12\n",
      "six                                      1.16.0\n",
      "sniffio                                  1.2.0\n",
      "soupsieve                                2.5\n",
      "SQLAlchemy                               2.0.27\n",
      "stack-data                               0.2.0\n",
      "starlette                                0.32.0.post1\n",
      "sympy                                    1.12\n",
      "tavily-python                            0.3.1\n",
      "tenacity                                 8.2.3\n",
      "termcolor                                2.4.0\n",
      "terminado                                0.17.1\n",
      "threadpoolctl                            3.2.0\n",
      "tiktoken                                 0.6.0\n",
      "tinycss2                                 1.2.1\n",
      "tokenizers                               0.15.0\n",
      "tomli                                    2.0.1\n",
      "torch                                    2.1.2\n",
      "torchvision                              0.16.2\n",
      "tornado                                  6.3.3\n",
      "tqdm                                     4.66.1\n",
      "traitlets                                5.14.0\n",
      "transformers                             4.36.2\n",
      "typer                                    0.9.0\n",
      "types-requests                           2.31.0.20240218\n",
      "typing_extensions                        4.9.0\n",
      "typing-inspect                           0.9.0\n",
      "tzdata                                   2023.4\n",
      "urllib3                                  2.2.1\n",
      "uvicorn                                  0.25.0\n",
      "watchfiles                               0.21.0\n",
      "wcwidth                                  0.2.12\n",
      "webencodings                             0.5.1\n",
      "websocket-client                         0.58.0\n",
      "websockets                               12.0\n",
      "wheel                                    0.41.2\n",
      "widgetsnbextension                       4.0.5\n",
      "win-inet-pton                            1.1.0\n",
      "wrapt                                    1.16.0\n",
      "xgboost                                  2.0.3\n",
      "yarl                                     1.9.4\n",
      "zipp                                     3.17.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# pip list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6087c065-c62e-42e5-b2ad-fcfd8056ee3a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-24T00:11:02.218143600Z",
     "start_time": "2024-02-24T00:11:02.201142500Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from uuid import uuid4\n",
    "\n",
    "unique_id = uuid4().hex[0:8]\n",
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_PROJECT\"] = f\"Tracing Walkthrough - {unique_id}\"\n",
    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = \"ls__eb24c4743a814171b07dc0591921a299\"  # Update to your API key\n",
    "\n",
    "# Used by the agent in this tutorial\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-OqlsJwGTMdD1ujTG02Bb0fE08b7f4b30B07d8e83012bA8A8\"\n",
    "os.environ[\"OPENAI_API_BASE\"] = \"https://oneapi.xty.app/v1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "65086c0765a2e3"
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e4fedacc-2494-4d55-a228-b319d8a04027",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-24T00:11:12.188278500Z",
     "start_time": "2024-02-24T00:11:11.913807900Z"
    }
   },
   "outputs": [],
   "source": [
    "from langsmith import Client\n",
    "\n",
    "client = Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "419f57b9-be19-4c1c-877a-06e1e686ec51",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-24T00:11:45.986558900Z",
     "start_time": "2024-02-24T00:11:43.949217500Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain import hub\n",
    "from langchain.agents import AgentExecutor\n",
    "from langchain.agents.format_scratchpad import format_to_openai_function_messages\n",
    "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
    "from langchain_community.tools import DuckDuckGoSearchResults\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "\n",
    "\n",
    "# Fetches the latest version of this prompt\n",
    "prompt = hub.pull(\"wfh/langsmith-agent-prompt:5d466cbc\")\n",
    "\n",
    "llm = ChatOpenAI(\n",
    "    model=\"gpt-3.5-turbo-16k\",\n",
    "    temperature=0,\n",
    ")\n",
    "\n",
    "tools = [\n",
    "    DuckDuckGoSearchResults(\n",
    "        name=\"duck_duck_go\"\n",
    "    ),  # General internet search using DuckDuckGo\n",
    "]\n",
    "\n",
    "llm_with_tools = llm.bind_functions(tools)\n",
    "\n",
    "runnable_agent = (\n",
    "    {\n",
    "        \"input\": lambda x: x[\"input\"],\n",
    "        \"agent_scratchpad\": lambda x: format_to_openai_function_messages(\n",
    "            x[\"intermediate_steps\"]\n",
    "        ),\n",
    "    }\n",
    "    | prompt\n",
    "    | llm_with_tools\n",
    "    | OpenAIFunctionsAgentOutputParser()\n",
    ")\n",
    "\n",
    "agent_executor = AgentExecutor(\n",
    "    agent=runnable_agent, tools=tools, handle_parsing_errors=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "798f98ed-62e4-490e-9ece-b58501aa41e5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-24T00:12:32.334477900Z",
     "start_time": "2024-02-24T00:12:21.044526100Z"
    }
   },
   "outputs": [],
   "source": [
    "inputs = [\n",
    "    \"What is LangChain?\",\n",
    "]\n",
    "from langchain.tools import DuckDuckGoSearchRun\n",
    "import os\n",
    "os.environ['http_proxy'] = \"127.0.0.1:7890\"\n",
    "os.environ['https_proxy'] = \"127.0.0.1:7890\"\n",
    "results = agent_executor.batch([{\"input\": x} for x in inputs], return_exceptions=True,verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "51f89155-3940-4b1a-8c8d-510b9198c5fc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-24T00:12:43.501945900Z",
     "start_time": "2024-02-24T00:12:43.478940Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "[{'input': 'What is LangChain?',\n  'output': \"LangChain is a tool that simplifies the creation of applications using large language models (LLMs) such as GPT-4. It provides a standard interface for chains, lots of integrations with other tools, and end-to-end chains for common applications. You can learn more about LangChain by visiting the following links:\\n\\n1. [Introduction to Langchain - GeeksforGeeks](https://www.geeksforgeeks.org/introduction-to-langchain/)\\n2. [LangChain: A Complete Guide & Tutorial - Nanonets](https://nanonets.com/blog/langchain/)\\n3. [Getting Started with LangChain: A Beginner's Guide to Building LLM-Powered Applications](https://towardsdatascience.com/getting-started-with-langchain-a-beginners-guide-to-building-llm-powered-applications-95fc8898732c)\\n4. [langchain · PyPI](https://pypi.org/project/langchain/)\"}]"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "88678abc-ecc6-4b9d-8722-a29b8ec7c85a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"France vs Argentina Match referee Alireza Faghani presents one of eight yellow cards during the match. The teams had faced each other in 11 previous matches, including two World Cup group stage matches, both won by Argentina ( 1-0 in 1930, and 2-1 in 1978). [5] The selection of Russia as host of the world cup makes them the largest country geographically to ever host the tournament, while Qatar—the succeeding host nation, scheduled to host the... Winning players A total of 471 players have been in the winning team in the World Cup. Brazil's Pelé is the only one to have won three times, while another 20 have won twice. Only Daniel Passarella (ARG) and players from Brazil and Italy have won the world Cup more than once. No player has won two World Cups both as captain. France won the tournament after defeating Croatia 4-2. [5] This was France's second World Cup title. Qualified teams Teams qualified for World Cup Teams did not qualify for World Cup Teams removed from the tournament by FIFA before playing a match Countries were not FIFA members The trophy has been won by eight national teams. Brazil, with five wins, are the only team to have played in every tournament. The other World Cup winners are Germany and Italy, with four titles each; Argentina, with three titles; France and inaugural winner Uruguay, each with two titles; and England and Spain, with one title each.\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:192: UserWarning: Curlm alread closed! quitting from process_data\n",
      "  warnings.warn(\"Curlm alread closed! quitting from process_data\")\n"
     ]
    }
   ],
   "source": [
    "from langchain.tools import DuckDuckGoSearchRun\n",
    "import os\n",
    "os.environ['http_proxy'] = \"127.0.0.1:7890\"\n",
    "os.environ['https_proxy'] = \"127.0.0.1:7890\"\n",
    "search = DuckDuckGoSearchRun()\n",
    "search.run(\"Who is winner of FIFA worldcup 2018?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02e10e80-87f7-45ac-9ac6-b0422fe6003b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting langgraph\n",
      "  Downloading langgraph-0.0.24-py3-none-any.whl.metadata (31 kB)\n",
      "Requirement already satisfied: langchain-core<0.2.0,>=0.1.16 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langgraph) (0.1.25)\n",
      "Requirement already satisfied: PyYAML>=5.3 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (6.0.1)\n",
      "Requirement already satisfied: anyio<5,>=3 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (3.5.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (1.33)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (0.1.5)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (23.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (2.5.3)\n",
      "Requirement already satisfied: requests<3,>=2 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from langchain-core<0.2.0,>=0.1.16->langgraph) (8.2.3)\n",
      "Requirement already satisfied: idna>=2.8 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from anyio<5,>=3->langchain-core<0.2.0,>=0.1.16->langgraph) (3.4)\n",
      "Requirement already satisfied: sniffio>=1.1 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from anyio<5,>=3->langchain-core<0.2.0,>=0.1.16->langgraph) (1.2.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2.0,>=0.1.16->langgraph) (2.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.16->langgraph) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.14.6 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.16->langgraph) (2.14.6)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.16->langgraph) (4.9.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from requests<3,>=2->langchain-core<0.2.0,>=0.1.16->langgraph) (2.0.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from requests<3,>=2->langchain-core<0.2.0,>=0.1.16->langgraph) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in e:\\anaconda\\envs\\pyautogen\\lib\\site-packages (from requests<3,>=2->langchain-core<0.2.0,>=0.1.16->langgraph) (2023.11.17)\n",
      "Downloading langgraph-0.0.24-py3-none-any.whl (40 kB)\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 0.0/40.3 kB ? eta -:--:--\n",
      "   ---------------------------------------- 40.3/40.3 kB ? eta 0:00:00\n",
      "Installing collected packages: langgraph\n",
      "Successfully installed langgraph-0.0.24\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "# pip install langgraph "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4d121f2c-4d11-4586-be28-93bb54460147",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = [\n",
    "    \"What is LangChain?\",\n",
    "    \"What's LangSmith?\",\n",
    "    \"When was Llama-v2 released?\",\n",
    "    \"What is the langsmith cookbook?\",\n",
    "    \"When did langchain first announce the hub?\",\n",
    "]\n",
    "outputs = [\n",
    "    \"LangChain is an open-source framework for building applications using large language models. It is also the name of the company building LangSmith.\",\n",
    "    \"LangSmith is a unified platform for debugging, testing, and monitoring language model applications and agents powered by LangChain\",\n",
    "    \"July 18, 2023\",\n",
    "    \"The langsmith cookbook is a github repository containing detailed examples of how to use LangSmith to debug, evaluate, and monitor large language model-powered applications.\",\n",
    "    \"September 5, 2023\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0a7700a9-d6b0-4192-9950-76961a05eb63",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = f\"agent-qa-{unique_id}\"\n",
    "\n",
    "dataset = client.create_dataset(\n",
    "    dataset_name,\n",
    "    description=\"An example dataset of questions over the LangSmith documentation.\",\n",
    ")\n",
    "\n",
    "client.create_examples(\n",
    "    inputs=[{\"input\": query} for query in inputs],\n",
    "    outputs=[{\"output\": answer} for answer in outputs],\n",
    "    dataset_id=dataset.id,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e9308147-ba40-4dfa-a0a4-1b4077863494",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import hub\n",
    "from langchain.agents import AgentExecutor, AgentType, initialize_agent, load_tools\n",
    "from langchain.agents.format_scratchpad import format_to_openai_function_messages\n",
    "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "\n",
    "# Since chains can be stateful (e.g. they can have memory), we provide\n",
    "# a way to initialize a new chain for each row in the dataset. This is done\n",
    "# by passing in a factory function that returns a new chain for each row.\n",
    "def create_agent(prompt, llm_with_tools):\n",
    "    runnable_agent = (\n",
    "        {\n",
    "            \"input\": lambda x: x[\"input\"],\n",
    "            \"agent_scratchpad\": lambda x: format_to_openai_function_messages(\n",
    "                x[\"intermediate_steps\"]\n",
    "            ),\n",
    "        }\n",
    "        | prompt\n",
    "        | llm_with_tools\n",
    "        | OpenAIFunctionsAgentOutputParser()\n",
    "    )\n",
    "    return AgentExecutor(agent=runnable_agent, tools=tools, handle_parsing_errors=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "194030ed-de4b-4faf-8294-c3777032dfb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langsmith.evaluation import EvaluationResult, run_evaluator\n",
    "from langsmith.schemas import Example, Run\n",
    "\n",
    "\n",
    "@run_evaluator\n",
    "def check_not_idk(run: Run, example: Example):\n",
    "    \"\"\"Illustration of a custom evaluator.\"\"\"\n",
    "    agent_response = run.outputs[\"output\"]\n",
    "    if \"don't know\" in agent_response or \"not sure\" in agent_response:\n",
    "        score = 0\n",
    "    else:\n",
    "        score = 1\n",
    "    # You can access the dataset labels in example.outputs[key]\n",
    "    # You can also access the model inputs in run.inputs[key]\n",
    "    return EvaluationResult(\n",
    "        key=\"not_uncertain\",\n",
    "        score=score,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "2154102aa57fd320"
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9b78cd0a-080e-4160-a899-b66f0237f596",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\langchain_core\\_api\\deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.openai.ChatOpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "from langchain.evaluation import EvaluatorType\n",
    "from langchain.smith import RunEvalConfig\n",
    "from langchain import chat_models, prompts, smith\n",
    "evaluation_config = RunEvalConfig(\n",
    "    # Evaluators can either be an evaluator type (e.g., \"qa\", \"criteria\", \"embedding_distance\", etc.) or a configuration for that evaluator\n",
    "    evaluators=[\n",
    "        # Measures whether a QA response is \"Correct\", based on a reference answer\n",
    "        # You can also select via the raw string \"qa\"\n",
    "        EvaluatorType.QA,\n",
    "        # Measure the embedding distance between the output and the reference answer\n",
    "        # Equivalent to: EvalConfig.EmbeddingDistance(embeddings=OpenAIEmbeddings())\n",
    "        EvaluatorType.EMBEDDING_DISTANCE,\n",
    "        # Grade whether the output satisfies the stated criteria.\n",
    "        # You can select a default one such as \"helpfulness\" or provide your own.\n",
    "        RunEvalConfig.LabeledCriteria(\"helpfulness\"),\n",
    "        # The LabeledScoreString evaluator outputs a score on a scale from 1-10.\n",
    "        # You can use default criteria or write our own rubric\n",
    "        RunEvalConfig.LabeledScoreString(\n",
    "            {\n",
    "                \"accuracy\": \"\"\"\n",
    "Score 1: The answer is completely unrelated to the reference.\n",
    "Score 3: The answer has minor relevance but does not align with the reference.\n",
    "Score 5: The answer has moderate relevance but contains inaccuracies.\n",
    "Score 7: The answer aligns with the reference but has minor errors or omissions.\n",
    "Score 10: The answer is completely accurate and aligns perfectly with the reference.\"\"\"\n",
    "            },\n",
    "            normalize_by=10,\n",
    "        ),\n",
    "    ],\n",
    "    # You can add custom StringEvaluator or RunEvaluator objects here as well, which will automatically be\n",
    "    # applied to each prediction. Check out the docs for examples.\n",
    "    custom_evaluators=[check_not_idk],\n",
    "    eval_llm=chat_models.ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4673ff61-8e1e-4c5c-a1f2-7fd25e777223",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import hub\n",
    "\n",
    "# We will test this version of the prompt\n",
    "prompt = hub.pull(\"wfh/langsmith-agent-prompt:798e7324\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3b3e7c50-9841-4505-be10-6cc208310751",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project 'evaluators-3' at:\n",
      "https://smith.langchain.com/o/703b146d-808c-511c-8d4d-df099f85290b/datasets/473c3d41-feab-49c8-9890-aa9140628f69/compare?selectedSessions=86efa52d-81de-4b67-9d2e-023c58061c55\n",
      "\n",
      "View all tests for Dataset agent-qa-20833026 at:\n",
      "https://smith.langchain.com/o/703b146d-808c-511c-8d4d-df099f85290b/datasets/473c3d41-feab-49c8-9890-aa9140628f69\n",
      "[>                                                 ] 0/5"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Task was destroyed but it is pending!\n",
      "task: <Task pending name='Task-166' coro=<AsyncCurl._force_timeout() running at E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:168> wait_for=<Future cancelled>>\n",
      "Task was destroyed but it is pending!\n",
      "task: <Task pending name='Task-169' coro=<AsyncCurl._force_timeout() running at E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:168> wait_for=<Future cancelled>>\n",
      "Task was destroyed but it is pending!\n",
      "task: <Task pending name='Task-172' coro=<AsyncCurl._force_timeout() running at E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:168> wait_for=<Future cancelled>>\n",
      "Task was destroyed but it is pending!\n",
      "task: <Task pending name='Task-175' coro=<AsyncCurl._force_timeout() running at E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:168> wait_for=<Future cancelled>>\n",
      "Task was destroyed but it is pending!\n",
      "task: <Task pending name='Task-202' coro=<AsyncCurl._force_timeout() running at E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\curl_cffi\\aio.py:168> wait_for=<Future cancelled>>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[------------------------------------------------->] 5/5"
     ]
    },
    {
     "data": {
      "text/html": [
       "<h3>Experiment Results:</h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feedback.correctness</th>\n",
       "      <th>feedback.embedding_cosine_distance</th>\n",
       "      <th>feedback.helpfulness</th>\n",
       "      <th>feedback.score_string:accuracy</th>\n",
       "      <th>feedback.not_uncertain</th>\n",
       "      <th>error</th>\n",
       "      <th>execution_time</th>\n",
       "      <th>run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e9b957e6-d4b0-4dd7-93be-516a6677ab7b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.101207</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.102859</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.050621</td>\n",
       "      <td>0.447214</td>\n",
       "      <td>0.164317</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.691004</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.056389</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.522663</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.064192</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.614773</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.072983</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.210582</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.153203</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.310856</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.159266</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.855421</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        feedback.correctness  feedback.embedding_cosine_distance  \\\n",
       "count                    5.0                            5.000000   \n",
       "unique                   NaN                                 NaN   \n",
       "top                      NaN                                 NaN   \n",
       "freq                     NaN                                 NaN   \n",
       "mean                     1.0                            0.101207   \n",
       "std                      0.0                            0.050621   \n",
       "min                      1.0                            0.056389   \n",
       "25%                      1.0                            0.064192   \n",
       "50%                      1.0                            0.072983   \n",
       "75%                      1.0                            0.153203   \n",
       "max                      1.0                            0.159266   \n",
       "\n",
       "        feedback.helpfulness  feedback.score_string:accuracy  \\\n",
       "count               5.000000                        5.000000   \n",
       "unique                   NaN                             NaN   \n",
       "top                      NaN                             NaN   \n",
       "freq                     NaN                             NaN   \n",
       "mean                0.800000                        0.820000   \n",
       "std                 0.447214                        0.164317   \n",
       "min                 0.000000                        0.700000   \n",
       "25%                 1.000000                        0.700000   \n",
       "50%                 1.000000                        0.700000   \n",
       "75%                 1.000000                        1.000000   \n",
       "max                 1.000000                        1.000000   \n",
       "\n",
       "        feedback.not_uncertain error  execution_time  \\\n",
       "count                      5.0     0        5.000000   \n",
       "unique                     NaN     0             NaN   \n",
       "top                        NaN   NaN             NaN   \n",
       "freq                       NaN   NaN             NaN   \n",
       "mean                       1.0   NaN        8.102859   \n",
       "std                        0.0   NaN        1.691004   \n",
       "min                        1.0   NaN        5.522663   \n",
       "25%                        1.0   NaN        7.614773   \n",
       "50%                        1.0   NaN        8.210582   \n",
       "75%                        1.0   NaN        9.310856   \n",
       "max                        1.0   NaN        9.855421   \n",
       "\n",
       "                                      run_id  \n",
       "count                                      5  \n",
       "unique                                     5  \n",
       "top     e9b957e6-d4b0-4dd7-93be-516a6677ab7b  \n",
       "freq                                       1  \n",
       "mean                                     NaN  \n",
       "std                                      NaN  \n",
       "min                                      NaN  \n",
       "25%                                      NaN  \n",
       "50%                                      NaN  \n",
       "75%                                      NaN  \n",
       "max                                      NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import functools\n",
    "\n",
    "from langchain.smith import arun_on_dataset, run_on_dataset\n",
    "\n",
    "chain_results = run_on_dataset(\n",
    "    dataset_name=dataset_name,\n",
    "    llm_or_chain_factory=functools.partial(\n",
    "        create_agent, prompt=prompt, llm_with_tools=llm_with_tools\n",
    "    ),\n",
    "    evaluation=evaluation_config,\n",
    "    verbose=True,\n",
    "    client=client,\n",
    "    project_name=f\"evaluators-3\",\n",
    "    # Project metadata communicates the experiment parameters,\n",
    "    # Useful for reviewing the test results\n",
    "    project_metadata={\n",
    "        \"env\": \"testing-notebook\",\n",
    "        \"model\": \"gpt-3.5-turbo\",\n",
    "        \"prompt\": \"5d466cbc\",\n",
    "    },\n",
    ")\n",
    "\n",
    "# Sometimes, the agent will error due to parsing issues, incompatible tool inputs, etc.\n",
    "# These are logged as warnings here and captured as errors in the tracing UI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0b11708b-0137-4ff8-b09f-6ec0d45aa165",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputs.input</th>\n",
       "      <th>outputs.input</th>\n",
       "      <th>outputs.output</th>\n",
       "      <th>reference.output</th>\n",
       "      <th>feedback.correctness</th>\n",
       "      <th>feedback.embedding_cosine_distance</th>\n",
       "      <th>feedback.helpfulness</th>\n",
       "      <th>feedback.score_string:accuracy</th>\n",
       "      <th>feedback.not_uncertain</th>\n",
       "      <th>error</th>\n",
       "      <th>execution_time</th>\n",
       "      <th>run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1f94c799-0124-4bd4-b900-824570b06f63</th>\n",
       "      <td>When did langchain first announce the hub?</td>\n",
       "      <td>When did langchain first announce the hub?</td>\n",
       "      <td>LangChain first announced the LangChain Hub on...</td>\n",
       "      <td>September 5, 2023</td>\n",
       "      <td>None</td>\n",
       "      <td>0.286927</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>9.208513</td>\n",
       "      <td>4008ffa6-24ac-4b4d-b1c9-204b91890626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35936138-b9e8-4bfe-9ca0-178c969de166</th>\n",
       "      <td>What is the langsmith cookbook?</td>\n",
       "      <td>What is the langsmith cookbook?</td>\n",
       "      <td>The LangSmith Cookbook is a part of LangSmith,...</td>\n",
       "      <td>The langsmith cookbook is a github repository ...</td>\n",
       "      <td>None</td>\n",
       "      <td>0.076693</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>9.316836</td>\n",
       "      <td>c08551b3-4c89-4944-a6c3-82730ed93115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8</th>\n",
       "      <td>When was Llama-v2 released?</td>\n",
       "      <td>When was Llama-v2 released?</td>\n",
       "      <td>LangChain Llama-v2 was released on July 18, 2023.</td>\n",
       "      <td>July 18, 2023</td>\n",
       "      <td>None</td>\n",
       "      <td>0.159439</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>8.802367</td>\n",
       "      <td>1aae1ae2-96de-4297-95cd-7ad5e2959444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ebff4cbf-0666-49a0-aa95-0833e1042952</th>\n",
       "      <td>What's LangSmith?</td>\n",
       "      <td>What's LangSmith?</td>\n",
       "      <td>LangSmith is a platform that helps developers ...</td>\n",
       "      <td>LangSmith is a unified platform for debugging,...</td>\n",
       "      <td>None</td>\n",
       "      <td>0.064192</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>9.865037</td>\n",
       "      <td>e7980cb5-0dd7-4522-86bd-7954500fc73a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eed120c6-eecd-42ba-9c74-999bc6de9aa2</th>\n",
       "      <td>What is LangChain?</td>\n",
       "      <td>What is LangChain?</td>\n",
       "      <td>LangChain is a framework for creating applicat...</td>\n",
       "      <td>LangChain is an open-source framework for buil...</td>\n",
       "      <td>None</td>\n",
       "      <td>0.065663</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>10.415497</td>\n",
       "      <td>d07c3caa-52a4-45ca-8bf1-0184298863df</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    inputs.input  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63  When did langchain first announce the hub?   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166             What is the langsmith cookbook?   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                 When was Llama-v2 released?   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                           What's LangSmith?   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                          What is LangChain?   \n",
       "\n",
       "                                                                   outputs.input  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63  When did langchain first announce the hub?   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166             What is the langsmith cookbook?   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                 When was Llama-v2 released?   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                           What's LangSmith?   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                          What is LangChain?   \n",
       "\n",
       "                                                                         outputs.output  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63  LangChain first announced the LangChain Hub on...   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166  The LangSmith Cookbook is a part of LangSmith,...   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8  LangChain Llama-v2 was released on July 18, 2023.   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952  LangSmith is a platform that helps developers ...   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2  LangChain is a framework for creating applicat...   \n",
       "\n",
       "                                                                       reference.output  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                                  September 5, 2023   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166  The langsmith cookbook is a github repository ...   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                                      July 18, 2023   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952  LangSmith is a unified platform for debugging,...   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2  LangChain is an open-source framework for buil...   \n",
       "\n",
       "                                     feedback.correctness  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                 None   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166                 None   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                 None   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                 None   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                 None   \n",
       "\n",
       "                                      feedback.embedding_cosine_distance  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                            0.286927   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166                            0.076693   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                            0.159439   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                            0.064192   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                            0.065663   \n",
       "\n",
       "                                     feedback.helpfulness  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                 None   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166                 None   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                 None   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                 None   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                 None   \n",
       "\n",
       "                                     feedback.score_string:accuracy  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                           None   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166                           None   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                           None   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                           None   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                           None   \n",
       "\n",
       "                                      feedback.not_uncertain error  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63                       1  None   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166                       1  None   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8                       1  None   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952                       1  None   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2                       1  None   \n",
       "\n",
       "                                      execution_time  \\\n",
       "1f94c799-0124-4bd4-b900-824570b06f63        9.208513   \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166        9.316836   \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8        8.802367   \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952        9.865037   \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2       10.415497   \n",
       "\n",
       "                                                                    run_id  \n",
       "1f94c799-0124-4bd4-b900-824570b06f63  4008ffa6-24ac-4b4d-b1c9-204b91890626  \n",
       "35936138-b9e8-4bfe-9ca0-178c969de166  c08551b3-4c89-4944-a6c3-82730ed93115  \n",
       "9827f8a4-2a41-4e1e-bb7e-4ad72c4c18e8  1aae1ae2-96de-4297-95cd-7ad5e2959444  \n",
       "ebff4cbf-0666-49a0-aa95-0833e1042952  e7980cb5-0dd7-4522-86bd-7954500fc73a  \n",
       "eed120c6-eecd-42ba-9c74-999bc6de9aa2  d07c3caa-52a4-45ca-8bf1-0184298863df  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chain_results.to_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "82b7a44a-3370-4552-b52a-33ea76e12c8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project 'runnable-agent-test-39f3bbd0-20833026' at:\n",
      "https://smith.langchain.com/o/703b146d-808c-511c-8d4d-df099f85290b/datasets/473c3d41-feab-49c8-9890-aa9140628f69/compare?selectedSessions=bda22c83-b6b9-4c20-86d2-4a5036b5a7ef\n",
      "\n",
      "View all tests for Dataset agent-qa-20833026 at:\n",
      "https://smith.langchain.com/o/703b146d-808c-511c-8d4d-df099f85290b/datasets/473c3d41-feab-49c8-9890-aa9140628f69\n",
      "[------------------------------------------------->] 5/5"
     ]
    },
    {
     "data": {
      "text/html": [
       "<h3>Experiment Results:</h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feedback.correctness</th>\n",
       "      <th>feedback.embedding_cosine_distance</th>\n",
       "      <th>feedback.helpfulness</th>\n",
       "      <th>feedback.score_string:accuracy</th>\n",
       "      <th>feedback.not_uncertain</th>\n",
       "      <th>error</th>\n",
       "      <th>execution_time</th>\n",
       "      <th>run_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1a421f4f-7334-4735-bd84-f7f04315412b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.119725</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.689420</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.102023</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.149202</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.047058</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.667260</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.059886</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.195550</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.062371</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.324686</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.138955</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.606992</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.290355</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.652611</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       feedback.correctness  feedback.embedding_cosine_distance  \\\n",
       "count                     0                            5.000000   \n",
       "unique                    0                                 NaN   \n",
       "top                     NaN                                 NaN   \n",
       "freq                    NaN                                 NaN   \n",
       "mean                    NaN                            0.119725   \n",
       "std                     NaN                            0.102023   \n",
       "min                     NaN                            0.047058   \n",
       "25%                     NaN                            0.059886   \n",
       "50%                     NaN                            0.062371   \n",
       "75%                     NaN                            0.138955   \n",
       "max                     NaN                            0.290355   \n",
       "\n",
       "       feedback.helpfulness feedback.score_string:accuracy  \\\n",
       "count                     0                              0   \n",
       "unique                    0                              0   \n",
       "top                     NaN                            NaN   \n",
       "freq                    NaN                            NaN   \n",
       "mean                    NaN                            NaN   \n",
       "std                     NaN                            NaN   \n",
       "min                     NaN                            NaN   \n",
       "25%                     NaN                            NaN   \n",
       "50%                     NaN                            NaN   \n",
       "75%                     NaN                            NaN   \n",
       "max                     NaN                            NaN   \n",
       "\n",
       "        feedback.not_uncertain error  execution_time  \\\n",
       "count                      5.0     0        5.000000   \n",
       "unique                     NaN     0             NaN   \n",
       "top                        NaN   NaN             NaN   \n",
       "freq                       NaN   NaN             NaN   \n",
       "mean                       1.0   NaN        8.689420   \n",
       "std                        0.0   NaN        1.149202   \n",
       "min                        1.0   NaN        7.667260   \n",
       "25%                        1.0   NaN        8.195550   \n",
       "50%                        1.0   NaN        8.324686   \n",
       "75%                        1.0   NaN        8.606992   \n",
       "max                        1.0   NaN       10.652611   \n",
       "\n",
       "                                      run_id  \n",
       "count                                      5  \n",
       "unique                                     5  \n",
       "top     1a421f4f-7334-4735-bd84-f7f04315412b  \n",
       "freq                                       1  \n",
       "mean                                     NaN  \n",
       "std                                      NaN  \n",
       "min                                      NaN  \n",
       "25%                                      NaN  \n",
       "50%                                      NaN  \n",
       "75%                                      NaN  \n",
       "max                                      NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidate_prompt = hub.pull(\"wfh/langsmith-agent-prompt:39f3bbd0\")\n",
    "\n",
    "chain_results = run_on_dataset(\n",
    "    dataset_name=dataset_name,\n",
    "    llm_or_chain_factory=functools.partial(\n",
    "        create_agent, prompt=candidate_prompt, llm_with_tools=llm_with_tools\n",
    "    ),\n",
    "    evaluation=evaluation_config,\n",
    "    verbose=True,\n",
    "    client=client,\n",
    "    project_name=f\"runnable-agent-test-39f3bbd0-{unique_id}\",\n",
    "    project_metadata={\n",
    "        \"env\": \"testing-notebook\",\n",
    "        \"model\": \"gpt-3.5-turbo\",\n",
    "        \"prompt\": \"39f3bbd0\",\n",
    "    },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b3221ea-e75e-4a36-b86d-74f91fd19f96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "619b009f-74a4-470c-8f72-45a049a9dec1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1029e22d-a981-490e-bdae-e9fc352c940b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f1cc8f15-b6c8-4971-9cdc-3a3d42784bcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from uuid import uuid4\n",
    "\n",
    "unique_id = uuid4().hex[0:8]\n",
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_PROJECT\"] = f\"Tracing Walkthrough - {unique_id}\"\n",
    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = \"ls__eb24c4743a814171b07dc0591921a299\"  # Update to your API key\n",
    "\n",
    "# Used by the agent in this tutorial\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-OqlsJwGTMdD1ujTG02Bb0fE08b7f4b30B07d8e83012bA8A8\"\n",
    "os.environ[\"OPENAI_API_BASE\"] = \"https://oneapi.xty.app/v1\"\n",
    "os.environ[\"TAVILY_API_KEY\"] = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f8e030b7-1403-42a7-9023-4eb7d02c3fbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
    "\n",
    "tools = [TavilySearchResults(max_results=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "44a493a8-8edb-4905-aa2b-f92f571c6a7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "# We will set streaming=True so that we can stream tokens\n",
    "# See the streaming section for more information on this.\n",
    "model = ChatOpenAI(temperature=0, streaming=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b98921ff-9ad9-4e68-a7ac-f80d4cf5d2cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda\\envs\\pyautogen\\lib\\site-packages\\langchain_core\\_api\\deprecation.py:117: LangChainDeprecationWarning: The function `format_tool_to_openai_function` was deprecated in LangChain 0.1.16 and will be removed in 0.2.0. Use langchain_core.utils.function_calling.convert_to_openai_function() instead.\n",
      "  warn_deprecated(\n"
     ]
    }
   ],
   "source": [
    "from langchain.tools.render import format_tool_to_openai_function\n",
    "\n",
    "functions = [format_tool_to_openai_function(t) for t in tools]\n",
    "model = model.bind_functions(functions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48756212-d287-4519-be38-60e6ec11b9c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import TypedDict, Annotated, Sequence\n",
    "import operator\n",
    "from langchain_core.messages import BaseMessage\n",
    "\n",
    "\n",
    "class AgentState(TypedDict):\n",
    "    messages: Annotated[Sequence[BaseMessage], operator.add]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9a1638cc-7f55-49de-930a-677664cb439c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langgraph.prebuilt import ToolInvocation\n",
    "import json\n",
    "from langchain_core.messages import FunctionMessage\n",
    "\n",
    "# Define the function that determines whether to continue or not\n",
    "def should_continue(state):\n",
    "    messages = state['messages']\n",
    "    last_message = messages[-1]\n",
    "    # If there is no function call, then we finish\n",
    "    if \"function_call\" not in last_message.additional_kwargs:\n",
    "        return \"end\"\n",
    "    # Otherwise if there is, we continue\n",
    "    else:\n",
    "        return \"continue\"\n",
    "\n",
    "# Define the function that calls the model\n",
    "def call_model(state):\n",
    "    messages = state['messages']\n",
    "    response = model.invoke(messages)\n",
    "    # We return a list, because this will get added to the existing list\n",
    "    return {\"messages\": [response]}\n",
    "\n",
    "# Define the function to execute tools\n",
    "def call_tool(state):\n",
    "    messages = state['messages']\n",
    "    # Based on the continue condition\n",
    "    # we know the last message involves a function call\n",
    "    last_message = messages[-1]\n",
    "    # We construct an ToolInvocation from the function_call\n",
    "    action = ToolInvocation(\n",
    "        tool=last_message.additional_kwargs[\"function_call\"][\"name\"],\n",
    "        tool_input=json.loads(last_message.additional_kwargs[\"function_call\"][\"arguments\"]),\n",
    "    )\n",
    "    # We call the tool_executor and get back a response\n",
    "    response = tool_executor.invoke(action)\n",
    "    # We use the response to create a FunctionMessage\n",
    "    function_message = FunctionMessage(content=str(response), name=action.tool)\n",
    "    # We return a list, because this will get added to the existing list\n",
    "    return {\"messages\": [function_message]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a2a1a210-7265-4195-afec-ed1f99571a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langgraph.graph import StateGraph, END\n",
    "# Define a new graph\n",
    "workflow = StateGraph(AgentState)\n",
    "\n",
    "# Define the two nodes we will cycle between\n",
    "workflow.add_node(\"agent\", call_model)\n",
    "workflow.add_node(\"action\", call_tool)\n",
    "\n",
    "# Set the entrypoint as `agent`\n",
    "# This means that this node is the first one called\n",
    "workflow.set_entry_point(\"agent\")\n",
    "\n",
    "# We now add a conditional edge\n",
    "workflow.add_conditional_edges(\n",
    "    # First, we define the start node. We use `agent`.\n",
    "    # This means these are the edges taken after the `agent` node is called.\n",
    "    \"agent\",\n",
    "    # Next, we pass in the function that will determine which node is called next.\n",
    "    should_continue,\n",
    "    # Finally we pass in a mapping.\n",
    "    # The keys are strings, and the values are other nodes.\n",
    "    # END is a special node marking that the graph should finish.\n",
    "    # What will happen is we will call `should_continue`, and then the output of that\n",
    "    # will be matched against the keys in this mapping.\n",
    "    # Based on which one it matches, that node will then be called.\n",
    "    {\n",
    "        # If `tools`, then we call the tool node.\n",
    "        \"continue\": \"action\",\n",
    "        # Otherwise we finish.\n",
    "        \"end\": END\n",
    "    }\n",
    ")\n",
    "\n",
    "# We now add a normal edge from `tools` to `agent`.\n",
    "# This means that after `tools` is called, `agent` node is called next.\n",
    "workflow.add_edge('action', 'agent')\n",
    "\n",
    "# Finally, we compile it!\n",
    "# This compiles it into a LangChain Runnable,\n",
    "# meaning you can use it as you would any other runnable\n",
    "app = workflow.compile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4fa692d8-4ff1-4e8a-9425-3d0d953a9ff2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'messages': [HumanMessage(content='what is the weather in sf'),\n",
       "  AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\"query\":\"weather in San Francisco\"}', 'name': 'tavily_search_results_json'}}),\n",
       "  FunctionMessage(content='[{\\'url\\': \\'https://www.weatherapi.com/\\', \\'content\\': \"Weather in San Francisco is {\\'location\\': {\\'name\\': \\'San Francisco\\', \\'region\\': \\'California\\', \\'country\\': \\'United States of America\\', \\'lat\\': 37.78, \\'lon\\': -122.42, \\'tz_id\\': \\'America/Los_Angeles\\', \\'localtime_epoch\\': 1708593536, \\'localtime\\': \\'2024-02-22 1:18\\'}, \\'current\\': {\\'last_updated_epoch\\': 1708593300, \\'last_updated\\': \\'2024-02-22 01:15\\', \\'temp_c\\': 10.0, \\'temp_f\\': 50.0, \\'is_day\\': 0, \\'condition\\': {\\'text\\': \\'Partly cloudy\\', \\'icon\\': \\'//cdn.weatherapi.com/weather/64x64/night/116.png\\', \\'code\\': 1003}, \\'wind_mph\\': 4.3, \\'wind_kph\\': 6.8, \\'wind_degree\\': 230, \\'wind_dir\\': \\'SW\\', \\'pressure_mb\\': 1024.0, \\'pressure_in\\': 30.25, \\'precip_mm\\': 0.0, \\'precip_in\\': 0.0, \\'humidity\\': 86, \\'cloud\\': 75, \\'feelslike_c\\': 10.3, \\'feelslike_f\\': 50.5, \\'vis_km\\': 16.0, \\'vis_miles\\': 9.0, \\'uv\\': 1.0, \\'gust_mph\\': 8.7, \\'gust_kph\\': 14.0}}\"}]', name='tavily_search_results_json'),\n",
       "  AIMessage(content='The current weather in San Francisco is partly cloudy with a temperature of 50°F (10°C). The wind speed is 6.8 km/h coming from the southwest direction. The humidity is at 86% and the visibility is 16.0 km.')]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.messages import HumanMessage\n",
    "from langgraph.prebuilt import ToolExecutor\n",
    "\n",
    "tool_executor = ToolExecutor(tools)\n",
    "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
    "app.invoke(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "16540079-d9fb-4ccf-83c8-57aa58700af0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output from node 'agent':\n",
      "---\n",
      "{'messages': [AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\"query\":\"weather in San Francisco\"}', 'name': 'tavily_search_results_json'}})]}\n",
      "\n",
      "---\n",
      "\n",
      "Output from node 'action':\n",
      "---\n",
      "{'messages': [FunctionMessage(content='[{\\'url\\': \\'https://www.weatherapi.com/\\', \\'content\\': \"Weather in San Francisco is {\\'location\\': {\\'name\\': \\'San Francisco\\', \\'region\\': \\'California\\', \\'country\\': \\'United States of America\\', \\'lat\\': 37.78, \\'lon\\': -122.42, \\'tz_id\\': \\'America/Los_Angeles\\', \\'localtime_epoch\\': 1708593536, \\'localtime\\': \\'2024-02-22 1:18\\'}, \\'current\\': {\\'last_updated_epoch\\': 1708593300, \\'last_updated\\': \\'2024-02-22 01:15\\', \\'temp_c\\': 10.0, \\'temp_f\\': 50.0, \\'is_day\\': 0, \\'condition\\': {\\'text\\': \\'Partly cloudy\\', \\'icon\\': \\'//cdn.weatherapi.com/weather/64x64/night/116.png\\', \\'code\\': 1003}, \\'wind_mph\\': 4.3, \\'wind_kph\\': 6.8, \\'wind_degree\\': 230, \\'wind_dir\\': \\'SW\\', \\'pressure_mb\\': 1024.0, \\'pressure_in\\': 30.25, \\'precip_mm\\': 0.0, \\'precip_in\\': 0.0, \\'humidity\\': 86, \\'cloud\\': 75, \\'feelslike_c\\': 10.3, \\'feelslike_f\\': 50.5, \\'vis_km\\': 16.0, \\'vis_miles\\': 9.0, \\'uv\\': 1.0, \\'gust_mph\\': 8.7, \\'gust_kph\\': 14.0}}\"}]', name='tavily_search_results_json')]}\n",
      "\n",
      "---\n",
      "\n",
      "Output from node 'agent':\n",
      "---\n",
      "{'messages': [AIMessage(content='The current weather in San Francisco is partly cloudy with a temperature of 10°C (50°F). The wind speed is 4.3 mph coming from the southwest direction. The humidity is at 86% and the visibility is 16.0 km.')]}\n",
      "\n",
      "---\n",
      "\n",
      "Output from node '__end__':\n",
      "---\n",
      "{'messages': [HumanMessage(content='what is the weather in sf'), AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\"query\":\"weather in San Francisco\"}', 'name': 'tavily_search_results_json'}}), FunctionMessage(content='[{\\'url\\': \\'https://www.weatherapi.com/\\', \\'content\\': \"Weather in San Francisco is {\\'location\\': {\\'name\\': \\'San Francisco\\', \\'region\\': \\'California\\', \\'country\\': \\'United States of America\\', \\'lat\\': 37.78, \\'lon\\': -122.42, \\'tz_id\\': \\'America/Los_Angeles\\', \\'localtime_epoch\\': 1708593536, \\'localtime\\': \\'2024-02-22 1:18\\'}, \\'current\\': {\\'last_updated_epoch\\': 1708593300, \\'last_updated\\': \\'2024-02-22 01:15\\', \\'temp_c\\': 10.0, \\'temp_f\\': 50.0, \\'is_day\\': 0, \\'condition\\': {\\'text\\': \\'Partly cloudy\\', \\'icon\\': \\'//cdn.weatherapi.com/weather/64x64/night/116.png\\', \\'code\\': 1003}, \\'wind_mph\\': 4.3, \\'wind_kph\\': 6.8, \\'wind_degree\\': 230, \\'wind_dir\\': \\'SW\\', \\'pressure_mb\\': 1024.0, \\'pressure_in\\': 30.25, \\'precip_mm\\': 0.0, \\'precip_in\\': 0.0, \\'humidity\\': 86, \\'cloud\\': 75, \\'feelslike_c\\': 10.3, \\'feelslike_f\\': 50.5, \\'vis_km\\': 16.0, \\'vis_miles\\': 9.0, \\'uv\\': 1.0, \\'gust_mph\\': 8.7, \\'gust_kph\\': 14.0}}\"}]', name='tavily_search_results_json'), AIMessage(content='The current weather in San Francisco is partly cloudy with a temperature of 10°C (50°F). The wind speed is 4.3 mph coming from the southwest direction. The humidity is at 86% and the visibility is 16.0 km.')]}\n",
      "\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
    "for output in app.stream(inputs):\n",
    "    # stream() yields dictionaries with output keyed by node name\n",
    "    for key, value in output.items():\n",
    "        print(f\"Output from node '{key}':\")\n",
    "        print(\"---\")\n",
    "        print(value)\n",
    "    print(\"\\n---\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "68d6a51a-f9a0-47fd-9cab-3dc136f29775",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
    "async for output in app.astream_log(inputs, include_types=[\"llm\"]):\n",
    "    # astream_log() yields the requested logs (here LLMs) in JSONPatch format\n",
    "    for op in output.ops:\n",
    "        if op[\"path\"] == \"/streamed_output/-\":\n",
    "            # this is the output from .stream()\n",
    "            ...\n",
    "        elif op[\"path\"].startswith(\"/logs/\") and op[\"path\"].endswith(\n",
    "            \"/streamed_output/-\"\n",
    "        ):\n",
    "            # because we chose to only include LLMs, these are LLM tokens\n",
    "            print(op[\"value\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cc7510f-cb50-42ed-98f4-483cf2d4f8f1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyautogen",
   "language": "python",
   "name": "pyautogen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}