From ecc56cf09809703e8e1260435ba956cd1b31aa7e Mon Sep 17 00:00:00 2001 From: Yun Kim <35776586+Yun-Kim@users.noreply.github.com> Date: Wed, 29 May 2024 16:39:08 -0400 Subject: [PATCH] chore(llmobs): extract and refactor langchain llmobs tests (#9407) This PR does a couple things: - Extract all langchain llmobs tests from `test_langchain.py`, `test_langchain_community.py` to `test_langchain_llmobs.py` to ease maintainability for LLMObs testing in the future - Refactors LLMObs langchain tests to be more readable and simplify tests. - Update tested versions of langchain from `0.0.192, 0.1.9` to `0.0.192 (start of ddtrace support), 0.1.20 (latest 0.1.x release), latest` and regenerates cassette/snapshot files accordingly. ## Checklist - [x] Change(s) are motivated and described in the PR description - [x] Testing strategy is described if automated tests are not included in the PR - [x] Risks are described (performance impact, potential for breakage, maintainability) - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] [Library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) are followed or label `changelog/no-changelog` is set - [x] Documentation is included (in-code, generated user docs, [public corp docs](https://github.com/DataDog/documentation/)) - [x] Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) - [x] If this PR changes the public interface, I've notified `@DataDog/apm-tees`. ## Reviewer Checklist - [x] Title is accurate - [x] All changes are related to the pull request's stated goal - [x] Description motivates each change - [x] Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - [x] Testing strategy adequately addresses listed risks - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] Release note makes sense to a user of the library - [x] Author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - [x] Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../requirements/{a186e90.txt => 1084a71.txt} | 50 +- .riot/requirements/17e8568.txt | 92 +++ .../requirements/{8ccebca.txt => 1ec5924.txt} | 50 +- .../requirements/{da7b0f6.txt => 1f6f978.txt} | 13 +- .../requirements/{13688bb.txt => 39161c9.txt} | 50 +- .../requirements/{6251000.txt => 76db01b.txt} | 13 +- .../requirements/{12739ce.txt => 8297334.txt} | 13 +- .riot/requirements/e17f33e.txt | 89 +++ .riot/requirements/ee6f953.txt | 87 +++ riotfile.py | 20 +- .../langchain/openai_completion_sync.yaml | 62 +- tests/contrib/langchain/conftest.py | 18 +- tests/contrib/langchain/test_langchain.py | 429 +----------- .../langchain/test_langchain_community.py | 571 +--------------- .../langchain/test_langchain_llmobs.py | 644 ++++++++++++++++++ .../contrib/langchain/test_langchain_patch.py | 3 + tests/contrib/langchain/utils.py | 11 + ...uential_chain_with_multiple_llm_async.json | 36 +- ...quential_chain_with_multiple_llm_sync.json | 36 +- ...ain_community.test_openai_integration.json | 2 +- ...y.test_openai_service_name[None-None].json | 2 +- ...ity.test_openai_service_name[None-v0].json | 2 +- ...ity.test_openai_service_name[None-v1].json | 2 +- ....test_openai_service_name[mysvc-None].json | 2 +- ...ty.test_openai_service_name[mysvc-v0].json | 2 +- ...ty.test_openai_service_name[mysvc-v1].json | 2 +- 26 files changed, 1136 insertions(+), 1165 deletions(-) rename .riot/requirements/{a186e90.txt => 1084a71.txt} (66%) create mode 100644 .riot/requirements/17e8568.txt rename .riot/requirements/{8ccebca.txt => 1ec5924.txt} (65%) rename .riot/requirements/{da7b0f6.txt => 1f6f978.txt} (89%) rename .riot/requirements/{13688bb.txt => 39161c9.txt} (66%) rename .riot/requirements/{6251000.txt => 76db01b.txt} (89%) rename .riot/requirements/{12739ce.txt => 8297334.txt} (89%) create mode 100644 .riot/requirements/e17f33e.txt create mode 100644 .riot/requirements/ee6f953.txt create mode 100644 tests/contrib/langchain/test_langchain_llmobs.py diff --git a/.riot/requirements/a186e90.txt b/.riot/requirements/1084a71.txt similarity index 66% rename from .riot/requirements/a186e90.txt rename to .riot/requirements/1084a71.txt index 01b0b62e5dc..d2efd941048 100644 --- a/.riot/requirements/a186e90.txt +++ b/.riot/requirements/1084a71.txt @@ -2,30 +2,30 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --no-annotate .riot/requirements/a186e90.in +# pip-compile --no-annotate .riot/requirements/1084a71.in # ai21==1.3.4 -aiohttp==3.9.3 +aiohttp==3.9.5 aiosignal==1.3.1 -anyio==4.3.0 +anyio==4.4.0 async-timeout==4.0.3 attrs==23.2.0 backoff==2.2.1 certifi==2024.2.2 charset-normalizer==3.3.2 -cohere==4.53 -coverage[toml]==7.4.3 +cohere==4.57 +coverage[toml]==7.5.3 dataclasses-json==0.5.14 dnspython==2.6.1 -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 fastavro==1.9.4 -filelock==3.13.1 +filelock==3.14.0 frozenlist==1.4.1 -fsspec==2024.2.0 +fsspec==2024.5.0 greenlet==3.0.3 -huggingface-hub==0.21.4 +huggingface-hub==0.23.2 hypothesis==6.45.0 -idna==3.6 +idna==3.7 importlib-metadata==6.11.0 iniconfig==2.0.0 jsonpatch==1.33 @@ -36,41 +36,41 @@ langchain-core==0.1.23 langchainplus-sdk==0.0.4 langsmith==0.0.87 loguru==0.7.2 -marshmallow==3.21.1 +marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 -numexpr==2.9.0 +numexpr==2.10.0 numpy==1.26.4 openai==0.27.8 openapi-schema-pydantic==1.2.4 opentracing==2.4.0 packaging==23.2 pinecone-client==2.2.4 -pluggy==1.4.0 +pluggy==1.5.0 psutil==5.9.8 -pydantic==1.10.14 -pytest==8.1.1 +pydantic==1.10.15 +pytest==8.2.1 pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 -regex==2023.12.25 -requests==2.31.0 +regex==2024.5.15 +requests==2.32.3 six==1.16.0 sniffio==1.3.1 sortedcontainers==2.4.0 -sqlalchemy==2.0.28 -tenacity==8.2.3 -tiktoken==0.6.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 tomli==2.0.1 -tqdm==4.66.2 -typing-extensions==4.10.0 +tqdm==4.66.4 +typing-extensions==4.12.0 typing-inspect==0.9.0 urllib3==2.2.1 vcrpy==6.0.1 wrapt==1.16.0 yarl==1.9.4 -zipp==3.17.0 +zipp==3.19.0 diff --git a/.riot/requirements/17e8568.txt b/.riot/requirements/17e8568.txt new file mode 100644 index 00000000000..9c1a89830fe --- /dev/null +++ b/.riot/requirements/17e8568.txt @@ -0,0 +1,92 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/17e8568.in +# +ai21==2.4.0 +ai21-tokenizer==0.9.1 +aiohttp==3.9.5 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.4.0 +async-timeout==4.0.3 +attrs==23.2.0 +boto3==1.34.114 +botocore==1.34.114 +certifi==2024.2.2 +charset-normalizer==3.3.2 +cohere==5.5.3 +coverage[toml]==7.5.3 +dataclasses-json==0.6.6 +distro==1.9.0 +exceptiongroup==1.2.1 +fastavro==1.9.4 +filelock==3.14.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +greenlet==3.0.3 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +httpx-sse==0.4.0 +huggingface-hub==0.23.2 +hypothesis==6.45.0 +idna==3.7 +importlib-metadata==7.1.0 +iniconfig==2.0.0 +jmespath==1.0.1 +jsonpatch==1.33 +jsonpointer==2.4 +langchain==0.2.1 +langchain-aws==0.1.6 +langchain-community==0.2.1 +langchain-core==0.2.1 +langchain-openai==0.1.7 +langchain-pinecone==0.1.1 +langchain-text-splitters==0.2.0 +langsmith==0.1.63 +marshmallow==3.21.2 +mock==5.1.0 +multidict==6.0.5 +mypy-extensions==1.0.0 +numexpr==2.10.0 +numpy==1.26.4 +openai==1.30.4 +opentracing==2.4.0 +orjson==3.10.3 +packaging==23.2 +pinecone-client==3.2.2 +pluggy==1.5.0 +psutil==5.9.8 +pydantic==2.7.2 +pydantic-core==2.18.3 +pytest==8.2.1 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +pyyaml==6.0.1 +regex==2024.5.15 +requests==2.32.3 +s3transfer==0.10.1 +sentencepiece==0.2.0 +six==1.16.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 +tokenizers==0.19.1 +tomli==2.0.1 +tqdm==4.66.4 +types-requests==2.31.0.6 +types-urllib3==1.26.25.14 +typing-extensions==4.12.0 +typing-inspect==0.9.0 +urllib3==1.26.18 +vcrpy==6.0.1 +wrapt==1.16.0 +yarl==1.9.4 +zipp==3.19.0 diff --git a/.riot/requirements/8ccebca.txt b/.riot/requirements/1ec5924.txt similarity index 65% rename from .riot/requirements/8ccebca.txt rename to .riot/requirements/1ec5924.txt index eaa0947d4ef..6cd3eef8c1b 100644 --- a/.riot/requirements/8ccebca.txt +++ b/.riot/requirements/1ec5924.txt @@ -2,29 +2,29 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-annotate .riot/requirements/8ccebca.in +# pip-compile --no-annotate .riot/requirements/1ec5924.in # ai21==1.3.4 -aiohttp==3.9.3 +aiohttp==3.9.5 aiosignal==1.3.1 -anyio==4.3.0 +anyio==4.4.0 attrs==23.2.0 backoff==2.2.1 certifi==2024.2.2 charset-normalizer==3.3.2 -cohere==4.53 -coverage[toml]==7.4.3 +cohere==4.57 +coverage[toml]==7.5.3 dataclasses-json==0.5.14 dnspython==2.6.1 -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 fastavro==1.9.4 -filelock==3.13.1 +filelock==3.14.0 frozenlist==1.4.1 -fsspec==2024.2.0 +fsspec==2024.5.0 greenlet==3.0.3 -huggingface-hub==0.21.4 +huggingface-hub==0.23.2 hypothesis==6.45.0 -idna==3.6 +idna==3.7 importlib-metadata==6.11.0 iniconfig==2.0.0 jsonpatch==1.33 @@ -35,40 +35,40 @@ langchain-core==0.1.23 langchainplus-sdk==0.0.4 langsmith==0.0.87 loguru==0.7.2 -marshmallow==3.21.1 +marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 -numexpr==2.9.0 +numexpr==2.10.0 numpy==1.26.4 openai==0.27.8 openapi-schema-pydantic==1.2.4 opentracing==2.4.0 packaging==23.2 pinecone-client==2.2.4 -pluggy==1.4.0 +pluggy==1.5.0 psutil==5.9.8 -pydantic==1.10.14 -pytest==8.1.1 +pydantic==1.10.15 +pytest==8.2.1 pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 -regex==2023.12.25 -requests==2.31.0 +regex==2024.5.15 +requests==2.32.3 six==1.16.0 sniffio==1.3.1 sortedcontainers==2.4.0 -sqlalchemy==2.0.28 -tenacity==8.2.3 -tiktoken==0.6.0 -tqdm==4.66.2 -typing-extensions==4.10.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 +tqdm==4.66.4 +typing-extensions==4.12.0 typing-inspect==0.9.0 urllib3==2.2.1 vcrpy==6.0.1 wrapt==1.16.0 yarl==1.9.4 -zipp==3.17.0 +zipp==3.19.0 diff --git a/.riot/requirements/da7b0f6.txt b/.riot/requirements/1f6f978.txt similarity index 89% rename from .riot/requirements/da7b0f6.txt rename to .riot/requirements/1f6f978.txt index f2c28511799..bf44d1459ea 100644 --- a/.riot/requirements/da7b0f6.txt +++ b/.riot/requirements/1f6f978.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --no-annotate .riot/requirements/da7b0f6.in +# pip-compile --no-annotate .riot/requirements/1f6f978.in # ai21==2.4.0 ai21-tokenizer==0.9.1 @@ -25,6 +25,7 @@ fastavro==1.9.4 filelock==3.14.0 frozenlist==1.4.1 fsspec==2024.5.0 +greenlet==3.0.3 h11==0.14.0 httpcore==1.0.5 httpx==0.27.0 @@ -40,17 +41,17 @@ langchain==0.1.20 langchain-aws==0.1.6 langchain-community==0.0.38 langchain-core==0.1.52 -langchain-openai==0.1.5 -langchain-pinecone==0.1.1 +langchain-openai==0.1.6 +langchain-pinecone==0.1.0 langchain-text-splitters==0.0.2 -langsmith==0.1.63 +langsmith==0.1.58 marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 numexpr==2.10.0 numpy==1.26.4 -openai==1.12.0 +openai==1.30.3 opentracing==2.4.0 orjson==3.10.3 packaging==23.2 @@ -67,7 +68,7 @@ pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 regex==2024.5.15 -requests==2.32.2 +requests==2.32.3 s3transfer==0.10.1 sentencepiece==0.2.0 six==1.16.0 diff --git a/.riot/requirements/13688bb.txt b/.riot/requirements/39161c9.txt similarity index 66% rename from .riot/requirements/13688bb.txt rename to .riot/requirements/39161c9.txt index 557e671c774..789b1dcc3c8 100644 --- a/.riot/requirements/13688bb.txt +++ b/.riot/requirements/39161c9.txt @@ -2,30 +2,30 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --no-annotate .riot/requirements/13688bb.in +# pip-compile --no-annotate .riot/requirements/39161c9.in # ai21==1.3.4 -aiohttp==3.9.3 +aiohttp==3.9.5 aiosignal==1.3.1 -anyio==4.3.0 +anyio==4.4.0 async-timeout==4.0.3 attrs==23.2.0 backoff==2.2.1 certifi==2024.2.2 charset-normalizer==3.3.2 -cohere==4.53 -coverage[toml]==7.4.3 +cohere==4.57 +coverage[toml]==7.5.3 dataclasses-json==0.5.14 dnspython==2.6.1 -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 fastavro==1.9.4 -filelock==3.13.1 +filelock==3.14.0 frozenlist==1.4.1 -fsspec==2024.2.0 +fsspec==2024.5.0 greenlet==3.0.3 -huggingface-hub==0.21.4 +huggingface-hub==0.23.2 hypothesis==6.45.0 -idna==3.6 +idna==3.7 importlib-metadata==6.11.0 iniconfig==2.0.0 jsonpatch==1.33 @@ -36,41 +36,41 @@ langchain-core==0.1.23 langchainplus-sdk==0.0.4 langsmith==0.0.87 loguru==0.7.2 -marshmallow==3.21.1 +marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 -numexpr==2.9.0 +numexpr==2.10.0 numpy==1.26.4 openai==0.27.8 openapi-schema-pydantic==1.2.4 opentracing==2.4.0 packaging==23.2 pinecone-client==2.2.4 -pluggy==1.4.0 +pluggy==1.5.0 psutil==5.9.8 -pydantic==1.10.14 -pytest==8.1.1 +pydantic==1.10.15 +pytest==8.2.1 pytest-asyncio==0.21.1 -pytest-cov==4.1.0 -pytest-mock==3.12.0 +pytest-cov==5.0.0 +pytest-mock==3.14.0 pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 -regex==2023.12.25 -requests==2.31.0 +regex==2024.5.15 +requests==2.32.3 six==1.16.0 sniffio==1.3.1 sortedcontainers==2.4.0 -sqlalchemy==2.0.28 -tenacity==8.2.3 -tiktoken==0.6.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 tomli==2.0.1 -tqdm==4.66.2 -typing-extensions==4.10.0 +tqdm==4.66.4 +typing-extensions==4.12.0 typing-inspect==0.9.0 urllib3==1.26.18 vcrpy==6.0.1 wrapt==1.16.0 yarl==1.9.4 -zipp==3.17.0 +zipp==3.19.0 diff --git a/.riot/requirements/6251000.txt b/.riot/requirements/76db01b.txt similarity index 89% rename from .riot/requirements/6251000.txt rename to .riot/requirements/76db01b.txt index 78abfa4a9c2..fcb99744e02 100644 --- a/.riot/requirements/6251000.txt +++ b/.riot/requirements/76db01b.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-annotate .riot/requirements/6251000.in +# pip-compile --no-annotate .riot/requirements/76db01b.in # ai21==2.4.0 ai21-tokenizer==0.9.1 @@ -24,6 +24,7 @@ fastavro==1.9.4 filelock==3.14.0 frozenlist==1.4.1 fsspec==2024.5.0 +greenlet==3.0.3 h11==0.14.0 httpcore==1.0.5 httpx==0.27.0 @@ -39,17 +40,17 @@ langchain==0.1.20 langchain-aws==0.1.6 langchain-community==0.0.38 langchain-core==0.1.52 -langchain-openai==0.1.5 -langchain-pinecone==0.1.1 +langchain-openai==0.1.6 +langchain-pinecone==0.1.0 langchain-text-splitters==0.0.2 -langsmith==0.1.63 +langsmith==0.1.58 marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 numexpr==2.10.0 numpy==1.26.4 -openai==1.12.0 +openai==1.30.3 opentracing==2.4.0 orjson==3.10.3 packaging==23.2 @@ -66,7 +67,7 @@ pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 regex==2024.5.15 -requests==2.32.2 +requests==2.32.3 s3transfer==0.10.1 sentencepiece==0.2.0 six==1.16.0 diff --git a/.riot/requirements/12739ce.txt b/.riot/requirements/8297334.txt similarity index 89% rename from .riot/requirements/12739ce.txt rename to .riot/requirements/8297334.txt index d0ceae82d1d..2e1342e47e6 100644 --- a/.riot/requirements/12739ce.txt +++ b/.riot/requirements/8297334.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.9 # by the following command: # -# pip-compile --no-annotate .riot/requirements/12739ce.in +# pip-compile --no-annotate .riot/requirements/8297334.in # ai21==2.4.0 ai21-tokenizer==0.9.1 @@ -25,6 +25,7 @@ fastavro==1.9.4 filelock==3.14.0 frozenlist==1.4.1 fsspec==2024.5.0 +greenlet==3.0.3 h11==0.14.0 httpcore==1.0.5 httpx==0.27.0 @@ -41,17 +42,17 @@ langchain==0.1.20 langchain-aws==0.1.6 langchain-community==0.0.38 langchain-core==0.1.52 -langchain-openai==0.1.5 -langchain-pinecone==0.1.1 +langchain-openai==0.1.6 +langchain-pinecone==0.1.0 langchain-text-splitters==0.0.2 -langsmith==0.1.63 +langsmith==0.1.58 marshmallow==3.21.2 mock==5.1.0 multidict==6.0.5 mypy-extensions==1.0.0 numexpr==2.10.0 numpy==1.26.4 -openai==1.12.0 +openai==1.30.3 opentracing==2.4.0 orjson==3.10.3 packaging==23.2 @@ -68,7 +69,7 @@ pytest-randomly==3.15.0 python-dateutil==2.9.0.post0 pyyaml==6.0.1 regex==2024.5.15 -requests==2.32.2 +requests==2.32.3 s3transfer==0.10.1 sentencepiece==0.2.0 six==1.16.0 diff --git a/.riot/requirements/e17f33e.txt b/.riot/requirements/e17f33e.txt new file mode 100644 index 00000000000..b8340758464 --- /dev/null +++ b/.riot/requirements/e17f33e.txt @@ -0,0 +1,89 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/e17f33e.in +# +ai21==2.4.0 +ai21-tokenizer==0.9.1 +aiohttp==3.9.5 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.4.0 +async-timeout==4.0.3 +attrs==23.2.0 +boto3==1.34.114 +botocore==1.34.114 +certifi==2024.2.2 +charset-normalizer==3.3.2 +cohere==5.5.3 +coverage[toml]==7.5.3 +dataclasses-json==0.6.6 +distro==1.9.0 +exceptiongroup==1.2.1 +fastavro==1.9.4 +filelock==3.14.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +greenlet==3.0.3 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +httpx-sse==0.4.0 +huggingface-hub==0.23.2 +hypothesis==6.45.0 +idna==3.7 +iniconfig==2.0.0 +jmespath==1.0.1 +jsonpatch==1.33 +jsonpointer==2.4 +langchain==0.2.1 +langchain-aws==0.1.6 +langchain-community==0.2.1 +langchain-core==0.2.1 +langchain-openai==0.1.7 +langchain-pinecone==0.1.1 +langchain-text-splitters==0.2.0 +langsmith==0.1.63 +marshmallow==3.21.2 +mock==5.1.0 +multidict==6.0.5 +mypy-extensions==1.0.0 +numexpr==2.10.0 +numpy==1.26.4 +openai==1.30.4 +opentracing==2.4.0 +orjson==3.10.3 +packaging==23.2 +pinecone-client==3.2.2 +pluggy==1.5.0 +psutil==5.9.8 +pydantic==2.7.2 +pydantic-core==2.18.3 +pytest==8.2.1 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +pyyaml==6.0.1 +regex==2024.5.15 +requests==2.32.3 +s3transfer==0.10.1 +sentencepiece==0.2.0 +six==1.16.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 +tokenizers==0.19.1 +tomli==2.0.1 +tqdm==4.66.4 +types-requests==2.32.0.20240523 +typing-extensions==4.12.0 +typing-inspect==0.9.0 +urllib3==2.2.1 +vcrpy==6.0.1 +wrapt==1.16.0 +yarl==1.9.4 diff --git a/.riot/requirements/ee6f953.txt b/.riot/requirements/ee6f953.txt new file mode 100644 index 00000000000..d2830024f5c --- /dev/null +++ b/.riot/requirements/ee6f953.txt @@ -0,0 +1,87 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/ee6f953.in +# +ai21==2.4.0 +ai21-tokenizer==0.9.1 +aiohttp==3.9.5 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.4.0 +attrs==23.2.0 +boto3==1.34.114 +botocore==1.34.114 +certifi==2024.2.2 +charset-normalizer==3.3.2 +cohere==5.5.3 +coverage[toml]==7.5.3 +dataclasses-json==0.6.6 +distro==1.9.0 +exceptiongroup==1.2.1 +fastavro==1.9.4 +filelock==3.14.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +greenlet==3.0.3 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +httpx-sse==0.4.0 +huggingface-hub==0.23.2 +hypothesis==6.45.0 +idna==3.7 +iniconfig==2.0.0 +jmespath==1.0.1 +jsonpatch==1.33 +jsonpointer==2.4 +langchain==0.2.1 +langchain-aws==0.1.6 +langchain-community==0.2.1 +langchain-core==0.2.1 +langchain-openai==0.1.7 +langchain-pinecone==0.1.1 +langchain-text-splitters==0.2.0 +langsmith==0.1.63 +marshmallow==3.21.2 +mock==5.1.0 +multidict==6.0.5 +mypy-extensions==1.0.0 +numexpr==2.10.0 +numpy==1.26.4 +openai==1.30.4 +opentracing==2.4.0 +orjson==3.10.3 +packaging==23.2 +pinecone-client==3.2.2 +pluggy==1.5.0 +psutil==5.9.8 +pydantic==2.7.2 +pydantic-core==2.18.3 +pytest==8.2.1 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +pyyaml==6.0.1 +regex==2024.5.15 +requests==2.32.3 +s3transfer==0.10.1 +sentencepiece==0.2.0 +six==1.16.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +sqlalchemy==2.0.30 +tenacity==8.3.0 +tiktoken==0.7.0 +tokenizers==0.19.1 +tqdm==4.66.4 +types-requests==2.32.0.20240523 +typing-extensions==4.12.0 +typing-inspect==0.9.0 +urllib3==2.2.1 +vcrpy==6.0.1 +wrapt==1.16.0 +yarl==1.9.4 diff --git a/riotfile.py b/riotfile.py index 17a34dffac4..1fb41058dbf 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2484,7 +2484,6 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): "vcrpy": latest, "pytest-asyncio": "==0.21.1", "tiktoken": latest, - "cohere": latest, "huggingface-hub": latest, "ai21": latest, "exceptiongroup": latest, @@ -2499,6 +2498,22 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): "langchain-community": "==0.0.14", "openai": "==0.27.8", "pinecone-client": "==2.2.4", + "cohere": "==4.57", + } + ), + Venv( + pkgs={ + "langchain": "==0.1.20", + "langchain-community": "==0.0.38", + "langchain-core": "==0.1.52", + "langchain-openai": "==0.1.6", + "langchain-pinecone": "==0.1.0", + "langsmith": "==0.1.58", + "openai": "==1.30.3", + "pinecone-client": latest, + "botocore": latest, + "langchain-aws": latest, + "cohere": latest, } ), Venv( @@ -2509,10 +2524,11 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION): "langchain-openai": latest, "langchain-pinecone": latest, "langsmith": latest, - "openai": "==1.12.0", + "openai": latest, "pinecone-client": latest, "botocore": latest, "langchain-aws": latest, + "cohere": latest, } ), ], diff --git a/tests/contrib/langchain/cassettes/langchain/openai_completion_sync.yaml b/tests/contrib/langchain/cassettes/langchain/openai_completion_sync.yaml index 763261c5e67..5647d2d5753 100644 --- a/tests/contrib/langchain/cassettes/langchain/openai_completion_sync.yaml +++ b/tests/contrib/langchain/cassettes/langchain/openai_completion_sync.yaml @@ -1,8 +1,8 @@ interactions: - request: body: '{"prompt": ["Can you explain what Descartes meant by ''I think, therefore - I am''?"], "model": "text-davinci-003", "temperature": 0.7, "max_tokens": 256, - "top_p": 1, "frequency_penalty": 0, "presence_penalty": 0, "n": 1, "logit_bias": + I am''?"], "model": "gpt-3.5-turbo-instruct", "temperature": 0.7, "max_tokens": + 256, "top_p": 1, "frequency_penalty": 0, "presence_penalty": 0, "n": 1, "logit_bias": {}}' headers: Accept: @@ -12,36 +12,48 @@ interactions: Connection: - keep-alive Content-Length: - - '235' + - '241' Content-Type: - application/json User-Agent: - OpenAI/v1 PythonBindings/0.27.8 X-OpenAI-Client-User-Agent: - '{"bindings_version": "0.27.8", "httplib": "requests", "lang": "python", "lang_version": - "3.10.5", "platform": "macOS-13.4-arm64-arm-64bit", "publisher": "openai", - "uname": "Darwin 22.5.0 Darwin Kernel Version 22.5.0: Mon Apr 24 20:52:24 - PDT 2023; root:xnu-8796.121.2~5/RELEASE_ARM64_T6000 arm64"}' + "3.11.1", "platform": "macOS-14.4.1-arm64-arm-64bit", "publisher": "openai", + "uname": "Darwin 23.4.0 Darwin Kernel Version 23.4.0: Fri Mar 15 00:10:42 + PDT 2024; root:xnu-10063.101.17~1/RELEASE_ARM64_T6000 arm64 arm"}' method: POST uri: https://api.openai.com/v1/completions response: + body: - string: "{\n \"id\": \"cmpl-7TZs9SoIjYLVNCYmAqIz9ihMX1grP\",\n \"object\": + string: "{\n \"id\": \"cmpl-7TZs9SoIjYLVNCYmAqIz9ihMX1grP\",\n \"object\": \"text_completion\",\n \"created\": 1687283761,\n \"model\": \"text-davinci-003\",\n \ \"choices\": [\n {\n \"text\": \"\\n\\nDescartes' famous phrase - \\\"I think, therefore I am\\\" is a fundamental statement of his philosophical - approach, known as Cartesian dualism. This phrase expresses his belief that - the very act of thinking proves one's existence. Descartes reasoned that even - if he was tricked by an evil genius into believing false ideas, he still must - exist in order to be tricked. Therefore, he concluded that his own existence - was certain, even if all else was uncertain.\",\n \"index\": 0,\n \"logprobs\": - null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 17,\n \"completion_tokens\": 95,\n \"total_tokens\": 112\n }\n}\n" + \\\"I think, therefore I am\\\" is a fundamental statement of his philosophical + approach, known as Cartesian logic. This phrase is often interpreted as a + statement that reflects his belief in the existence of the self and one's + consciousness.\\n\\nIn his philosophical work, Descartes was searching for + a solid foundation for knowledge and truth. He doubted everything he had learned + and believed, even the existence of the external world and his own body. However, + he reached a point where he realized that even if everything else could be + doubted, there was one thing that he could not doubt - the fact that he was + doubting. This led him to the conclusion that he must exist in order to have + thoughts and doubts.\\n\\nTherefore, by saying \\\"I think,\\\" Descartes + is asserting that he is a thinking being. And by saying \\\"therefore I am,\\\" + he is affirming that his existence is inseparable from his ability to think. + In other words, the very act of thinking proves his existence as a conscious + being.\\n\\nThis statement also implies that the mind and consciousness are + essential to one's existence, rather than the physical body. While the body + can be doubted and can deceive the senses, the mind's ability to think and\",\n + \ \"index\": 0,\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 17,\n \"completion_tokens\": + 95,\n \"total_tokens\": 112\n }\n}\n" headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 7da5e2d45d89a211-YYZ + - 88b1dec92bf84414-EWR Cache-Control: - no-cache, must-revalidate Connection: @@ -51,9 +63,15 @@ interactions: Content-Type: - application/json Date: - - Tue, 20 Jun 2023 17:56:03 GMT + - Tue, 28 May 2024 23:02:09 GMT Server: - cloudflare + Set-Cookie: + - __cf_bm=GO1qX6goSJ8IacyvMS7tLMf0y55kYhL6qoz7yWHzVaw-1716937329-1.0.1.1-KvV3UvXRKv112CoSczDnRzcxKh8yFACOJH3Ze3RU6UqLAbH9hU09.k9UMn37Jjz4lL0vp_QtsV86A5CZ6pj0xg; + path=/; expires=Tue, 28-May-24 23:32:09 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=2wZlGKfUHYLv2py8qBP8822KGpT_cuKDJqqhhCMyEAs-1716937329373-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked access-control-allow-origin: @@ -61,11 +79,11 @@ interactions: alt-svc: - h3=":443"; ma=86400 openai-model: - - text-davinci-003 + - gpt-3.5-turbo-instruct openai-organization: - datadog-4 openai-processing-ms: - - '2066' + - '4278' openai-version: - '2020-10-01' strict-transport-security: @@ -79,11 +97,11 @@ interactions: x-ratelimit-remaining-tokens: - '249744' x-ratelimit-reset-requests: - - 20ms + - 17ms x-ratelimit-reset-tokens: - - 61ms + - 181ms x-request-id: - - 0f687e05c8e1f21d8431c83ea86acb0f + - req_9c5e9d6c980fce68872534cb21e92939 status: code: 200 message: OK diff --git a/tests/contrib/langchain/conftest.py b/tests/contrib/langchain/conftest.py index c6595e295c0..5d7a9db0b4e 100644 --- a/tests/contrib/langchain/conftest.py +++ b/tests/contrib/langchain/conftest.py @@ -14,12 +14,6 @@ from tests.utils import override_global_config -@pytest.fixture -def ddtrace_global_config(): - config = {} - return config - - def default_global_config(): return {"_dd_api_key": ""} @@ -91,10 +85,8 @@ def mock_llmobs_span_writer(): @pytest.fixture -def langchain(ddtrace_global_config, ddtrace_config_langchain, mock_logs, mock_metrics): - global_config = default_global_config() - global_config.update(ddtrace_global_config) - with override_global_config(global_config): +def langchain(ddtrace_config_langchain, mock_logs, mock_metrics): + with override_global_config(default_global_config()): with override_config("langchain", ddtrace_config_langchain): with override_env( dict( @@ -115,14 +107,14 @@ def langchain(ddtrace_global_config, ddtrace_config_langchain, mock_logs, mock_m @pytest.fixture -def langchain_community(ddtrace_global_config, ddtrace_config_langchain, mock_logs, mock_metrics, langchain): +def langchain_community(ddtrace_config_langchain, mock_logs, mock_metrics, langchain): import langchain_community yield langchain_community @pytest.fixture -def langchain_core(ddtrace_global_config, ddtrace_config_langchain, mock_logs, mock_metrics, langchain): +def langchain_core(ddtrace_config_langchain, mock_logs, mock_metrics, langchain): import langchain_core import langchain_core.prompts # noqa: F401 @@ -130,7 +122,7 @@ def langchain_core(ddtrace_global_config, ddtrace_config_langchain, mock_logs, m @pytest.fixture -def langchain_openai(ddtrace_global_config, ddtrace_config_langchain, mock_logs, mock_metrics, langchain): +def langchain_openai(ddtrace_config_langchain, mock_logs, mock_metrics, langchain): try: import langchain_openai diff --git a/tests/contrib/langchain/test_langchain.py b/tests/contrib/langchain/test_langchain.py index eb07c7e0b04..af449f0a90e 100644 --- a/tests/contrib/langchain/test_langchain.py +++ b/tests/contrib/langchain/test_langchain.py @@ -1,4 +1,3 @@ -import json import os import re import sys @@ -9,10 +8,8 @@ from ddtrace.contrib.langchain.patch import BASE_LANGCHAIN_MODULE_NAME from ddtrace.contrib.langchain.patch import SHOULD_PATCH_LANGCHAIN_COMMUNITY from ddtrace.internal.utils.version import parse_version -from ddtrace.llmobs import LLMObs from tests.contrib.langchain.utils import get_request_vcr -from tests.llmobs._utils import _expected_llmobs_llm_span_event -from tests.llmobs._utils import _expected_llmobs_non_llm_span_event +from tests.contrib.langchain.utils import long_input_text from tests.utils import override_global_config @@ -670,17 +667,8 @@ def test_openai_sequential_chain_with_multiple_llm_sync(langchain, request_vcr): output_variables=["final_output"], ) - input_text = """ - I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no - bodies. Does it now follow that I too do not exist? No: if I convinced myself of something then I certainly - existed. But there is a deceiver of supreme power and cunning who is deliberately and constantly deceiving - me. In that case I too undoubtedly exist, if he is deceiving me; and let him deceive me as much as he can, - he will never bring it about that I am nothing so long as I think that I am something. So after considering - everything very thoroughly, I must finally conclude that this proposition, I am, I exist, is necessarily - true whenever it is put forward by me or conceived in my mind. - """ with request_vcr.use_cassette("openai_sequential_paraphrase_and_rhyme_sync.yaml"): - sequential_chain.run({"input_text": input_text}) + sequential_chain.run({"input_text": long_input_text}) @pytest.mark.asyncio @@ -707,18 +695,8 @@ async def test_openai_sequential_chain_with_multiple_llm_async(langchain, langch input_variables=["input_text"], output_variables=["final_output"], ) - - input_text = """ - I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no - bodies. Does it now follow that I too do not exist? No: if I convinced myself of something then I certainly - existed. But there is a deceiver of supreme power and cunning who is deliberately and constantly deceiving - me. In that case I too undoubtedly exist, if he is deceiving me; and let him deceive me as much as he can, - he will never bring it about that I am nothing so long as I think that I am something. So after considering - everything very thoroughly, I must finally conclude that this proposition, I am, I exist, is necessarily - true whenever it is put forward by me or conceived in my mind. - """ with request_vcr.use_cassette("openai_sequential_paraphrase_and_rhyme_async.yaml"): - await sequential_chain.acall({"input_text": input_text}) + await sequential_chain.acall({"input_text": long_input_text}) def test_openai_chain_metrics(langchain, langchain_openai, request_vcr, mock_metrics, mock_logs, snapshot_tracer): @@ -1260,404 +1238,3 @@ def test_vectorstore_logs_error(langchain, ddtrace_config_langchain, mock_logs, "documents": [], } ) - - -@pytest.mark.parametrize( - "ddtrace_global_config", [dict(_llmobs_enabled=True, _llmobs_sample_rate=1.0, _llmobs_ml_app="langchain_test")] -) -class TestLLMObsLangchain: - @staticmethod - def _expected_llmobs_chain_calls(trace, expected_spans_data: list): - expected_llmobs_writer_calls = [mock.call.start()] - - for idx, span in enumerate(trace): - kind, kwargs = expected_spans_data[idx] - expected_span_event = None - if kind == "chain": - expected_span_event = TestLLMObsLangchain._expected_llmobs_chain_call(span, **kwargs) - else: - expected_span_event = TestLLMObsLangchain._expected_llmobs_llm_call(span, **kwargs) - - expected_llmobs_writer_calls += [mock.call.enqueue(expected_span_event)] - - return expected_llmobs_writer_calls - - @staticmethod - def _expected_llmobs_chain_call(span, metadata=None, input_value=None, output_value=None): - return _expected_llmobs_non_llm_span_event( - span, - span_kind="workflow", - metadata=metadata, - input_value=input_value, - output_value=output_value, - tags={ - "ml_app": "langchain_test", - }, - integration="langchain", - ) - - @staticmethod - def _expected_llmobs_llm_call(span, provider="openai", input_roles=[None], output_role=None): - input_meta = [{"content": mock.ANY} for _ in input_roles] - for idx, role in enumerate(input_roles): - if role is not None: - input_meta[idx]["role"] = role - - output_meta = {"content": mock.ANY} - if output_role is not None: - output_meta["role"] = output_role - - temperature_key = "temperature" - if provider == "huggingface_hub": - max_tokens_key = "model_kwargs.max_tokens" - temperature_key = "model_kwargs.temperature" - elif provider == "ai21": - max_tokens_key = "maxTokens" - else: - max_tokens_key = "max_tokens" - - metadata = {} - temperature = span.get_tag(f"langchain.request.{provider}.parameters.{temperature_key}") - max_tokens = span.get_tag(f"langchain.request.{provider}.parameters.{max_tokens_key}") - if temperature is not None: - metadata["temperature"] = float(temperature) - if max_tokens is not None: - metadata["max_tokens"] = int(max_tokens) - - return _expected_llmobs_llm_span_event( - span, - model_name=span.get_tag("langchain.request.model"), - model_provider=span.get_tag("langchain.request.provider"), - input_messages=input_meta, - output_messages=[output_meta], - metadata=metadata, - token_metrics={}, - tags={ - "ml_app": "langchain_test", - }, - integration="langchain", - ) - - @classmethod - def _test_llmobs_llm_invoke( - cls, - provider, - generate_trace, - request_vcr, - mock_llmobs_span_writer, - mock_tracer, - cassette_name, - input_roles=[None], - output_role=None, - different_py39_cassette=False, - ): - LLMObs.disable() - LLMObs.enable(_tracer=mock_tracer, integrations_enabled=False) # only want langchain patched - - if sys.version_info < (3, 10, 0) and different_py39_cassette: - cassette_name = cassette_name.replace(".yaml", "_39.yaml") - with request_vcr.use_cassette(cassette_name): - generate_trace("Can you explain what an LLM chain is?") - span = mock_tracer.pop_traces()[0][0] - - expected_llmons_writer_calls = [ - mock.call.start(), - mock.call.enqueue( - cls._expected_llmobs_llm_call( - span, - provider=provider, - input_roles=input_roles, - output_role=output_role, - ) - ), - ] - - assert mock_llmobs_span_writer.enqueue.call_count == 1 - mock_llmobs_span_writer.assert_has_calls(expected_llmons_writer_calls) - - @classmethod - def _test_llmobs_chain_invoke( - cls, - generate_trace, - request_vcr, - mock_llmobs_span_writer, - mock_tracer, - cassette_name, - expected_spans_data=[("llm", {"provider": "openai", "input_roles": [None], "output_role": None})], - different_py39_cassette=False, - ): - # disable the service before re-enabling it, as it was enabled in another test - LLMObs.disable() - LLMObs.enable(_tracer=mock_tracer, integrations_enabled=False) # only want langchain patched - - if sys.version_info < (3, 10, 0) and different_py39_cassette: - cassette_name = cassette_name.replace(".yaml", "_39.yaml") - with request_vcr.use_cassette(cassette_name): - generate_trace("Can you explain what an LLM chain is?") - trace = mock_tracer.pop_traces()[0] - - expected_llmobs_writer_calls = cls._expected_llmobs_chain_calls( - trace=trace, expected_spans_data=expected_spans_data - ) - assert mock_llmobs_span_writer.enqueue.call_count == len(expected_spans_data) - mock_llmobs_span_writer.assert_has_calls(expected_llmobs_writer_calls) - - def test_llmobs_openai_llm(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain.llms.OpenAI() - - self._test_llmobs_llm_invoke( - generate_trace=llm, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_completion_sync.yaml", - different_py39_cassette=True, - provider="openai", - ) - - def test_llmobs_cohere_llm(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain.llms.Cohere(model="cohere.command-light-text-v14") - - self._test_llmobs_llm_invoke( - generate_trace=llm, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="cohere_completion_sync.yaml", - provider="cohere", - ) - - def test_llmobs_ai21_llm(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain.llms.AI21() - - self._test_llmobs_llm_invoke( - generate_trace=llm, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="ai21_completion_sync.yaml", - provider="ai21", - different_py39_cassette=True, - ) - - def test_llmobs_huggingfacehub_llm(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain.llms.HuggingFaceHub( - repo_id="google/flan-t5-xxl", - model_kwargs={"temperature": 0.0, "max_tokens": 256}, - huggingfacehub_api_token=os.getenv("HUGGINGFACEHUB_API_TOKEN", ""), - ) - - self._test_llmobs_llm_invoke( - generate_trace=llm, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="huggingfacehub_completion_sync.yaml", - provider="huggingface_hub", - ) - - def test_llmobs_openai_chat_model(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - chat = langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) - - self._test_llmobs_llm_invoke( - generate_trace=lambda prompt: chat([langchain.schema.HumanMessage(content=prompt)]), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_chat_completion_sync_call.yaml", - provider="openai", - input_roles=["user"], - output_role="assistant", - different_py39_cassette=True, - ) - - def test_llmobs_openai_chat_model_custom_role(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - chat = langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) - - self._test_llmobs_llm_invoke( - generate_trace=lambda prompt: chat([langchain.schema.ChatMessage(content=prompt, role="custom")]), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_chat_completion_sync_call.yaml", - provider="openai", - input_roles=["custom"], - output_role="assistant", - different_py39_cassette=True, - ) - - def test_llmobs_chain(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - chain = langchain.chains.LLMMathChain(llm=langchain.llms.OpenAI(temperature=0, max_tokens=256)) - - self._test_llmobs_chain_invoke( - generate_trace=lambda prompt: chain.run("what is two raised to the fifty-fourth power?"), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_math_chain_sync.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps({"question": "what is two raised to the fifty-fourth power?"}), - "output_value": json.dumps( - { - "question": "what is two raised to the fifty-fourth power?", - "answer": "Answer: 18014398509481984", - } - ), - }, - ), - ( - "chain", - { - "input_value": json.dumps( - { - "question": "what is two raised to the fifty-fourth power?", - "stop": ["```output"], - } - ), - "output_value": json.dumps( - { - "question": "what is two raised to the fifty-fourth power?", - "stop": ["```output"], - "text": '\n```text\n2**54\n```\n...numexpr.evaluate("2**54")...\n', - } - ), - }, - ), - ("llm", {"provider": "openai", "input_roles": [None], "output_role": None}), - ], - different_py39_cassette=True, - ) - - @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") - def test_llmobs_chain_nested(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - template = """Paraphrase this text: - - {input_text} - - Paraphrase: """ - prompt = langchain.PromptTemplate(input_variables=["input_text"], template=template) - style_paraphrase_chain = langchain.chains.LLMChain( - llm=langchain.llms.OpenAI(model="gpt-3.5-turbo-instruct"), prompt=prompt, output_key="paraphrased_output" - ) - rhyme_template = """Make this text rhyme: - - {paraphrased_output} - - Rhyme: """ - rhyme_prompt = langchain.PromptTemplate(input_variables=["paraphrased_output"], template=rhyme_template) - rhyme_chain = langchain.chains.LLMChain( - llm=langchain.llms.OpenAI(model="gpt-3.5-turbo-instruct"), prompt=rhyme_prompt, output_key="final_output" - ) - sequential_chain = langchain.chains.SequentialChain( - chains=[style_paraphrase_chain, rhyme_chain], - input_variables=["input_text"], - output_variables=["final_output"], - ) - - input_text = """ - I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no - bodies. Does it now follow that I too do not exist? No: if I convinced myself of something then I certainly - existed. But there is a deceiver of supreme power and cunning who is deliberately and constantly deceiving - me. In that case I too undoubtedly exist, if he is deceiving me; and let him deceive me as much as he can, - he will never bring it about that I am nothing so long as I think that I am something. So after considering - everything very thoroughly, I must finally conclude that this proposition, I am, I exist, is necessarily - true whenever it is put forward by me or conceived in my mind. - """ - - self._test_llmobs_chain_invoke( - generate_trace=lambda prompt: sequential_chain.run({"input_text": input_text}), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_sequential_paraphrase_and_rhyme_sync.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps({"input_text": input_text}), - "output_value": mock.ANY, - }, - ), - ( - "chain", - { - "input_value": json.dumps({"input_text": input_text}), - "output_value": mock.ANY, - }, - ), - ("llm", {"provider": "openai", "input_roles": [None], "output_role": None}), - ( - "chain", - { - "input_value": mock.ANY, - "output_value": mock.ANY, - }, - ), - ("llm", {"provider": "openai", "input_roles": [None], "output_role": None}), - ], - ) - - @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") - def test_llmobs_chain_schema_io(self, langchain, mock_llmobs_span_writer, mock_tracer, request_vcr): - model = langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) - prompt = langchain.prompts.ChatPromptTemplate.from_messages( - [ - langchain.prompts.SystemMessagePromptTemplate.from_template( - "You're an assistant who's good at {ability}. Respond in 20 words or fewer" - ), - langchain.prompts.MessagesPlaceholder(variable_name="history"), - langchain.prompts.HumanMessagePromptTemplate.from_template("{input}"), - ] - ) - - chain = langchain.chains.LLMChain(prompt=prompt, llm=model) - - self._test_llmobs_chain_invoke( - generate_trace=lambda input_text: chain.run( - { - "ability": "world capitals", - "history": [ - langchain.schema.HumanMessage(content="Can you be my science teacher instead?"), - langchain.schema.AIMessage(content="Yes"), - ], - "input": "What's the powerhouse of the cell?", - } - ), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_chain_schema_io.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps( - { - "ability": "world capitals", - "history": [["user", "Can you be my science teacher instead?"], ["assistant", "Yes"]], - "input": "What's the powerhouse of the cell?", - } - ), - "output_value": json.dumps( - { - "ability": "world capitals", - "history": [["user", "Can you be my science teacher instead?"], ["assistant", "Yes"]], - "input": "What's the powerhouse of the cell?", - "text": "Mitochondria.", - } - ), - }, - ), - ( - "llm", - { - "provider": "openai", - "input_roles": ["system", "user", "assistant", "user"], - "output_role": "assistant", - }, - ), - ], - ) diff --git a/tests/contrib/langchain/test_langchain_community.py b/tests/contrib/langchain/test_langchain_community.py index cf946f9d981..47e4d01bedb 100644 --- a/tests/contrib/langchain/test_langchain_community.py +++ b/tests/contrib/langchain/test_langchain_community.py @@ -1,4 +1,3 @@ -import json from operator import itemgetter import os import re @@ -10,12 +9,7 @@ import pytest from ddtrace.internal.utils.version import parse_version -from ddtrace.llmobs import LLMObs from tests.contrib.langchain.utils import get_request_vcr -from tests.llmobs._utils import _expected_llmobs_llm_span_event -from tests.llmobs._utils import _expected_llmobs_non_llm_span_event -from tests.subprocesstest import SubprocessTestCase -from tests.subprocesstest import run_in_subprocess from tests.utils import flaky from tests.utils import override_global_config @@ -34,7 +28,6 @@ def request_vcr(): yield get_request_vcr(subdirectory_name="langchain_community") -@flaky(1735812000) @pytest.mark.parametrize("ddtrace_config_langchain", [dict(logs_enabled=True, log_prompt_completion_sample_rate=1.0)]) def test_global_tags( ddtrace_config_langchain, langchain, langchain_openai, request_vcr, mock_metrics, mock_logs, mock_tracer @@ -86,7 +79,6 @@ def test_global_tags( ) -@flaky(1735812000) @pytest.mark.snapshot(ignores=["metrics.langchain.tokens.total_cost", "resource"]) def test_openai_llm_sync(langchain, langchain_openai, request_vcr): llm = langchain_openai.OpenAI() @@ -94,7 +86,6 @@ def test_openai_llm_sync(langchain, langchain_openai, request_vcr): llm.invoke("Can you explain what Descartes meant by 'I think, therefore I am'?") -@flaky(1735812000) @pytest.mark.snapshot def test_openai_llm_sync_multiple_prompts(langchain, langchain_openai, request_vcr): llm = langchain_openai.OpenAI() @@ -130,7 +121,6 @@ async def test_openai_llm_async_stream(langchain, langchain_openai, request_vcr) await llm.agenerate(["Why is Spongebob so bad at driving?"]) -@flaky(1735812000) @pytest.mark.snapshot(ignores=["meta.error.stack", "resource"]) def test_openai_llm_error(langchain, langchain_openai, request_vcr): import openai # Imported here because the os env OPENAI_API_KEY needs to be set via langchain fixture before import @@ -226,7 +216,6 @@ def test_llm_logs( mock_metrics.count.assert_not_called() -@flaky(1735812000) @pytest.mark.snapshot def test_openai_chat_model_sync_call_langchain_openai(langchain, langchain_openai, request_vcr): chat = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256) @@ -234,7 +223,6 @@ def test_openai_chat_model_sync_call_langchain_openai(langchain, langchain_opena chat.invoke(input=[langchain.schema.HumanMessage(content="When do you use 'whom' instead of 'who'?")]) -@flaky(1735812000) @pytest.mark.snapshot def test_openai_chat_model_sync_generate(langchain, langchain_openai, request_vcr): chat = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256) @@ -328,7 +316,6 @@ async def test_openai_chat_model_async_stream(langchain, langchain_openai, reque await chat.agenerate([[langchain.schema.HumanMessage(content="What is the secret Krabby Patty recipe?")]]) -@flaky(1735812000) def test_chat_model_metrics(langchain, langchain_openai, request_vcr, mock_metrics, mock_logs, snapshot_tracer): chat = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256) with request_vcr.use_cassette("openai_chat_completion_sync_call.yaml"): @@ -356,7 +343,6 @@ def test_chat_model_metrics(langchain, langchain_openai, request_vcr, mock_metri mock_logs.assert_not_called() -@flaky(1735812000) @pytest.mark.parametrize( "ddtrace_config_langchain", [dict(metrics_enabled=False, logs_enabled=True, log_prompt_completion_sample_rate=1.0)], @@ -487,7 +473,6 @@ def test_embedding_logs(langchain_openai, ddtrace_config_langchain, request_vcr, mock_metrics.count.assert_not_called() -@flaky(1735812000) @pytest.mark.snapshot def test_openai_math_chain_sync(langchain, langchain_openai, request_vcr): """ @@ -499,7 +484,6 @@ def test_openai_math_chain_sync(langchain, langchain_openai, request_vcr): chain.invoke("what is two raised to the fifty-fourth power?") -@flaky(1735812000) @pytest.mark.snapshot(token="tests.contrib.langchain.test_langchain_community.test_chain_invoke") def test_chain_invoke_dict_input(langchain, langchain_openai, request_vcr): prompt_template = "what is {base} raised to the fifty-fourth power?" @@ -509,7 +493,6 @@ def test_chain_invoke_dict_input(langchain, langchain_openai, request_vcr): chain.invoke(input={"base": "two"}) -@flaky(1735812000) @pytest.mark.snapshot(token="tests.contrib.langchain.test_langchain_community.test_chain_invoke") def test_chain_invoke_str_input(langchain, langchain_openai, request_vcr): prompt_template = "what is {base} raised to the fifty-fourth power?" @@ -597,7 +580,6 @@ def _transform_func(inputs): sequential_chain.invoke({"text": input_text, "style": "a 90s rapper"}) -@flaky(1735812000) @pytest.mark.snapshot def test_openai_sequential_chain_with_multiple_llm_sync(langchain, langchain_openai, request_vcr): template = """Paraphrase this text: @@ -677,7 +659,6 @@ async def test_openai_sequential_chain_with_multiple_llm_async(langchain, langch await sequential_chain.ainvoke({"input_text": input_text}) -@flaky(1735812000) @pytest.mark.parametrize( "ddtrace_config_langchain", [dict(metrics_enabled=False, logs_enabled=True, log_prompt_completion_sample_rate=1.0)], @@ -928,7 +909,7 @@ def test_vectorstore_logs( mock_metrics.count.assert_not_called() -@pytest.mark.snapshot(ignores=["metrics.langchain.tokens.total_cost", "resource"]) +@pytest.mark.snapshot(ignores=["metrics.langchain.tokens.total_cost", "meta.http.useragent", "resource"]) def test_openai_integration(langchain, request_vcr, ddtrace_run_python_code_in_subprocess): env = os.environ.copy() pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] @@ -959,7 +940,7 @@ def test_openai_integration(langchain, request_vcr, ddtrace_run_python_code_in_s assert err == b"" -@pytest.mark.snapshot +@pytest.mark.snapshot(ignores=["meta.http.useragent"]) @pytest.mark.parametrize("schema_version", [None, "v0", "v1"]) @pytest.mark.parametrize("service_name", [None, "mysvc"]) def test_openai_service_name( @@ -1001,7 +982,6 @@ def test_openai_service_name( assert err == b"" -@flaky(1735812000) @pytest.mark.parametrize( "ddtrace_config_langchain", [dict(metrics_enabled=False, logs_enabled=True, log_prompt_completion_sample_rate=1.0)], @@ -1120,7 +1100,6 @@ def test_embedding_logs_when_response_not_completed( ) -@flaky(1735812000) @pytest.mark.snapshot def test_lcel_chain_simple(langchain_core, langchain_openai, request_vcr): prompt = langchain_core.prompts.ChatPromptTemplate.from_messages( @@ -1133,7 +1112,6 @@ def test_lcel_chain_simple(langchain_core, langchain_openai, request_vcr): chain.invoke({"input": "how can langsmith help with testing?"}) -@flaky(1735812000) @pytest.mark.snapshot def test_lcel_chain_complicated(langchain_core, langchain_openai, request_vcr): prompt = langchain_core.prompts.ChatPromptTemplate.from_template( @@ -1176,7 +1154,7 @@ async def test_lcel_chain_simple_async(langchain_core, langchain_openai, request await chain.ainvoke({"input": "how can langsmith help with testing?"}) -@flaky(1735812000) +@flaky(1735812000, reason="batch() is non-deterministic in which order it processes inputs") @pytest.mark.snapshot @pytest.mark.skipif(sys.version_info >= (3, 11, 0), reason="Python <3.11 test") def test_lcel_chain_batch(langchain_core, langchain_openai, request_vcr): @@ -1193,7 +1171,7 @@ def test_lcel_chain_batch(langchain_core, langchain_openai, request_vcr): chain.batch(inputs=["chickens", "pigs"]) -@flaky(1735812000) +@flaky(1735812000, reason="batch() is non-deterministic in which order it processes inputs") @pytest.mark.snapshot @pytest.mark.skipif(sys.version_info < (3, 11, 0), reason="Python 3.11+ required") def test_lcel_chain_batch_311(langchain_core, langchain_openai, request_vcr): @@ -1210,7 +1188,6 @@ def test_lcel_chain_batch_311(langchain_core, langchain_openai, request_vcr): chain.batch(inputs=["chickens", "pigs"]) -@flaky(1735812000) @pytest.mark.snapshot def test_lcel_chain_nested(langchain_core, langchain_openai, request_vcr): """ @@ -1234,7 +1211,7 @@ def test_lcel_chain_nested(langchain_core, langchain_openai, request_vcr): complete_chain.invoke({"person": "Spongebob Squarepants", "language": "Spanish"}) -@flaky(1735812000) +@flaky(1735812000, reason="batch() is non-deterministic in which order it processes inputs") @pytest.mark.asyncio @pytest.mark.snapshot async def test_lcel_chain_batch_async(langchain_core, langchain_openai, request_vcr): @@ -1249,541 +1226,3 @@ async def test_lcel_chain_batch_async(langchain_core, langchain_openai, request_ with request_vcr.use_cassette("lcel_openai_chain_batch_async.yaml"): await chain.abatch(inputs=["chickens", "pigs"]) - - -@pytest.mark.parametrize( - "ddtrace_global_config", - [dict(_llmobs_enabled=True, _llmobs_sample_rate=1.0, _llmobs_ml_app="langchain_community_test")], -) -class TestLLMObsLangchain: - @staticmethod - def _expected_llmobs_chain_calls(trace, expected_spans_data: list): - expected_llmobs_writer_calls = [mock.call.start()] - - for idx, span in enumerate(trace): - kind, kwargs = expected_spans_data[idx] - expected_span_event = None - if kind == "chain": - expected_span_event = TestLLMObsLangchain._expected_llmobs_chain_call(span, **kwargs) - else: - expected_span_event = TestLLMObsLangchain._expected_llmobs_llm_call(span, **kwargs) - - expected_llmobs_writer_calls += [mock.call.enqueue(expected_span_event)] - - return expected_llmobs_writer_calls - - @staticmethod - def _expected_llmobs_chain_call(span, input_parameters=None, input_value=None, output_value=None): - return _expected_llmobs_non_llm_span_event( - span, - span_kind="workflow", - parameters=input_parameters, - input_value=input_value, - output_value=output_value, - tags={ - "ml_app": "langchain_community_test", - }, - integration="langchain", - ) - - @staticmethod - def _expected_llmobs_llm_call(span, provider="openai", input_roles=[None], output_role=None): - input_meta = [{"content": mock.ANY} for _ in input_roles] - for idx, role in enumerate(input_roles): - if role is not None: - input_meta[idx]["role"] = role - - output_meta = {"content": mock.ANY} - if output_role is not None: - output_meta["role"] = output_role - - temperature_key = "temperature" - if provider == "huggingface_hub": - max_tokens_key = "model_kwargs.max_tokens" - temperature_key = "model_kwargs.temperature" - elif provider == "ai21": - max_tokens_key = "maxTokens" - else: - max_tokens_key = "max_tokens" - - metadata = {} - temperature = span.get_tag(f"langchain.request.{provider}.parameters.{temperature_key}") - max_tokens = span.get_tag(f"langchain.request.{provider}.parameters.{max_tokens_key}") - if temperature is not None: - metadata["temperature"] = float(temperature) - if max_tokens is not None: - metadata["max_tokens"] = int(max_tokens) - - return _expected_llmobs_llm_span_event( - span, - model_name=span.get_tag("langchain.request.model"), - model_provider=span.get_tag("langchain.request.provider"), - input_messages=input_meta, - output_messages=[output_meta], - metadata=metadata, - token_metrics={}, - tags={ - "ml_app": "langchain_community_test", - }, - integration="langchain", - ) - - @classmethod - def _test_llmobs_llm_invoke( - cls, - provider, - generate_trace, - request_vcr, - mock_llmobs_span_writer, - mock_tracer, - cassette_name, - input_roles=[None], - output_role=None, - ): - LLMObs.disable() - LLMObs.enable(_tracer=mock_tracer, integrations_enabled=False) # only want langchain patched - - with request_vcr.use_cassette(cassette_name): - generate_trace("Can you explain what an LLM chain is?") - span = mock_tracer.pop_traces()[0][0] - - expected_llmons_writer_calls = [ - mock.call.start(), - mock.call.enqueue( - cls._expected_llmobs_llm_call( - span, - provider=provider, - input_roles=input_roles, - output_role=output_role, - ) - ), - ] - - assert mock_llmobs_span_writer.enqueue.call_count == 1 - mock_llmobs_span_writer.assert_has_calls(expected_llmons_writer_calls) - - @classmethod - def _test_llmobs_chain_invoke( - cls, - generate_trace, - request_vcr, - mock_llmobs_span_writer, - mock_tracer, - cassette_name, - expected_spans_data=[("llm", {"provider": "openai", "input_roles": [None], "output_role": None})], - ): - # disable the service before re-enabling it, as it was enabled in another test - LLMObs.disable() - LLMObs.enable(_tracer=mock_tracer, integrations_enabled=False) # only want langchain patched - - with request_vcr.use_cassette(cassette_name): - generate_trace("Can you explain what an LLM chain is?") - trace = mock_tracer.pop_traces()[0] - - expected_llmobs_writer_calls = cls._expected_llmobs_chain_calls( - trace=trace, expected_spans_data=expected_spans_data - ) - assert mock_llmobs_span_writer.enqueue.call_count == len(expected_spans_data) - mock_llmobs_span_writer.assert_has_calls(expected_llmobs_writer_calls) - - @flaky(1735812000) - def test_llmobs_openai_llm(self, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain_openai.OpenAI() - - self._test_llmobs_llm_invoke( - generate_trace=llm.invoke, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_completion_sync.yaml", - provider="openai", - ) - - def test_llmobs_cohere_llm(self, langchain_community, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain_community.llms.Cohere(model="cohere.command-light-text-v14") - - self._test_llmobs_llm_invoke( - generate_trace=llm.invoke, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="cohere_completion_sync.yaml", - provider="cohere", - ) - - def test_llmobs_ai21_llm(self, langchain_community, mock_llmobs_span_writer, mock_tracer, request_vcr): - llm = langchain_community.llms.AI21() - - self._test_llmobs_llm_invoke( - generate_trace=llm.invoke, - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="ai21_completion_sync.yaml", - provider="ai21", - ) - - @flaky(1735812000) - def test_llmobs_openai_chat_model(self, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr): - chat = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256) - - self._test_llmobs_llm_invoke( - generate_trace=lambda prompt: chat.invoke([langchain.schema.HumanMessage(content=prompt)]), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_chat_completion_sync_call.yaml", - provider="openai", - input_roles=["user"], - output_role="assistant", - ) - - @flaky(1735812000) - def test_llmobs_openai_chat_model_custom_role( - self, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr - ): - chat = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256) - - self._test_llmobs_llm_invoke( - generate_trace=lambda prompt: chat.invoke([langchain.schema.ChatMessage(content=prompt, role="custom")]), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="openai_chat_completion_sync_call.yaml", - provider="openai", - input_roles=["custom"], - output_role="assistant", - ) - - @flaky(1735812000) - def test_llmobs_chain(self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr): - prompt = langchain_core.prompts.ChatPromptTemplate.from_messages( - [("system", "You are world class technical documentation writer."), ("user", "{input}")] - ) - llm = langchain_openai.OpenAI() - - chain = prompt | llm - - expected_output = ( - "\nSystem: Langsmith can help with testing in several ways. " - "First, it can generate automated tests based on your technical documentation, " - "ensuring that your code matches the documented specifications. " - "This can save you time and effort in testing your code manually. " - "Additionally, Langsmith can also analyze your technical documentation for completeness and accuracy, " - "helping you identify any potential gaps or errors before testing begins. " - "Finally, Langsmith can assist with creating test cases and scenarios based on your documentation, " - "making the testing process more efficient and effective." - ) - - self._test_llmobs_chain_invoke( - generate_trace=lambda prompt: chain.invoke({"input": prompt}), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="lcel_openai_chain_call.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps([{"input": "Can you explain what an LLM chain is?"}]), - "output_value": expected_output, - }, - ), - ("llm", {"provider": "openai", "input_roles": [None], "output_role": None}), - ], - ) - - def test_llmobs_chain_nested( - self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr - ): - prompt1 = langchain_core.prompts.ChatPromptTemplate.from_template("what is the city {person} is from?") - prompt2 = langchain_core.prompts.ChatPromptTemplate.from_template( - "what country is the city {city} in? respond in {language}" - ) - - model = langchain_openai.ChatOpenAI() - - chain1 = prompt1 | model | langchain_core.output_parsers.StrOutputParser() - chain2 = prompt2 | model | langchain_core.output_parsers.StrOutputParser() - - complete_chain = {"city": chain1, "language": itemgetter("language")} | chain2 - - self._test_llmobs_chain_invoke( - generate_trace=lambda inputs: complete_chain.invoke( - {"person": "Spongebob Squarepants", "language": "Spanish"} - ), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="lcel_openai_chain_nested.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps([{"person": "Spongebob Squarepants", "language": "Spanish"}]), - "output_value": mock.ANY, - }, - ), - ( - "chain", - { - "input_value": json.dumps([{"person": "Spongebob Squarepants", "language": "Spanish"}]), - "output_value": mock.ANY, - }, - ), - ("llm", {"provider": "openai", "input_roles": ["user"], "output_role": "assistant"}), - ("llm", {"provider": "openai", "input_roles": ["user"], "output_role": "assistant"}), - ], - ) - - @pytest.mark.skipif(sys.version_info >= (3, 11, 0), reason="Python <3.11 required") - def test_llmobs_chain_batch( - self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr - ): - prompt = langchain_core.prompts.ChatPromptTemplate.from_template("Tell me a short joke about {topic}") - output_parser = langchain_core.output_parsers.StrOutputParser() - model = langchain_openai.ChatOpenAI() - chain = {"topic": langchain_core.runnables.RunnablePassthrough()} | prompt | model | output_parser - - self._test_llmobs_chain_invoke( - generate_trace=lambda inputs: chain.batch(inputs=["chickens", "pigs"]), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="lcel_openai_chain_batch.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps(["chickens", "pigs"]), - "output_value": mock.ANY, - }, - ), - ("llm", {"provider": "openai", "input_roles": ["user"], "output_role": "assistant"}), - ("llm", {"provider": "openai", "input_roles": ["user"], "output_role": "assistant"}), - ], - ) - - @flaky(1735812000) - def test_llmobs_chain_schema_io( - self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer, request_vcr - ): - model = langchain_openai.ChatOpenAI() - prompt = langchain_core.prompts.ChatPromptTemplate.from_messages( - [ - ("system", "You're an assistant who's good at {ability}. Respond in 20 words or fewer"), - langchain_core.prompts.MessagesPlaceholder(variable_name="history"), - ("human", "{input}"), - ] - ) - - chain = prompt | model - - self._test_llmobs_chain_invoke( - generate_trace=lambda inputs: chain.invoke( - { - "ability": "world capitals", - "history": [ - langchain.schema.HumanMessage(content="Can you be my science teacher instead?"), - langchain.schema.AIMessage(content="Yes"), - ], - "input": "What's the powerhouse of the cell?", - } - ), - request_vcr=request_vcr, - mock_llmobs_span_writer=mock_llmobs_span_writer, - mock_tracer=mock_tracer, - cassette_name="lcel_openai_chain_schema_io.yaml", - expected_spans_data=[ - ( - "chain", - { - "input_value": json.dumps( - [ - { - "ability": "world capitals", - "history": [ - ["user", "Can you be my science teacher instead?"], - ["assistant", "Yes"], - ], - "input": "What's the powerhouse of the cell?", - } - ] - ), - "output_value": json.dumps(["assistant", "Mitochondria."]), - }, - ), - ( - "llm", - { - "provider": "openai", - "input_roles": ["system", "user", "assistant", "user"], - "output_role": "assistant", - }, - ), - ], - ) - - -class TestLangchainTraceStructureWithLlmIntegrations(SubprocessTestCase): - bedrock_env_config = dict( - AWS_ACCESS_KEY_ID="testing", - AWS_SECRET_ACCESS_KEY="testing", - AWS_SECURITY_TOKEN="testing", - AWS_SESSION_TOKEN="testing", - AWS_DEFAULT_REGION="us-east-1", - DD_LANGCHAIN_METRICS_ENABLED="false", - DD_API_KEY="", - ) - - openai_env_config = dict( - OPENAI_API_KEY="testing", - DD_API_KEY="", - ) - - def setUp(self): - patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter") - LLMObsSpanWriterMock = patcher.start() - mock_llmobs_span_writer = mock.MagicMock() - LLMObsSpanWriterMock.return_value = mock_llmobs_span_writer - - self.mock_llmobs_span_writer = mock_llmobs_span_writer - - super(TestLangchainTraceStructureWithLlmIntegrations, self).setUp() - - def _assert_trace_structure_from_writer_call_args(self, span_kinds): - assert self.mock_llmobs_span_writer.enqueue.call_count == len(span_kinds) - - calls = self.mock_llmobs_span_writer.enqueue.call_args_list - - for span_kind, call in zip(span_kinds, calls): - call_args = call.args[0] - - assert call_args["meta"]["span.kind"] == span_kind - if span_kind == "workflow": - assert len(call_args["meta"]["input"]["value"]) > 0 - assert len(call_args["meta"]["output"]["value"]) > 0 - elif span_kind == "llm": - assert len(call_args["meta"]["input"]["messages"]) > 0 - assert len(call_args["meta"]["output"]["messages"]) > 0 - - def _call_bedrock_chat_model(self, ChatBedrock, HumanMessage): - chat = ChatBedrock( - model_id="amazon.titan-tg1-large", - model_kwargs={"max_tokens": 50, "temperature": 0}, - ) - messages = [HumanMessage(content="summarize the plot to the lord of the rings in a dozen words")] - with get_request_vcr(subdirectory_name="langchain_community").use_cassette("bedrock_amazon_chat_invoke.yaml"): - chat.invoke(messages) - - def _call_bedrock_llm(self, Bedrock, ConversationChain, ConversationBufferMemory): - llm = Bedrock( - model_id="amazon.titan-tg1-large", - region_name="us-east-1", - model_kwargs={"temperature": 0, "topP": 0.9, "stopSequences": [], "maxTokens": 50}, - ) - - conversation = ConversationChain(llm=llm, verbose=True, memory=ConversationBufferMemory()) - - with get_request_vcr(subdirectory_name="langchain_community").use_cassette("bedrock_amazon_invoke.yaml"): - conversation.predict(input="can you explain what Datadog is to someone not in the tech industry?") - - def _call_openai_llm(self, OpenAI): - llm = OpenAI() - with get_request_vcr(subdirectory_name="langchain_community").use_cassette("openai_completion_sync.yaml"): - llm.invoke("Can you explain what Descartes meant by 'I think, therefore I am'?") - - @run_in_subprocess(env_overrides=bedrock_env_config) - def test_llmobs_with_chat_model_bedrock_enabled(self): - from langchain_aws import ChatBedrock - from langchain_core.messages import HumanMessage - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True, botocore=True) - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - - self._call_bedrock_chat_model(ChatBedrock, HumanMessage) - - self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) - - LLMObs.disable() - - @run_in_subprocess(env_overrides=bedrock_env_config) - def test_llmobs_with_chat_model_bedrock_disabled(self): - from langchain_aws import ChatBedrock - from langchain_core.messages import HumanMessage - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True) - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - - self._call_bedrock_chat_model(ChatBedrock, HumanMessage) - - self._assert_trace_structure_from_writer_call_args(["llm"]) - - LLMObs.disable() - - @run_in_subprocess(env_overrides=bedrock_env_config) - def test_llmobs_with_llm_model_bedrock_enabled(self): - from langchain.chains import ConversationChain - from langchain.memory import ConversationBufferMemory - from langchain_community.llms import Bedrock - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True, botocore=True) - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - self._call_bedrock_llm(Bedrock, ConversationChain, ConversationBufferMemory) - self._assert_trace_structure_from_writer_call_args(["workflow", "workflow", "llm"]) - - LLMObs.disable() - - @run_in_subprocess(env_overrides=bedrock_env_config) - def test_llmobs_with_llm_model_bedrock_disabled(self): - from langchain.chains import ConversationChain - from langchain.memory import ConversationBufferMemory - from langchain_community.llms import Bedrock - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True) - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - self._call_bedrock_llm(Bedrock, ConversationChain, ConversationBufferMemory) - self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) - - LLMObs.disable() - - @run_in_subprocess(env_overrides=openai_env_config) - def test_llmobs_langchain_with_openai_enabled(self): - from langchain_openai import OpenAI - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True, openai=True) - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - self._call_openai_llm(OpenAI) - self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) - - LLMObs.disable() - - @run_in_subprocess(env_overrides=openai_env_config) - def test_llmobs_langchain_with_openai_disabled(self): - from langchain_openai import OpenAI - - from ddtrace import patch - from ddtrace.llmobs import LLMObs - - patch(langchain=True) - - LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) - self._call_openai_llm(OpenAI) - self._assert_trace_structure_from_writer_call_args(["llm"]) - - LLMObs.disable() diff --git a/tests/contrib/langchain/test_langchain_llmobs.py b/tests/contrib/langchain/test_langchain_llmobs.py new file mode 100644 index 00000000000..c8cb72009b0 --- /dev/null +++ b/tests/contrib/langchain/test_langchain_llmobs.py @@ -0,0 +1,644 @@ +import json +from operator import itemgetter +import os +import sys + +import mock +import pytest + +from ddtrace import patch +from ddtrace.contrib.langchain.patch import SHOULD_PATCH_LANGCHAIN_COMMUNITY +from ddtrace.llmobs import LLMObs +from tests.contrib.langchain.utils import get_request_vcr +from tests.contrib.langchain.utils import long_input_text +from tests.llmobs._utils import _expected_llmobs_llm_span_event +from tests.llmobs._utils import _expected_llmobs_non_llm_span_event +from tests.subprocesstest import SubprocessTestCase +from tests.subprocesstest import run_in_subprocess + + +if SHOULD_PATCH_LANGCHAIN_COMMUNITY: + from langchain_core.messages import AIMessage + from langchain_core.messages import ChatMessage + from langchain_core.messages import HumanMessage +else: + from langchain.schema import AIMessage + from langchain.schema import ChatMessage + from langchain.schema import HumanMessage + + +def _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer, input_role=None, mock_io=False): + provider = span.get_tag("langchain.request.provider") + + metadata = {} + temperature_key = "temperature" + if provider == "huggingface_hub": + temperature_key = "model_kwargs.temperature" + max_tokens_key = "model_kwargs.max_tokens" + elif provider == "ai21": + max_tokens_key = "maxTokens" + else: + max_tokens_key = "max_tokens" + temperature = span.get_tag(f"langchain.request.{provider}.parameters.{temperature_key}") + max_tokens = span.get_tag(f"langchain.request.{provider}.parameters.{max_tokens_key}") + if temperature is not None: + metadata["temperature"] = float(temperature) + if max_tokens is not None: + metadata["max_tokens"] = int(max_tokens) + + input_messages = [{"content": mock.ANY}] + output_messages = [{"content": mock.ANY}] + if input_role is not None: + input_messages[0]["role"] = input_role + output_messages[0]["role"] = "assistant" + + mock_llmobs_span_writer.enqueue.assert_any_call( + _expected_llmobs_llm_span_event( + span, + model_name=span.get_tag("langchain.request.model"), + model_provider=span.get_tag("langchain.request.provider"), + input_messages=input_messages if not mock_io else mock.ANY, + output_messages=output_messages if not mock_io else mock.ANY, + metadata=metadata, + token_metrics={}, + tags={"ml_app": "langchain_test"}, + integration="langchain", + ) + ) + + +def _assert_expected_llmobs_chain_span(span, mock_llmobs_span_writer, input_value=None, output_value=None): + expected_chain_span_event = _expected_llmobs_non_llm_span_event( + span, + "workflow", + input_value=input_value if input_value is not None else mock.ANY, + output_value=output_value if output_value is not None else mock.ANY, + tags={"ml_app": "langchain_test"}, + integration="langchain", + ) + mock_llmobs_span_writer.enqueue.assert_any_call(expected_chain_span_event) + + +class BaseTestLLMObsLangchain: + cassette_subdirectory_name = "langchain" + ml_app = "langchain_test" + + @classmethod + def _invoke_llm(cls, llm, prompt, mock_tracer, cassette_name): + LLMObs.enable(ml_app=cls.ml_app, integrations_enabled=False, _tracer=mock_tracer) + with get_request_vcr(subdirectory_name=cls.cassette_subdirectory_name).use_cassette(cassette_name): + if SHOULD_PATCH_LANGCHAIN_COMMUNITY: + llm.invoke(prompt) + else: + llm(prompt) + LLMObs.disable() + return mock_tracer.pop_traces()[0][0] + + @classmethod + def _invoke_chat(cls, chat_model, prompt, mock_tracer, cassette_name, role="user"): + LLMObs.enable(ml_app=cls.ml_app, integrations_enabled=False, _tracer=mock_tracer) + with get_request_vcr(subdirectory_name=cls.cassette_subdirectory_name).use_cassette(cassette_name): + if role == "user": + messages = [HumanMessage(content=prompt)] + else: + messages = [ChatMessage(content=prompt, role="custom")] + if SHOULD_PATCH_LANGCHAIN_COMMUNITY: + chat_model.invoke(messages) + else: + chat_model(messages) + LLMObs.disable() + return mock_tracer.pop_traces()[0][0] + + @classmethod + def _invoke_chain(cls, chain, prompt, mock_tracer, cassette_name, batch=False): + LLMObs.enable(ml_app=cls.ml_app, integrations_enabled=False, _tracer=mock_tracer) + with get_request_vcr(subdirectory_name=cls.cassette_subdirectory_name).use_cassette(cassette_name): + if batch: + chain.batch(inputs=prompt) + elif SHOULD_PATCH_LANGCHAIN_COMMUNITY: + chain.invoke(prompt) + else: + chain.run(prompt) + LLMObs.disable() + return mock_tracer.pop_traces()[0] + + +@pytest.mark.skipif(SHOULD_PATCH_LANGCHAIN_COMMUNITY, reason="These tests are for langchain < 0.1.0") +class TestLLMObsLangchain(BaseTestLLMObsLangchain): + cassette_subdirectory_name = "langchain" + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_openai_llm(self, langchain, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_llm( + llm=langchain.llms.OpenAI(model="gpt-3.5-turbo-instruct"), + prompt="Can you explain what Descartes meant by 'I think, therefore I am'?", + mock_tracer=mock_tracer, + cassette_name="openai_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + def test_llmobs_cohere_llm(self, langchain, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_llm( + llm=langchain.llms.Cohere(model="cohere.command-light-text-v14"), + prompt="Can you explain what Descartes meant by 'I think, therefore I am'?", + mock_tracer=mock_tracer, + cassette_name="cohere_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_ai21_llm(self, langchain, mock_llmobs_span_writer, mock_tracer): + llm = langchain.llms.AI21() + span = self._invoke_llm( + llm=llm, + prompt="Can you explain what Descartes meant by 'I think, therefore I am'?", + mock_tracer=mock_tracer, + cassette_name="ai21_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + def test_llmobs_huggingfacehub_llm(self, langchain, mock_llmobs_span_writer, mock_tracer): + llm = langchain.llms.HuggingFaceHub( + repo_id="google/flan-t5-xxl", + model_kwargs={"temperature": 0.0, "max_tokens": 256}, + huggingfacehub_api_token=os.getenv("HUGGINGFACEHUB_API_TOKEN", ""), + ) + span = self._invoke_llm( + llm=llm, + prompt="Can you explain what Descartes meant by 'I think, therefore I am'?", + mock_tracer=mock_tracer, + cassette_name="huggingfacehub_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_openai_chat_model(self, langchain, mock_llmobs_span_writer, mock_tracer): + chat = langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) + span = self._invoke_chat( + chat_model=chat, + prompt="When do you use 'whom' instead of 'who'?", + mock_tracer=mock_tracer, + cassette_name="openai_chat_completion_sync_call.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer, input_role="user") + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_openai_chat_model_custom_role(self, langchain, mock_llmobs_span_writer, mock_tracer): + chat = langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) + span = self._invoke_chat( + chat_model=chat, + prompt="When do you use 'whom' instead of 'who'?", + mock_tracer=mock_tracer, + cassette_name="openai_chat_completion_sync_call.yaml", + role="custom", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer, input_role="custom") + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_chain(self, langchain, mock_llmobs_span_writer, mock_tracer): + chain = langchain.chains.LLMMathChain(llm=langchain.llms.OpenAI(temperature=0, max_tokens=256)) + + trace = self._invoke_chain( + chain=chain, + prompt="what is two raised to the fifty-fourth power?", + mock_tracer=mock_tracer, + cassette_name="openai_math_chain_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 3 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps({"question": "what is two raised to the fifty-fourth power?"}), + output_value=json.dumps( + {"question": "what is two raised to the fifty-fourth power?", "answer": "Answer: 18014398509481984"} + ), + ) + _assert_expected_llmobs_chain_span( + trace[1], + mock_llmobs_span_writer, + input_value=json.dumps( + {"question": "what is two raised to the fifty-fourth power?", "stop": ["```output"]} + ), + output_value=json.dumps( + { + "question": "what is two raised to the fifty-fourth power?", + "stop": ["```output"], + "text": '\n```text\n2**54\n```\n...numexpr.evaluate("2**54")...\n', + } + ), + ) + _assert_expected_llmobs_llm_span(trace[2], mock_llmobs_span_writer) + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_chain_nested(self, langchain, mock_llmobs_span_writer, mock_tracer): + template = "Paraphrase this text:\n{input_text}\nParaphrase: " + prompt = langchain.PromptTemplate(input_variables=["input_text"], template=template) + style_paraphrase_chain = langchain.chains.LLMChain( + llm=langchain.llms.OpenAI(model="gpt-3.5-turbo-instruct"), prompt=prompt, output_key="paraphrased_output" + ) + rhyme_template = "Make this text rhyme:\n{paraphrased_output}\nRhyme: " + rhyme_prompt = langchain.PromptTemplate(input_variables=["paraphrased_output"], template=rhyme_template) + rhyme_chain = langchain.chains.LLMChain( + llm=langchain.llms.OpenAI(model="gpt-3.5-turbo-instruct"), prompt=rhyme_prompt, output_key="final_output" + ) + sequential_chain = langchain.chains.SequentialChain( + chains=[style_paraphrase_chain, rhyme_chain], + input_variables=["input_text"], + output_variables=["final_output"], + ) + input_text = long_input_text + trace = self._invoke_chain( + chain=sequential_chain, + prompt={"input_text": input_text}, + mock_tracer=mock_tracer, + cassette_name="openai_sequential_paraphrase_and_rhyme_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 5 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps({"input_text": input_text}), + ) + _assert_expected_llmobs_chain_span( + trace[1], + mock_llmobs_span_writer, + input_value=json.dumps({"input_text": input_text}), + ) + _assert_expected_llmobs_llm_span(trace[2], mock_llmobs_span_writer) + _assert_expected_llmobs_chain_span(trace[3], mock_llmobs_span_writer) + _assert_expected_llmobs_llm_span(trace[4], mock_llmobs_span_writer) + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_chain_schema_io(self, langchain, mock_llmobs_span_writer, mock_tracer): + prompt = langchain.prompts.ChatPromptTemplate.from_messages( + [ + langchain.prompts.SystemMessagePromptTemplate.from_template( + "You're an assistant who's good at {ability}. Respond in 20 words or fewer" + ), + langchain.prompts.MessagesPlaceholder(variable_name="history"), + langchain.prompts.HumanMessagePromptTemplate.from_template("{input}"), + ] + ) + chain = langchain.chains.LLMChain( + prompt=prompt, llm=langchain.chat_models.ChatOpenAI(temperature=0, max_tokens=256) + ) + trace = self._invoke_chain( + chain=chain, + prompt={ + "ability": "world capitals", + "history": [ + HumanMessage(content="Can you be my science teacher instead?"), + AIMessage(content="Yes"), + ], + "input": "What's the powerhouse of the cell?", + }, + mock_tracer=mock_tracer, + cassette_name="openai_chain_schema_io.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 2 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps( + { + "ability": "world capitals", + "history": [["user", "Can you be my science teacher instead?"], ["assistant", "Yes"]], + "input": "What's the powerhouse of the cell?", + } + ), + output_value=json.dumps( + { + "ability": "world capitals", + "history": [["user", "Can you be my science teacher instead?"], ["assistant", "Yes"]], + "input": "What's the powerhouse of the cell?", + "text": "Mitochondria.", + } + ), + ) + _assert_expected_llmobs_llm_span(trace[1], mock_llmobs_span_writer, mock_io=True) + + +@pytest.mark.skipif(not SHOULD_PATCH_LANGCHAIN_COMMUNITY, reason="These tests are for langchain >= 0.1.0") +class TestLLMObsLangchainCommunity(BaseTestLLMObsLangchain): + cassette_subdirectory_name = "langchain_community" + + def test_llmobs_openai_llm(self, langchain_openai, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_llm( + llm=langchain_openai.OpenAI(), + prompt="Can you explain what Descartes meant by 'I think, therefore I am'?", + mock_tracer=mock_tracer, + cassette_name="openai_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + def test_llmobs_cohere_llm(self, langchain_community, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_llm( + llm=langchain_community.llms.Cohere(model="cohere.command-light-text-v14"), + prompt="What is the secret Krabby Patty recipe?", + mock_tracer=mock_tracer, + cassette_name="cohere_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + @pytest.mark.skipif(sys.version_info < (3, 10, 0), reason="Requires unnecessary cassette file for Python 3.9") + def test_llmobs_ai21_llm(self, langchain_community, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_llm( + llm=langchain_community.llms.AI21(), + prompt="Why does everyone in Bikini Bottom hate Plankton?", + mock_tracer=mock_tracer, + cassette_name="ai21_completion_sync.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer) + + def test_llmobs_openai_chat_model(self, langchain_openai, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_chat( + chat_model=langchain_openai.ChatOpenAI(temperature=0, max_tokens=256), + prompt="When do you use 'who' instead of 'whom'?", + mock_tracer=mock_tracer, + cassette_name="openai_chat_completion_sync_call.yaml", + role="user", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer, input_role="user") + + def test_llmobs_openai_chat_model_custom_role(self, langchain_openai, mock_llmobs_span_writer, mock_tracer): + span = self._invoke_chat( + chat_model=langchain_openai.ChatOpenAI(temperature=0, max_tokens=256), + prompt="When do you use 'who' instead of 'whom'?", + mock_tracer=mock_tracer, + cassette_name="openai_chat_completion_sync_call.yaml", + role="custom", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 1 + _assert_expected_llmobs_llm_span(span, mock_llmobs_span_writer, input_role="custom") + + def test_llmobs_chain(self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer): + prompt = langchain_core.prompts.ChatPromptTemplate.from_messages( + [("system", "You are world class technical documentation writer."), ("user", "{input}")] + ) + chain = prompt | langchain_openai.OpenAI() + expected_output = ( + "\nSystem: Langsmith can help with testing in several ways. " + "First, it can generate automated tests based on your technical documentation, " + "ensuring that your code matches the documented specifications. " + "This can save you time and effort in testing your code manually. " + "Additionally, Langsmith can also analyze your technical documentation for completeness and accuracy, " + "helping you identify any potential gaps or errors before testing begins. " + "Finally, Langsmith can assist with creating test cases and scenarios based on your documentation, " + "making the testing process more efficient and effective." + ) + trace = self._invoke_chain( + chain=chain, + prompt={"input": "Can you explain what an LLM chain is?"}, + mock_tracer=mock_tracer, + cassette_name="lcel_openai_chain_call.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 2 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps([{"input": "Can you explain what an LLM chain is?"}]), + output_value=expected_output, + ) + _assert_expected_llmobs_llm_span(trace[1], mock_llmobs_span_writer) + + def test_llmobs_chain_nested(self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer): + prompt1 = langchain_core.prompts.ChatPromptTemplate.from_template("what is the city {person} is from?") + prompt2 = langchain_core.prompts.ChatPromptTemplate.from_template( + "what country is the city {city} in? respond in {language}" + ) + model = langchain_openai.ChatOpenAI() + chain1 = prompt1 | model | langchain_core.output_parsers.StrOutputParser() + chain2 = prompt2 | model | langchain_core.output_parsers.StrOutputParser() + complete_chain = {"city": chain1, "language": itemgetter("language")} | chain2 + trace = self._invoke_chain( + chain=complete_chain, + prompt={"person": "Spongebob Squarepants", "language": "Spanish"}, + mock_tracer=mock_tracer, + cassette_name="lcel_openai_chain_nested.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 4 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps([{"person": "Spongebob Squarepants", "language": "Spanish"}]), + output_value=mock.ANY, + ) + _assert_expected_llmobs_chain_span( + trace[1], + mock_llmobs_span_writer, + input_value=json.dumps([{"person": "Spongebob Squarepants", "language": "Spanish"}]), + output_value=mock.ANY, + ) + _assert_expected_llmobs_llm_span(trace[2], mock_llmobs_span_writer, input_role="user") + _assert_expected_llmobs_llm_span(trace[3], mock_llmobs_span_writer, input_role="user") + + @pytest.mark.skipif(sys.version_info >= (3, 11, 0), reason="Python <3.11 required") + def test_llmobs_chain_batch(self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer): + prompt = langchain_core.prompts.ChatPromptTemplate.from_template("Tell me a short joke about {topic}") + output_parser = langchain_core.output_parsers.StrOutputParser() + model = langchain_openai.ChatOpenAI() + chain = {"topic": langchain_core.runnables.RunnablePassthrough()} | prompt | model | output_parser + + trace = self._invoke_chain( + chain=chain, + prompt=["chickens", "pigs"], + mock_tracer=mock_tracer, + cassette_name="lcel_openai_chain_batch.yaml", + batch=True, + ) + assert mock_llmobs_span_writer.enqueue.call_count == 3 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps(["chickens", "pigs"]), + output_value=mock.ANY, + ) + _assert_expected_llmobs_llm_span(trace[1], mock_llmobs_span_writer, input_role="user") + _assert_expected_llmobs_llm_span(trace[2], mock_llmobs_span_writer, input_role="user") + + def test_llmobs_chain_schema_io(self, langchain_core, langchain_openai, mock_llmobs_span_writer, mock_tracer): + prompt = langchain_core.prompts.ChatPromptTemplate.from_messages( + [ + ("system", "You're an assistant who's good at {ability}. Respond in 20 words or fewer"), + langchain_core.prompts.MessagesPlaceholder(variable_name="history"), + ("human", "{input}"), + ] + ) + chain = prompt | langchain_openai.ChatOpenAI() + trace = self._invoke_chain( + chain=chain, + prompt={ + "ability": "world capitals", + "history": [HumanMessage(content="Can you be my science teacher instead?"), AIMessage(content="Yes")], + "input": "What's the powerhouse of the cell?", + }, + mock_tracer=mock_tracer, + cassette_name="lcel_openai_chain_schema_io.yaml", + ) + assert mock_llmobs_span_writer.enqueue.call_count == 2 + _assert_expected_llmobs_chain_span( + trace[0], + mock_llmobs_span_writer, + input_value=json.dumps( + [ + { + "ability": "world capitals", + "history": [["user", "Can you be my science teacher instead?"], ["assistant", "Yes"]], + "input": "What's the powerhouse of the cell?", + } + ] + ), + output_value=json.dumps(["assistant", "Mitochondria."]), + ) + _assert_expected_llmobs_llm_span(trace[1], mock_llmobs_span_writer, mock_io=True) + + +@pytest.mark.skipif(not SHOULD_PATCH_LANGCHAIN_COMMUNITY, reason="These tests are for langchain >= 0.1.0") +class TestLangchainTraceStructureWithLlmIntegrations(SubprocessTestCase): + bedrock_env_config = dict( + AWS_ACCESS_KEY_ID="testing", + AWS_SECRET_ACCESS_KEY="testing", + AWS_SECURITY_TOKEN="testing", + AWS_SESSION_TOKEN="testing", + AWS_DEFAULT_REGION="us-east-1", + DD_LANGCHAIN_METRICS_ENABLED="false", + DD_API_KEY="", + ) + + openai_env_config = dict( + OPENAI_API_KEY="testing", + DD_API_KEY="", + ) + + def setUp(self): + patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter") + LLMObsSpanWriterMock = patcher.start() + mock_llmobs_span_writer = mock.MagicMock() + LLMObsSpanWriterMock.return_value = mock_llmobs_span_writer + + self.mock_llmobs_span_writer = mock_llmobs_span_writer + + super(TestLangchainTraceStructureWithLlmIntegrations, self).setUp() + + def tearDown(self): + LLMObs.disable() + + def _assert_trace_structure_from_writer_call_args(self, span_kinds): + assert self.mock_llmobs_span_writer.enqueue.call_count == len(span_kinds) + + calls = self.mock_llmobs_span_writer.enqueue.call_args_list + + for span_kind, call in zip(span_kinds, calls): + call_args = call.args[0] + + assert call_args["meta"]["span.kind"] == span_kind + if span_kind == "workflow": + assert len(call_args["meta"]["input"]["value"]) > 0 + assert len(call_args["meta"]["output"]["value"]) > 0 + elif span_kind == "llm": + assert len(call_args["meta"]["input"]["messages"]) > 0 + assert len(call_args["meta"]["output"]["messages"]) > 0 + + @staticmethod + def _call_bedrock_chat_model(ChatBedrock, HumanMessage): + chat = ChatBedrock( + model_id="amazon.titan-tg1-large", + model_kwargs={"max_tokens": 50, "temperature": 0}, + ) + messages = [HumanMessage(content="summarize the plot to the lord of the rings in a dozen words")] + with get_request_vcr(subdirectory_name="langchain_community").use_cassette("bedrock_amazon_chat_invoke.yaml"): + chat.invoke(messages) + + @staticmethod + def _call_bedrock_llm(Bedrock, ConversationChain, ConversationBufferMemory): + llm = Bedrock( + model_id="amazon.titan-tg1-large", + region_name="us-east-1", + model_kwargs={"temperature": 0, "topP": 0.9, "stopSequences": [], "maxTokens": 50}, + ) + + conversation = ConversationChain(llm=llm, verbose=True, memory=ConversationBufferMemory()) + + with get_request_vcr(subdirectory_name="langchain_community").use_cassette("bedrock_amazon_invoke.yaml"): + conversation.predict(input="can you explain what Datadog is to someone not in the tech industry?") + + @staticmethod + def _call_openai_llm(OpenAI): + llm = OpenAI() + with get_request_vcr(subdirectory_name="langchain_community").use_cassette("openai_completion_sync.yaml"): + llm.invoke("Can you explain what Descartes meant by 'I think, therefore I am'?") + + @run_in_subprocess(env_overrides=bedrock_env_config) + def test_llmobs_with_chat_model_bedrock_enabled(self): + from langchain_aws import ChatBedrock + from langchain_core.messages import HumanMessage + + patch(langchain=True, botocore=True) + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + + self._call_bedrock_chat_model(ChatBedrock, HumanMessage) + + self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) + + @run_in_subprocess(env_overrides=bedrock_env_config) + def test_llmobs_with_chat_model_bedrock_disabled(self): + from langchain_aws import ChatBedrock + from langchain_core.messages import HumanMessage + + patch(langchain=True) + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + + self._call_bedrock_chat_model(ChatBedrock, HumanMessage) + + self._assert_trace_structure_from_writer_call_args(["llm"]) + + @run_in_subprocess(env_overrides=bedrock_env_config) + def test_llmobs_with_llm_model_bedrock_enabled(self): + from langchain.chains import ConversationChain + from langchain.memory import ConversationBufferMemory + from langchain_community.llms import Bedrock + + patch(langchain=True, botocore=True) + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + self._call_bedrock_llm(Bedrock, ConversationChain, ConversationBufferMemory) + self._assert_trace_structure_from_writer_call_args(["workflow", "workflow", "llm"]) + + @run_in_subprocess(env_overrides=bedrock_env_config) + def test_llmobs_with_llm_model_bedrock_disabled(self): + from langchain.chains import ConversationChain + from langchain.memory import ConversationBufferMemory + from langchain_community.llms import Bedrock + + patch(langchain=True) + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + self._call_bedrock_llm(Bedrock, ConversationChain, ConversationBufferMemory) + self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) + + @run_in_subprocess(env_overrides=openai_env_config) + def test_llmobs_langchain_with_openai_enabled(self): + from langchain_openai import OpenAI + + patch(langchain=True, openai=True) + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + self._call_openai_llm(OpenAI) + self._assert_trace_structure_from_writer_call_args(["workflow", "llm"]) + + @run_in_subprocess(env_overrides=openai_env_config) + def test_llmobs_langchain_with_openai_disabled(self): + from langchain_openai import OpenAI + + patch(langchain=True) + + LLMObs.enable(ml_app="", integrations_enabled=False, agentless_enabled=True) + self._call_openai_llm(OpenAI) + self._assert_trace_structure_from_writer_call_args(["llm"]) diff --git a/tests/contrib/langchain/test_langchain_patch.py b/tests/contrib/langchain/test_langchain_patch.py index dd17e6e7781..1d707d63e72 100644 --- a/tests/contrib/langchain/test_langchain_patch.py +++ b/tests/contrib/langchain/test_langchain_patch.py @@ -76,6 +76,9 @@ def assert_not_module_patched(self, langchain): self.assert_not_wrapped(langchain_openai.OpenAIEmbeddings.embed_documents) self.assert_not_wrapped(langchain_pinecone.PineconeVectorStore.similarity_search) else: + from langchain import embeddings # noqa: F401 + from langchain import vectorstores # noqa: F401 + gated_langchain = langchain self.assert_not_wrapped(langchain.llms.base.BaseLLM.generate) self.assert_not_wrapped(langchain.llms.base.BaseLLM.agenerate) diff --git a/tests/contrib/langchain/utils.py b/tests/contrib/langchain/utils.py index 1aadcf58066..629fca145d6 100644 --- a/tests/contrib/langchain/utils.py +++ b/tests/contrib/langchain/utils.py @@ -3,6 +3,17 @@ import vcr +long_input_text = """ +I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no +bodies. Does it now follow that I too do not exist? No: if I convinced myself of something then I certainly +existed. But there is a deceiver of supreme power and cunning who is deliberately and constantly deceiving +me. In that case I too undoubtedly exist, if he is deceiving me; and let him deceive me as much as he can, +he will never bring it about that I am nothing so long as I think that I am something. So after considering +everything very thoroughly, I must finally conclude that this proposition, I am, I exist, is necessarily +true whenever it is put forward by me or conceived in my mind. +""" + + # VCR is used to capture and store network requests made to OpenAI and other APIs. # This is done to avoid making real calls to the API which could introduce # flakiness and cost. diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_async.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_async.json index 4ed7c04315d..d5771069c40 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_async.json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_async.json @@ -10,13 +10,13 @@ "error": 0, "meta": { "_dd.p.dm": "-0", - "_dd.p.tid": "6615aa1200000000", - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "_dd.p.tid": "66566a3300000000", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.type": "chain", "langchain.response.outputs.final_output": "\\nI've convinced my mind, no physical world's there, no sky, no earth, no minds, no bodies there. Does this mean I don't exist, ...", - "langchain.response.outputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.response.outputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "language": "python", - "runtime-id": "5bdc2648fed64ec2a49138caab5a0bf0" + "runtime-id": "103801afb5f54bf4b14c4af40585cea5" }, "metrics": { "_dd.measured": 1, @@ -27,10 +27,10 @@ "langchain.tokens.prompt_tokens": 304, "langchain.tokens.total_cost": 0.01004, "langchain.tokens.total_tokens": 502, - "process_id": 45546 + "process_id": 69928 }, - "duration": 2437000, - "start": 1712695826646228000 + "duration": 39665000, + "start": 1716939315688098000 }, { "name": "langchain.request", @@ -42,7 +42,7 @@ "type": "llm", "error": 0, "meta": { - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.prompt": "Paraphrase this text:\\n\\n {input_text}\\n\\n Paraphrase: ", "langchain.request.type": "chain", "langchain.response.outputs.paraphrased_output": "\\nI have convinced myself that there is no physical world - no sky, earth, minds, or bodies. Does this mean that I don't exist e..." @@ -54,8 +54,8 @@ "langchain.tokens.total_cost": 0.00578, "langchain.tokens.total_tokens": 289 }, - "duration": 1268000, - "start": 1712695826646374000 + "duration": 37649000, + "start": 1716939315688241000 }, { "name": "langchain.request", @@ -77,7 +77,7 @@ "langchain.request.openai.parameters.request_timeout": "None", "langchain.request.openai.parameters.temperature": "0.7", "langchain.request.openai.parameters.top_p": "1", - "langchain.request.prompts.0": "Paraphrase this text:\\n\\n \\n I have convinced myself that there is absolutely nothing in the world, no sky, no...", + "langchain.request.prompts.0": "Paraphrase this text:\\n\\n \\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no m...", "langchain.request.provider": "openai", "langchain.request.type": "llm", "langchain.response.completions.0.finish_reason": "stop", @@ -91,8 +91,8 @@ "langchain.tokens.total_cost": 0.00578, "langchain.tokens.total_tokens": 289 }, - "duration": 1048000, - "start": 1712695826646530000 + "duration": 37325000, + "start": 1716939315688408000 }, { "name": "langchain.request", @@ -104,7 +104,7 @@ "type": "llm", "error": 0, "meta": { - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.inputs.paraphrased_output": "\\nI have convinced myself that there is no physical world - no sky, earth, minds, or bodies. Does this mean that I don't exist e...", "langchain.request.prompt": "Make this text rhyme:\\n\\n {paraphrased_output}\\n\\n Rhyme: ", "langchain.request.type": "chain", @@ -117,8 +117,8 @@ "langchain.tokens.total_cost": 0.00426, "langchain.tokens.total_tokens": 213 }, - "duration": 952000, - "start": 1712695826647681000 + "duration": 1727000, + "start": 1716939315725990000 }, { "name": "langchain.request", @@ -154,6 +154,6 @@ "langchain.tokens.total_cost": 0.00426, "langchain.tokens.total_tokens": 213 }, - "duration": 734000, - "start": 1712695826647863000 + "duration": 1248000, + "start": 1716939315726416000 }]] diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_sync.json b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_sync.json index 0f18b2efa65..3c8fd74f046 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_sync.json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain.test_openai_sequential_chain_with_multiple_llm_sync.json @@ -10,13 +10,13 @@ "error": 0, "meta": { "_dd.p.dm": "-0", - "_dd.p.tid": "6615aa1a00000000", - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "_dd.p.tid": "66566a3300000000", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.type": "chain", "langchain.response.outputs.final_output": "\\nI've come to the conclusion it's true,\\nThere's nothing in the world like me and you.\\nDoes this mean I don't exist? No!\\nFor ...", - "langchain.response.outputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.response.outputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "language": "python", - "runtime-id": "5bdc2648fed64ec2a49138caab5a0bf0" + "runtime-id": "103801afb5f54bf4b14c4af40585cea5" }, "metrics": { "_dd.measured": 1, @@ -27,10 +27,10 @@ "langchain.tokens.prompt_tokens": 331, "langchain.tokens.total_cost": 0.011720000000000001, "langchain.tokens.total_tokens": 586, - "process_id": 45546 + "process_id": 69928 }, - "duration": 6313000, - "start": 1712695834803226000 + "duration": 8782000, + "start": 1716939315772875000 }, { "name": "langchain.request", @@ -42,7 +42,7 @@ "type": "llm", "error": 0, "meta": { - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.prompt": "Paraphrase this text:\\n\\n {input_text}\\n\\n Paraphrase: ", "langchain.request.type": "chain", "langchain.response.outputs.paraphrased_output": "\\nI have come to the conclusion that there is nothing in the world such as the sky, the earth, minds, or bodies. Does this mean ..." @@ -54,8 +54,8 @@ "langchain.tokens.total_cost": 0.006240000000000001, "langchain.tokens.total_tokens": 312 }, - "duration": 2738000, - "start": 1712695834803367000 + "duration": 6067000, + "start": 1716939315773033000 }, { "name": "langchain.request", @@ -77,7 +77,7 @@ "langchain.request.openai.parameters.request_timeout": "None", "langchain.request.openai.parameters.temperature": "0.7", "langchain.request.openai.parameters.top_p": "1", - "langchain.request.prompts.0": "Paraphrase this text:\\n\\n \\n I have convinced myself that there is absolutely nothing in the world, no sky, no...", + "langchain.request.prompts.0": "Paraphrase this text:\\n\\n \\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no m...", "langchain.request.provider": "openai", "langchain.request.type": "llm", "langchain.response.completions.0.finish_reason": "stop", @@ -91,8 +91,8 @@ "langchain.tokens.total_cost": 0.006240000000000001, "langchain.tokens.total_tokens": 312 }, - "duration": 2513000, - "start": 1712695834803528000 + "duration": 5836000, + "start": 1716939315773210000 }, { "name": "langchain.request", @@ -104,7 +104,7 @@ "type": "llm", "error": 0, "meta": { - "langchain.request.inputs.input_text": "\\n I have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\n ...", + "langchain.request.inputs.input_text": "\\nI have convinced myself that there is absolutely nothing in the world, no sky, no earth, no minds, no\\nbodies. Does it now fol...", "langchain.request.inputs.paraphrased_output": "\\nI have come to the conclusion that there is nothing in the world such as the sky, the earth, minds, or bodies. Does this mean ...", "langchain.request.prompt": "Make this text rhyme:\\n\\n {paraphrased_output}\\n\\n Rhyme: ", "langchain.request.type": "chain", @@ -117,8 +117,8 @@ "langchain.tokens.total_cost": 0.0054800000000000005, "langchain.tokens.total_tokens": 274 }, - "duration": 3305000, - "start": 1712695834806198000 + "duration": 2486000, + "start": 1716939315779141000 }, { "name": "langchain.request", @@ -154,6 +154,6 @@ "langchain.tokens.total_cost": 0.0054800000000000005, "langchain.tokens.total_tokens": 274 }, - "duration": 3083000, - "start": 1712695834806377000 + "duration": 2254000, + "start": 1716939315779332000 }]] diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_integration.json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_integration.json index fbaeffdf5c1..14b61dac41c 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_integration.json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_integration.json @@ -97,7 +97,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "span.kind": "client" }, diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-None].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-None].json index b0165550edf..8ef8b8ec5f2 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-None].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-None].json @@ -97,7 +97,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "span.kind": "client" }, diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v0].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v0].json index 347567903bd..c0e3ac0f6e8 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v0].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v0].json @@ -97,7 +97,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "span.kind": "client" }, diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v1].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v1].json index f3632148c3f..533b8a4458f 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v1].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[None-v1].json @@ -98,7 +98,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "peer.service": "api.openai.com", "span.kind": "client" diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-None].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-None].json index 523f11bb824..a159b6e60fd 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-None].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-None].json @@ -97,7 +97,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "span.kind": "client" }, diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v0].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v0].json index c1b29c96ed6..9bb0cc6d005 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v0].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v0].json @@ -97,7 +97,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "span.kind": "client" }, diff --git a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v1].json b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v1].json index 4a9eca88a41..7bc04096149 100644 --- a/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v1].json +++ b/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_openai_service_name[mysvc-v1].json @@ -98,7 +98,7 @@ "http.method": "POST", "http.status_code": "200", "http.url": "https://api.openai.com/v1/completions", - "http.useragent": "OpenAI/Python 1.12.0", + "http.useragent": "OpenAI/Python 1.30.3", "out.host": "api.openai.com", "peer.service": "api.openai.com", "span.kind": "client"