From 700d54e5e769f578444783dc5ec8c822ae6c1994 Mon Sep 17 00:00:00 2001 From: Rathijit Paul Date: Tue, 26 Mar 2024 20:23:24 +0600 Subject: [PATCH] :sparkles: phoenix tracing added for llm --- backend/.env.template | 7 +- backend/app/config.py | 5 +- backend/app/main.py | 11 +- backend/app/services/tracing.py | 21 +++- backend/poetry.lock | 200 +++++++++++++++++++++++++++++++- backend/pyproject.toml | 3 + 6 files changed, 235 insertions(+), 12 deletions(-) diff --git a/backend/.env.template b/backend/.env.template index 6518ea5f..d953243d 100644 --- a/backend/.env.template +++ b/backend/.env.template @@ -83,5 +83,10 @@ POSTHOG_API_KEY= POSTHOG_HOST= # QDRANT API Configuration +## need to be set before running the server QDRANT_API_KEY= -QDRANT_TOP_K= \ No newline at end of file +QDRANT_TOP_K= + +# Phoenix Configuration +## default values set in the code for local development +PHOENIX_API_ENDPOINT= \ No newline at end of file diff --git a/backend/app/config.py b/backend/app/config.py index e8c37ae2..de8e67b3 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -135,4 +135,7 @@ # Dspy Integration Configuration CLINICAL_TRIAL_SQL_PROGRAM: str = "app/dspy_integration/dspy_programs/clinical_trials_sql_generation.json" CLINICAL_TRIALS_RESPONSE_REFINEMENT_PROGRAM: str = "app/dspy_integration/dspy_programs/clinical_trials_response_refinement.json" -ORCHESRATOR_ROUTER_PROMPT_PROGRAM: str = "app/dspy_integration/dspy_programs/orchestrator_router_prompt.json" \ No newline at end of file +ORCHESRATOR_ROUTER_PROMPT_PROGRAM: str = "app/dspy_integration/dspy_programs/orchestrator_router_prompt.json" + +# Phoenix Configuration +PHOENIX_API_ENDPOINT: str = config("PHOENIX_API_ENDPOINT", default="http://127.0.0.1:6007/v1/traces") \ No newline at end of file diff --git a/backend/app/main.py b/backend/app/main.py index 197755d0..2c461071 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,7 +1,10 @@ +# setup tracing +from app.services.tracing import setup_tracing +setup_tracing() + from fastapi import FastAPI, HTTPException from fastapi.responses import RedirectResponse -# from authx import AuthX, AuthXConfig from app.database.redis import Redis from app import config @@ -11,9 +14,6 @@ from app.middleware.process_time import ProcessTimeHeaderMiddleware -from app.services.tracing import setup_tracing - - def get_application() -> FastAPI: application = FastAPI( title=config.PROJECT_NAME, debug=config.DEBUG, version=config.VERSION @@ -27,9 +27,6 @@ def redirect_to_docs() -> RedirectResponse: # pylint: disable=W0612 async def startup(): # pylint: disable=W0612 print() - # setup tracing - setup_tracing() - # connect to redis cache = Redis() await cache.connect() diff --git a/backend/app/services/tracing.py b/backend/app/services/tracing.py index 4736beae..6d0fbb85 100644 --- a/backend/app/services/tracing.py +++ b/backend/app/services/tracing.py @@ -5,7 +5,14 @@ from sentry_sdk.integrations.starlette import StarletteIntegration from sentry_sdk.integrations.fastapi import FastApiIntegration -from app.config import SENTRY_DSN, SENTRY_ENABLE_TRACING +from openinference.instrumentation.llama_index import LlamaIndexInstrumentor +from opentelemetry import trace as trace_api +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +from app.config import SENTRY_DSN, SENTRY_ENABLE_TRACING, PHOENIX_API_ENDPOINT, ENVIRONMENT def setup_tracing(): sentry_sdk.init( @@ -24,4 +31,14 @@ def setup_tracing(): event_level=logging.WARNING ), ], - ) \ No newline at end of file + ) + + + if ENVIRONMENT == 'production': + resource = Resource(attributes={}) + tracer_provider = trace_sdk.TracerProvider(resource=resource) + span_exporter = OTLPSpanExporter(endpoint=PHOENIX_API_ENDPOINT) + span_processor = SimpleSpanProcessor(span_exporter=span_exporter) + tracer_provider.add_span_processor(span_processor=span_processor) + trace_api.set_tracer_provider(tracer_provider=tracer_provider) + LlamaIndexInstrumentor().instrument() \ No newline at end of file diff --git a/backend/poetry.lock b/backend/poetry.lock index af7fe410..d36f95b7 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1205,6 +1205,23 @@ files = [ {file = "future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05"}, ] +[[package]] +name = "googleapis-common-protos" +version = "1.63.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"}, + {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + [[package]] name = "greenlet" version = "3.0.3" @@ -2931,6 +2948,187 @@ typing-extensions = ">=4.7,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "openinference-instrumentation-llama-index" +version = "1.2.1" +description = "OpenInference LlamaIndex Instrumentation" +optional = false +python-versions = "<3.13,>=3.8" +files = [ + {file = "openinference_instrumentation_llama_index-1.2.1-py3-none-any.whl", hash = "sha256:e71ea261603e359cd198817049d5048a7bc779b06d0aef7973a56d77830e4149"}, + {file = "openinference_instrumentation_llama_index-1.2.1.tar.gz", hash = "sha256:ef918a1769f2c3bcc08ba23088cf5e49748d657557b93e509757357c2c2e0e08"}, +] + +[package.dependencies] +openinference-semantic-conventions = "*" +opentelemetry-api = "*" +opentelemetry-instrumentation = "*" +opentelemetry-semantic-conventions = "*" +typing-extensions = "*" +wrapt = "*" + +[package.extras] +instruments = ["llama-index (>=0.10.5)"] +test = ["llama-index (==0.10.5)", "llama-index-llms-openai", "opentelemetry-sdk", "respx"] + +[[package]] +name = "openinference-semantic-conventions" +version = "0.1.5" +description = "OpenInference Semantic Conventions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "openinference_semantic_conventions-0.1.5-py3-none-any.whl", hash = "sha256:d974532aecfb973835b4204b4f6651547dca132b90dd28cbca9e73be23b464ba"}, + {file = "openinference_semantic_conventions-0.1.5.tar.gz", hash = "sha256:d674b25044ca251f276c9f6230db18979c833d06e4998223437746dbba86aa3f"}, +] + +[[package]] +name = "opentelemetry-api" +version = "1.23.0" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_api-1.23.0-py3-none-any.whl", hash = "sha256:cc03ea4025353048aadb9c64919099663664672ea1c6be6ddd8fee8e4cd5e774"}, + {file = "opentelemetry_api-1.23.0.tar.gz", hash = "sha256:14a766548c8dd2eb4dfc349739eb4c3893712a0daa996e5dbf945f9da665da9d"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +importlib-metadata = ">=6.0,<7.0" + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.23.0" +description = "OpenTelemetry Collector Exporters" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp-1.23.0-py3-none-any.whl", hash = "sha256:92371fdc8d7803465a45801fe30cd8c522ef355a385b0a1d5346d32f77511ea2"}, + {file = "opentelemetry_exporter_otlp-1.23.0.tar.gz", hash = "sha256:4af8798f9bc3bddb92fcbb5b4aa9d0e955d962aa1d9bceaab08891c355a9f907"}, +] + +[package.dependencies] +opentelemetry-exporter-otlp-proto-grpc = "1.23.0" +opentelemetry-exporter-otlp-proto-http = "1.23.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.23.0" +description = "OpenTelemetry Protobuf encoding" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp_proto_common-1.23.0-py3-none-any.whl", hash = "sha256:2a9e7e9d5a8b026b572684b6b24dcdefcaa58613d5ce3d644130b0c373c056c1"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.23.0.tar.gz", hash = "sha256:35e4ea909e7a0b24235bd0aaf17fba49676527feb1823b46565ff246d5a1ab18"}, +] + +[package.dependencies] +opentelemetry-proto = "1.23.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.23.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.23.0-py3-none-any.whl", hash = "sha256:40f9e3e7761eb34f2a1001f4543028783ac26e2db27e420d5374f2cca0182dad"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.23.0.tar.gz", hash = "sha256:aa1a012eea5342bfef51fcf3f7f22601dcb0f0984a07ffe6025b2fbb6d91a2a9"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.23.0" +opentelemetry-proto = "1.23.0" +opentelemetry-sdk = ">=1.23.0,<1.24.0" + +[package.extras] +test = ["pytest-grpc"] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.23.0" +description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp_proto_http-1.23.0-py3-none-any.whl", hash = "sha256:ad853b58681df8efcb2cfc93be2b5fd86351c99ff4ab47dc917da384b8650d91"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.23.0.tar.gz", hash = "sha256:088eac2320f4a604e2d9ff71aced71fdae601ac6457005fb0303d6bbbf44e6ca"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +googleapis-common-protos = ">=1.52,<2.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.23.0" +opentelemetry-proto = "1.23.0" +opentelemetry-sdk = ">=1.23.0,<1.24.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["responses (>=0.22.0,<0.25)"] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.44b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation-0.44b0-py3-none-any.whl", hash = "sha256:79560f386425176bcc60c59190064597096114c4a8e5154f1cb281bb4e47d2fc"}, + {file = "opentelemetry_instrumentation-0.44b0.tar.gz", hash = "sha256:8213d02d8c0987b9b26386ae3e091e0477d6331673123df736479322e1a50b48"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-proto" +version = "1.23.0" +description = "OpenTelemetry Python Proto" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_proto-1.23.0-py3-none-any.whl", hash = "sha256:4c017deca052cb287a6003b7c989ed8b47af65baeb5d57ebf93dde0793f78509"}, + {file = "opentelemetry_proto-1.23.0.tar.gz", hash = "sha256:e6aaf8b7ace8d021942d546161401b83eed90f9f2cc6f13275008cea730e4651"}, +] + +[package.dependencies] +protobuf = ">=3.19,<5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.23.0" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_sdk-1.23.0-py3-none-any.whl", hash = "sha256:a93c96990ac0f07c6d679e2f1015864ff7a4f5587122dd5af968034436efb1fd"}, + {file = "opentelemetry_sdk-1.23.0.tar.gz", hash = "sha256:9ddf60195837b59e72fd2033d6a47e2b59a0f74f0ec37d89387d89e3da8cab7f"}, +] + +[package.dependencies] +opentelemetry-api = "1.23.0" +opentelemetry-semantic-conventions = "0.44b0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.44b0" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_semantic_conventions-0.44b0-py3-none-any.whl", hash = "sha256:7c434546c9cbd797ab980cc88bf9ff3f4a5a28f941117cad21694e43d5d92019"}, + {file = "opentelemetry_semantic_conventions-0.44b0.tar.gz", hash = "sha256:2e997cb28cd4ca81a25a9a43365f593d0c2b76be0685015349a89abdf1aa4ffa"}, +] + [[package]] name = "optuna" version = "3.6.0" @@ -5551,4 +5749,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "9e718f9f8a76e61f557e885819df84ed5d55785f918417e3ec68b91197b878e6" +content-hash = "ce646c7c4d5c1b1922c5ba0f962add72af7ac64cf78cd384e550394cacf8a4c8" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1d4ddd94..e832cd2f 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -31,6 +31,9 @@ llama-index-llms-together = "^0.1.3" llama-index-postprocessor-cohere-rerank = "^0.1.2" together = "^0.2.11" llama-index-vector-stores-qdrant = "^0.1.4" +openinference-semantic-conventions = "^0.1.5" +openinference-instrumentation-llama-index = "^1.2.1" +opentelemetry-exporter-otlp = "^1.23.0" [tool.poetry.group.dev.dependencies] pytest = "^8.1.1"