From f68281982cb0ec4f184078c78bc12d8e9481b3eb Mon Sep 17 00:00:00 2001 From: raahulmona <157174139+raahulmona@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:30:25 +0100 Subject: [PATCH] Commit Logging --- .editorconfig | 31 ++ .flake8 | 93 +++++ .github/workflows/tests.yaml | 48 +++ .gitignore copy | 144 +++++++ .pre-commit-config.yaml | 62 +++ .vscode/launch.json | 24 ++ .vscode/settings.json | 21 + README copy.md | 112 ++++++ app/api/endpoints/search_endpoint.py | 10 +- app/main.py | 1 + app/rag/generation/response_synthesis.py | 5 +- app/rag/reranker/response_reranker.py | 7 +- .../clinical_trial_sql_query_engine.py | 29 +- app/router/orchestrator.py | 40 +- docker-compose copy.yml | 37 ++ pip copy | 0 poetry copy.lock | 216 ++++++++++ pytest copy.ini | 3 + requirements copy.txt | 367 +++++++++++++++++ text2sql_dag copy.html | 369 ++++++++++++++++++ 20 files changed, 1584 insertions(+), 35 deletions(-) create mode 100644 .editorconfig create mode 100644 .flake8 create mode 100644 .github/workflows/tests.yaml create mode 100644 .gitignore copy create mode 100644 .pre-commit-config.yaml create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 README copy.md create mode 100644 docker-compose copy.yml create mode 100644 pip copy create mode 100644 poetry copy.lock create mode 100644 pytest copy.ini create mode 100644 requirements copy.txt create mode 100644 text2sql_dag copy.html diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..eb831fe6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,31 @@ +root = true + +[*] +tab_width = 4 +end_of_line = lf +max_line_length = 88 +ij_visual_guides = 88 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{js,py,html}] +charset = utf-8 + +[*.md] +trim_trailing_whitespace = false + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 + +[Makefile] +indent_style = tab + +[.flake8] +indent_style = space +indent_size = 2 + +[*.py] +indent_style = space +indent_size = 4 +ij_python_from_import_parentheses_force_if_multiline = true diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..8b6458bc --- /dev/null +++ b/.flake8 @@ -0,0 +1,93 @@ +[flake8] +max-complexity = 6 +inline-quotes = double +max-line-length = 88 +extend-ignore = E203 +docstring_style=sphinx + +ignore = + ; Found `f` string + WPS305, + ; Missing docstring in public module + D100, + ; Missing docstring in magic method + D105, + ; Missing docstring in __init__ + D107, + ; Found `__init__.py` module with logic + WPS412, + ; Found class without a base class + WPS306, + ; Missing docstring in public nested class + D106, + ; First line should be in imperative mood + D401, + ; Found `__init__.py` module with logic + WPS326, + ; Found string constant over-use + WPS226, + ; Found upper-case constant in a class + WPS115, + ; Found nested function + WPS602, + ; Found method without arguments + WPS605, + ; Found overused expression + WPS204, + ; Found too many module members + WPS202, + ; Found too high module cognitive complexity + WPS232, + ; line break before binary operator + W503, + ; Found module with too many imports + WPS201, + ; Inline strong start-string without end-string. + RST210, + ; Found nested class + WPS431, + ; Found wrong module name + WPS100, + ; Found too many methods + WPS214, + ; Found too long ``try`` body + WPS229, + ; Found unpythonic getter or setter + WPS615, + ; Found a line that starts with a dot + WPS348, + ; Found complex default value (for dependency injection) + WPS404, + ; not perform function calls in argument defaults (for dependency injection) + B008, + ; line to long + E501, + +per-file-ignores = + ; all tests + test_*.py,tests.py,tests_*.py,*/tests/*,conftest.py: + ; Use of assert detected + S101, + ; Found outer scope names shadowing + WPS442, + ; Found too many local variables + WPS210, + ; Found magic number + WPS432, + ; Missing parameter(s) in Docstring + DAR101, + + ; all init files + __init__.py: + ; ignore not used imports + F401, + ; ignore import with wildcard + F403, + ; Found wrong metadata variable + WPS410, + +exclude = + ./.git, + ./venv, + migrations, + ./var, diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000..dd836f35 --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,48 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: ["3.10", "3.11"] + + services: + postgres: + image: postgres:14 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: db_test + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v2 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/requirements/development.txt' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r app/requirements/development.txt + + - name: Run tests + env: + DB_HOST: localhost + DB_PORT: 5432 + DB_USER: postgres + DB_PASS: postgres + DB_BASE: db_test + run: cd app && pytest . diff --git a/.gitignore copy b/.gitignore copy new file mode 100644 index 00000000..9d38591f --- /dev/null +++ b/.gitignore copy @@ -0,0 +1,144 @@ +### Python template + +.idea/ +.vscode/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +.python-version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..0cacf892 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,62 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-ast + - id: trailing-whitespace + - id: check-toml + - id: end-of-file-fixer + +- repo: https://github.com/asottile/add-trailing-comma + rev: v2.1.0 + hooks: + - id: add-trailing-comma + +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.1.0 + hooks: + - id: pretty-format-yaml + args: + - --autofix + - --preserve-quotes + - --indent=2 + +- repo: local + hooks: + - id: black + name: Format with Black + entry: black + language: system + types: [python] + + - id: isort + name: isort + entry: isort + language: system + types: [python] + + - id: flake8 + name: Check with Flake8 + entry: flake8 + language: system + pass_filenames: false + types: [python] + args: [--count, .] + + - id: mypy + name: Validate types with MyPy + entry: mypy + language: system + types: [python] + pass_filenames: false + args: + - "app" + - "--ignore-missing-imports" + + - id: yesqa + name: Remove usless noqa + entry: yesqa + language: system + types: [python] diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..cd3d11be --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: fastapi", + "type": "debugpy", + "request": "launch", + "console": "integratedTerminal", + "justMyCode": false, + "module": "uvicorn", + "args": ["app.main:app","--reload"] + }, + { + "name": "Python: Current File", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..dfaba624 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,21 @@ +{ + "cSpell.words": [ + "AACT", + "arize", + "bravesearch", + "Curieo", + "llms", + "mistralai", + "openai", + "perticipents", + "pubmed", + "pydantic", + "pyvis", + "Rerank", + "reranked", + "reranker", + "sqlalchemy", + "starlette", + "togetherai" + ] +} \ No newline at end of file diff --git a/README copy.md b/README copy.md new file mode 100644 index 00000000..3b3aa204 --- /dev/null +++ b/README copy.md @@ -0,0 +1,112 @@ +# FastAPI Starter Project + +Project includes: + +- `fastapi` +- `sqlmodel` +- `alembic` + +## + +## Models + +Check db/models and migrations, there is one example. + +## Using docker + +Setup env variables in `app/core/.env` using `app/core/.env-example` + +#### Install and run + +```bash +docker-compose up -d web + +# you can track logs with: +docker-compose logs -f --tail=100 web +``` + +Go to: http://localhost:8000/api/docs/ + +#### Migrations + +Create migrations + +```bash +docker-compose exec web alembic revision --autogenerate -m "Example model" +``` + +Apply migrations + +```bash +docker-compose exec web alembic upgrade head +``` + +#### Tests + +Run tests + +```bash +docker-compose exec web pytest . +``` + +## Without docker + +#### Install + +```bash +cd app/ +pip install -r requirements/development.txt +``` + +Setup env variables in `app/core/.env`. + +#### Run + +```bash +cd app/ +python app/server.py +``` + +Go to: http://localhost:8000/api/docs/ + +#### Migrations + +Create migrations + +```bash +alembic revision --autogenerate -m "Example model" +``` + +Apply migrations + +```bash +alembic upgrade head +``` + +#### Tests + +Run tests + +```bash +pytest . +``` + +## Environment Variables + +To run this project, you will need to add the following environment variables to your app/core/.env file + +`BASE_URL` - default: http://localhost:8000 + +`RELOAD` - default: false + +`DB_HOST` - default: localhost + +`DB_PORT` - default: 5432 + +`DB_USER` - default: postgres + +`DB_PASS` - default: postgres + +`DB_BASE` - default: db + +`DB_ECHO` - default: false diff --git a/app/api/endpoints/search_endpoint.py b/app/api/endpoints/search_endpoint.py index f9926262..54bce5ed 100644 --- a/app/api/endpoints/search_endpoint.py +++ b/app/api/endpoints/search_endpoint.py @@ -12,6 +12,7 @@ from app.api.router.gzip import GzipRoute from app.router.orchestrator import Orchestrator from app.config import config, JWT_SECRET_KEY, JWT_ALGORITHM +from app.services.search_utility import setup_logger router = APIRouter() router.route_class = GzipRoute @@ -23,6 +24,9 @@ security = AuthX(config=auth_config) orchestrator = Orchestrator(config) +logger = setup_logger('Search_Endpoint') + + @router.get('/login') def login(username: str, password: str): if username == "curieo" and password == "curieo": @@ -30,18 +34,20 @@ def login(username: str, password: str): return {"access_token": token} raise HTTPException(401, detail={"message": "Bad credentials"}) + @router.get( "/Search/", summary="List all Search Results", description="List all Search Results", - dependencies=[Depends(security.access_token_required)], - #responses=get_responses(201, 400, 401, 403) + dependencies=[Depends(security.access_token_required)] ) @cache(expire=31536000) @version(1, 0) async def get_search_results( query: str = "" ) -> JSONResponse: + logger.debug(f"Search_Endpoint.get_search_results. query: {query}") data = await orchestrator.query_and_get_answer(search_text=query) + logger.debug(f"Search_Endpoint.get_search_results. result: {data}") return JSONResponse(status_code=200, content=data) \ No newline at end of file diff --git a/app/main.py b/app/main.py index 382c6099..fabd9f33 100644 --- a/app/main.py +++ b/app/main.py @@ -8,6 +8,7 @@ from app.api import api from app.api.errors.http_error import http_error_handler from app.api.errors.if_none_match import IfNoneMatch, if_none_match_handler +from app.services.search_utility import setup_logger from app.middleware.process_time import ProcessTimeHeaderMiddleware diff --git a/app/rag/generation/response_synthesis.py b/app/rag/generation/response_synthesis.py index 6ce221f8..6f15573e 100644 --- a/app/rag/generation/response_synthesis.py +++ b/app/rag/generation/response_synthesis.py @@ -1,11 +1,9 @@ import collections import json -import os import requests -import re from urllib.parse import urlparse -from app.services.search_utility import setup_logger, get_project_root, storage_cached +from app.services.search_utility import setup_logger from app.config import TOGETHER_API, TOGETHER_KEY, TOGETHER_MODEL, TOGETHER_PROMPT_CONFIG, PROMPT_LANGUAGE logger = setup_logger('ResponseSynthesisEngine') @@ -88,6 +86,7 @@ def call_llm_service_api(self, search_text: str, reranked_results: collections.d response = requests.request("POST", TOGETHER_API, headers=headers, data=payload) except Exception as ex: + logger.exception("ResponseSynthesisEngine.call_llm_service_api Exception -", exc_info = ex, stack_info=True) raise ex return { "result" : self.clean_response_text(response.json()['choices'][0]['text']), diff --git a/app/rag/reranker/response_reranker.py b/app/rag/reranker/response_reranker.py index c347e56e..0aa4436d 100644 --- a/app/rag/reranker/response_reranker.py +++ b/app/rag/reranker/response_reranker.py @@ -1,11 +1,9 @@ import collections -import json -import os import requests import re -from app.services.search_utility import setup_logger, get_project_root, storage_cached -from app.config import BRAVE_RESULT_COUNT, BRAVE_SEARCH_API, EMBEDDING_RERANK_API, EMBEDDING_CHUNK_SIZE +from app.services.search_utility import setup_logger +from app.config import EMBEDDING_RERANK_API, EMBEDDING_CHUNK_SIZE logger = setup_logger('Reranking') TAG_RE = re.compile(r'<[^>]+>') @@ -54,5 +52,6 @@ def call_embedding_api(self, search_text: str, retrieval_results: collections.de rerank_orders = response.json() results = [retrieval_results[order.get('index')] for order in rerank_orders] except Exception as ex: + logger.exception("ReRankEngine.call_embedding_api Exception -", exc_info = ex, stack_info=True) raise ex return results diff --git a/app/rag/retrieval/clinical_trials/clinical_trial_sql_query_engine.py b/app/rag/retrieval/clinical_trials/clinical_trial_sql_query_engine.py index 416bf033..b2fdfa6c 100644 --- a/app/rag/retrieval/clinical_trials/clinical_trial_sql_query_engine.py +++ b/app/rag/retrieval/clinical_trials/clinical_trial_sql_query_engine.py @@ -1,15 +1,11 @@ -from llama_index.core.query_pipeline import (QueryPipeline as QP, Link, InputComponent, FnComponent) -from llama_index.core.llms import ChatMessage, ChatResponse +from llama_index.core.query_pipeline import (QueryPipeline as QP, InputComponent, FnComponent) +from llama_index.core.llms import ChatResponse from llama_index.llms.openai import OpenAI from llama_index.core.prompts import PromptTemplate from llama_index.core import VectorStoreIndex, load_index_from_storage, SQLDatabase -from llama_index.core.schema import TextNode -from llama_index.core.storage import StorageContext from llama_index.core.prompts.default_prompts import DEFAULT_TEXT_TO_SQL_PROMPT -from llama_index.core.program import LLMTextCompletionProgram from llama_index.core.bridge.pydantic import BaseModel, Field -from llama_index.core.service_context import ServiceContext from llama_index.core.objects import ( SQLTableNodeMapping, ObjectIndex, @@ -20,20 +16,22 @@ import os from pathlib import Path -from typing import Dict, List -import pandas as pd -from sqlalchemy import create_engine, MetaData, Table, Column, String, Integer, text +from typing import List +from sqlalchemy import create_engine from pyvis.network import Network import phoenix as px -import llama_index from app.config import OPENAPI_KEY, CLINICAL_TRIALS_TABLE_INFO_DIR, POSTGRES_ENGINE, EMBEDDING_MODEL_API, EMBEDDING_MODEL_NAME +from app.services.search_utility import setup_logger + +logger = setup_logger('ClinicalTrialText2SQLEngine') #px.launch_app(port=6060) class TableInfo(BaseModel): - """Information regarding a structured table.""" - + """ + Information regarding a structured table. + """ table_name: str = Field( ..., description="table name (must be underscores and NO spaces)" ) @@ -95,6 +93,7 @@ def get_all_table_info(self): for i in range(file_counts): table_info = self._get_table_info_with_index(i) table_infos.append(table_info) + logger.debug(f"ClinicalTrialText2SQLEngine.get_all_table_info table_infos: {len(table_infos)}") return table_infos def get_table_context_str(self, table_schema_objs: List[SQLTableSchema]): @@ -121,6 +120,7 @@ def parse_response_to_sql(self, response: ChatResponse) -> str: sql_result_start = response.find("SQLResult:") if sql_result_start != -1: response = response[:sql_result_start] + logger.debug(f"ClinicalTrialText2SQLEngine.parse_response_to_sql sql: {response}") return response.strip().strip("```").strip() def get_response_synthesis_prompt(self, query_str, sql_query, context_str) -> PromptTemplate: @@ -172,9 +172,14 @@ def build_query_pipeline(self): def call_text2sql(self, search_text:str): try: + logger.debug(f"ClinicalTrialText2SQLEngine.call_text2sql search_text: {search_text}") response = self.qp.run(query=search_text) + logger.debug(f"ClinicalTrialText2SQLEngine.call_text2sql response: {str(response)}") + except Exception as ex: + logger.exception("ClinicalTrialText2SQLEngine.call_text2sql Exception -", exc_info = ex, stack_info=True) raise ex + return { "result" : str(response) } diff --git a/app/router/orchestrator.py b/app/router/orchestrator.py index bb483a7e..db6a4e60 100644 --- a/app/router/orchestrator.py +++ b/app/router/orchestrator.py @@ -1,10 +1,5 @@ -import os -import re - from llama_index.core.tools import ToolMetadata from llama_index.core.selectors import LLMSingleSelector -from llama_index.core.prompts.base import PromptTemplate -from llama_index.core.prompts.prompt_type import PromptType from llama_index.llms.openai import OpenAI from app.rag.retrieval.web.brave_search import BraveSearchQueryEngine @@ -13,17 +8,28 @@ from app.rag.generation.response_synthesis import ResponseSynthesisEngine from app.config import config, OPENAPI_KEY, RERANK_TOP_COUNT +from app.services.search_utility import setup_logger + +logger = setup_logger('Orchestrator') + class Orchestrator: + """ + Orchestrator is responsible for routing the search engine query. + It routes the query into three routes now.The first one is clinical trails, second one is drug related information, + and third one is pubmed brave. + """ def __init__(self, config): self.config = config self.choices = [ - ToolMetadata( - description="useful for retrieving only the clinical trials information like adverse effects,eligibility details of clinical trials perticipents, sponsor details, death count, condition of many healthcare problems", - name="clinical_trial_choice"), - ToolMetadata(description="useful only for retrieving the drug related information like molecular weights, similarities,smile codes, target medicines, effects on other medicine", + ToolMetadata(description=f"""useful for retrieving only the clinical trials information like adverse effects,eligibility details + of clinical trials perticipents, sponsor details, death count, condition of many healthcare problems""", + name="clinical_trial_choice"), + ToolMetadata(description=f"""useful only for retrieving the drug related information like molecular weights, + similarities,smile codes, target medicines, effects on other medicine""", name="drug_information_choice"), - ToolMetadata(description="useful for retrieving general information about healthcare data.", name="pubmed_brave_choice") + ToolMetadata(description=f"""useful for retrieving general information about healthcare data.""", + name="pubmed_brave_choice") ] self.ROUTER_PROMPT = "You are working as router of a healthcare search engine.Some choices are given below. It is provided in a numbered list (1 to {num_choices}) where each item in the list corresponds to a summary.\n---------------------\n{context_list}\n---------------------\nIf you are not super confident then please use choice 3 as default choice.Using only the choices above and not prior knowledge, return the choice that is most relevant to the question: '{query_str}'\n" @@ -34,8 +40,10 @@ def __init__(self, config): async def query_and_get_answer(self, search_text): # search router call + logger.debug(f"Orchestrator.query_and_get_answer.router_id search_text: {search_text}") selector_result = self.selector.select(self.choices, query=search_text) router_id = selector_result.selections[0].index + logger.debug(f"Orchestrator.query_and_get_answer.router_id router_id: {router_id}") breaks_sql = False @@ -45,8 +53,10 @@ async def query_and_get_answer(self, search_text): try: sqlResponse = clinicalTrialSearch.call_text2sql(search_text=search_text) result = str(sqlResponse) - except: + logger.debug(f"Orchestrator.query_and_get_answer.sqlResponse sqlResponse: {result}") + except Exception as e: breaks_sql = True + logger.exception("Orchestrator.query_and_get_answer.sqlResponse Exception -", exc_info = e, stack_info=True) pass elif router_id == 1: @@ -56,10 +66,10 @@ async def query_and_get_answer(self, search_text): print() if router_id == 2 or breaks_sql == True: + logger.debug(f"Orchestrator.query_and_get_answer.router_id Fallback Entered.") bravesearch = BraveSearchQueryEngine(config) - webResponse = bravesearch.call_brave_search_api(search_text=search_text) - - extracted_retrieved_results = webResponse + extracted_retrieved_results = bravesearch.call_brave_search_api(search_text=search_text) + logger.debug(f"Orchestrator.query_and_get_answer.extracted_retrieved_results: {extracted_retrieved_results}") #rerank call rerank = ReRankEngine(config) @@ -68,6 +78,7 @@ async def query_and_get_answer(self, search_text): retrieval_results=extracted_retrieved_results ) rerankResponse_sliced = rerankResponse[:RERANK_TOP_COUNT] + logger.debug(f"Orchestrator.query_and_get_answer.rerankResponse_sliced: {rerankResponse_sliced}") #generation call response_synthesis = ResponseSynthesisEngine(config) @@ -75,5 +86,6 @@ async def query_and_get_answer(self, search_text): search_text=search_text, reranked_results=rerankResponse_sliced ) + logger.debug(f"Orchestrator.query_and_get_answer.response_synthesis: {result}") return result diff --git a/docker-compose copy.yml b/docker-compose copy.yml new file mode 100644 index 00000000..ac9dfa8c --- /dev/null +++ b/docker-compose copy.yml @@ -0,0 +1,37 @@ +version: '3.8' + +services: + + web: + build: + context: . + dockerfile: docker/Dockerfile.local + command: ./scripts/docker-entrypoint.sh + volumes: + - ./app:/usr/src/app + ports: + - 8000:8000 + depends_on: + db: + condition: service_healthy + links: + - db + + db: + image: postgres:14-alpine + ports: + - 5432:5432 + volumes: + - db_data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + db_data: diff --git a/pip copy b/pip copy new file mode 100644 index 00000000..e69de29b diff --git a/poetry copy.lock b/poetry copy.lock new file mode 100644 index 00000000..567ef8d8 --- /dev/null +++ b/poetry copy.lock @@ -0,0 +1,216 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "anyio" +version = "4.3.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"}, + {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "fastapi" +version = "0.110.0" +description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fastapi-0.110.0-py3-none-any.whl", hash = "sha256:87a1f6fb632a218222c5984be540055346a8f5d8a68e8f6fb647b1dc9934de4b"}, + {file = "fastapi-0.110.0.tar.gz", hash = "sha256:266775f0dcc95af9d3ef39bad55cff525329a931d5fd51930aadd4f428bf7ff3"}, +] + +[package.dependencies] +pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" +starlette = ">=0.36.3,<0.37.0" +typing-extensions = ">=4.8.0" + +[package.extras] +all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "pydantic" +version = "2.6.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.6.3-py3-none-any.whl", hash = "sha256:72c6034df47f46ccdf81869fddb81aade68056003900a8724a4f160700016a2a"}, + {file = "pydantic-2.6.3.tar.gz", hash = "sha256:e07805c4c7f5c6826e33a1d4c9d47950d7eaf34868e2690f8594d2e30241f11f"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.16.3" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.16.3" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"}, + {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"}, + {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"}, + {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"}, + {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"}, + {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"}, + {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"}, + {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"}, + {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"}, + {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"}, + {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"}, + {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"}, + {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"}, + {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"}, + {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"}, + {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"}, + {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"}, + {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"}, + {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"}, + {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"}, + {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"}, + {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"}, + {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"}, + {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"}, + {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"}, + {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"}, + {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"}, + {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"}, + {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"}, + {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "starlette" +version = "0.36.3" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.36.3-py3-none-any.whl", hash = "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044"}, + {file = "starlette-0.36.3.tar.gz", hash = "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] + +[[package]] +name = "typing-extensions" +version = "4.10.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "738915f0b1326463eab01db5066fb0e7157431d29cb658948bda7b3a0eb5d14f" diff --git a/pytest copy.ini b/pytest copy.ini new file mode 100644 index 00000000..90415015 --- /dev/null +++ b/pytest copy.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + load_db_fixtures: load db fixtures only \ No newline at end of file diff --git a/requirements copy.txt b/requirements copy.txt new file mode 100644 index 00000000..88aa62b1 --- /dev/null +++ b/requirements copy.txt @@ -0,0 +1,367 @@ +aiofiles==23.2.1 +aiohttp==3.9.3 +aiosignal==1.3.1 +aiosmtplib==2.0.1 +alembic==1.8.1 +altair==5.2.0 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +appnope==0.1.3 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +arize-phoenix==3.5.0 +arrow==1.3.0 +asgiref==3.7.2 +asttokens==2.4.1 +async-lru==2.0.4 +async-timeout==4.0.3 +asyncpg==0.27.0 +attrs==23.2.0 +Authlib==0.15.5 +authx==1.0.0b0 +authx_extra==1.0.0 +Babel==2.14.0 +backoff==2.2.1 +bcrypt==4.0.1 +beautifulsoup4==4.12.3 +bidict==0.23.1 +black==22.6.0 +bleach==6.1.0 +bs4==0.0.2 +build==1.1.1 +CacheControl==0.14.0 +cachetools==5.3.3 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==2.1.1 +chembl_downloader==0.4.5 +chroma-hnswlib==0.7.3 +chromadb==0.4.24 +cleo==2.1.0 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +comm==0.2.1 +contourpy==1.2.0 +courlan==1.0.0 +crashtest==0.4.1 +cryptography==41.0.0 +cycler==0.12.1 +Cython==0.29.37 +databases==0.7.0 +dataclasses-json==0.6.4 +dateparser==1.2.0 +ddsketch==2.0.4 +debugpy==1.8.0 +decorator==5.1.1 +defusedxml==0.7.1 +Deprecated==1.2.14 +dirtyjson==1.0.8 +distlib==0.3.8 +distro==1.9.0 +dnspython==2.6.1 +dulwich==0.21.7 +ecdsa==0.18.0 +email-validator==2.0.0.post2 +executing==2.0.1 +fastapi==0.98.0 +fastapi-versioning==0.10.0 +fastjsonschema==2.19.1 +ffmpy==0.3.1 +filelock==3.13.1 +flake8==7.0.0 +Flask==2.2.3 +flatbuffers==23.5.26 +fonttools==4.47.2 +fqdn==1.5.1 +frozenlist==1.4.1 +fsspec==2024.2.0 +google-auth==2.28.1 +googleapis-common-protos==1.62.0 +gradio==4.17.0 +gradio_client==0.9.0 +graphql-core==3.2.3 +greenlet==3.0.3 +grpcio==1.62.0 +gunicorn==20.1.0 +h11==0.12.0 +hdbscan==0.8.33 +html-table-parser-python3==0.3.1 +htmldate==1.7.0 +httpcore==0.15.0 +httptools==0.6.1 +httpx==0.24.1 +huggingface-hub==0.20.3 +humanfriendly==10.0 +idna==3.6 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 +iniconfig==2.0.0 +install==1.3.5 +installer==0.7.0 +ipykernel==6.29.0 +ipython==8.21.0 +ipywidgets==8.1.1 +isoduration==20.11.0 +isort==5.10.1 +itsdangerous==2.1.2 +jaraco.classes==3.3.1 +jedi==0.19.1 +Jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonpickle==3.0.2 +jsonpointer==2.4 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +jupyter==1.0.0 +jupyter-console==6.6.3 +jupyter-events==0.9.0 +jupyter-lsp==2.2.2 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +jupyter_server==2.12.5 +jupyter_server_terminals==0.5.2 +jupyterlab==4.0.12 +jupyterlab-widgets==3.0.9 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.2 +jusText==3.0.0 +keyring==24.3.1 +kiwisolver==1.4.5 +kubernetes==29.0.0 +lab==8.0 +langcodes==3.3.0 +llama-index==0.10.16 +llama-index-agent-openai==0.1.5 +llama-index-callbacks-arize-phoenix==0.1.4 +llama-index-cli==0.1.7 +llama-index-core==0.10.16.post1 +llama-index-embeddings-huggingface==0.1.4 +llama-index-embeddings-openai==0.1.6 +llama-index-indices-managed-llama-cloud==0.1.3 +llama-index-legacy==0.9.48 +llama-index-llms-ollama==0.1.1 +llama-index-llms-openai==0.1.7 +llama-index-llms-replicate==0.1.3 +llama-index-multi-modal-llms-openai==0.1.4 +llama-index-program-openai==0.1.4 +llama-index-question-gen-openai==0.1.3 +llama-index-readers-file==0.1.8 +llama-index-readers-llama-parse==0.1.3 +llama-index-readers-wikipedia==0.1.2 +llama-index-vector-stores-chroma==0.1.5 +llama-parse==0.3.6 +llamaindex-py-client==0.1.13 +llvmlite==0.42.0 +lxml==5.1.0 +Mako==1.3.2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +marshmallow==3.20.2 +matplotlib==3.7.1 +matplotlib-inline==0.1.6 +mccabe==0.7.0 +mdurl==0.1.2 +mistune==3.0.2 +mmh3==4.1.0 +monotonic==1.6 +more-click==0.1.2 +more-itertools==10.2.0 +motor==3.1.2 +mpmath==1.3.0 +msgpack==1.0.8 +multidict==6.0.5 +mypy==0.961 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.14.2 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==3.2.1 +nltk==3.8.1 +notebook==7.0.7 +notebook_shim==0.2.3 +numba==0.59.0 +numpy==1.26.3 +oauthlib==3.2.2 +onnxruntime==1.17.1 +openai==1.12.0 +openinference-instrumentation-langchain==0.1.11 +openinference-instrumentation-llama-index==1.1.1 +openinference-instrumentation-openai==0.1.3 +openinference-semantic-conventions==0.1.4 +opentelemetry-api==1.23.0 +opentelemetry-exporter-otlp==1.23.0 +opentelemetry-exporter-otlp-proto-common==1.23.0 +opentelemetry-exporter-otlp-proto-grpc==1.23.0 +opentelemetry-exporter-otlp-proto-http==1.23.0 +opentelemetry-instrumentation==0.44b0 +opentelemetry-instrumentation-asgi==0.44b0 +opentelemetry-instrumentation-fastapi==0.44b0 +opentelemetry-proto==1.23.0 +opentelemetry-sdk==1.23.0 +opentelemetry-semantic-conventions==0.44b0 +opentelemetry-util-http==0.44b0 +opentracing==2.4.0 +orjson==3.9.13 +outcome==1.3.0.post0 +overrides==7.7.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +passlib==1.7.4 +pathspec==0.12.1 +pbr==6.0.0 +pexpect==4.9.0 +pgvector==0.2.5 +pillow==10.2.0 +pkginfo==1.10.0 +platformdirs==4.2.0 +plotly==5.13.1 +pluggy==1.4.0 +poetry==1.8.2 +poetry-core==1.9.0 +poetry-plugin-export==1.6.0 +posthog==3.5.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.25.3 +psutil==5.9.4 +psycopg2-binary==2.9.9 +ptyprocess==0.7.0 +PubChemPy==1.0.4 +pulsar-client==3.4.0 +pure-eval==0.2.2 +py==1.11.0 +pyarrow==15.0.0 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pycodestyle==2.11.1 +pycparser==2.21 +pydantic==1.10.7 +pydantic_core==2.16.2 +pydub==0.25.1 +pyee==8.2.2 +pyflakes==3.2.0 +Pygments==2.17.2 +pyinstrument==4.4.0 +PyJWT==2.7.0 +pymongo==4.6.2 +PyMuPDF==1.23.22 +PyMuPDFb==1.23.22 +pynndescent==0.5.11 +pyparsing==3.1.1 +pypdf==4.0.1 +PyPika==0.48.9 +pyppeteer==1.0.2 +pyproject_hooks==1.0.0 +PySocks==1.7.1 +pystow==0.5.2 +pytest==6.2.5 +pytest-asyncio==0.19.0 +python-dateutil==2.8.2 +python-docx==0.8.11 +python-dotenv==0.21.0 +python-engineio==4.9.0 +python-jose==3.3.0 +python-json-logger==2.0.7 +python-multipart==0.0.7 +python-pptx==0.6.21 +python-socketio==5.8.0 +pytz==2023.3 +pyvis==0.3.2 +PyYAML==6.0.1 +pyzmq==25.1.2 +qtconsole==5.5.1 +QtPy==2.4.1 +rapidfuzz==3.6.2 +redis==4.5.5 +referencing==0.33.0 +regex==2023.12.25 +requests==2.31.0 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +rfc3339-validator==0.1.4 +rfc3986==1.5.0 +rfc3986-validator==0.1.1 +rich==13.7.0 +rpds-py==0.17.1 +rsa==4.9 +ruff==0.2.1 +safetensors==0.4.2 +scikit-learn==1.2.1 +scipy==1.10.1 +selenium==4.17.2 +semantic-version==2.10.0 +Send2Trash==1.8.2 +shellingham==1.5.4 +simple-websocket==1.0.0 +simplejson==3.19.2 +six==1.16.0 +sniffio==1.3.0 +sortedcontainers==2.4.0 +soupsieve==2.5 +SQLAlchemy==1.4.52 +SQLAlchemy-Utils==0.38.3 +sqlalchemy2-stubs==0.0.2a38 +sqlmodel==0.0.8 +stack-data==0.6.3 +starlette==0.27.0 +Starlette-OpenTracing==0.1.0 +stevedore==5.2.0 +strawberry-graphql==0.208.2 +sympy==1.12 +tenacity==8.2.3 +terminado==0.18.0 +threadpoolctl==3.2.0 +tiktoken==0.6.0 +tinycss2==1.2.1 +tld==0.13 +tokenize-rt==5.2.0 +tokenizers==0.15.2 +toml==0.10.2 +tomlkit==0.12.0 +toolz==0.12.1 +torch==2.2.1 +tornado==6.4 +tqdm==4.66.1 +trafilatura==1.4.1 +traitlets==5.14.1 +transformers==4.38.2 +trio==0.24.0 +trio-websocket==0.11.1 +trove-classifiers==2024.3.3 +txt2tags==3.9 +typer==0.9.0 +types-python-dateutil==2.8.19.20240106 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzlocal==5.2 +ujson==5.5.0 +umap-learn==0.5.5 +uri-template==1.3.0 +urllib3==1.26.18 +uvicorn==0.20.0 +uvloop==0.19.0 +virtualenv==20.25.0 +virtualenv-clone==0.5.7 +virtualenvwrapper==6.1.0 +watchfiles==0.21.0 +wcwidth==0.2.13 +webcolors==1.13 +webdriver-manager==4.0.1 +webencodings==0.5.1 +websocket-client==1.7.0 +websockets==10.4 +Werkzeug==2.2.2 +widgetsnbextension==4.0.9 +wrapt==1.16.0 +wsproto==1.2.0 +xattr==1.1.0 +XlsxWriter==3.1.9 +yarl==1.9.4 +yesqa==1.3.0 +zipp==3.17.0 diff --git a/text2sql_dag copy.html b/text2sql_dag copy.html new file mode 100644 index 00000000..afe24b22 --- /dev/null +++ b/text2sql_dag copy.html @@ -0,0 +1,369 @@ + + + + + + + + + +
+

+
+ + + + + + +
+

+
+ + + + + +
+ + +
+
+ + + + + + + \ No newline at end of file