diff --git a/.gitignore b/.gitignore index bd4c8a7..97d500c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,168 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + + env/ __pycache__/ node_modules housewatch.sqlite3 .DS_Store -yarn.lock \ No newline at end of file +yarn.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..aa28a28 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.275 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black diff --git a/Dockerfile b/Dockerfile index 422099d..40ba989 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,20 @@ ENV PYTHONUNBUFFERED 1 WORKDIR /code COPY requirements.txt ./ -RUN pip install -r requirements.txt --compile --no-cache-dir + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + "build-essential" \ + "git" \ + "libpq-dev" \ + "libxmlsec1" \ + "libxmlsec1-dev" \ + "libffi-dev" \ + "pkg-config" \ + && \ + rm -rf /var/lib/apt/lists/* && \ + pip install -r requirements.txt --compile --no-cache-dir + USER root @@ -14,4 +27,3 @@ COPY housewatch housewatch/ COPY bin bin/ RUN DEBUG=1 python manage.py collectstatic --noinput - diff --git a/README.md b/README.md index 4457cc3..7615c36 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ To deploy HouseWatch, clone this repo and then run the following, substituting the environment variables for the relevant values of one of your ClickHouse instances: ```bash +SITE_ADDRESS= \ CLICKHOUSE_HOST=localhost \ CLICKHOUSE_CLUSTER=mycluster \ CLICKHOUSE_USER=default \ @@ -30,7 +31,9 @@ CLICKHOUSE_PASSWORD=xxxxxxxxxxx \ docker compose -f docker-compose.yml up ``` -After running the above, the UI will be running on [http://localhost:3000](http://localhost:3000). For production installs, you might want to setup something like [Caddy](https://caddyserver.com/) or [NGINX](https://nginx.org/en/) with a [Let's Encrypt](https://letsencrypt.org/) TLS certificate. +`SITE_ADDRESS` here is the address that the UI will be running on. It can be a domain name or simply a port like `:80`. + +After running the above, the UI will be running on the address you specified. This will be something like http://localhost if you used `:80` for your `SITE_ADDRESS` above. I would think twice about exposing this to the internet, as it is not currently secured in any way.
@@ -49,13 +52,13 @@ The following are the supported environment variables for configuring your House - `CLICKHOUSE_VERIFY`: Optional - see [clickhouse-driver docs](https://clickhouse-driver.readthedocs.io/en/latest/index.html) for more information - `CLICKHOUSE_CA`: Optional - see [clickhouse-driver docs](https://clickhouse-driver.readthedocs.io/en/latest/index.html) for more information - `OPENAI_API_KEY`: Optional - enables the experimental "AI Tools" page, which currently features a natural language query editor -- `OPENAI_MODEL`: Optional - a valid OpenAI model (e.g. `gpt-3.5-turbo`, `gpt-4`) that you have access to with the key above to be used for the AI features +- `OPENAI_MODEL`: Optional - a valid OpenAI model (e.g. `gpt-3.5-turbo`, `gpt-4`) that you have access to with the key above to be used for the AI features
## 💡 Motivation -At PostHog we manage a few large ClickHouse clusters and found ourselves in need of a tool to monitor and manage these more easily. +At PostHog we manage a few large ClickHouse clusters and found ourselves in need of a tool to monitor and manage these more easily. ClickHouse is fantastic at introspection, providing a lot of metadata about the system in its system tables so that it can be easily queried. However, knowing exactly how to query and parse the available information can be a difficult task. Over the years at PostHog, we've developed great intuition for how to debug ClickHouse issues using ClickHouse, and HouseWatch is the compilation of this knowledge into a tool. @@ -65,7 +68,7 @@ As a result, we felt it was appropriate to have these tools live in one place. U ## 🏗️ Status of the project -HouseWatch is in its early days and we have a lot more features in mind that we'd like to build into it going forward. The code could also use some cleaning up :) As of right now, it is considered Beta software and you should exercise caution when using it in production. +HouseWatch is in its early days and we have a lot more features in mind that we'd like to build into it going forward. The code could also use some cleaning up :) As of right now, it is considered Beta software and you should exercise caution when using it in production. One potential approach is to connect HouseWatch to ClickHouse using a read-only user. In this case, the cluster management features will not work (e.g. operations, query editor), but the analysis toolset will function normally. @@ -175,7 +178,7 @@ A public list of things we intend to do with HouseWatch in the near future. Cleanup -- [ ] Extract README images out of repo +- [ ] Extract README images out of repo - [ ] Make banner subtitle work on dark mode - [ ] Fetch data independently on the query analyzer - [ ] Breakpoint for logs search diff --git a/bin/celery b/bin/celery index 13b4bb0..4445a6f 100755 --- a/bin/celery +++ b/bin/celery @@ -1,4 +1,4 @@ #!/bin/bash set -e -celery -A housewatch worker \ No newline at end of file +celery -A housewatch worker diff --git a/bin/lint b/bin/lint deleted file mode 100755 index 7423f0a..0000000 --- a/bin/lint +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -python -m isort housewatch -python -m black . -python -m flake8 housewatch -python -m mypy . diff --git a/bin/start b/bin/start index 5bbc7e0..8475910 100755 --- a/bin/start +++ b/bin/start @@ -4,4 +4,4 @@ set -e export DEBUG=1 -./bin/celery & python manage.py runserver \ No newline at end of file +./bin/celery & python manage.py runserver diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..4820dbe --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,96 @@ +version: "3" + +services: + app: + build: . + environment: &django_env + DEBUG: 1 + REDIS_URL: redis://redis:6379 + DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch + CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_DATABASE: default + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: "" + CLICKHOUSE_CLUSTER: parallel_replicas + CLICKHOUSE_SECURE: false + CLICKHOUSE_VERIFY: false + CLICKHOUSE_CA: "" + command: + - bash + - -c + - | + python manage.py migrate + python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + depends_on: + - clickhouse + - db + - redis + + web: + build: + context: ./frontend + dockerfile: Dockerfile.dev + volumes: + - ./frontend/public:/frontend/public + - ./frontend/src:/frontend/src + ports: + - "3000:3000" + + db: + image: postgres:14-alpine + restart: on-failure + environment: + POSTGRES_USER: housewatch + POSTGRES_DB: housewatch + POSTGRES_PASSWORD: housewatch + + healthcheck: + test: ["CMD-SHELL", "pg_isready -U housewatch"] + interval: 5s + timeout: 5s + + redis: + image: redis:6.2.7-alpine + restart: on-failure + ports: + - "6388:6379" + command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb + + worker: + build: . + environment: + <<: *django_env + command: + - ./bin/celery + volumes: + - .:/code + depends_on: + - clickhouse + - db + - redis + + clickhouse: + image: ${CLICKHOUSE_SERVER_IMAGE:-clickhouse/clickhouse-server:23.4.2.11} + restart: on-failure + depends_on: + - zookeeper + + zookeeper: + image: zookeeper:3.7.0 + restart: on-failure + + caddy: + image: caddy:2.6.1 + ports: + - "8888:8888" + environment: + SITE_ADDRESS: ":8888" + volumes: + - ./docker/Caddyfile:/etc/caddy/Caddyfile + depends_on: + - web + - app diff --git a/docker-compose.yml b/docker-compose.yml index 2f8dc47..fc10d6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,61 +1,67 @@ -version: '3' +version: "3" services: - web: - build: - context: frontend/ - dockerfile: Dockerfile.dev - command: ["npm", "run", "start"] - ports: - - "3000:3000" + app: + build: . + environment: &django_env + DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch + REDIS_URL: redis://redis:6379 + CLICKHOUSE_HOST: $CLICKHOUSE_HOST + CLICKHOUSE_DATABASE: $CLICKHOUSE_DATABASE + CLICKHOUSE_USER: $CLICKHOUSE_USER + CLICKHOUSE_PASSWORD: $CLICKHOUSE_PASSWORD + CLICKHOUSE_CLUSTER: $CLICKHOUSE_CLUSTER + CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE + CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY + CLICKHOUSE_CA: $CLICKHOUSE_CA + command: + - bash + - -c + - | + python manage.py migrate + python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + web: + build: ./frontend + ports: + - "3000:3000" - app: - build: . - environment: - REDIS_URL: redis://redis:6379 - CLICKHOUSE_HOST: $CLICKHOUSE_HOST - CLICKHOUSE_DATABASE: $CLICKHOUSE_DATABASE - CLICKHOUSE_USER: $CLICKHOUSE_USER - CLICKHOUSE_PASSWORD: $CLICKHOUSE_PASSWORD - CLICKHOUSE_CLUSTER: $CLICKHOUSE_CLUSTER - CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE - CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY - CLICKHOUSE_CA: $CLICKHOUSE_CA - command: - - bash - - -c - - | - python manage.py migrate - python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/code -<<<<<<< HEAD - ports: - - '8000:8000' -======= - network_mode: "service:web" ->>>>>>> 0921a8c (Add helm chart) + worker: + build: . + environment: + <<: *django_env + command: + - ./bin/celery + volumes: + - .:/code + redis: + image: redis:6.2.7-alpine + restart: on-failure + ports: + - "6388:6379" + command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb - redis: - image: redis:6.2.7-alpine - restart: on-failure - ports: - - '6388:6379' - command: redis-server --maxmemory-policy allkeys-lru --maxmemory 200mb + db: + image: postgres:14-alpine + restart: on-failure + environment: + POSTGRES_USER: housewatch + POSTGRES_DB: housewatch + POSTGRES_PASSWORD: housewatch - worker: - build: . - environment: - REDIS_URL: redis://redis:6379 - CLICKHOUSE_HOST: $CLICKHOUSE_HOST - CLICKHOUSE_DATABASE: $CLICKHOUSE_DATABASE - CLICKHOUSE_USER: $CLICKHOUSE_USER - CLICKHOUSE_PASSWORD: $CLICKHOUSE_PASSWORD - CLICKHOUSE_CLUSTER: $CLICKHOUSE_CLUSTER - CLICKHOUSE_SECURE: $CLICKHOUSE_SECURE - CLICKHOUSE_VERIFY: $CLICKHOUSE_VERIFY - CLICKHOUSE_CA: $CLICKHOUSE_CA - command: - - ./bin/celery - volumes: - - .:/code \ No newline at end of file + caddy: + image: caddy:2.6.1 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + environment: + SITE_ADDRESS: $SITE_ADDRESS + volumes: + - ./docker/Caddyfile:/etc/caddy/Caddyfile + depends_on: + - web + - app diff --git a/docker/Caddyfile b/docker/Caddyfile new file mode 100644 index 0000000..928ad4e --- /dev/null +++ b/docker/Caddyfile @@ -0,0 +1,10 @@ +{ + debug +} + +{$SITE_ADDRESS} { + reverse_proxy web:3000 + reverse_proxy /api/* app:8000 + reverse_proxy /logout app:8000 + reverse_proxy /admin/ app:8000 +} diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 4a2b9bc..1ce372a 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,4 +4,4 @@ WORKDIR /frontend COPY build/ build/ -CMD ["echo", "Serve the files from /frontend/build, don't run this container directly"] +CMD ["echo", "Serve the files from /frontend/build, don't run this container directly"] \ No newline at end of file diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev index 8c63fcd..cbc99f4 100644 --- a/frontend/Dockerfile.dev +++ b/frontend/Dockerfile.dev @@ -6,9 +6,8 @@ COPY . . ENV NODE_OPTIONS --openssl-legacy-provider RUN npm install -RUN npm run build RUN npm install -g serve EXPOSE 3000 -CMD ["serve", "-s", "build"] +CMD ["npm", "run", "start"] diff --git a/frontend/public/fonts/OFL.txt b/frontend/public/fonts/OFL.txt index 1c8d125..5a06ae3 100644 --- a/frontend/public/fonts/OFL.txt +++ b/frontend/public/fonts/OFL.txt @@ -18,7 +18,7 @@ with others. The OFL allows the licensed fonts to be used, studied, modified and redistributed freely as long as they are not sold by themselves. The -fonts, including any derivative works, can be bundled, embedded, +fonts, including any derivative works, can be bundled, embedded, redistributed and/or sold with any software provided that any reserved names are not used by derivative works. The fonts and derivatives, however, cannot be released under any other type of license. The diff --git a/frontend/src/pages/QueryEditor/Benchmark.tsx b/frontend/src/pages/QueryEditor/Benchmark.tsx index fdacd5b..26a076a 100644 --- a/frontend/src/pages/QueryEditor/Benchmark.tsx +++ b/frontend/src/pages/QueryEditor/Benchmark.tsx @@ -22,7 +22,7 @@ export interface BenchmarkingData { const DEFAULT_QUERY1 = `SELECT number FROM system.errors errors JOIN ( SELECT * FROM system.numbers LIMIT 1000 -) numbers +) numbers ON numbers.number = toUInt64(errors.code) SETTINGS join_algorithm = 'default' ` @@ -30,7 +30,7 @@ SETTINGS join_algorithm = 'default' const DEFAULT_QUERY2 = `SELECT number FROM system.errors errors JOIN ( SELECT * FROM system.numbers LIMIT 1000 -) numbers +) numbers ON numbers.number = toUInt64(errors.code) SETTINGS join_algorithm = 'parallel_hash' ` diff --git a/frontend/src/utils/dateUtils.ts b/frontend/src/utils/dateUtils.ts index cae459e..8bda77f 100644 --- a/frontend/src/utils/dateUtils.ts +++ b/frontend/src/utils/dateUtils.ts @@ -11,11 +11,10 @@ export function isoTimestampToHumanReadable(isoDate: string): string { // Prepare the date format const formattedDate = monthNames[date.getMonth()] + ' ' - + date.getDate() + + date.getDate() + year + ' ' + ('0' + date.getHours()).slice(-2) + ':' + ('0' + date.getMinutes()).slice(-2) return formattedDate } - diff --git a/housewatch/admin.py b/housewatch/admin.py index 723e08e..6eb1ec0 100644 --- a/housewatch/admin.py +++ b/housewatch/admin.py @@ -1,8 +1,4 @@ -from django.contrib import admin from django.utils.html import format_html -from django.utils.translation import gettext_lazy as _ - -from housewatch import settings def html_link(url, text, new_tab=False): diff --git a/housewatch/ai/templates.py b/housewatch/ai/templates.py index ef84a63..d71c175 100644 --- a/housewatch/ai/templates.py +++ b/housewatch/ai/templates.py @@ -78,4 +78,4 @@ # {database}.{table} {create_table_query} -""" \ No newline at end of file +""" diff --git a/housewatch/api/analyze.py b/housewatch/api/analyze.py index 9bda071..36a8e95 100644 --- a/housewatch/api/analyze.py +++ b/housewatch/api/analyze.py @@ -2,7 +2,10 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.decorators import action -from housewatch.clickhouse.client import run_query, ch_host, existing_system_tables + +from django.conf import settings + +from housewatch.clickhouse.client import run_query, existing_system_tables from housewatch.clickhouse.queries.sql import ( SLOW_QUERIES_SQL, SCHEMA_SQL, @@ -11,7 +14,6 @@ ERRORS_SQL, QUERY_MEMORY_USAGE_SQL, QUERY_READ_BYTES_SQL, - TABLES_SQL, RUNNING_QUERIES_SQL, KILL_QUERY_SQL, PARTS_SQL, @@ -23,14 +25,18 @@ EXPLAIN_QUERY, BENCHMARKING_SQL, AVAILABLE_TABLES_SQL, - TABLE_SCHEMAS_SQL + TABLE_SCHEMAS_SQL, ) from uuid import uuid4 import json from time import sleep import os import openai -from housewatch.ai.templates import NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT, NATURAL_LANGUAGE_QUERY_USER_PROMPT, TABLE_PROMPT +from housewatch.ai.templates import ( + NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT, + NATURAL_LANGUAGE_QUERY_USER_PROMPT, + TABLE_PROMPT, +) openai.api_key = os.getenv("OPENAI_API_KEY") OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-3.5-turbo") @@ -44,13 +50,14 @@ "-3m": "INTERVAL 3 MONTH", } + class AnalyzeViewset(GenericViewSet): def list(self, request: Request) -> Response: pass @action(detail=False, methods=["GET"]) def slow_queries(self, request: Request): - ch_interval = TIME_RANGE_TO_CLICKHOUSE_INTERVAL[request.GET.get('time_range', '-1w')] + ch_interval = TIME_RANGE_TO_CLICKHOUSE_INTERVAL[request.GET.get("time_range", "-1w")] params = {"limit": 100, "date_from": f"now() - {ch_interval}"} query_result = run_query(SLOW_QUERIES_SQL, params) return Response(query_result) @@ -113,11 +120,6 @@ def query_graphs(self, request: Request): {"execution_count": execution_count, "memory_usage": memory_usage, "read_bytes": read_bytes, "cpu": cpu} ) - @action(detail=False, methods=["GET"]) - def tables(self, request: Request): - query_result = run_query(TABLES_SQL) - return Response(query_result) - @action(detail=False, methods=["POST"]) def logs(self, request: Request): if "text_log" not in existing_system_tables: @@ -148,7 +150,7 @@ def query(self, request: Request): @action(detail=False, methods=["GET"]) def hostname(self, request: Request): - return Response({"hostname": ch_host}) + return Response({"hostname": settings.CLICKHOUSE_HOST}) @action(detail=True, methods=["GET"]) def schema(self, request: Request, pk: str): @@ -216,12 +218,18 @@ def benchmark(self, request: Request): try: error_location = "Control" query1_result = run_query( - request.data["query1"], settings={"log_comment": query1_tag, "min_bytes_to_use_direct_io": 1 }, use_cache=False, substitute_params=False + request.data["query1"], + settings={"log_comment": query1_tag, "min_bytes_to_use_direct_io": 1}, + use_cache=False, + substitute_params=False, ) error_location = "Test" query2_result = run_query( - request.data["query2"], settings={"log_comment": query2_tag, "min_bytes_to_use_direct_io": 1 }, use_cache=False, substitute_params=False + request.data["query2"], + settings={"log_comment": query2_tag, "min_bytes_to_use_direct_io": 1}, + use_cache=False, + substitute_params=False, ) error_location = "benchmark" @@ -243,56 +251,64 @@ def benchmark(self, request: Request): @action(detail=False, methods=["GET"]) def ai_tools_available(self, request: Request): - openai_api_key = os.getenv("OPENAI_API_KEY") + openai_api_key = os.getenv("OPENAI_API_KEY") if not openai_api_key: - return Response(status=400, data={ "error": "OPENAI_API_KEY not set. To use the AI toolset you must pass in an OpenAI API key via the OPENAI_API_KEY environment variable." }) - return Response({ "status": "ok" }) - + return Response( + status=400, + data={ + "error": "OPENAI_API_KEY not set. To use the AI toolset you must pass in an OpenAI API key via the OPENAI_API_KEY environment variable." + }, + ) + return Response({"status": "ok"}) + @action(detail=False, methods=["GET"]) def tables(self, request: Request): query_result = run_query(AVAILABLE_TABLES_SQL, use_cache=False) - return Response(query_result) - + @action(detail=False, methods=["POST"]) def natural_language_query(self, request: Request): - + table_schema_sql_conditions = [] for full_table_name in request.data["tables_to_query"]: database, table = full_table_name.split(">>>>>") condition = f"(database = '{database}' AND table = '{table}')" table_schema_sql_conditions.append(condition) - - table_schemas = run_query(TABLE_SCHEMAS_SQL, { "conditions": " OR ".join(table_schema_sql_conditions)}) - + + table_schemas = run_query(TABLE_SCHEMAS_SQL, {"conditions": " OR ".join(table_schema_sql_conditions)}) + user_prompt_tables = "" for row in table_schemas: - user_prompt_tables += TABLE_PROMPT.format(database=row['database'], table=row['table'], create_table_query=row['create_table_query']) - - final_user_prompt = NATURAL_LANGUAGE_QUERY_USER_PROMPT % { "tables_to_query": user_prompt_tables, "query": request.data['query'] } - + user_prompt_tables += TABLE_PROMPT.format( + database=row["database"], table=row["table"], create_table_query=row["create_table_query"] + ) + + final_user_prompt = NATURAL_LANGUAGE_QUERY_USER_PROMPT % { + "tables_to_query": user_prompt_tables, + "query": request.data["query"], + } + try: completion = openai.ChatCompletion.create( model=OPENAI_MODEL, messages=[ {"role": "system", "content": NATURAL_LANGUAGE_QUERY_SYSTEM_PROMPT}, - {"role": "user", "content": final_user_prompt} - ] + {"role": "user", "content": final_user_prompt}, + ], ) except Exception as e: - return Response(status=418, data={"error": str(e), "sql": None}) - - response_json = json.loads(completion.choices[0].message['content']) - sql = response_json['sql'] - error = response_json['error'] + return Response(status=418, data={"error": str(e), "sql": None}) + + response_json = json.loads(completion.choices[0].message["content"]) + sql = response_json["sql"] + error = response_json["error"] if error: - return Response(status=418, data={ "error": error, "sql": sql }) - - settings = { "readonly": 1 } if request.data.get('readonly', False) else {} - + return Response(status=418, data={"error": error, "sql": sql}) + + settings = {"readonly": 1} if request.data.get("readonly", False) else {} + try: query_result = run_query(sql, use_cache=False, substitute_params=False, settings=settings) - return Response({ "result": query_result, "sql": sql, "error": None }) + return Response({"result": query_result, "sql": sql, "error": None}) except Exception as e: return Response(status=418, data={"error": str(e), "sql": sql}) - \ No newline at end of file diff --git a/housewatch/api/instance.py b/housewatch/api/instance.py index a3ea212..cbfc30b 100644 --- a/housewatch/api/instance.py +++ b/housewatch/api/instance.py @@ -1,14 +1,6 @@ -from typing import cast - import structlog -from django.http import JsonResponse -from rest_framework import status -from rest_framework.decorators import action -from rest_framework.exceptions import APIException, ValidationError -from rest_framework.request import Request -from rest_framework.viewsets import GenericViewSet, ModelViewSet +from rest_framework.viewsets import ModelViewSet from rest_framework.serializers import ModelSerializer -from sentry_sdk import capture_exception from housewatch.models import Instance diff --git a/housewatch/async_migrations/async_migration_utils.py b/housewatch/async_migrations/async_migration_utils.py index b2c7789..5483011 100644 --- a/housewatch/async_migrations/async_migration_utils.py +++ b/housewatch/async_migrations/async_migration_utils.py @@ -1,9 +1,7 @@ -import asyncio from datetime import datetime -from typing import Callable, Optional +from typing import Optional import structlog -from django.conf import settings from django.db import transaction from django.utils.timezone import now from housewatch.models.async_migration import AsyncMigration, MigrationStatus @@ -27,7 +25,7 @@ def execute_op(sql: str, args=None, *, query_id: str, timeout_seconds: int = 600 try: run_query(sql, args, settings=settings, use_cache=False) except Exception as e: - raise Exception(f"Failed to execute ClickHouse op: sql={sql},\nquery_id={query_id},\nexception={str(e)}") + raise Exception(f"Failed to execute ClickHouse op: sql={sql},\nquery_id={query_id},\nexception={str(e)}") from e def mark_async_migration_as_running(migration: AsyncMigration) -> bool: diff --git a/housewatch/async_migrations/runner.py b/housewatch/async_migrations/runner.py index 6e74fa2..d432d63 100644 --- a/housewatch/async_migrations/runner.py +++ b/housewatch/async_migrations/runner.py @@ -1,5 +1,3 @@ -from typing import List, Optional, Tuple - import structlog from sentry_sdk.api import capture_exception @@ -140,7 +138,7 @@ def update_migration_progress(migration: AsyncMigration): update_async_migration( migration=migration, progress=int((migration.current_operation_index / len(migration.operations)) * 100) ) - except: + except Exception: pass diff --git a/housewatch/celery.py b/housewatch/celery.py index 91a8395..83f6219 100644 --- a/housewatch/celery.py +++ b/housewatch/celery.py @@ -1,7 +1,6 @@ import os from celery import Celery -from celery.schedules import crontab from django_structlog.celery.steps import DjangoStructLogInitStep # set the default Django settings module for the 'celery' program. diff --git a/housewatch/clickhouse/client.py b/housewatch/clickhouse/client.py index 582fe7a..ab2fa7a 100644 --- a/housewatch/clickhouse/client.py +++ b/housewatch/clickhouse/client.py @@ -1,13 +1,13 @@ +import os from typing import Dict, Optional from clickhouse_pool import ChPool -import os from housewatch.clickhouse.queries.sql import EXISTING_TABLES_SQL +from housewatch.utils import str_to_bool from django.core.cache import cache +from django.conf import settings import hashlib import json -def str_to_bool(s: str) -> bool: - return str(s).lower() in ("y", "yes", "t", "true", "on", "1") ch_host = os.getenv("CLICKHOUSE_HOST", "localhost") ch_verify = os.getenv("CLICKHOUSE_VERIFY", "true").lower() not in ("false", "0") @@ -15,17 +15,18 @@ def str_to_bool(s: str) -> bool: ch_secure = os.getenv("CLICKHOUSE_SECURE", "true").lower() not in ("false", "0") pool = ChPool( - host=ch_host, - database=os.getenv("CLICKHOUSE_DATABASE", "default"), - user=os.getenv("CLICKHOUSE_USER", "default"), - password=os.getenv("CLICKHOUSE_PASSWORD", ""), - secure=str_to_bool(ch_secure) if ch_secure != "" else True, - ca_certs=ch_ca if ch_ca != "" else None, - verify=ch_verify if ch_verify != "" else True, + host=settings.CLICKHOUSE_HOST, + database=settings.CLICKHOUSE_DATABASE, + user=settings.CLICKHOUSE_USER, + secure=settings.CLICKHOUSE_SECURE, + ca_certs=settings.CLICKHOUSE_CA, + verify=settings.CLICKHOUSE_VERIFY, settings={"max_result_rows": "2000"}, send_receive_timeout=30, + password=settings.CLICKHOUSE_PASSWORD, ) + def run_query( query: str, params: Dict[str, str | int] = {}, diff --git a/housewatch/clickhouse/queries/sql.py b/housewatch/clickhouse/queries/sql.py index eb558fa..2f8052d 100644 --- a/housewatch/clickhouse/queries/sql.py +++ b/housewatch/clickhouse/queries/sql.py @@ -20,7 +20,7 @@ (sum(read_bytes)/(sum(sum(read_bytes)) over ()))*100 AS percentage_iops, (sum(query_duration_ms)/(sum(sum(query_duration_ms)) over ()))*100 AS percentage_runtime, toString(normalized_query_hash) AS normalized_query_hash -FROM {QUERY_LOG_SYSTEM_TABLE} +FROM {QUERY_LOG_SYSTEM_TABLE} WHERE is_initial_query AND event_time > %(date_from)s AND type = 2 GROUP BY normalized_query_hash, normalizeQuery(query) ORDER BY sum(read_bytes) DESC @@ -33,7 +33,7 @@ FROM {QUERY_LOG_SYSTEM_TABLE} WHERE event_time >= toDateTime(%(date_from)s) - AND event_time <= toDateTime(%(date_to)s) + AND event_time <= toDateTime(%(date_to)s) GROUP BY day ORDER BY day """ @@ -75,7 +75,7 @@ SELECT name AS part, data_compressed_bytes AS compressed, formatReadableSize(data_compressed_bytes) AS compressed_readable, formatReadableSize(data_uncompressed_bytes) AS uncompressed FROM system.parts WHERE table = '%(table)s' -ORDER BY data_compressed_bytes DESC +ORDER BY data_compressed_bytes DESC LIMIT 100 """ @@ -176,7 +176,7 @@ """ RUNNING_QUERIES_SQL = """ -SELECT query, elapsed, read_rows, total_rows_approx, formatReadableSize(memory_usage) AS memory_usage, query_id +SELECT query, elapsed, read_rows, total_rows_approx, formatReadableSize(memory_usage) AS memory_usage, query_id FROM system.processes WHERE Settings['log_comment'] != 'running_queries_lookup' ORDER BY elapsed DESC @@ -188,10 +188,10 @@ """ NODE_STORAGE_SQL = f""" -SELECT - hostName() node, - sum(total_space) space_used, - sum(free_space) free_space, +SELECT + hostName() node, + sum(total_space) space_used, + sum(free_space) free_space, (space_used + free_space) total_space_available, formatReadableSize(total_space_available) readable_total_space_available, formatReadableSize(space_used) readable_space_used, @@ -205,7 +205,7 @@ NODE_DATA_TRANSFER_ACROSS_SHARDS_SQL = f""" SELECT hostName() node, sum(read_bytes) total_bytes_transferred, formatReadableSize(total_bytes_transferred) AS readable_bytes_transferred FROM {QUERY_LOG_SYSTEM_TABLE} -WHERE is_initial_query != 0 AND type = 2 +WHERE is_initial_query != 0 AND type = 2 GROUP BY node ORDER BY node """ @@ -244,16 +244,16 @@ BENCHMARKING_SQL = f""" SELECT if(log_comment = '%(query1_tag)s', 'Control', 'Test') AS query_version, - sumIf(query_duration_ms, is_initial_query) AS duration_ms, - sumIf(memory_usage, is_initial_query) AS memory_usage, + sumIf(query_duration_ms, is_initial_query) AS duration_ms, + sumIf(memory_usage, is_initial_query) AS memory_usage, sumIf(ProfileEvents['OSCPUVirtualTimeMicroseconds'], is_initial_query) AS cpu, sumIf(read_bytes, is_initial_query) AS read_bytes, sumIf(read_rows, NOT is_initial_query) AS read_bytes_from_other_shards, sumIf(ProfileEvents['NetworkReceiveBytes'], is_initial_query) AS network_receive_bytes FROM {QUERY_LOG_SYSTEM_TABLE} -WHERE - type = 2 - AND event_time > now() - INTERVAL 10 MINUTE +WHERE + type = 2 + AND event_time > now() - INTERVAL 10 MINUTE AND (log_comment = '%(query1_tag)s' OR log_comment = '%(query2_tag)s') GROUP BY query_version ORDER BY query_version @@ -261,7 +261,7 @@ AVAILABLE_TABLES_SQL = """ SELECT database, table -FROM system.tables +FROM system.tables WHERE database NOT ILIKE 'information_schema' """ @@ -269,4 +269,4 @@ SELECT database, table, create_table_query FROM system.tables WHERE %(conditions)s -""" \ No newline at end of file +""" diff --git a/housewatch/migrations/0001_initial.py b/housewatch/migrations/0001_initial.py index b2ea95b..3c9f5ab 100644 --- a/housewatch/migrations/0001_initial.py +++ b/housewatch/migrations/0001_initial.py @@ -8,19 +8,18 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Instance', + name="Instance", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created_at', models.DateTimeField(default=django.utils.timezone.now)), - ('username', models.CharField(max_length=200)), - ('password', models.CharField(max_length=200)), - ('host', models.CharField(max_length=200)), - ('port', models.IntegerField()), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("username", models.CharField(max_length=200)), + ("password", models.CharField(max_length=200)), + ("host", models.CharField(max_length=200)), + ("port", models.IntegerField()), ], ), ] diff --git a/housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py b/housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py index 6175baa..937308e 100644 --- a/housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py +++ b/housewatch/migrations/0002_asyncmigration_asyncmigration_unique name.py @@ -6,26 +6,26 @@ class Migration(migrations.Migration): dependencies = [ - ('housewatch', '0001_initial'), + ("housewatch", "0001_initial"), ] operations = [ migrations.CreateModel( - name='AsyncMigration', + name="AsyncMigration", fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=50)), - ('description', models.CharField(blank=True, max_length=400, null=True)), - ('progress', models.PositiveSmallIntegerField(default=0)), - ('status', models.PositiveSmallIntegerField(default=0)), - ('current_operation_index', models.PositiveSmallIntegerField(default=0)), - ('current_query_id', models.CharField(default='', max_length=100)), - ('task_id', models.CharField(blank=True, default='', max_length=100, null=True)), - ('started_at', models.DateTimeField(blank=True, null=True)), + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=50)), + ("description", models.CharField(blank=True, max_length=400, null=True)), + ("progress", models.PositiveSmallIntegerField(default=0)), + ("status", models.PositiveSmallIntegerField(default=0)), + ("current_operation_index", models.PositiveSmallIntegerField(default=0)), + ("current_query_id", models.CharField(default="", max_length=100)), + ("task_id", models.CharField(blank=True, default="", max_length=100, null=True)), + ("started_at", models.DateTimeField(blank=True, null=True)), ], ), migrations.AddConstraint( - model_name='asyncmigration', - constraint=models.UniqueConstraint(fields=('name',), name='unique name'), + model_name="asyncmigration", + constraint=models.UniqueConstraint(fields=("name",), name="unique name"), ), ] diff --git a/housewatch/migrations/0003_asyncmigration_operations_and_more.py b/housewatch/migrations/0003_asyncmigration_operations_and_more.py index bfe03de..d96c104 100644 --- a/housewatch/migrations/0003_asyncmigration_operations_and_more.py +++ b/housewatch/migrations/0003_asyncmigration_operations_and_more.py @@ -6,19 +6,19 @@ class Migration(migrations.Migration): dependencies = [ - ('housewatch', '0002_asyncmigration_asyncmigration_unique name'), + ("housewatch", "0002_asyncmigration_asyncmigration_unique name"), ] operations = [ migrations.AddField( - model_name='asyncmigration', - name='operations', + model_name="asyncmigration", + name="operations", field=models.JSONField(default=[]), preserve_default=False, ), migrations.AddField( - model_name='asyncmigration', - name='rollback_operations', + model_name="asyncmigration", + name="rollback_operations", field=models.JSONField(blank=True, null=True), ), ] diff --git a/housewatch/migrations/0004_asyncmigration_last_error.py b/housewatch/migrations/0004_asyncmigration_last_error.py index 9db379f..f7984f8 100644 --- a/housewatch/migrations/0004_asyncmigration_last_error.py +++ b/housewatch/migrations/0004_asyncmigration_last_error.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('housewatch', '0003_asyncmigration_operations_and_more'), + ("housewatch", "0003_asyncmigration_operations_and_more"), ] operations = [ migrations.AddField( - model_name='asyncmigration', - name='last_error', - field=models.CharField(blank=True, default='', max_length=800, null=True), + model_name="asyncmigration", + name="last_error", + field=models.CharField(blank=True, default="", max_length=800, null=True), ), ] diff --git a/housewatch/migrations/0005_asyncmigration_finished_at.py b/housewatch/migrations/0005_asyncmigration_finished_at.py index 8d4c74f..7175df2 100644 --- a/housewatch/migrations/0005_asyncmigration_finished_at.py +++ b/housewatch/migrations/0005_asyncmigration_finished_at.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('housewatch', '0004_asyncmigration_last_error'), + ("housewatch", "0004_asyncmigration_last_error"), ] operations = [ migrations.AddField( - model_name='asyncmigration', - name='finished_at', + model_name="asyncmigration", + name="finished_at", field=models.DateTimeField(blank=True, null=True), ), ] diff --git a/housewatch/migrations/0006_savedquery.py b/housewatch/migrations/0006_savedquery.py index bec42f2..7539e85 100644 --- a/housewatch/migrations/0006_savedquery.py +++ b/housewatch/migrations/0006_savedquery.py @@ -6,17 +6,17 @@ class Migration(migrations.Migration): dependencies = [ - ('housewatch', '0005_asyncmigration_finished_at'), + ("housewatch", "0005_asyncmigration_finished_at"), ] operations = [ migrations.CreateModel( - name='SavedQuery', + name="SavedQuery", fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=200)), - ('query', models.CharField(max_length=2000)), - ('created_at', models.DateTimeField(auto_now=True)), + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("name", models.CharField(max_length=200)), + ("query", models.CharField(max_length=2000)), + ("created_at", models.DateTimeField(auto_now=True)), ], ), ] diff --git a/housewatch/settings.py b/housewatch/settings/__init__.py similarity index 92% rename from housewatch/settings.py rename to housewatch/settings/__init__.py index a8b2e8c..180c3f2 100644 --- a/housewatch/settings.py +++ b/housewatch/settings/__init__.py @@ -15,11 +15,13 @@ from datetime import timedelta from pathlib import Path from typing import Any, Callable, Optional - import dj_database_url + from django.core.exceptions import ImproperlyConfigured from kombu import Exchange, Queue +from housewatch.utils import str_to_bool + # TODO: Figure out why things dont work on cloud without debug DEBUG = os.getenv("DEBUG", "false").lower() in ["true", "1"] @@ -132,13 +134,8 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_ if DATABASE_URL: DATABASES = {"default": dj_database_url.config(default=DATABASE_URL, conn_max_age=600)} -else: - DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.path.join(BASE_DIR, "housewatch.sqlite3"), - } - } +elif not DEBUG: + raise ImproperlyConfigured("DATABASE_URL environment variable not set!") # Password validation @@ -243,3 +240,14 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_ POSTHOG_PROJECT_API_KEY = get_from_env("POSTHOG_PROJECT_API_KEY", "123456789") + + +# ClickHouse + +CLICKHOUSE_HOST = get_from_env("CLICKHOUSE_HOST", "localhost") +CLICKHOUSE_VERIFY = str_to_bool(get_from_env("CLICKHOUSE_VERIFY", "True")) +CLICKHOUSE_CA = get_from_env("CLICKHOUSE_CA", "") +CLICKHOUSE_SECURE = str_to_bool(get_from_env("CLICKHOUSE_SECURE", "True")) +CLICKHOUSE_DATABASE = get_from_env("CLICKHOUSE_DATABASE", "defaul") +CLICKHOUSE_USER = get_from_env("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = get_from_env("CLICKHOUSE_PASSWORD", "") diff --git a/housewatch/settings/utils.py b/housewatch/settings/utils.py new file mode 100644 index 0000000..d81a754 --- /dev/null +++ b/housewatch/settings/utils.py @@ -0,0 +1,28 @@ +import os +from typing import Any, Callable, List, Optional + +from django.core.exceptions import ImproperlyConfigured + +from housewatch.utils import str_to_bool + +__all__ = ["get_from_env", "get_list", "str_to_bool"] + + +def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_cast: Optional[Callable] = None) -> Any: + value = os.getenv(key) + if value is None or value == "": + if optional: + return None + if default is not None: + return default + else: + raise ImproperlyConfigured(f'The environment variable "{key}" is required to run HouseWatch!') + if type_cast is not None: + return type_cast(value) + return value + + +def get_list(text: str) -> List[str]: + if not text: + return [] + return [item.strip() for item in text.split(",")] diff --git a/housewatch/utils.py b/housewatch/utils.py new file mode 100644 index 0000000..aae9e3a --- /dev/null +++ b/housewatch/utils.py @@ -0,0 +1,10 @@ +from typing import Any + + +def str_to_bool(value: Any) -> bool: + """Return whether the provided string (or any value really) represents true. Otherwise false. + Just like plugin server stringToBoolean. + """ + if not value: + return False + return str(value).lower() in ("y", "yes", "t", "true", "on", "1") diff --git a/mypy.ini b/mypy.ini index 3e39a50..2231c4a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,4 +12,4 @@ strict_equality = True ignore_missing_imports = True [mypy.plugins.django-stubs] -django_settings_module = housewatch.settings \ No newline at end of file +django_settings_module = housewatch.settings diff --git a/pyproject.toml b/pyproject.toml index 6683389..1c19186 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,3 +18,18 @@ include_trailing_comma = true force_grid_wrap = 8 ensure_newline_before_comments = true line_length = 120 + +[tool.ruff] +# Enable flake8-bugbear (`B`) rules. +select = ["E", "F", "B"] + +# Never enforce `E501` (line length violations). +ignore = ["E501"] + +# Avoid trying to fix flake8-bugbear (`B`) violations. +unfixable = ["B"] + +# Ignore `E402` (import violations) in all `__init__.py` files, and in `path/to/file.py`. +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402"] +"path/to/file.py" = ["E402"] diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..721a87f --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,4 @@ +black==23.3.0 +ruff==0.0.275 +pip-tools==6.13.0 +pre-commit==3.3.3 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..486e1f3 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,58 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements-dev.in +# +black==23.3.0 + # via -r requirements-dev.in +build==0.10.0 + # via pip-tools +cfgv==3.3.1 + # via pre-commit +click==8.1.3 + # via + # black + # pip-tools +distlib==0.3.6 + # via virtualenv +filelock==3.12.2 + # via virtualenv +identify==2.5.24 + # via pre-commit +mypy-extensions==1.0.0 + # via black +nodeenv==1.8.0 + # via pre-commit +packaging==23.1 + # via + # black + # build +pathspec==0.11.1 + # via black +pip-tools==6.13.0 + # via -r requirements-dev.in +platformdirs==3.8.0 + # via + # black + # virtualenv +pre-commit==3.3.3 + # via -r requirements-dev.in +pyproject-hooks==1.0.0 + # via build +pyyaml==6.0 + # via pre-commit +ruff==0.0.275 + # via -r requirements-dev.in +tomli==2.0.1 + # via + # black + # build +virtualenv==20.23.1 + # via pre-commit +wheel==0.40.0 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..4ee4202 --- /dev/null +++ b/requirements.in @@ -0,0 +1,55 @@ +aiohttp==3.8.4 +aiosignal==1.3.1 +amqp==5.1.1 +asgiref==3.6.0 +async-timeout==4.0.2 +attrs==22.2.0 +billiard==3.6.4.0 +celery==5.2.7 +certifi==2022.12.7 +charset-normalizer==3.1.0 +click==8.1.3 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.2.0 +clickhouse-driver==0.2.5 +clickhouse-pool==0.5.3 +dj-database-url==1.0.0 +Django==4.1.1 +django-cors-headers==3.13.0 +django-ipware==5.0.0 +django-redis==5.2.0 +django-structlog==3.0.1 +djangorestframework==3.14.0 +drf-exceptions-hog==0.2.0 +drf-extensions==0.7.1 +drf-spectacular==0.24.2 +frozenlist==1.3.3 +gunicorn==20.1.0 +idna==3.4 +inflection==0.5.1 +jsonschema==4.17.3 +kombu==5.2.4 +multidict==6.0.4 +openai==0.27.8 +prompt-toolkit==3.0.38 +psycopg2-binary==2.9.7 +pyrsistent==0.19.3 +pytz==2023.2 +pytz-deprecation-shim==0.1.0.post0 +PyYAML==6.0 +redis==4.5.3 +requests==2.31.0 +sentry-sdk==1.9.8 +six==1.16.0 +sqlparse==0.4.3 +structlog==22.3.0 +tqdm==4.65.0 +tzdata==2023.3 +tzlocal==4.3 +uritemplate==4.1.1 +urllib3==1.26.15 +vine==5.0.0 +wcwidth==0.2.6 +whitenoise==5.2.0 +yarl==1.9.2 diff --git a/requirements.txt b/requirements.txt index bc62b9a..cc9a9fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,55 +1,223 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile requirements.in +# aiohttp==3.8.4 + # via + # -r requirements.in + # openai aiosignal==1.3.1 + # via + # -r requirements.in + # aiohttp amqp==5.1.1 + # via + # -r requirements.in + # kombu asgiref==3.6.0 + # via + # -r requirements.in + # django async-timeout==4.0.2 + # via + # -r requirements.in + # aiohttp + # redis attrs==22.2.0 + # via + # -r requirements.in + # aiohttp + # jsonschema billiard==3.6.4.0 + # via + # -r requirements.in + # celery celery==5.2.7 + # via -r requirements.in certifi==2022.12.7 + # via + # -r requirements.in + # requests + # sentry-sdk charset-normalizer==3.1.0 + # via + # -r requirements.in + # aiohttp + # requests click==8.1.3 + # via + # -r requirements.in + # celery + # click-didyoumean + # click-plugins + # click-repl click-didyoumean==0.3.0 + # via + # -r requirements.in + # celery click-plugins==1.1.1 + # via + # -r requirements.in + # celery click-repl==0.2.0 + # via + # -r requirements.in + # celery clickhouse-driver==0.2.5 + # via + # -r requirements.in + # clickhouse-pool clickhouse-pool==0.5.3 + # via -r requirements.in dj-database-url==1.0.0 -Django==4.1.1 + # via -r requirements.in +django==4.1.1 + # via + # -r requirements.in + # dj-database-url + # django-cors-headers + # django-redis + # django-structlog + # djangorestframework + # drf-spectacular django-cors-headers==3.13.0 + # via -r requirements.in django-ipware==5.0.0 + # via + # -r requirements.in + # django-structlog django-redis==5.2.0 + # via -r requirements.in django-structlog==3.0.1 + # via -r requirements.in djangorestframework==3.14.0 + # via + # -r requirements.in + # drf-exceptions-hog + # drf-extensions + # drf-spectacular drf-exceptions-hog==0.2.0 + # via -r requirements.in drf-extensions==0.7.1 + # via -r requirements.in drf-spectacular==0.24.2 + # via -r requirements.in frozenlist==1.3.3 + # via + # -r requirements.in + # aiohttp + # aiosignal gunicorn==20.1.0 + # via -r requirements.in idna==3.4 + # via + # -r requirements.in + # requests + # yarl inflection==0.5.1 + # via + # -r requirements.in + # drf-spectacular jsonschema==4.17.3 + # via + # -r requirements.in + # drf-spectacular kombu==5.2.4 + # via + # -r requirements.in + # celery multidict==6.0.4 + # via + # -r requirements.in + # aiohttp + # yarl openai==0.27.8 + # via -r requirements.in prompt-toolkit==3.0.38 -psycopg2-binary==2.9.3 + # via + # -r requirements.in + # click-repl +psycopg2-binary==2.9.7 + # via -r requirements.in pyrsistent==0.19.3 + # via + # -r requirements.in + # jsonschema pytz==2023.2 + # via + # -r requirements.in + # celery + # clickhouse-driver + # djangorestframework pytz-deprecation-shim==0.1.0.post0 -PyYAML==6.0 + # via + # -r requirements.in + # tzlocal +pyyaml==6.0 + # via + # -r requirements.in + # drf-spectacular redis==4.5.3 + # via + # -r requirements.in + # django-redis requests==2.31.0 + # via + # -r requirements.in + # openai sentry-sdk==1.9.8 + # via -r requirements.in six==1.16.0 + # via + # -r requirements.in + # click-repl sqlparse==0.4.3 + # via + # -r requirements.in + # django structlog==22.3.0 + # via + # -r requirements.in + # django-structlog tqdm==4.65.0 + # via + # -r requirements.in + # openai tzdata==2023.3 + # via + # -r requirements.in + # pytz-deprecation-shim tzlocal==4.3 + # via + # -r requirements.in + # clickhouse-driver uritemplate==4.1.1 + # via + # -r requirements.in + # drf-spectacular urllib3==1.26.15 + # via + # -r requirements.in + # requests + # sentry-sdk vine==5.0.0 + # via + # -r requirements.in + # amqp + # celery + # kombu wcwidth==0.2.6 + # via + # -r requirements.in + # prompt-toolkit whitenoise==5.2.0 + # via -r requirements.in yarl==1.9.2 + # via + # -r requirements.in + # aiohttp + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/runtime.txt b/runtime.txt index f96c0fa..a48890e 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.10 \ No newline at end of file +python-3.8.10