From d22e5fbc6ee64cd7fd3683213ebd3f8feb658d00 Mon Sep 17 00:00:00 2001 From: harishmohanraj Date: Wed, 9 Aug 2023 05:03:07 +0000 Subject: [PATCH 1/2] Save the generated plan in a file --- .gitignore | 5 +- .../_code_generator/plan_generator.py | 24 ++- fastkafka_gen/_modidx.py | 2 + fastkafka_gen/cli.py | 10 +- nbs/CLI.ipynb | 10 +- nbs/Helper.ipynb | 56 ++++- nbs/Plan_Generator.ipynb | 201 +++++++++++++++--- 7 files changed, 269 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 49f946d..55ad474 100644 --- a/.gitignore +++ b/.gitignore @@ -172,4 +172,7 @@ mkdocs/site/ # nbdev_mkdocs mkdocs/docs/ -mkdocs/site/ \ No newline at end of file +mkdocs/site/ + +# fastkafka-gen +fastkafka-gen/ diff --git a/fastkafka_gen/_code_generator/plan_generator.py b/fastkafka_gen/_code_generator/plan_generator.py index cc90a9a..629ce46 100644 --- a/fastkafka_gen/_code_generator/plan_generator.py +++ b/fastkafka_gen/_code_generator/plan_generator.py @@ -8,6 +8,7 @@ from typing import * import time import json +from pathlib import Path from yaspin import yaspin @@ -255,11 +256,25 @@ def _validate_response(response: str) -> List[str]: return ["JSON decoding failed. Please send JSON response only."] # %% ../../nbs/Plan_Generator.ipynb 44 -def generate_plan(description: str) -> Tuple[str, str]: +def _save_plan_as_json(contents: str, output_file: str) -> None: + """Save the contents as JSON to the specified output file. + + Args: + contents: A JSON-formatted string to be saved as a JSON file. + output_file: The path to the output file where the JSON content will be saved. + """ + Path(output_file).parent.mkdir(exist_ok=True) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(json.loads(contents), f, indent=4) + +# %% ../../nbs/Plan_Generator.ipynb 46 +def generate_plan(description: str, output_path: str) -> Tuple[str, str]: """Generate a plan from user's application description Args: description: Validated User application description + output_path: Path to the output directory where generated files will be saved. This path should be relative to the current working directory Returns: The plan generated by OpenAI as a dictionary @@ -273,6 +288,9 @@ def generate_plan(description: str) -> Tuple[str, str]: plan_validator = ValidateAndFixResponse(plan_generator, _validate_response) validated_plan, total_tokens = plan_validator.fix(description) + output_file = f"{output_path}/plan.json" + _save_plan_as_json(validated_plan, output_file) + sp.text = "" - sp.ok(" ✔ Plan generated") - return validated_plan, total_tokens + sp.ok(f" ✔ Plan generated and saved at: {output_file}") + return total_tokens diff --git a/fastkafka_gen/_modidx.py b/fastkafka_gen/_modidx.py index 991a46d..c29fe54 100644 --- a/fastkafka_gen/_modidx.py +++ b/fastkafka_gen/_modidx.py @@ -37,6 +37,8 @@ 'fastkafka_gen/_code_generator/helper.py')}, 'fastkafka_gen._code_generator.plan_generator': { 'fastkafka_gen._code_generator.plan_generator._get_error_msgs_and_expected_keys': ( 'plan_generator.html#_get_error_msgs_and_expected_keys', 'fastkafka_gen/_code_generator/plan_generator.py'), + 'fastkafka_gen._code_generator.plan_generator._save_plan_as_json': ( 'plan_generator.html#_save_plan_as_json', + 'fastkafka_gen/_code_generator/plan_generator.py'), 'fastkafka_gen._code_generator.plan_generator._vaidate_plan': ( 'plan_generator.html#_vaidate_plan', 'fastkafka_gen/_code_generator/plan_generator.py'), 'fastkafka_gen._code_generator.plan_generator._validate_apps': ( 'plan_generator.html#_validate_apps', diff --git a/fastkafka_gen/cli.py b/fastkafka_gen/cli.py index 4db1633..976179f 100644 --- a/fastkafka_gen/cli.py +++ b/fastkafka_gen/cli.py @@ -72,18 +72,24 @@ def generate_fastkafka_app( \n""" ), + output_path: str = typer.Option( + "./fastkafka-gen", + "--output_path", + "-o", + help="Path to the output directory where generated files will be saved. This path should be relative to the current working directory.", + ), ) -> None: """Generate a new FastKafka app(s) effortlessly with advanced AI assistance""" try: _ensure_openai_api_key_set() validated_description, description_token = validate_app_description(description) -# validated_plan, plan_token = generate_plan(validated_description) + plan_token = generate_plan(validated_description, output_path) # code = generate_app(validated_plan, validated_description) # test = generate_test(code) # total_token_usage = description_token + plan_token # typer.secho(f" ▶ Total tokens usage: {total_token_usage}", fg=typer.colors.CYAN) - typer.secho("✨ All files were successfully generated.!", fg=typer.colors.CYAN) + typer.secho("✨ All files were successfully generated!", fg=typer.colors.CYAN) except (ValueError, KeyError) as e: typer.secho(e, err=True, fg=typer.colors.RED) diff --git a/nbs/CLI.ipynb b/nbs/CLI.ipynb index 0969912..54dccf3 100644 --- a/nbs/CLI.ipynb +++ b/nbs/CLI.ipynb @@ -226,18 +226,24 @@ "\n", "\\n\"\"\"\n", " ),\n", + " output_path: str = typer.Option(\n", + " \"./fastkafka-gen\",\n", + " \"--output_path\",\n", + " \"-o\",\n", + " help=\"Path to the output directory where generated files will be saved. This path should be relative to the current working directory.\",\n", + " ),\n", ") -> None:\n", " \"\"\"Generate a new FastKafka app(s) effortlessly with advanced AI assistance\"\"\"\n", " try:\n", " _ensure_openai_api_key_set()\n", " validated_description, description_token = validate_app_description(description)\n", - "# validated_plan, plan_token = generate_plan(validated_description)\n", + " plan_token = generate_plan(validated_description, output_path)\n", "# code = generate_app(validated_plan, validated_description)\n", "# test = generate_test(code)\n", " \n", "# total_token_usage = description_token + plan_token\n", "# typer.secho(f\" ▶ Total tokens usage: {total_token_usage}\", fg=typer.colors.CYAN)\n", - " typer.secho(\"✨ All files were successfully generated.!\", fg=typer.colors.CYAN)\n", + " typer.secho(\"✨ All files were successfully generated!\", fg=typer.colors.CYAN)\n", " \n", " except (ValueError, KeyError) as e:\n", " typer.secho(e, err=True, fg=typer.colors.RED)\n", diff --git a/nbs/Helper.ipynb b/nbs/Helper.ipynb index da7e2fe..bcad7c7 100644 --- a/nbs/Helper.ipynb +++ b/nbs/Helper.ipynb @@ -425,7 +425,26 @@ "execution_count": null, "id": "7a39d512", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "some prompt\n", + "\n", + "==== RESPONSE WITH ISSUES ====\n", + "\n", + "some response\n", + "\n", + "Read the contents of ==== RESPONSE WITH ISSUES ==== section and fix the below mentioned issues:\n", + "\n", + "error 1\n", + "error 2\n", + "error 3\n", + "\n" + ] + } + ], "source": [ "def fixture_generate(initial_prompt):\n", " return \"some response\"\n", @@ -504,7 +523,15 @@ "execution_count": null, "id": "e74e159a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Some Valid response\n" + ] + } + ], "source": [ "fixture_initial_prompt = \"some valid prompt\"\n", "expected = \"Some Valid response\"\n", @@ -526,7 +553,16 @@ "execution_count": null, "id": "efbaf85e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error 1\n", + "error 2\n" + ] + } + ], "source": [ "fixture_initial_prompt = \"some invalid prompt\"\n", "max_attempts = 2\n", @@ -582,9 +618,21 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" } }, "nbformat": 4, diff --git a/nbs/Plan_Generator.ipynb b/nbs/Plan_Generator.ipynb index dce29d4..76fd944 100644 --- a/nbs/Plan_Generator.ipynb +++ b/nbs/Plan_Generator.ipynb @@ -22,6 +22,7 @@ "from typing import *\n", "import time\n", "import json\n", + "from pathlib import Path\n", "\n", "from yaspin import yaspin\n", "\n", @@ -37,6 +38,8 @@ "metadata": {}, "outputs": [], "source": [ + "from tempfile import TemporaryDirectory\n", + "\n", "from fastkafka_gen._components.logger import suppress_timestamps" ] }, @@ -1307,7 +1310,7 @@ } ], "source": [ - "response = \"\"\"{\n", + "fixture_response = \"\"\"{\n", " \"entities\": [\n", " {\n", " \"name\": \"StoreProduct\",\n", @@ -1355,11 +1358,109 @@ "}\"\"\"\n", "\n", "expected = []\n", - "actual = _validate_response(response)\n", + "actual = _validate_response(fixture_response)\n", "print(actual)\n", "assert actual == expected" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1881cb0", + "metadata": {}, + "outputs": [], + "source": [ + "# | export\n", + "\n", + "\n", + "def _save_plan_as_json(contents: str, output_file: str) -> None:\n", + " \"\"\"Save the contents as JSON to the specified output file.\n", + "\n", + " Args:\n", + " contents: A JSON-formatted string to be saved as a JSON file.\n", + " output_file: The path to the output file where the JSON content will be saved.\n", + " \"\"\"\n", + " Path(output_file).parent.mkdir(exist_ok=True)\n", + "\n", + " with open(output_file, \"w\", encoding=\"utf-8\") as f:\n", + " json.dump(json.loads(contents), f, indent=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85a937f7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"StoreProduct\",\n", + " \"arguments\": {\n", + " \"product_name\": \"str\",\n", + " \"currency\": \"str\",\n", + " \"price\": \"float\"\n", + " }\n", + " }\n", + " ],\n", + " \"apps\": [\n", + " {\n", + " \"app_name\": \"store_app\",\n", + " \"kafka_brokers\": {\n", + " \"localhost\": {\n", + " \"url\": \"localhost\",\n", + " \"description\": \"local development kafka broker\",\n", + " \"port\": 9092\n", + " }\n", + " },\n", + " \"title\": \"Store Kafka App\",\n", + " \"consumes_functions\": {\n", + " \"on_store_product\": {\n", + " \"topic\": \"store_product\",\n", + " \"prefix\": \"on\",\n", + " \"parameters\": {\n", + " \"msg\": \"StoreProduct\"\n", + " },\n", + " \"description\": \"This function will listen to the 'store_product' topic, it will consume the messages posted on the 'store_product' topic. The message should be of type 'StoreProduct' which contains the attributes 'product_name', 'currency', and 'price'. After consuming the data, it will forward the store product details to the 'change_currency' topic.\"\n", + " }\n", + " },\n", + " \"produces_functions\": {\n", + " \"to_change_currency\": {\n", + " \"topic\": \"change_currency\",\n", + " \"prefix\": \"to\",\n", + " \"parameters\": {\n", + " \"store_product\": \"StoreProduct\"\n", + " },\n", + " \"description\": \"This function will be triggered when a store product is received from the 'store_product' topic. It will take the store product as input and will produce a message to the 'change_currency' topic. If the currency in the input store product is 'HRK', the currency will be set to 'EUR', and the price will be divided by 7.5. After producing the message, it will return the transformed store product.\",\n", + " \"returns\": \"StoreProduct\"\n", + " }\n", + " }\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "with TemporaryDirectory() as d:\n", + " output_path = f\"{str(d)}/fastkafka-gen\"\n", + " json_file_path = f\"{output_path}/plan.json\"\n", + " \n", + " _save_plan_as_json(fixture_response, json_file_path)\n", + " \n", + " assert Path(output_path).exists()\n", + " actual = [file for file in Path(output_path).iterdir()]\n", + " assert str(actual[0]) == json_file_path\n", + " \n", + " with open(json_file_path, 'r') as f:\n", + " contents = f.read()\n", + " print(contents)\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1370,11 +1471,12 @@ "# | export\n", "\n", "\n", - "def generate_plan(description: str) -> Tuple[str, str]:\n", + "def generate_plan(description: str, output_path: str) -> Tuple[str, str]:\n", " \"\"\"Generate a plan from user's application description\n", "\n", " Args:\n", " description: Validated User application description\n", + " output_path: Path to the output directory where generated files will be saved. This path should be relative to the current working directory\n", "\n", " Returns:\n", " The plan generated by OpenAI as a dictionary\n", @@ -1388,9 +1490,12 @@ " plan_validator = ValidateAndFixResponse(plan_generator, _validate_response)\n", " validated_plan, total_tokens = plan_validator.fix(description)\n", " \n", + " output_file = f\"{output_path}/plan.json\"\n", + " _save_plan_as_json(validated_plan, output_file)\n", + "\n", " sp.text = \"\"\n", - " sp.ok(\" ✔ Plan generated\")\n", - " return validated_plan, total_tokens" + " sp.ok(f\" ✔ Plan generated and saved at: {output_file}\")\n", + " return total_tokens" ] }, { @@ -1403,26 +1508,54 @@ "name": "stdout", "output_type": "stream", "text": [ - "[INFO] fastkafka_gen._code_generator.helper: logger.info\n", - "⠋ Generating plan[WARNING] fastkafka_gen._code_generator.helper: logger.warning\n", - "⠹ Generating plan " - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/harish/.local/lib/python3.11/site-packages/yaspin/core.py:59: UserWarning: color, on_color and attrs are not supported when running in jupyter\n", - " self._color = self._set_color(color) if color else color\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " ✔ Plan generated \n", - "{'entities': [{'name': 'StoreProduct', 'arguments': {'product_name': 'str', 'currency': 'str', 'price': 'float'}}], 'apps': [{'app_name': 'store_app', 'kafka_brokers': {'localhost': {'url': 'localhost', 'description': 'local kafka broker', 'port': 9092}}, 'title': 'Store Kafka App', 'consumes_functions': {'on_store_product': {'topic': 'store_product', 'prefix': 'on', 'parameters': {'msg': 'StoreProduct'}, 'description': \"This function will listen to the 'store_product' topic and consume messages with attributes 'product_name', 'currency', and 'price'. The message should be of type 'StoreProduct'. After consuming the message, it will produce a message to the 'change_currency' topic.\"}}, 'produces_functions': {'to_change_currency': {'topic': 'change_currency', 'prefix': 'to', 'parameters': {'store_product': 'StoreProduct'}, 'description': \"This function will be triggered when a message is consumed from the 'store_product' topic. It takes a 'StoreProduct' object as input and produces a message to the 'change_currency' topic. If the currency in the input 'StoreProduct' object is 'HRK', the currency will be changed to 'EUR' and the price will be divided by 7.5. The function returns the modified 'StoreProduct' object.\", 'returns': 'StoreProduct'}}}]}\n", - "4246\n" + " ✔ Plan generated and saved at: /tmp/tmpvqqjetxg/fastkafka-gen/plan.json \n", + "{\n", + " \"entities\": [\n", + " {\n", + " \"name\": \"StoreProduct\",\n", + " \"arguments\": {\n", + " \"product_name\": \"str\",\n", + " \"currency\": \"str\",\n", + " \"price\": \"float\"\n", + " }\n", + " }\n", + " ],\n", + " \"apps\": [\n", + " {\n", + " \"app_name\": \"product_app\",\n", + " \"kafka_brokers\": {\n", + " \"localhost\": {\n", + " \"url\": \"localhost\",\n", + " \"description\": \"local kafka broker\",\n", + " \"port\": 9092\n", + " }\n", + " },\n", + " \"title\": \"Product Kafka App\",\n", + " \"consumes_functions\": {\n", + " \"on_store_product\": {\n", + " \"topic\": \"store_product\",\n", + " \"prefix\": \"on\",\n", + " \"parameters\": {\n", + " \"msg\": \"StoreProduct\"\n", + " },\n", + " \"description\": \"This function will listen to the 'store_product' topic. It will consume the messages posted on the 'store_product' topic. The messages should have three attributes: 'product_name', 'currency', and 'price'. After consuming the data, it will produce a message to the 'change_currency' topic.\"\n", + " }\n", + " },\n", + " \"produces_functions\": {\n", + " \"to_change_currency\": {\n", + " \"topic\": \"change_currency\",\n", + " \"prefix\": \"to\",\n", + " \"parameters\": {\n", + " \"store_product\": \"StoreProduct\"\n", + " },\n", + " \"description\": \"This function will be triggered when a message is received from the 'store_product' topic. It takes a 'store_product' object as input and produces a message to the 'change_currency' topic. If the currency in the input 'store_product' is 'HRK', the currency will be set to 'EUR' and the price will be divided by 7.5. The function returns the updated 'store_product' object.\",\n", + " \"returns\": \"StoreProduct\"\n", + " }\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "5337\n" ] } ], @@ -1431,8 +1564,22 @@ "Create FastKafka application which consumes messages from the store_product topic, it consumes messages with three attributes: product_name, currency and price. While consuming, it should produce a message to the change_currency topic. input parameters for this producing function should be store_product object and function should return store_product. produces function should check if the currency in the input store_product parameter is \"HRK\", currency should be set to \"EUR\" and the price should be divided with 7.5.\n", "app should use localhost broker\n", "\"\"\"\n", - "plan, total_tokens = generate_plan(app_description)\n", - "print(json.loads(plan))\n", + "\n", + "\n", + "with TemporaryDirectory() as d:\n", + " output_path = f\"{str(d)}/fastkafka-gen\"\n", + " json_file_path = f\"{output_path}/plan.json\"\n", + " \n", + " total_tokens = generate_plan(app_description, output_path)\n", + " \n", + " assert Path(output_path).exists()\n", + " actual = [file for file in Path(output_path).iterdir()]\n", + " assert str(actual[0]) == json_file_path\n", + " \n", + " with open(json_file_path, 'r') as f:\n", + " contents = f.read()\n", + " print(json.loads(contents))\n", + "\n", "assert int(total_tokens) > 0\n", "print(total_tokens)" ] From 644016334b6cd43aecadc641b4fc99ad8aef4366 Mon Sep 17 00:00:00 2001 From: harishmohanraj Date: Wed, 9 Aug 2023 06:06:55 +0000 Subject: [PATCH 2/2] Save the generated plan in a file --- .../_code_generator/plan_generator.py | 10 +-- fastkafka_gen/cli.py | 4 +- nbs/CLI.ipynb | 4 +- nbs/Helper.ipynb | 14 +--- nbs/Plan_Generator.ipynb | 77 ++++--------------- 5 files changed, 26 insertions(+), 83 deletions(-) diff --git a/fastkafka_gen/_code_generator/plan_generator.py b/fastkafka_gen/_code_generator/plan_generator.py index 629ce46..76acf6f 100644 --- a/fastkafka_gen/_code_generator/plan_generator.py +++ b/fastkafka_gen/_code_generator/plan_generator.py @@ -263,21 +263,20 @@ def _save_plan_as_json(contents: str, output_file: str) -> None: contents: A JSON-formatted string to be saved as a JSON file. output_file: The path to the output file where the JSON content will be saved. """ - Path(output_file).parent.mkdir(exist_ok=True) - + Path(output_file).parent.mkdir(parents=True, exist_ok=True) with open(output_file, "w", encoding="utf-8") as f: json.dump(json.loads(contents), f, indent=4) # %% ../../nbs/Plan_Generator.ipynb 46 -def generate_plan(description: str, output_path: str) -> Tuple[str, str]: +def generate_plan(description: str, output_file: str) -> str: """Generate a plan from user's application description Args: description: Validated User application description - output_path: Path to the output directory where generated files will be saved. This path should be relative to the current working directory + output_path: The path to the output file where the generated plan JSON will be saved. Returns: - The plan generated by OpenAI as a dictionary + The total token used to generate the plan """ with yaspin( text="Generating plan", # (slowest step, usually takes 30 to 90 seconds)... @@ -288,7 +287,6 @@ def generate_plan(description: str, output_path: str) -> Tuple[str, str]: plan_validator = ValidateAndFixResponse(plan_generator, _validate_response) validated_plan, total_tokens = plan_validator.fix(description) - output_file = f"{output_path}/plan.json" _save_plan_as_json(validated_plan, output_file) sp.text = "" diff --git a/fastkafka_gen/cli.py b/fastkafka_gen/cli.py index 976179f..d2bf16e 100644 --- a/fastkafka_gen/cli.py +++ b/fastkafka_gen/cli.py @@ -83,7 +83,9 @@ def generate_fastkafka_app( try: _ensure_openai_api_key_set() validated_description, description_token = validate_app_description(description) - plan_token = generate_plan(validated_description, output_path) + + output_file = f"{output_path}/plan.json" + plan_token = generate_plan(validated_description, output_file) # code = generate_app(validated_plan, validated_description) # test = generate_test(code) diff --git a/nbs/CLI.ipynb b/nbs/CLI.ipynb index 54dccf3..d11c458 100644 --- a/nbs/CLI.ipynb +++ b/nbs/CLI.ipynb @@ -237,7 +237,9 @@ " try:\n", " _ensure_openai_api_key_set()\n", " validated_description, description_token = validate_app_description(description)\n", - " plan_token = generate_plan(validated_description, output_path)\n", + " \n", + " output_file = f\"{output_path}/plan.json\"\n", + " plan_token = generate_plan(validated_description, output_file)\n", "# code = generate_app(validated_plan, validated_description)\n", "# test = generate_test(code)\n", " \n", diff --git a/nbs/Helper.ipynb b/nbs/Helper.ipynb index bcad7c7..e76ef3a 100644 --- a/nbs/Helper.ipynb +++ b/nbs/Helper.ipynb @@ -618,21 +618,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python3", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" } }, "nbformat": 4, diff --git a/nbs/Plan_Generator.ipynb b/nbs/Plan_Generator.ipynb index 76fd944..8fd93a7 100644 --- a/nbs/Plan_Generator.ipynb +++ b/nbs/Plan_Generator.ipynb @@ -1366,7 +1366,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f1881cb0", + "id": "30091dcb", "metadata": {}, "outputs": [], "source": [ @@ -1380,8 +1380,7 @@ " contents: A JSON-formatted string to be saved as a JSON file.\n", " output_file: The path to the output file where the JSON content will be saved.\n", " \"\"\"\n", - " Path(output_file).parent.mkdir(exist_ok=True)\n", - "\n", + " Path(output_file).parent.mkdir(parents=True, exist_ok=True)\n", " with open(output_file, \"w\", encoding=\"utf-8\") as f:\n", " json.dump(json.loads(contents), f, indent=4)" ] @@ -1389,7 +1388,7 @@ { "cell_type": "code", "execution_count": null, - "id": "85a937f7", + "id": "c9f43be4", "metadata": {}, "outputs": [ { @@ -1447,16 +1446,16 @@ ], "source": [ "with TemporaryDirectory() as d:\n", - " output_path = f\"{str(d)}/fastkafka-gen\"\n", - " json_file_path = f\"{output_path}/plan.json\"\n", + " output_path = f\"{str(d)}/parent/child\"\n", + " output_file = f\"{output_path}/plan.json\"\n", " \n", - " _save_plan_as_json(fixture_response, json_file_path)\n", + " _save_plan_as_json(fixture_response, output_file)\n", " \n", " assert Path(output_path).exists()\n", " actual = [file for file in Path(output_path).iterdir()]\n", - " assert str(actual[0]) == json_file_path\n", + " assert str(actual[0]) == output_file\n", " \n", - " with open(json_file_path, 'r') as f:\n", + " with open(output_file, 'r') as f:\n", " contents = f.read()\n", " print(contents)\n" ] @@ -1471,15 +1470,15 @@ "# | export\n", "\n", "\n", - "def generate_plan(description: str, output_path: str) -> Tuple[str, str]:\n", + "def generate_plan(description: str, output_file: str) -> str:\n", " \"\"\"Generate a plan from user's application description\n", "\n", " Args:\n", " description: Validated User application description\n", - " output_path: Path to the output directory where generated files will be saved. This path should be relative to the current working directory\n", + " output_path: The path to the output file where the generated plan JSON will be saved.\n", "\n", " Returns:\n", - " The plan generated by OpenAI as a dictionary\n", + " The total token used to generate the plan\n", " \"\"\"\n", " with yaspin(\n", " text=\"Generating plan\", # (slowest step, usually takes 30 to 90 seconds)...\n", @@ -1490,7 +1489,6 @@ " plan_validator = ValidateAndFixResponse(plan_generator, _validate_response)\n", " validated_plan, total_tokens = plan_validator.fix(description)\n", " \n", - " output_file = f\"{output_path}/plan.json\"\n", " _save_plan_as_json(validated_plan, output_file)\n", "\n", " sp.text = \"\"\n", @@ -1508,54 +1506,9 @@ "name": "stdout", "output_type": "stream", "text": [ - " ✔ Plan generated and saved at: /tmp/tmpvqqjetxg/fastkafka-gen/plan.json \n", - "{\n", - " \"entities\": [\n", - " {\n", - " \"name\": \"StoreProduct\",\n", - " \"arguments\": {\n", - " \"product_name\": \"str\",\n", - " \"currency\": \"str\",\n", - " \"price\": \"float\"\n", - " }\n", - " }\n", - " ],\n", - " \"apps\": [\n", - " {\n", - " \"app_name\": \"product_app\",\n", - " \"kafka_brokers\": {\n", - " \"localhost\": {\n", - " \"url\": \"localhost\",\n", - " \"description\": \"local kafka broker\",\n", - " \"port\": 9092\n", - " }\n", - " },\n", - " \"title\": \"Product Kafka App\",\n", - " \"consumes_functions\": {\n", - " \"on_store_product\": {\n", - " \"topic\": \"store_product\",\n", - " \"prefix\": \"on\",\n", - " \"parameters\": {\n", - " \"msg\": \"StoreProduct\"\n", - " },\n", - " \"description\": \"This function will listen to the 'store_product' topic. It will consume the messages posted on the 'store_product' topic. The messages should have three attributes: 'product_name', 'currency', and 'price'. After consuming the data, it will produce a message to the 'change_currency' topic.\"\n", - " }\n", - " },\n", - " \"produces_functions\": {\n", - " \"to_change_currency\": {\n", - " \"topic\": \"change_currency\",\n", - " \"prefix\": \"to\",\n", - " \"parameters\": {\n", - " \"store_product\": \"StoreProduct\"\n", - " },\n", - " \"description\": \"This function will be triggered when a message is received from the 'store_product' topic. It takes a 'store_product' object as input and produces a message to the 'change_currency' topic. If the currency in the input 'store_product' is 'HRK', the currency will be set to 'EUR' and the price will be divided by 7.5. The function returns the updated 'store_product' object.\",\n", - " \"returns\": \"StoreProduct\"\n", - " }\n", - " }\n", - " }\n", - " ]\n", - "}\n", - "5337\n" + " ✔ Plan generated and saved at: /tmp/tmphqygrc39/fastkafka-gen/plan.json \n", + "{'entities': [{'name': 'StoreProduct', 'arguments': {'product_name': 'str', 'currency': 'str', 'price': 'float'}}], 'apps': [{'app_name': 'store_app', 'kafka_brokers': {'localhost': {'url': 'localhost', 'description': 'local kafka broker', 'port': 9092}}, 'title': 'Store Kafka App', 'consumes_functions': {'on_store_product': {'topic': 'store_product', 'prefix': 'on', 'parameters': {'msg': 'StoreProduct'}, 'description': \"This function listens to the 'store_product' topic and consumes messages with attributes 'product_name', 'currency', and 'price'. The consumed message should be of type 'StoreProduct'. After consuming the message, it produces a message to the 'change_currency' topic.\"}}, 'produces_functions': {'to_change_currency': {'topic': 'change_currency', 'prefix': 'to', 'parameters': {'store_product': 'StoreProduct'}, 'description': \"This function is triggered when a message is consumed from the 'store_product' topic. It takes a 'StoreProduct' object as input and produces a message to the 'change_currency' topic. If the currency in the input 'store_product' is 'HRK', the currency is set to 'EUR' and the price is divided by 7.5. The function returns the modified 'StoreProduct' object.\", 'returns': 'StoreProduct'}}}]}\n", + "5306\n" ] } ], @@ -1570,7 +1523,7 @@ " output_path = f\"{str(d)}/fastkafka-gen\"\n", " json_file_path = f\"{output_path}/plan.json\"\n", " \n", - " total_tokens = generate_plan(app_description, output_path)\n", + " total_tokens = generate_plan(app_description, json_file_path)\n", " \n", " assert Path(output_path).exists()\n", " actual = [file for file in Path(output_path).iterdir()]\n",