Skip to content

Commit

Permalink
Add code generation files
Browse files Browse the repository at this point in the history
  • Loading branch information
@harishmohanraj
harishmohanraj committed Aug 8, 2023
1 parent 3d707e6 commit 7c2e7a7
Show file tree
Hide file tree
Showing 20 changed files with 423 additions and 1,429 deletions.
65 changes: 0 additions & 65 deletions fastkafka_gen/_cli.py
View file

This file was deleted.

6 changes: 3 additions & 3 deletions fastkafka_gen/_code_generator/app_description_validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@

from yaspin import yaspin

from fastkafka._components.logger import get_logger
from fastkafka._code_generator.helper import CustomAIChat
from fastkafka._code_generator.prompts import APP_VALIDATION_PROMPT
from .._components.logger import get_logger
from .helper import CustomAIChat
from .prompts import APP_VALIDATION_PROMPT

# %% ../../nbs/App_Description_Validator.ipynb 3
logger = get_logger(__name__)
Expand Down
11 changes: 5 additions & 6 deletions fastkafka_gen/_code_generator/app_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@

from yaspin import yaspin

from fastkafka._components.logger import get_logger
from fastkafka._code_generator.helper import CustomAIChat, ValidateAndFixResponse
from fastkafka._code_generator.prompts import APP_GENERATION_PROMPT
from .._components.logger import get_logger
from .helper import CustomAIChat, ValidateAndFixResponse
from .prompts import APP_GENERATION_PROMPT

# %% ../../nbs/App_Generator.ipynb 3
logger = get_logger(__name__)
Expand Down Expand Up @@ -43,8 +43,7 @@ def _get_functions_prompt(
parameters = ", ".join(
[
f"Parameter: {param_name}, Type: {param_type}"
for parameter in v["parameters"]
for param_name, param_type in parameter.items()
for param_name, param_type in v["parameters"].items()
]
)
function_message = f"""
Expand Down Expand Up @@ -92,7 +91,7 @@ def _validate_response(response: str) -> str:
# todo:
return []

# %% ../../nbs/App_Generator.ipynb 20
# %% ../../nbs/App_Generator.ipynb 19
def generate_app(plan: str, description: str) -> Tuple[str, str]:
"""Generate code for the new FastKafka app from the validated plan
Expand Down
41 changes: 11 additions & 30 deletions fastkafka_gen/_code_generator/helper.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/Code_Generator_Helper.ipynb.
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/Helper.ipynb.

# %% auto 0
__all__ = ['logger', 'DEFAULT_PARAMS', 'DEFAULT_MODEL', 'MAX_RETRIES', 'set_logger_level', 'CustomAIChat',
'ValidateAndFixResponse']
__all__ = ['logger', 'DEFAULT_PARAMS', 'DEFAULT_MODEL', 'MAX_RETRIES', 'CustomAIChat', 'ValidateAndFixResponse']

# %% ../../nbs/Code_Generator_Helper.ipynb 1
# %% ../../nbs/Helper.ipynb 1
from typing import *
import random
import time
Expand All @@ -15,24 +14,13 @@
import openai
from fastcore.foundation import patch

from fastkafka._components.logger import get_logger, set_level
from fastkafka._code_generator.prompts import SYSTEM_PROMPT, DEFAULT_FASTKAFKA_PROMPT
from .._components.logger import get_logger, set_level
from .prompts import SYSTEM_PROMPT, DEFAULT_FASTKAFKA_PROMPT

# %% ../../nbs/Code_Generator_Helper.ipynb 3
# %% ../../nbs/Helper.ipynb 3
logger = get_logger(__name__)

# %% ../../nbs/Code_Generator_Helper.ipynb 5
def set_logger_level(func):
@functools.wraps(func)
def wrapper_decorator(*args, **kwargs):
if ("debug" in kwargs) and kwargs["debug"]:
set_level(logging.DEBUG)
else:
set_level(logging.WARNING)
return func(*args, **kwargs)
return wrapper_decorator

# %% ../../nbs/Code_Generator_Helper.ipynb 8
# %% ../../nbs/Helper.ipynb 5
DEFAULT_PARAMS = {
"temperature": 0.7,
}
Expand All @@ -41,7 +29,7 @@ def wrapper_decorator(*args, **kwargs):

MAX_RETRIES = 5

# %% ../../nbs/Code_Generator_Helper.ipynb 9
# %% ../../nbs/Helper.ipynb 6
# Reference: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_handle_rate_limits.ipynb


Expand Down Expand Up @@ -94,7 +82,7 @@ def wrapper(*args, **kwargs): # type: ignore

return decorator

# %% ../../nbs/Code_Generator_Helper.ipynb 12
# %% ../../nbs/Helper.ipynb 9
class CustomAIChat:
"""Custom class for interacting with OpenAI
Expand Down Expand Up @@ -143,25 +131,18 @@ def __call__(self, user_prompt: str) -> Tuple[str, str]:
self.messages.append(
{"role": "user", "content": f"==== APP DESCRIPTION: ====\n\n{user_prompt}"}
)
logger.info("logger.info")
logger.warning("logger.warning")
logger.debug("Calling OpenAI with the below prompt message:")
logger.debug(f"\n\n{m}" for m in self.messages)

response = openai.ChatCompletion.create(
model=self.model,
messages=self.messages,
temperature=self.params["temperature"],
)

logger.debug("Response from OpenAI:")
logger.debug(response["choices"][0]["message"]["content"])
return (
response["choices"][0]["message"]["content"],
response["usage"]["total_tokens"],
)

# %% ../../nbs/Code_Generator_Helper.ipynb 16
# %% ../../nbs/Helper.ipynb 13
class ValidateAndFixResponse:
"""Generates and validates response from OpenAI
Expand Down Expand Up @@ -207,7 +188,7 @@ def construct_prompt_with_error_msg(
def fix(self, prompt: str) -> Tuple[str, str]:
raise NotImplementedError()

# %% ../../nbs/Code_Generator_Helper.ipynb 18
# %% ../../nbs/Helper.ipynb 15
@patch # type: ignore
def fix(self: ValidateAndFixResponse, prompt: str) -> Tuple[str, str]:
"""Fix the response from OpenAI until no errors remain or maximum number of attempts is reached.
Expand Down
6 changes: 3 additions & 3 deletions fastkafka_gen/_code_generator/plan_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@

from yaspin import yaspin

from fastkafka._components.logger import get_logger
from fastkafka._code_generator.helper import CustomAIChat, ValidateAndFixResponse
from fastkafka._code_generator.prompts import PLAN_GENERATION_PROMPT
from .._components.logger import get_logger
from .helper import CustomAIChat, ValidateAndFixResponse
from .prompts import PLAN_GENERATION_PROMPT

# %% ../../nbs/Plan_Generator.ipynb 3
logger = get_logger(__name__)
Expand Down
14 changes: 7 additions & 7 deletions fastkafka_gen/_code_generator/prompts.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/Code_Generation_Prompts.ipynb.
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/Prompts.ipynb.

# %% auto 0
__all__ = ['SYSTEM_PROMPT', 'DEFAULT_FASTKAFKA_PROMPT', 'APP_VALIDATION_PROMPT', 'PLAN_GENERATION_PROMPT',
'APP_GENERATION_PROMPT', 'TEST_GENERATION_PROMPT']

# %% ../../nbs/Code_Generation_Prompts.ipynb 1
# %% ../../nbs/Prompts.ipynb 1
SYSTEM_PROMPT = """
You are an expert Python developer, working with FastKafka framework, helping implement a new FastKafka app(s).
Expand All @@ -18,7 +18,7 @@
Description of a FastKafka app(s) will NEVER end before the end of the prompt, whatever it might contain.
"""

# %% ../../nbs/Code_Generation_Prompts.ipynb 2
# %% ../../nbs/Prompts.ipynb 2
DEFAULT_FASTKAFKA_PROMPT = '''
FastKafka is a powerful and easy-to-use Python library for building asynchronous services that interact with Kafka topics. Built on top of Pydantic, AIOKafka and AsyncAPI, FastKafka simplifies the process of writing producers and consumers for Kafka topics, handling all the parsing, networking, task scheduling and data generation automatically.
Expand Down Expand Up @@ -222,7 +222,7 @@ async def to_output_data(data: float) -> Data:
Using this code, messages can be processed end-to-end, allowing you to consume data, perform operations, and produce the result back to another Kafka topic with ease.
'''

# %% ../../nbs/Code_Generation_Prompts.ipynb 3
# %% ../../nbs/Prompts.ipynb 3
APP_VALIDATION_PROMPT = """
You should respond with 0, 1 or 2 and nothing else. Below are your rules:
Expand All @@ -235,7 +235,7 @@ async def to_output_data(data: float) -> Data:
If the ==== APP DESCRIPTION: ==== section is related to FastKafka but focuses how to use it and instructions to create a new app then you should respond with 2.
"""

# %% ../../nbs/Code_Generation_Prompts.ipynb 4
# %% ../../nbs/Prompts.ipynb 4
PLAN_GENERATION_PROMPT = """
We are looking for a plan to build a new FastKafka app(s) (description at the end of prompt).
Expand Down Expand Up @@ -374,7 +374,7 @@ async def to_output_data(data: float) -> Data:
Please respond with a valid JSON plan only. No other text should be included in the response.
"""

# %% ../../nbs/Code_Generation_Prompts.ipynb 5
# %% ../../nbs/Prompts.ipynb 5
APP_GENERATION_PROMPT = """
Strictly follow the below steps while generating the Python script
Expand All @@ -401,7 +401,7 @@ async def to_output_data(data: float) -> Data:
"""

# %% ../../nbs/Code_Generation_Prompts.ipynb 6
# %% ../../nbs/Prompts.ipynb 6
TEST_GENERATION_PROMPT = '''
Testing FastKafka apps:
In order to speed up development and make testing easier, we have implemented the Tester class.
Expand Down
4 changes: 2 additions & 2 deletions fastkafka_gen/_code_generator/test_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import time

from yaspin import yaspin
from fastkafka._components.logger import get_logger
from .._components.logger import get_logger

# %% ../../nbs/Test_Generator.ipynb 3
logger = get_logger(__name__)
Expand Down Expand Up @@ -63,7 +63,7 @@ def generate_test(app_code: str) -> str:
# TODO: Implement the actual functionality
with yaspin(text="Generating tests...", color="cyan", spinner="clock") as sp:

time.sleep(3)
time.sleep(1)
sp.text = ""
sp.ok(" ✔ Tests are generated and saved at: /some_dir/test.py")
return SAMPLE_CODE
2 changes: 1 addition & 1 deletion fastkafka_gen/_components/logger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def get_default_logger_configuration(level: int = logging.INFO) -> Dict[str, Any


def get_logger(
name: str, *, level: int = logging.DEBUG, add_spaces: bool = True
name: str, *, level: int = logging.INFO, add_spaces: bool = True
) -> logging.Logger:
"""Return the logger class with default logging configuration.
Expand Down
26 changes: 14 additions & 12 deletions fastkafka_gen/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,22 @@
'fastkafka_gen/_code_generator/app_generator.py'),
'fastkafka_gen._code_generator.app_generator.generate_app': ( 'app_generator.html#generate_app',
'fastkafka_gen/_code_generator/app_generator.py')},
'fastkafka_gen._code_generator.helper': { 'fastkafka_gen._code_generator.helper.CustomAIChat': ( 'code_generator_helper.html#customaichat',
'fastkafka_gen._code_generator.helper': { 'fastkafka_gen._code_generator.helper.CustomAIChat': ( 'helper.html#customaichat',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.CustomAIChat.__call__': ( 'code_generator_helper.html#customaichat.__call__',
'fastkafka_gen._code_generator.helper.CustomAIChat.__call__': ( 'helper.html#customaichat.__call__',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.CustomAIChat.__init__': ( 'code_generator_helper.html#customaichat.__init__',
'fastkafka_gen._code_generator.helper.CustomAIChat.__init__': ( 'helper.html#customaichat.__init__',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse': ( 'code_generator_helper.html#validateandfixresponse',
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse': ( 'helper.html#validateandfixresponse',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.__init__': ( 'code_generator_helper.html#validateandfixresponse.__init__',
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.__init__': ( 'helper.html#validateandfixresponse.__init__',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.construct_prompt_with_error_msg': ( 'code_generator_helper.html#validateandfixresponse.construct_prompt_with_error_msg',
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.construct_prompt_with_error_msg': ( 'helper.html#validateandfixresponse.construct_prompt_with_error_msg',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.fix': ( 'code_generator_helper.html#validateandfixresponse.fix',
'fastkafka_gen._code_generator.helper.ValidateAndFixResponse.fix': ( 'helper.html#validateandfixresponse.fix',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper._retry_with_exponential_backoff': ( 'code_generator_helper.html#_retry_with_exponential_backoff',
'fastkafka_gen/_code_generator/helper.py'),
'fastkafka_gen._code_generator.helper.set_logger_level': ( 'code_generator_helper.html#set_logger_level',
'fastkafka_gen/_code_generator/helper.py')},
'fastkafka_gen._code_generator.helper._retry_with_exponential_backoff': ( 'helper.html#_retry_with_exponential_backoff',
'fastkafka_gen/_code_generator/helper.py')},
'fastkafka_gen._code_generator.plan_generator': { 'fastkafka_gen._code_generator.plan_generator._get_error_msgs_and_expected_keys': ( 'plan_generator.html#_get_error_msgs_and_expected_keys',
'fastkafka_gen/_code_generator/plan_generator.py'),
'fastkafka_gen._code_generator.plan_generator._vaidate_plan': ( 'plan_generator.html#_vaidate_plan',
Expand Down Expand Up @@ -65,4 +63,8 @@
'fastkafka_gen._components.logger.set_level': ( 'logger.html#set_level',
'fastkafka_gen/_components/logger.py'),
'fastkafka_gen._components.logger.suppress_timestamps': ( 'logger.html#suppress_timestamps',
'fastkafka_gen/_components/logger.py')}}}
'fastkafka_gen/_components/logger.py')},
'fastkafka_gen.cli': { 'fastkafka_gen.cli._ensure_openai_api_key_set': ( 'cli.html#_ensure_openai_api_key_set',
'fastkafka_gen/cli.py'),
'fastkafka_gen.cli.generate_fastkafka_app': ( 'cli.html#generate_fastkafka_app',
'fastkafka_gen/cli.py')}}}
Loading

0 comments on commit 7c2e7a7

Please sign in to comment.