Add code generator files

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+
+
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v4.4.0"
+    hooks:
+      #    -   id: trailing-whitespace
+      #    -   id: end-of-file-fixer
+      #    -   id: check-yaml
+    -   id: check-added-large-files
+
+- repo: https://github.com/PyCQA/bandit
+  rev: '1.7.5'
+  hooks:
+  - id: bandit
+    #args: ["-r", "nbdev_mkdocs"]
+
+- repo: https://github.com/returntocorp/semgrep
+  rev: "v1.14.0"
+  hooks:
+    - id: semgrep
+      name: Semgrep 
+      args: ["--config", "auto", "--error"]
+
+- repo: https://github.com/Yelp/detect-secrets
+  rev: v1.4.0
+  hooks:
+    - id: detect-secrets
+      args: ["--baseline", ".secrets.baseline"]

.secrets.baseline

@@ -0,0 +1,116 @@
+{
+  "version": "1.4.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {},
+  "generated_at": "2023-08-08T10:13:31Z"
+}

fastkafka_gen/_cli.py

@@ -0,0 +1,65 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/CLI.ipynb.
+
+# %% auto 0
+__all__ = ['logger', 'run']
+
+# %% ../nbs/CLI.ipynb 1
+from typing import *
+
+import typer
+
+from fastkafka._components.logger import get_logger
+
+# %% ../nbs/CLI.ipynb 5
+logger = get_logger(__name__, level=20)
+
+# %% ../nbs/CLI.ipynb 8
+_app = typer.Typer(help="")
+
+# %% ../nbs/CLI.ipynb 9
+@_app.command(
+    help="Runs Fast Kafka API application",
+)
+def run(
+    num_workers: int = typer.Option(
+        multiprocessing.cpu_count(),
+        help="Number of FastKafka instances to run, defaults to number of CPU cores.",
+    ),
+    app: str = typer.Argument(
+        ...,
+        help="input in the form of 'path:app', where **path** is the path to a python file and **app** is an object of type **FastKafka**.",
+    ),
+    kafka_broker: str = typer.Option(
+        "localhost",
+        help="kafka_broker, one of the keys of the kafka_brokers dictionary passed in the constructor of FastaKafka class.",
+    ),
+) -> None:
+    """
+    Runs FastKafka application.
+
+    Args:
+        num_workers (int): Number of FastKafka instances to run, defaults to the number of CPU cores.
+        app (str): Input in the form of 'path:app', where **path** is the path to a python file and **app** is an object of type **FastKafka**.
+        kafka_broker (str): Kafka broker, one of the keys of the kafka_brokers dictionary passed in the constructor of FastKafka class.
+
+    Raises:
+        typer.Exit: If there is an unexpected internal error.
+    """
+    try:
+        asyncio.run(
+            run_fastkafka_server(
+                num_workers=num_workers, app=app, kafka_broker=kafka_broker
+            )
+        )
+    except Exception as e:
+        typer.secho(f"Unexpected internal error: {e}", err=True, fg=typer.colors.RED)
+        raise typer.Exit(1)
+
+# %% ../nbs/CLI.ipynb 12
+_app.add_typer(_cli_docs._docs_app, name="docs")
+
+# %% ../nbs/CLI.ipynb 20
+_app.add_typer(_cli_testing._testing_app, name="testing")
+
+# %% ../nbs/CLI.ipynb 23
+_app.add_typer(_cli_code_generator._code_generator_app, name="code_generator")

fastkafka_gen/_cli_code_generator.py

@@ -0,0 +1,101 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/Code_Generator.ipynb.
+
+# %% auto 0
+__all__ = ['logger', 'OPENAI_KEY_EMPTY_ERROR', 'OPENAI_KEY_NOT_SET_ERROR', 'generate_fastkafka_app']
+
+# %% ../nbs/Code_Generator.ipynb 1
+from typing import *
+import os
+
+import typer
+
+from fastkafka._components.logger import get_logger
+from fastkafka._code_generator.app_description_validator import validate_app_description
+from fastkafka._code_generator.plan_generator import generate_plan
+from fastkafka._code_generator.app_generator import generate_app
+from fastkafka._code_generator.test_generator import generate_test
+from fastkafka._code_generator.helper import set_logger_level
+
+# %% ../nbs/Code_Generator.ipynb 3
+logger = get_logger(__name__)
+
+# %% ../nbs/Code_Generator.ipynb 6
+OPENAI_KEY_EMPTY_ERROR = "Error: OPENAI_API_KEY cannot be empty. Please set a valid OpenAI API key in OPENAI_API_KEY environment variable and try again.\nYou can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details."
+OPENAI_KEY_NOT_SET_ERROR = "Error: OPENAI_API_KEY not found in environment variables. Set a valid OpenAI API key in OPENAI_API_KEY environment variable and try again. You can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details."
+
+
+def _ensure_openai_api_key_set() -> None:
+    """Ensure the 'OPENAI_API_KEY' environment variable is set and is not empty.
+
+    Raises:
+        KeyError: If the 'OPENAI_API_KEY' environment variable is not found.
+        ValueError: If the 'OPENAI_API_KEY' environment variable is found but its value is empty.
+    """
+    try:
+        openai_api_key = os.environ["OPENAI_API_KEY"]
+        if openai_api_key == "":
+            raise ValueError(OPENAI_KEY_EMPTY_ERROR)
+    except KeyError:
+        raise KeyError(OPENAI_KEY_NOT_SET_ERROR)
+
+# %% ../nbs/Code_Generator.ipynb 10
+_code_generator_app = typer.Typer(
+    short_help="Commands for accelerating FastKafka app creation using advanced AI technology",
+     help="""Commands for accelerating FastKafka app creation using advanced AI technology.
+
+These commands use a combination of OpenAI's gpt-3.5-turbo and gpt-3.5-turbo-16k models to generate FastKafka code. To access this feature, kindly sign up if you haven't already and create an API key with OpenAI. You can generate API keys in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.
+
+Once you have the key, please set it in the OPENAI_API_KEY environment variable before executing the code generation commands.
+
+Note: Accessing OpenAI API incurs charges. However, when you sign up for the first time, you usually get free credits that are more than enough to generate multiple FastKafka applications. For further information on pricing and free credicts, check this link: https://openai.com/pricing
+    """,
+)
+
+# %% ../nbs/Code_Generator.ipynb 11
+@_code_generator_app.command(
+    "generate",
+    help="Generate a new FastKafka app(s) effortlessly with advanced AI assistance",
+)
+@set_logger_level
+def generate_fastkafka_app(
+    description: str = typer.Argument(
+        ...,
+        help="""Summarize your FastKafka app in a few sentences!
+
+
+\nInclude details about message classes, FastKafka app configuration (e.g., kafka_brokers), consumer and producer functions, and specify the business logic to be implemented. 
+
+
+\nThe simpler and more specific the app description is, the better the generated app will be. Please refer to the below example for inspiration:
+
+
+\nCreate a FastKafka application that consumes messages from the "store_product" topic. These messages should have three attributes: "product_name," "currency," and "price". While consuming, the app needs to produce a message to the "change_currency" topic. The function responsible for producing should take a "store_product" object as input and return the same object. Additionally, this function should check if the currency in the input "store_product" is "HRK." If it is, then the currency should be changed to "EUR," and the price should be divided by 7.5. Remember, the app should use a "localhost" broker.
+
+
+\n"""
+    ),
+    debug: bool = typer.Option(
+        False,
+        "--debug",
+        "-d",
+        help="Enable verbose logging by setting the logger level to DEBUG.",
+    ),
+) -> None:
+    """Generate a new FastKafka app(s) effortlessly with advanced AI assistance"""
+    try:
+        _ensure_openai_api_key_set()
+        validated_description, description_token = validate_app_description(description)
+#         validated_plan, plan_token = generate_plan(validated_description)
+#         code = generate_app(validated_plan, validated_description)
+#         test = generate_test(code)
+
+#         total_token_usage = description_token + plan_token
+#         typer.secho(f" ▶ Total tokens usage: {total_token_usage}", fg=typer.colors.CYAN)
+        typer.secho("✨  All files were successfully generated.!", fg=typer.colors.CYAN)
+
+    except (ValueError, KeyError) as e:
+        typer.secho(e, err=True, fg=typer.colors.RED)
+        raise typer.Exit(code=1)
+    except Exception as e:
+        typer.secho(f"Unexpected internal error: {e}", err=True, fg=typer.colors.RED)
+        raise typer.Exit(code=1)

fastkafka_gen/_code_generator/app_description_validator.py

@@ -0,0 +1,52 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/App_Description_Validator.ipynb.
+
+# %% auto 0
+__all__ = ['logger', 'ERROR_RESPONSE', 'GENERAL_FASTKAFKA_RESPONSE', 'validate_app_description']
+
+# %% ../../nbs/App_Description_Validator.ipynb 1
+from typing import *
+import time
+
+from yaspin import yaspin
+
+from fastkafka._components.logger import get_logger
+from fastkafka._code_generator.helper import CustomAIChat
+from fastkafka._code_generator.prompts import APP_VALIDATION_PROMPT
+
+# %% ../../nbs/App_Description_Validator.ipynb 3
+logger = get_logger(__name__)
+
+# %% ../../nbs/App_Description_Validator.ipynb 5
+ERROR_RESPONSE = "I apologize, but I can only respond to queries related to FastKafka code generation. Feel free to ask me about using FastKafka, and I'll do my best to help you with that!"
+GENERAL_FASTKAFKA_RESPONSE = "Great to see your interest in FastKafka! Unfortunately, I can only generate FastKafka code and offer assistance in that area. For general information about FastKafka, please visit https://fastkafka.airt.ai/"
+
+# %% ../../nbs/App_Description_Validator.ipynb 6
+def validate_app_description(description: str) -> Tuple[str, str]:
+    """Validate the user's application description
+
+    If the description is unrelated to FastKafka or contains insensitive/inappropriate language, show an error
+    message and exit the program. Otherwise, display the success message in the terminal.
+
+    Args:
+        description: User's application description
+        
+    Raises:
+        ValueError: If the application description is invalid
+    """
+
+    print("✨  Generating a new FastKafka application!")
+    with yaspin(
+        text="Validating the application description...", color="cyan", spinner="clock"
+    ) as sp:
+
+        ai = CustomAIChat(model = "gpt-3.5-turbo", user_prompt=APP_VALIDATION_PROMPT)
+        response, total_tokens = ai(description)
+
+        sp.text = ""
+        if response == "0":
+            raise ValueError(f"✘ Error: Application description validation failed.\n{ERROR_RESPONSE}")
+        elif response == "1":
+            raise ValueError(f"✘ Error: Application description validation failed.\n{GENERAL_FASTKAFKA_RESPONSE}")
+        else:
+            sp.ok(" ✔ Application description validated")
+            return description, total_tokens