improve code structure

run-llama · Mar 25, 2024 · 6d347dc · 6d347dc
1 parent e49af1b
commit 6d347dc
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 84 deletions.
diff --git a/create-app.ts b/create-app.ts
@@ -130,8 +130,8 @@ export async function createApp({
     console.log(
       yellow(
         `You have selected tools that require configuration. Please configure them in the ${terminalLink(
-          "tools_config.json",
-          `file://${root}/tools_config.json`,
+          "config/tools.json",
+          `file://${root}/config/tools.json`,
         )} file.`,
       ),
     );

diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts
@@ -1,7 +1,6 @@
 import fs from "fs/promises";
 import path from "path";
 import {
-  FileSourceConfig,
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
@@ -99,29 +98,6 @@ const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => {
   }
 };
 
-const getDataSourceEnvs = (
-  dataSource: TemplateDataSource,
-  llamaCloudKey?: string,
-) => {
-  switch (dataSource.type) {
-    case "file":
-    case "folder":
-      return [
-        ...((dataSource?.config as FileSourceConfig).useLlamaParse
-          ? [
-              {
-                name: "LLAMA_CLOUD_API_KEY",
-                description: `The Llama Cloud API key.`,
-                value: llamaCloudKey,
-              },
-            ]
-          : []),
-      ];
-    default:
-      return [];
-  }
-};
-
 export const createBackendEnvFile = async (
   root: string,
   opts: {
@@ -153,9 +129,15 @@ export const createBackendEnvFile = async (
 
     // Add vector database environment variables
     ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []),
-    // Add data source environment variables
-    ...(opts.dataSource
-      ? getDataSourceEnvs(opts.dataSource, opts.llamaCloudKey)
+    // Add LlamaCloud API key
+    ...(opts.llamaCloudKey
+      ? [
+          {
+            name: "LLAMA_CLOUD_API_KEY",
+            description: `The Llama Cloud API key.`,
+            value: opts.llamaCloudKey,
+          },
+        ]
       : []),
   ];
   let envVars: EnvVar[] = [];

diff --git a/helpers/python.ts b/helpers/python.ts
@@ -238,12 +238,13 @@ export const installPythonTemplate = async ({
         parents: true,
         cwd: path.join(compPath, "engines", "python", "agent"),
       });
-      // Write tools_config.json
+      // Write tool configs
       const configContent: Record<string, any> = {};
       tools.forEach((tool) => {
         configContent[tool.name] = tool.config ?? {};
       });
-      const configFilePath = path.join(root, "tools_config.json");
+      const configFilePath = path.join(root, "config/tools.json");
+      await fs.mkdir(path.join(root, "config"), { recursive: true });
       await fs.writeFile(
         configFilePath,
         JSON.stringify(configContent, null, 2),
@@ -255,22 +256,7 @@ export const installPythonTemplate = async ({
       });
     }
 
-    const dataSourceType = dataSource?.type;
-    if (dataSourceType !== undefined && dataSourceType !== "none") {
-      let loaderFolder: string;
-      if (dataSourceType === "file" || dataSourceType === "folder") {
-        const dataSourceConfig = dataSource?.config as FileSourceConfig;
-        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
-      } else {
-        loaderFolder = dataSourceType;
-      }
-      await copy("**", enginePath, {
-        parents: true,
-        cwd: path.join(compPath, "loaders", "python", loaderFolder),
-      });
-    }
-
-    // Generate loader configs
+    // Write loader configs
     if (dataSource?.type === "web") {
       const config = dataSource.config as WebSourceConfig[];
       const webLoaderConfig = config.map((c) => {
@@ -280,7 +266,8 @@ export const installPythonTemplate = async ({
           depth: c.depth || 1,
         };
       });
-      const loaderConfigPath = path.join(root, "loaders.json");
+      const loaderConfigPath = path.join(root, "config/loaders.json");
+      await fs.mkdir(path.join(root, "config"), { recursive: true });
       await fs.writeFile(
         loaderConfigPath,
         JSON.stringify(
@@ -292,6 +279,21 @@ export const installPythonTemplate = async ({
         ),
       );
     }
+
+    const dataSourceType = dataSource?.type;
+    if (dataSourceType !== undefined && dataSourceType !== "none") {
+      let loaderFolder: string;
+      if (dataSourceType === "file" || dataSourceType === "folder") {
+        const dataSourceConfig = dataSource?.config as FileSourceConfig;
+        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
+      } else {
+        loaderFolder = dataSourceType;
+      }
+      await copy("**", enginePath, {
+        parents: true,
+        cwd: path.join(compPath, "loaders", "python", loaderFolder),
+      });
+    }
   }
 
   const addOnDependencies = getAdditionalDependencies(

diff --git a/questions.ts b/questions.ts
@@ -11,6 +11,7 @@ import {
   FileSourceConfig,
   TemplateDataSourceType,
   TemplateFramework,
+  WebSourceConfig,
 } from "./helpers";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
 import { templatesDir } from "./helpers/dir";
@@ -755,19 +756,6 @@ export const askQuestions = async (
   }
 
   if (program.dataSource?.type === "web" && program.framework === "fastapi") {
-    const validateUrl = (value: string) => {
-      for (let url of value.split(",")) {
-        if (!url.includes("://")) {
-          url = `https://${url}`;
-        }
-        const urlObj = new URL(url);
-        if (urlObj.protocol !== "https:" && urlObj.protocol !== "http:") {
-          return `URL=${url} has invalid protocol, only allow http or https`;
-        }
-      }
-      return true;
-    };
-
     program.dataSource.config = [];
 
     while (true) {
@@ -777,15 +765,24 @@ export const askQuestions = async (
           name: "baseUrl",
           message: "Please provide base URL of the website: ",
           initial: "https://www.llamaindex.ai",
-          validate: validateUrl,
-        },
-        {
-          type: "text",
-          name: "prefix",
-          message:
-            "Please provide prefix for the URLs you would like to crawl: ",
-          initial: "https://www.llamaindex.ai/enterprise",
-          validate: validateUrl,
+          validate: (value: string) => {
+            if (!value.includes("://")) {
+              value = `https://${value}`;
+            }
+            const urlObj = new URL(value);
+            if (urlObj.protocol !== "https:" && urlObj.protocol !== "http:") {
+              return `URL=${value} has invalid protocol, only allow http or https`;
+            }
+            // Check duplicated URL
+            if (
+              (program.dataSource?.config as WebSourceConfig[]).some(
+                (c) => c.baseUrl === value,
+              )
+            ) {
+              return `URL=${value} is already added. Please provide a different URL.`;
+            }
+            return true;
+          },
         },
         {
           type: "toggle",
@@ -796,11 +793,12 @@ export const askQuestions = async (
           inactive: "No",
         },
       ];
-      let { shouldContinue, baseUrl, prefix } = await prompts(
-        questions,
-        handlers,
-      );
-      program.dataSource.config.push({ baseUrl, prefix, depth: 1 });
+      let { shouldContinue, baseUrl } = await prompts(questions, handlers);
+      program.dataSource.config.push({
+        baseUrl: baseUrl,
+        prefix: baseUrl,
+        depth: 1,
+      });
       if (!shouldContinue) {
         break;
       }

diff --git a/templates/components/engines/python/agent/tools.py b/templates/components/engines/python/agent/tools.py
@@ -26,7 +26,7 @@ def create_tool(tool_name: str, **kwargs) -> list[FunctionTool]:
     @staticmethod
     def from_env() -> list[FunctionTool]:
         tools = []
-        with open("tools_config.json", "r") as f:
+        with open("config/tools.json", "r") as f:
             tool_configs = json.load(f)
             for name, config in tool_configs.items():
                 tools += ToolFactory.create_tool(name, **config)

diff --git a/templates/components/loaders/python/web/loader.py b/templates/components/loaders/python/web/loader.py
@@ -4,29 +4,28 @@
 from llama_index.readers.web import WholeSiteReader
 
 
-class LoaderConfig(BaseModel):
+class WebLoaderConfig(BaseModel):
     base_url: str
     prefix: str
     max_depth: int = Field(default=1, ge=0)
 
 
 def load_configs():
-    with open("loaders.json") as f:
+    with open("config/loaders.json") as f:
         configs = json.load(f)
     web_config = configs.get("web", None)
     if web_config is None:
         raise ValueError("No web config found in loaders.json")
-    print(web_config)
-    return [LoaderConfig(**config) for config in web_config]
+    return [WebLoaderConfig(**config) for config in web_config]
 
 
 def get_documents():
     web_config = load_configs()
     documents = []
-    for config in web_config:
+    for entry in web_config:
         scraper = WholeSiteReader(
-            prefix=config.prefix,
-            max_depth=config.max_depth,
+            prefix=entry.prefix,
+            max_depth=entry.max_depth,
         )
-        documents.extend(scraper.load_data(config.base_url))
+        documents.extend(scraper.load_data(entry.base_url))
     return documents
diff --git a/templates/types/simple/fastapi/README-template.md b/templates/types/simple/fastapi/README-template.md
@@ -19,6 +19,8 @@ Example `backend/.env` file:
 OPENAI_API_KEY=<openai_api_key>
 ```
 
+If you are using any tools or data sources, you can update their config files in the `config` folder.
+
 Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
 
 ```

diff --git a/templates/types/streaming/fastapi/README-template.md b/templates/types/streaming/fastapi/README-template.md
@@ -19,6 +19,8 @@ Example `.env` file:
 OPENAI_API_KEY=<openai_api_key>
 ```
 
+If you are using any tools or data sources, you can update their config files in the `config` folder.
+
 Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
 
 ```