feat: Add multiple data sources

create-app.ts

@@ -41,8 +41,9 @@ export async function createApp({
   vectorDb,
   externalPort,
   postInstallAction,
-  dataSource,
+  dataSources,
   tools,
+  useLlamaParse,
   observability,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
@@ -89,8 +90,9 @@ export async function createApp({
     vectorDb,
     externalPort,
     postInstallAction,
-    dataSource,
+    dataSources,
     tools,
+    useLlamaParse,
     observability,
   };
 

helpers/env-variables.ts

@@ -107,7 +107,7 @@ export const createBackendEnvFile = async (
     model?: string;
     embeddingModel?: string;
     framework?: TemplateFramework;
-    dataSource?: TemplateDataSource;
+    dataSources?: TemplateDataSource[];
     port?: number;
   },
 ) => {

helpers/index.ts

@@ -27,8 +27,8 @@ async function generateContextData(
   packageManager?: PackageManager,
   openAiKey?: string,
   vectorDb?: TemplateVectorDB,
-  dataSource?: TemplateDataSource,
   llamaCloudKey?: string,
+  useLlamaParse?: boolean,
 ) {
   if (packageManager) {
     const runGenerate = `${cyan(
@@ -37,8 +37,7 @@ async function generateContextData(
         : `${packageManager} run generate`,
     )}`;
     const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"];
-    const llamaCloudKeyConfigured = (dataSource?.config as FileSourceConfig)
-      ?.useLlamaParse
+    const llamaCloudKeyConfigured = useLlamaParse
       ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
       : true;
     const hasVectorDb = vectorDb && vectorDb !== "none";
@@ -97,7 +96,6 @@ const copyContextData = async (
       console.log("Missing file path in config");
       process.exit(1);
     }
-    return;
   }
 
   // Copy folder
@@ -116,7 +114,6 @@ const copyContextData = async (
         cwd: p,
       });
     }
-    return;
   }
 };
 
@@ -166,12 +163,17 @@ export const installTemplate = async (
       model: props.model,
       embeddingModel: props.embeddingModel,
       framework: props.framework,
-      dataSource: props.dataSource,
+      dataSources: props.dataSources,
       port: props.externalPort,
     });
 
     if (props.engine === "context") {
-      await copyContextData(props.root, props.dataSource);
+      console.log("\nGenerating context data...\n");
+      for (const ds of props.dataSources) {
+        if (ds.type === "file" || ds.type === "folder") {
+          await copyContextData(props.root, ds);
+        }
+      }
       if (
         props.postInstallAction === "runApp" ||
         props.postInstallAction === "dependencies"
@@ -181,14 +183,14 @@ export const installTemplate = async (
           props.packageManager,
           props.openAiKey,
           props.vectorDb,
-          props.dataSource,
           props.llamaCloudKey,
+          props.useLlamaParse,
         );
       }
     }
   } else {
     // this is a frontend for a full-stack app, create .env file with model information
-    createFrontendEnvFile(props.root, {
+    await createFrontendEnvFile(props.root, {
       model: props.model,
       customApiPath: props.customApiPath,
     });

helpers/python.ts

@@ -8,7 +8,6 @@ import { templatesDir } from "./dir";
 import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { Tool } from "./tools";
 import {
-  FileSourceConfig,
   InstallTemplateArgs,
   TemplateDataSource,
   TemplateVectorDB,
@@ -180,18 +179,20 @@ export const installPythonTemplate = async ({
   framework,
   engine,
   vectorDb,
-  dataSource,
+  dataSources,
   tools,
   postInstallAction,
+  useLlamaParse,
 }: Pick<
   InstallTemplateArgs,
   | "root"
   | "framework"
   | "template"
   | "engine"
   | "vectorDb"
-  | "dataSource"
+  | "dataSources"
   | "tools"
+  | "useLlamaParse"
   | "postInstallAction"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
@@ -256,51 +257,62 @@ export const installPythonTemplate = async ({
       });
     }
 
-    // Write loader configs
-    if (dataSource?.type === "web") {
-      const config = dataSource.config as WebSourceConfig[];
-      const webLoaderConfig = config.map((c) => {
-        return {
-          base_url: c.baseUrl,
-          prefix: c.prefix || c.baseUrl,
-          depth: c.depth || 1,
-        };
-      });
-      const loaderConfigPath = path.join(root, "config/loaders.json");
-      await fs.mkdir(path.join(root, "config"), { recursive: true });
-      await fs.writeFile(
-        loaderConfigPath,
-        JSON.stringify(
-          {
-            web: webLoaderConfig,
-          },
-          null,
-          2,
-        ),
-      );
-    }
+    if (dataSources.length > 0 && dataSources[0].type !== "none") {
+      const loaderConfigs: Record<string, any> = {};
+      const loaderPath = path.join(enginePath, "loaders");
 
-    const dataSourceType = dataSource?.type;
-    if (dataSourceType !== undefined && dataSourceType !== "none") {
-      let loaderFolder: string;
-      if (dataSourceType === "file" || dataSourceType === "folder") {
-        const dataSourceConfig = dataSource?.config as FileSourceConfig;
-        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
-      } else {
-        loaderFolder = dataSourceType;
-      }
-      await copy("**", enginePath, {
+      // Copy loader.py file to enginePath
+      await copy("loader.py", enginePath, {
         parents: true,
-        cwd: path.join(compPath, "loaders", "python", loaderFolder),
+        cwd: path.join(compPath, "loaders", "python"),
       });
+
+      for (const dataSource of dataSources) {
+        const sourceType = dataSource.type;
+        switch (sourceType) {
+          case "file":
+          case "folder": {
+            const loaderFolder = useLlamaParse ? "llama_parse" : "file";
+            await copy("**", loaderPath, {
+              parents: true,
+              cwd: path.join(compPath, "loaders", "python", loaderFolder),
+            });
+            break;
+          }
+          case "web": {
+            const config = dataSource.config as WebSourceConfig[];
+            // Append web loader config
+            const webLoaderConfig = config.map((c) => {
+              return {
+                base_url: c.baseUrl,
+                prefix: c.prefix || c.baseUrl,
+                depth: c.depth || 1,
+              };
+            });
+            loaderConfigs["web"] = webLoaderConfig;
+            await copy("**", loaderPath, {
+              parents: true,
+              cwd: path.join(compPath, "loaders", "python", sourceType),
+            });
+            break;
+          }
+        }
+      }
+      // Write loaders config
+      if (Object.keys(loaderConfigs).length > 0) {
+        const loaderConfigPath = path.join(root, "config/loaders.json");
+        await fs.mkdir(path.join(root, "config"), { recursive: true });
+        await fs.writeFile(
+          loaderConfigPath,
+          JSON.stringify(loaderConfigs, null, 2),
+        );
+      }
     }
   }
 
-  const addOnDependencies = getAdditionalDependencies(
-    vectorDb,
-    dataSource,
-    tools,
-  );
+  const addOnDependencies = dataSources
+    .map((ds) => getAdditionalDependencies(vectorDb, ds, tools))
+    .flat();
   await addDependencies(root, addOnDependencies);
 
   if (postInstallAction === "runApp" || postInstallAction === "dependencies") {

helpers/types.ts

@@ -46,11 +46,12 @@ export interface InstallTemplateArgs {
   framework: TemplateFramework;
   engine: TemplateEngine;
   ui: TemplateUI;
-  dataSource?: TemplateDataSource;
+  dataSources: TemplateDataSource[];
   eslint: boolean;
   customApiPath?: string;
   openAiKey?: string;
   llamaCloudKey?: string;
+  useLlamaParse?: boolean;
   model: string;
   embeddingModel: string;
   communityProjectConfig?: CommunityProjectConfig;

helpers/typescript.ts

@@ -6,7 +6,7 @@ import { copy } from "../helpers/copy";
 import { callPackageManager } from "../helpers/install";
 import { templatesDir } from "./dir";
 import { PackageManager } from "./get-pkg-manager";
-import { FileSourceConfig, InstallTemplateArgs } from "./types";
+import { InstallTemplateArgs } from "./types";
 
 const rename = (name: string) => {
   switch (name) {
@@ -65,7 +65,8 @@ export const installTSTemplate = async ({
   backend,
   observability,
   tools,
-  dataSource,
+  dataSources,
+  useLlamaParse,
 }: InstallTemplateArgs & { backend: boolean }) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -173,12 +174,11 @@ export const installTSTemplate = async ({
     });
 
     // copy loader component
-    const dataSourceType = dataSource?.type;
+    const dataSourceType = dataSources[0]?.type;
     if (dataSourceType && dataSourceType !== "none") {
       let loaderFolder: string;
       if (dataSourceType === "file" || dataSourceType === "folder") {
-        const dataSourceConfig = dataSource?.config as FileSourceConfig;
-        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
+        loaderFolder = useLlamaParse ? "llama_parse" : "file";
       } else {
         loaderFolder = dataSourceType;
       }

index.ts

@@ -302,8 +302,9 @@ async function run(): Promise<void> {
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
     postInstallAction: program.postInstallAction,
-    dataSource: program.dataSource,
+    dataSources: program.dataSources,
     tools: program.tools,
+    useLlamaParse: program.useLlamaParse,
     observability: program.observability,
   });
   conf.set("preferences", preferences);