feat: Add multiple data sources

helpers/env-variables.ts

@@ -126,19 +126,13 @@ export const createBackendEnvFile = async (
       description: "The OpenAI API key to use.",
       value: opts.openAiKey,
     },
-
+    {
+      name: "LLAMA_CLOUD_API_KEY",
+      description: `The Llama Cloud API key.`,
+      value: opts.llamaCloudKey,
+    },
     // Add vector database environment variables
     ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []),
-    // Add LlamaCloud API key
-    ...(opts.llamaCloudKey
-      ? [
-          {
-            name: "LLAMA_CLOUD_API_KEY",
-            description: `The Llama Cloud API key.`,
-            value: opts.llamaCloudKey,
-          },
-        ]
-      : []),
   ];
   let envVars: EnvVar[] = [];
   if (opts.framework === "fastapi") {

helpers/index.ts

@@ -1,10 +1,9 @@
-import { copy } from "./copy";
 import { callPackageManager } from "./install";
 
-import fs from "fs/promises";
 import path from "path";
 import { cyan } from "picocolors";
 
+import fsExtra from "fs-extra";
 import { templatesDir } from "./dir";
 import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables";
 import { PackageManager } from "./get-pkg-manager";
@@ -75,45 +74,16 @@ async function generateContextData(
 
 const copyContextData = async (
   root: string,
-  dataSource?: TemplateDataSource,
+  dataSources: TemplateDataSource[],
 ) => {
-  const destPath = path.join(root, "data");
-
-  const dataSourceConfig = dataSource?.config as FileSourceConfig;
-
-  // Copy file
-  if (dataSource?.type === "file") {
-    if (dataSourceConfig.paths) {
-      await fs.mkdir(destPath, { recursive: true });
-      console.log(
-        "Copying data from files:",
-        dataSourceConfig.paths.toString(),
-      );
-      for (const p of dataSourceConfig.paths) {
-        await fs.copyFile(p, path.join(destPath, path.basename(p)));
-      }
-    } else {
-      console.log("Missing file path in config");
-      process.exit(1);
-    }
-  }
-
-  // Copy folder
-  if (dataSource?.type === "folder") {
-    // Example data does not have path config, set the default path
-    const srcPaths = dataSourceConfig.paths ?? [
-      path.join(templatesDir, "components", "data"),
-    ];
-    console.log("Copying data from folders: ", srcPaths);
-    for (const p of srcPaths) {
-      const folderName = path.basename(p);
-      const destFolderPath = path.join(destPath, folderName);
-      await fs.mkdir(destFolderPath, { recursive: true });
-      await copy("**", destFolderPath, {
-        parents: true,
-        cwd: p,
-      });
-    }
+  for (const dataSource of dataSources) {
+    const dataSourceConfig = dataSource?.config as FileSourceConfig;
+    // Copy local data
+    const dataPath =
+      dataSourceConfig.path ?? path.join(templatesDir, "components", "data");
+    const destPath = path.join(root, "data", path.basename(dataPath));
+    console.log("Copying data from path:", dataPath);
+    await fsExtra.copy(dataPath, destPath);
   }
 };
 
@@ -169,11 +139,7 @@ export const installTemplate = async (
 
     if (props.engine === "context") {
       console.log("\nGenerating context data...\n");
-      for (const ds of props.dataSources) {
-        if (ds.type === "file" || ds.type === "folder") {
-          await copyContextData(props.root, ds);
-        }
-      }
+      await copyContextData(props.root, props.dataSources);
       if (
         props.postInstallAction === "runApp" ||
         props.postInstallAction === "dependencies"

helpers/python.ts

@@ -64,7 +64,7 @@ const getAdditionalDependencies = (
 
   // Add data source dependencies
   const dataSourceType = dataSource?.type;
-  if (dataSourceType === "file" || dataSourceType === "folder") {
+  if (dataSourceType === "file") {
     // llama-index-readers-file (pdf, excel, csv) is already included in llama_index package
     dependencies.push({
       name: "docx2txt",
@@ -283,9 +283,7 @@ export const installPythonTemplate = async ({
         loaderConfigs["web"] = webLoaderConfig;
       }
       // File loader config
-      if (
-        dataSources.some((ds) => ds.type === "file" || ds.type === "folder")
-      ) {
+      if (dataSources.some((ds) => ds.type === "file")) {
         loaderConfigs["file"] = {
           use_llama_parse: useLlamaParse,
         };

helpers/types.ts

@@ -15,11 +15,11 @@ export type TemplateDataSource = {
   type: TemplateDataSourceType;
   config: TemplateDataSourceConfig;
 };
-export type TemplateDataSourceType = "file" | "folder" | "web";
+export type TemplateDataSourceType = "file" | "web";
 export type TemplateObservability = "none" | "opentelemetry";
 // Config for both file and folder
 export type FileSourceConfig = {
-  paths?: string[];
+  path?: string;
 };
 export type WebSourceConfig = {
   baseUrl?: string;

helpers/typescript.ts

@@ -177,11 +177,7 @@ export const installTSTemplate = async ({
     const dataSourceType = dataSources[0]?.type;
     if (dataSourceType) {
       let loaderFolder: string;
-      if (dataSourceType === "file" || dataSourceType === "folder") {
-        loaderFolder = useLlamaParse ? "llama_parse" : "file";
-      } else {
-        loaderFolder = dataSourceType;
-      }
+      loaderFolder = useLlamaParse ? "llama_parse" : dataSourceType;
       await copy("**", enginePath, {
         parents: true,
         cwd: path.join(compPath, "loaders", "typescript", loaderFolder),

package.json

@@ -43,6 +43,7 @@
     "@types/prompts": "2.0.1",
     "@types/tar": "6.1.5",
     "@types/validate-npm-package-name": "3.0.0",
+    "@types/fs-extra": "11.0.4",
     "@vercel/ncc": "0.38.1",
     "async-retry": "1.3.1",
     "async-sema": "3.0.1",
@@ -68,7 +69,8 @@
     "prettier-plugin-organize-imports": "^3.2.4",
     "typescript": "^5.3.3",
     "eslint-config-prettier": "^8.10.0",
-    "ora": "^8.0.1"
+    "ora": "^8.0.1",
+    "fs-extra": "11.2.0"
   },
   "engines": {
     "node": ">=16.14.0"

questions.ts

@@ -193,9 +193,10 @@ const selectLocalContextData = async (type: TemplateDataSourceType) => {
       process.platform === "win32"
         ? selectedPath.split("\r\n")
         : selectedPath.split(", ");
+
     for (const p of paths) {
       if (
-        type == "file" &&
+        fs.statSync(p).isFile() &&
         !supportedContextFileTypes.includes(path.extname(p))
       ) {
         console.log(
@@ -630,19 +631,14 @@ export const askQuestions = async (
   if (program.files) {
     // If user specified files option, then the program should use context engine
     program.engine = "context";
-    if (!fs.existsSync(program.files)) {
-      console.log("File or folder not found");
-      process.exit(1);
-    } else {
-      program.dataSources = [
-        {
-          type: fs.lstatSync(program.files).isDirectory() ? "folder" : "file",
-          config: {
-            paths: program.files.split(","),
-          },
+    program.files.split(",").forEach((filePath) => {
+      program.dataSources.push({
+        type: "file",
+        config: {
+          path: filePath,
         },
-      ];
-    }
+      });
+    });
   }
 
   if (!program.engine) {
@@ -682,21 +678,20 @@ export const askQuestions = async (
 
         if (selectedSource === "exampleFile") {
           program.dataSources.push({
-            type: "folder",
+            type: "file",
             config: {},
           });
         } else if (selectedSource === "file" || selectedSource === "folder") {
-          // Select local file or folder
-          const dataSource = {
-            type: selectedSource,
-            config: {},
-          };
-
+          // Select local data source
           const selectedPaths = await selectLocalContextData(selectedSource);
-          dataSource.config = {
-            paths: selectedPaths,
-          };
-          program.dataSources.push(dataSource);
+          for (const p of selectedPaths) {
+            program.dataSources.push({
+              type: "file",
+              config: {
+                path: p,
+              },
+            });
+          }
         } else if (selectedSource === "web") {
           // Selected web data source
           const { baseUrl } = await prompts(
@@ -731,13 +726,6 @@ export const askQuestions = async (
             },
           });
         }
-
-        if (
-          program.framework !== "fastapi" ||
-          selectedSource === "exampleFile"
-        ) {
-          break;
-        }
       }
 
       if (program.dataSources.length === 0) {
@@ -751,7 +739,7 @@ export const askQuestions = async (
     if (program.engine === "context") {
       program.dataSources = [
         {
-          type: "folder",
+          type: "file",
           config: {},
         },
       ];
@@ -762,9 +750,7 @@ export const askQuestions = async (
 
   // Asking for LlamaParse if user selected file or folder data source
   if (
-    program.dataSources.some(
-      (ds) => ds.type === "file" || ds.type === "folder",
-    ) &&
+    program.dataSources.some((ds) => ds.type === "file") &&
     !program.useLlamaParse
   ) {
     if (ciInfo.isCI) {