feat: ability to download community submodules (#3)

create-app.ts

@@ -36,7 +36,7 @@ export async function createApp({
   llamaCloudKey,
   model,
   embeddingModel,
-  communityProjectPath,
+  communityProjectConfig,
   llamapack,
   vectorDb,
   externalPort,
@@ -84,7 +84,7 @@ export async function createApp({
     llamaCloudKey,
     model,
     embeddingModel,
-    communityProjectPath,
+    communityProjectConfig,
     llamapack,
     vectorDb,
     externalPort,

helpers/index.ts

@@ -5,7 +5,6 @@ import fs from "fs/promises";
 import path from "path";
 import { cyan } from "picocolors";
 
-import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant";
 import { templatesDir } from "./dir";
 import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables";
 import { PackageManager } from "./get-pkg-manager";
@@ -14,6 +13,7 @@ import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
 import {
+  CommunityProjectConfig,
   FileSourceConfig,
   InstallTemplateArgs,
   TemplateDataSource,
@@ -117,14 +117,15 @@ const copyContextData = async (
 
 const installCommunityProject = async ({
   root,
-  communityProjectPath,
-}: Pick<InstallTemplateArgs, "root" | "communityProjectPath">) => {
-  console.log("\nInstalling community project:", communityProjectPath!);
+  communityProjectConfig,
+}: Pick<InstallTemplateArgs, "root" | "communityProjectConfig">) => {
+  const { owner, repo, branch, filePath } = communityProjectConfig!;
+  console.log("\nInstalling community project:", filePath || repo);
   await downloadAndExtractRepo(root, {
-    username: COMMUNITY_OWNER,
-    name: COMMUNITY_REPO,
-    branch: "main",
-    filePath: communityProjectPath!,
+    username: owner,
+    name: repo,
+    branch,
+    filePath: filePath || "",
   });
 };
 
@@ -133,7 +134,7 @@ export const installTemplate = async (
 ) => {
   process.chdir(props.root);
 
-  if (props.template === "community" && props.communityProjectPath) {
+  if (props.template === "community" && props.communityProjectConfig) {
     await installCommunityProject(props);
     return;
   }

helpers/repo.ts

@@ -6,6 +6,7 @@ import { Stream } from "stream";
 import tar from "tar";
 import { promisify } from "util";
 import { makeDir } from "./make-dir";
+import { CommunityProjectConfig } from "./types";
 
 export type RepoInfo = {
   username: string;
@@ -47,19 +48,81 @@ export async function downloadAndExtractRepo(
   await promises.unlink(tempFile);
 }
 
-export async function getRepoRootFolders(
+const getRepoInfo = async (owner: string, repo: string) => {
+  const repoInfoRes = await got(
+    `https://api.github.com/repos/${owner}/${repo}`,
+    {
+      responseType: "json",
+    },
+  );
+  const data = repoInfoRes.body as any;
+  return data;
+};
+
+export async function getProjectOptions(
   owner: string,
   repo: string,
-): Promise<string[]> {
-  const url = `https://api.github.com/repos/${owner}/${repo}/contents`;
+): Promise<
+  {
+    value: CommunityProjectConfig;
+    title: string;
+  }[]
+> {
+  // TODO: consider using octokit (https://github.com/octokit) if more changes are needed in the future
+  const getCommunityProjectConfig = async (
+    item: any,
+  ): Promise<CommunityProjectConfig | null> => {
+    // if item is a folder, return the path with default owner, repo, and main branch
+    if (item.type === "dir")
+      return {
+        owner,
+        repo,
+        branch: "main",
+        filePath: item.path,
+      };
+
+    // check if it's a submodule (has size = 0 and different owner & repo)
+    if (item.type === "file") {
+      if (item.size !== 0) return null; // submodules have size = 0
+
+      // get owner and repo from git_url
+      const { git_url } = item;
+      const startIndex = git_url.indexOf("repos/") + 6;
+      const endIndex = git_url.indexOf("/git");
+      const ownerRepoStr = git_url.substring(startIndex, endIndex);
+      const [owner, repo] = ownerRepoStr.split("/");
+
+      // quick fetch repo info to get the default branch
+      const { default_branch } = await getRepoInfo(owner, repo);
 
+      // return the path with default owner, repo, and main branch (path is empty for submodules)
+      return {
+        owner,
+        repo,
+        branch: default_branch,
+      };
+    }
+
+    return null;
+  };
+
+  const url = `https://api.github.com/repos/${owner}/${repo}/contents`;
   const response = await got(url, {
     responseType: "json",
   });
-
   const data = response.body as any[];
-  const folders = data.filter((item) => item.type === "dir");
-  return folders.map((item) => item.name);
+
+  const projectConfigs: CommunityProjectConfig[] = [];
+  for (const item of data) {
+    const communityProjectConfig = await getCommunityProjectConfig(item);
+    if (communityProjectConfig) projectConfigs.push(communityProjectConfig);
+  }
+  return projectConfigs.map((config) => {
+    return {
+      value: config,
+      title: config.filePath || config.repo, // for submodules, use repo name as title
+    };
+  });
 }
 
 export async function getRepoRawContent(repoFilePath: string) {

helpers/types.ts

@@ -28,6 +28,13 @@ export type WebSourceConfig = {
 };
 export type TemplateDataSourceConfig = FileSourceConfig | WebSourceConfig;
 
+export type CommunityProjectConfig = {
+  owner: string;
+  repo: string;
+  branch: string;
+  filePath?: string;
+};
+
 export interface InstallTemplateArgs {
   appName: string;
   root: string;
@@ -44,7 +51,7 @@ export interface InstallTemplateArgs {
   llamaCloudKey?: string;
   model: string;
   embeddingModel: string;
-  communityProjectPath?: string;
+  communityProjectConfig?: CommunityProjectConfig;
   llamapack?: string;
   vectorDb?: TemplateVectorDB;
   externalPort?: number;

index.ts

@@ -290,7 +290,7 @@ async function run(): Promise<void> {
     llamaCloudKey: program.llamaCloudKey,
     model: program.model,
     embeddingModel: program.embeddingModel,
-    communityProjectPath: program.communityProjectPath,
+    communityProjectConfig: program.communityProjectConfig,
     llamapack: program.llamapack,
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,

questions.ts

@@ -13,7 +13,7 @@ import {
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
 import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
-import { getRepoRootFolders } from "./helpers/repo";
+import { getProjectOptions } from "./helpers/repo";
 import { supportedTools, toolsRequireConfig } from "./helpers/tools";
 
 export type QuestionArgs = Omit<
@@ -70,7 +70,7 @@ const defaults: QuestionArgs = {
   llamaCloudKey: "",
   model: "gpt-3.5-turbo",
   embeddingModel: "text-embedding-ada-002",
-  communityProjectPath: "",
+  communityProjectConfig: undefined,
   llamapack: "",
   postInstallAction: "dependencies",
   dataSource: {
@@ -299,25 +299,26 @@ export const askQuestions = async (
   }
 
   if (program.template === "community") {
-    const rootFolderNames = await getRepoRootFolders(
+    const projectOptions = await getProjectOptions(
       COMMUNITY_OWNER,
       COMMUNITY_REPO,
     );
-    const { communityProjectPath } = await prompts(
+    const { communityProjectConfig } = await prompts(
       {
         type: "select",
-        name: "communityProjectPath",
+        name: "communityProjectConfig",
         message: "Select community template",
-        choices: rootFolderNames.map((name) => ({
-          title: name,
-          value: name,
+        choices: projectOptions.map(({ title, value }) => ({
+          title,
+          value: JSON.stringify(value), // serialize value to string in terminal
         })),
         initial: 0,
       },
       handlers,
     );
-    program.communityProjectPath = communityProjectPath;
-    preferences.communityProjectPath = communityProjectPath;
+    const projectConfig = JSON.parse(communityProjectConfig);
+    program.communityProjectConfig = projectConfig;
+    preferences.communityProjectConfig = projectConfig;
     return; // early return - no further questions needed for community projects
   }