diff --git a/create-app.ts b/create-app.ts index 8cb6f287..e453deb1 100644 --- a/create-app.ts +++ b/create-app.ts @@ -36,7 +36,7 @@ export async function createApp({ llamaCloudKey, model, embeddingModel, - communityProjectPath, + communityProjectConfig, llamapack, vectorDb, externalPort, @@ -84,7 +84,7 @@ export async function createApp({ llamaCloudKey, model, embeddingModel, - communityProjectPath, + communityProjectConfig, llamapack, vectorDb, externalPort, diff --git a/helpers/index.ts b/helpers/index.ts index 91280af3..7e1b2d31 100644 --- a/helpers/index.ts +++ b/helpers/index.ts @@ -5,7 +5,6 @@ import fs from "fs/promises"; import path from "path"; import { cyan } from "picocolors"; -import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./constant"; import { templatesDir } from "./dir"; import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables"; import { PackageManager } from "./get-pkg-manager"; @@ -14,6 +13,7 @@ import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry"; import { installPythonTemplate } from "./python"; import { downloadAndExtractRepo } from "./repo"; import { + CommunityProjectConfig, FileSourceConfig, InstallTemplateArgs, TemplateDataSource, @@ -117,14 +117,15 @@ const copyContextData = async ( const installCommunityProject = async ({ root, - communityProjectPath, -}: Pick) => { - console.log("\nInstalling community project:", communityProjectPath!); + communityProjectConfig, +}: Pick) => { + const { owner, repo, branch, filePath } = communityProjectConfig!; + console.log("\nInstalling community project:", filePath || repo); await downloadAndExtractRepo(root, { - username: COMMUNITY_OWNER, - name: COMMUNITY_REPO, - branch: "main", - filePath: communityProjectPath!, + username: owner, + name: repo, + branch, + filePath: filePath || "", }); }; @@ -133,7 +134,7 @@ export const installTemplate = async ( ) => { process.chdir(props.root); - if (props.template === "community" && props.communityProjectPath) { + if (props.template === "community" && props.communityProjectConfig) { await installCommunityProject(props); return; } diff --git a/helpers/repo.ts b/helpers/repo.ts index 3942c28c..7a33ecb3 100644 --- a/helpers/repo.ts +++ b/helpers/repo.ts @@ -6,6 +6,7 @@ import { Stream } from "stream"; import tar from "tar"; import { promisify } from "util"; import { makeDir } from "./make-dir"; +import { CommunityProjectConfig } from "./types"; export type RepoInfo = { username: string; @@ -47,19 +48,81 @@ export async function downloadAndExtractRepo( await promises.unlink(tempFile); } -export async function getRepoRootFolders( +const getRepoInfo = async (owner: string, repo: string) => { + const repoInfoRes = await got( + `https://api.github.com/repos/${owner}/${repo}`, + { + responseType: "json", + }, + ); + const data = repoInfoRes.body as any; + return data; +}; + +export async function getProjectOptions( owner: string, repo: string, -): Promise { - const url = `https://api.github.com/repos/${owner}/${repo}/contents`; +): Promise< + { + value: CommunityProjectConfig; + title: string; + }[] +> { + // TODO: consider using octokit (https://github.com/octokit) if more changes are needed in the future + const getCommunityProjectConfig = async ( + item: any, + ): Promise => { + // if item is a folder, return the path with default owner, repo, and main branch + if (item.type === "dir") + return { + owner, + repo, + branch: "main", + filePath: item.path, + }; + + // check if it's a submodule (has size = 0 and different owner & repo) + if (item.type === "file") { + if (item.size !== 0) return null; // submodules have size = 0 + + // get owner and repo from git_url + const { git_url } = item; + const startIndex = git_url.indexOf("repos/") + 6; + const endIndex = git_url.indexOf("/git"); + const ownerRepoStr = git_url.substring(startIndex, endIndex); + const [owner, repo] = ownerRepoStr.split("/"); + + // quick fetch repo info to get the default branch + const { default_branch } = await getRepoInfo(owner, repo); + // return the path with default owner, repo, and main branch (path is empty for submodules) + return { + owner, + repo, + branch: default_branch, + }; + } + + return null; + }; + + const url = `https://api.github.com/repos/${owner}/${repo}/contents`; const response = await got(url, { responseType: "json", }); - const data = response.body as any[]; - const folders = data.filter((item) => item.type === "dir"); - return folders.map((item) => item.name); + + const projectConfigs: CommunityProjectConfig[] = []; + for (const item of data) { + const communityProjectConfig = await getCommunityProjectConfig(item); + if (communityProjectConfig) projectConfigs.push(communityProjectConfig); + } + return projectConfigs.map((config) => { + return { + value: config, + title: config.filePath || config.repo, // for submodules, use repo name as title + }; + }); } export async function getRepoRawContent(repoFilePath: string) { diff --git a/helpers/types.ts b/helpers/types.ts index 0d359423..dee6a685 100644 --- a/helpers/types.ts +++ b/helpers/types.ts @@ -28,6 +28,13 @@ export type WebSourceConfig = { }; export type TemplateDataSourceConfig = FileSourceConfig | WebSourceConfig; +export type CommunityProjectConfig = { + owner: string; + repo: string; + branch: string; + filePath?: string; +}; + export interface InstallTemplateArgs { appName: string; root: string; @@ -44,7 +51,7 @@ export interface InstallTemplateArgs { llamaCloudKey?: string; model: string; embeddingModel: string; - communityProjectPath?: string; + communityProjectConfig?: CommunityProjectConfig; llamapack?: string; vectorDb?: TemplateVectorDB; externalPort?: number; diff --git a/index.ts b/index.ts index 131d6eba..057b30ce 100644 --- a/index.ts +++ b/index.ts @@ -290,7 +290,7 @@ async function run(): Promise { llamaCloudKey: program.llamaCloudKey, model: program.model, embeddingModel: program.embeddingModel, - communityProjectPath: program.communityProjectPath, + communityProjectConfig: program.communityProjectConfig, llamapack: program.llamapack, vectorDb: program.vectorDb, externalPort: program.externalPort, diff --git a/questions.ts b/questions.ts index 275c2d0a..f9eb2539 100644 --- a/questions.ts +++ b/questions.ts @@ -13,7 +13,7 @@ import { import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant"; import { templatesDir } from "./helpers/dir"; import { getAvailableLlamapackOptions } from "./helpers/llama-pack"; -import { getRepoRootFolders } from "./helpers/repo"; +import { getProjectOptions } from "./helpers/repo"; import { supportedTools, toolsRequireConfig } from "./helpers/tools"; export type QuestionArgs = Omit< @@ -70,7 +70,7 @@ const defaults: QuestionArgs = { llamaCloudKey: "", model: "gpt-3.5-turbo", embeddingModel: "text-embedding-ada-002", - communityProjectPath: "", + communityProjectConfig: undefined, llamapack: "", postInstallAction: "dependencies", dataSource: { @@ -299,25 +299,26 @@ export const askQuestions = async ( } if (program.template === "community") { - const rootFolderNames = await getRepoRootFolders( + const projectOptions = await getProjectOptions( COMMUNITY_OWNER, COMMUNITY_REPO, ); - const { communityProjectPath } = await prompts( + const { communityProjectConfig } = await prompts( { type: "select", - name: "communityProjectPath", + name: "communityProjectConfig", message: "Select community template", - choices: rootFolderNames.map((name) => ({ - title: name, - value: name, + choices: projectOptions.map(({ title, value }) => ({ + title, + value: JSON.stringify(value), // serialize value to string in terminal })), initial: 0, }, handlers, ); - program.communityProjectPath = communityProjectPath; - preferences.communityProjectPath = communityProjectPath; + const projectConfig = JSON.parse(communityProjectConfig); + program.communityProjectConfig = projectConfig; + preferences.communityProjectConfig = projectConfig; return; // early return - no further questions needed for community projects }