diff --git a/connectors/package-lock.json b/connectors/package-lock.json index 2d73c42d39a8..8ae8944c7df0 100644 --- a/connectors/package-lock.json +++ b/connectors/package-lock.json @@ -49,6 +49,7 @@ "redis": "^4.6.10", "sequelize": "^6.31.0", "talisman": "^1.1.4", + "tar": "^6.2.0", "uuid": "^9.0.0" }, "devDependencies": { @@ -2660,6 +2661,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, "node_modules/chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", @@ -3666,6 +3675,28 @@ "node": ">=14.14" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/fs-monkey": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/fs-monkey/-/fs-monkey-1.0.3.tgz", @@ -5467,6 +5498,48 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mnemonist": { "version": "0.38.5", "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.5.tgz", @@ -6799,6 +6872,22 @@ "node": ">=6" } }, + "node_modules/tar": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", + "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/terser": { "version": "5.17.1", "resolved": "https://registry.npmjs.org/terser/-/terser-5.17.1.tgz", diff --git a/connectors/package.json b/connectors/package.json index 5491a9b251c2..c3155004f356 100644 --- a/connectors/package.json +++ b/connectors/package.json @@ -54,6 +54,7 @@ "redis": "^4.6.10", "sequelize": "^6.31.0", "talisman": "^1.1.4", + "tar": "^6.2.0", "uuid": "^9.0.0" }, "devDependencies": { diff --git a/connectors/src/connectors/github/lib/github_api.ts b/connectors/src/connectors/github/lib/github_api.ts index b78edbc64671..1b01e19f2de4 100644 --- a/connectors/src/connectors/github/lib/github_api.ts +++ b/connectors/src/connectors/github/lib/github_api.ts @@ -1,8 +1,15 @@ import { createAppAuth } from "@octokit/auth-app"; import { isLeft } from "fp-ts/lib/Either"; +import { createWriteStream } from "fs"; +import { mkdtemp, readdir, rmdir } from "fs/promises"; import fs from "fs-extra"; import * as reporter from "io-ts-reporters"; import { Octokit } from "octokit"; +import { tmpdir } from "os"; +import { join, resolve } from "path"; +import { pipeline } from "stream"; +import { extract } from "tar"; +import { promisify } from "util"; import { DiscussionCommentNode, @@ -526,3 +533,49 @@ async function getOctokit(installationId: string): Promise { }, }); } + +const asyncPipeline = promisify(pipeline); + +export async function downloadRepository( + installationId: string, + login: string, + repoName: string +) { + const octokit = await getOctokit(installationId); + + const { data: tarballStream } = await octokit.request( + "GET /repos/{owner}/{repo}/tarball", + { + owner: login, + repo: repoName, + } + ); + + // Create a temp directory. + const tempDir = await mkdtemp(join(tmpdir(), "repo-")); + const tarPath = resolve(tempDir, "repo.tar.gz"); + + // Save the tarball to the temp directory. + await asyncPipeline(tarballStream, createWriteStream(tarPath)); + console.log("Downloaded: ", tarPath); + + // Extract the tarball. + await extract({ + file: tarPath, + cwd: tempDir, + }); + console.log("Extracted: ", tarPath); + + // Delete the tarball. + await fs.unlink(tarPath); + + // Iterate over the files in the temp directory. + const files = await readdir(tempDir); + for (const file of files) { + console.log("FILE: ", file); + } + + // Delete the temp directory. + await rmdir(tempDir, { recursive: true }); + console.log("Cleaned up: ", tempDir); +}