Add seed data

.gitignore

@@ -132,3 +132,6 @@ dist
 # Local config
 towtruck.private-key.pem
 .mygitignore
+
+# Sensitive repo data
+/data

README.md

@@ -64,3 +64,9 @@ In order for Towtruck to communicate with the GitHub API, it needs several piece
 - `CLIENT_ID`: A unique alphanumeric ID assigned to the GitHub App.
 - `CLIENT_SECRET`: A token used to authenticate API requests. These are generated by GitHub in the app settings.
 - `WEBHOOK_SECRET`: A user-defined secret used to authenticate GitHub to Towtruck for receiving webhooks. This must be exactly the same as it is entered in the app settings on GitHub.
+
+
+### Seeding
+
+Once all the other setup steps have been completed run `script/seed` or `script/bootstrap --seed` to seed the data. 
+This will call the Github API which is rate limit so take care not to to run the script too often. 

dataScripts/fetchAllRepos.js

@@ -0,0 +1,51 @@
+import { OctokitApp } from "../octokitApp.js";
+import { writeFile, mkdir } from "fs/promises";
+import { mapRepoFromApiForStorage } from "../utils.js";
+import path from "path";
+import { getDependenciesForRepo } from "../renovate/dependencyDashboard.js";
+
+const fetchAllRepos = async () => {
+  const repos = [];
+
+  await OctokitApp.app.eachRepository(async ({ repository, octokit }) => {
+    if (repository.archived) return;
+
+    repository.dependencies = await getDependenciesForRepo({
+      repository,
+      octokit,
+    });
+
+    repos.push(mapRepoFromApiForStorage(repository));
+  });
+
+  return repos;
+};
+
+const installationOctokit = await OctokitApp.app.octokit.request(
+  "GET /app/installations"
+);
+
+const saveAllRepos = async () => {
+  console.info("Fetching all repos...");
+  const repos = await fetchAllRepos();
+
+  try {
+    const dir = path.dirname("./data/repos.json");
+    await mkdir(dir, { recursive: true });
+
+    console.info("Saving all repos...");
+    const toSave = {
+      org: installationOctokit.data[0].account.login,
+      repos,
+    };
+
+    await writeFile("./data/repos.json", JSON.stringify(toSave), {
+      encoding: "utf-8",
+      flag: "w",
+    });
+  } catch (error) {
+    console.error("Error saving all repos", error);
+  }
+};
+
+await saveAllRepos();

index.js

@@ -1,8 +1,7 @@
 import { createServer } from "http";
 import nunjucks from "nunjucks";
+import { getReposFromJson, mapRepoFromStorageToUi } from "./utils.js";
 import { OctokitApp } from "./octokitApp.js";
-import { orgRespositoriesToUiRepositories } from "./utils.js";
-import { getDependenciesForRepo } from "./renovate/dependencyDashboard.js";
 
 nunjucks.configure({
   autoescape: true,
@@ -12,47 +11,17 @@ nunjucks.configure({
 const httpServer = createServer(async (request, response) => {
   if (await OctokitApp.middleware(request, response)) return;
 
-  // Currently we only want to support single-account installations.
-  // There doesn't seem to be a neat way to get the installation ID from an account name,
-  // so we will use `eachInstallation` to loop (hopefully once) and just take the first (hopefully only)
-  // element from `installations` so that we can have more meaningful template names in Nunjucks.
-  //
-  // We can enforce this one-installation approach through GitHub by configuring the app to be
-  // "Only on this account" when registering the app.
+  const pathToRepos = "./data/repos.json";
+  const persistedData = await getReposFromJson(pathToRepos);
 
-  const installations = [];
-  await OctokitApp.app.eachInstallation(async (octokit) => {
-    const name = octokit.installation.account.login;
-
-    const { repos, totalRepos } = await getReposForInstallation(octokit);
-
-    for (const repo of repos) {
-      repo.dependencies = await getDependenciesForRepo(octokit, repo);
-    }
-
-    installations.push({
-      name,
-      repos,
-      totalRepos,
-    });
-  });
-
-  const template = nunjucks.render("index.njk", installations[0]);
+  const template = nunjucks.render(
+    "index.njk",
+    mapRepoFromStorageToUi(persistedData)
+  );
 
   return response.end(template);
 });
 
-const getReposForInstallation = async ({ octokit, installation }) => {
-  return octokit
-    .request(installation.repositories_url)
-    .then(({ data }) => {
-      return orgRespositoriesToUiRepositories(data);
-    })
-    .catch((error) => {
-      console.error(error);
-    });
-};
-
 const PORT = 3000;
 httpServer.listen(PORT, () => {
   console.info(`Server is running on port ${PORT}`);

index.njk

@@ -24,24 +24,30 @@
           {% else %} 
           There are <span class="font-bold">{{ totalRepos }}</span> repositories
           {% endif %}
-          that Towtruck is tracking for <span class="font-bold">{{ name }}</span>.
+          that Towtruck is tracking for <span class="font-bold">{{ org }}</span>.
         </p>
         <div class="mx-4">
           <table class="table-auto w-full mb-4 border-y border-stone-200">
             <thead class="bg-white border-b border-stone-200 text-left">
               <tr class="space-x-2">
                 <th class="py-2 pl-2" scope="col">Name</th>
                 <th class="py-2 pl-2" scope="col">Description</th>
+                <th class="py-2 pl-2" scope="col">Language</th>
+                <th class="py-2 pl-2" scope="col">Topics</th>
+                <th class="py-2 pl-2" scope="col">Open issues count</th>
                 <th class="py-2 pl-2 last:pr-2" scope="col">Last updated</th>
                 <th class="py-2 pl-2 last:pr-2" scope="col">Dependencies</th>
               </tr>
             </thead>
             <tbody>
               {% for repo in repos %}
               <tr class="even:bg-white">
-                <td class="py-2 pl-2 align-text-top"><a target="_blank" class="text-blue-600 dark:text-blue-500 hover:underline" href="{{repo.url}}">{{ repo.name }}</a></td>
-                <td class="py-2 pl-2 align-text-top">{{ repo.description }}</td>
-                <td class="py-2 pl-2 last:pr-2 align-text-top">{{ repo.updatedAt }}</td>
+                 <td class="py-2 pl-2"><a target="_blank" class="text-blue-600 dark:text-blue-500 hover:underline" href="{{repo.url}}">{{ repo.name }}</a></td>
+                <td class="py-2 pl-2">{{ repo.description }}</td>
+                <td class="py-2 pl-2">{{ repo.language }}</td>
+                <td class="py-2 pl-2">{{ repo.topics | join(", ")}}</td>
+                <td class="py-2 pl-2">{{ repo.openIssues }}</td>
+                <td class="py-2 pl-2 last:pr-2">{{ repo.updatedAt }}</td>
 
                 {% if repo.dependencies.length %}
                 <td class="group py-2 pl-2 last:pr-2 align-text-top space-y-2">
@@ -75,6 +81,7 @@
                     Please make sure that Renovate has been configured on the repository to produce a dependency dashboard.
                   </td>
                 {% endif %}
+
               </tr>
               {% else %}
               <tr>

package.json

@@ -10,7 +10,8 @@
     "test": "node --test",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
-    "format": "prettier --write ."
+    "format": "prettier --write .",
+    "seed": "node --env-file=.env ./dataScripts/fetchAllRepos.js"
   },
   "author": "dxw",
   "license": "MIT",

renovate/dependencyDashboard.js

@@ -55,6 +55,6 @@ export const handleIssuesApiResponse = (response) => {
 
 export const getDependenciesForRepo = ({ octokit }, repo) => {
   return octokit
-    .request(repo.issuesUrl)
+    .request(repo.issues_url)
     .then(handleIssuesApiResponse);
 }

script/bootstrap

@@ -17,3 +17,10 @@ nvm install
 
 echo "==> Installing node modules..."
 npm install
+
+if [ -n "$1" ]; then
+    echo "==> Seeding data..."
+    npm run seed
+else
+    echo "==> Skipping seeding data..."
+fi

script/seed

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# script/seed: Seeds the necessary data
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+echo "==> Seeding data..."
+npm run seed

utils.js

@@ -1,12 +1,34 @@
-export const orgRespositoriesToUiRepositories = (apiResponse) => {
-  const repos = apiResponse.repositories.map((repo) => ({
-    name: repo.name,
-    updatedAt: new Date(repo.updated_at).toLocaleDateString(),
-    url: repo.html_url,
-    issuesUrl: repo.issues_url,
-    description: repo.description,
-  }));
-  const totalRepos = apiResponse.total_count;
-
-  return { repos, totalRepos };
+import { readFile } from "fs/promises";
+export const mapRepoFromStorageToUi = (persistedData) => {
+  const mappedRepos = persistedData.repos.map((repo) => {
+    const newDate = new Date(repo.updatedAt).toLocaleDateString();
+    return {
+      ...repo,
+      updatedAt: newDate,
+    };
+  });
+
+  const totalRepos = mappedRepos.length;
+
+  return { ...persistedData, repos: mappedRepos, totalRepos };
+};
+
+export const getReposFromJson = async (filePath) => {
+  const reposJson = await readFile(filePath, { encoding: "utf-8" });
+  const persistedData = JSON.parse(reposJson);
+
+  return persistedData;
 };
+
+export const mapRepoFromApiForStorage = (repo) => ({
+  name: repo.name,
+  description: repo.description,
+  htmlUrl: repo.html_url,
+  apiUrl: repo.url,
+  pullsUrl: repo.pulls_url,
+  issuesUrl: repo.issues_url,
+  updatedAt: repo.updated_at,
+  language: repo.language,
+  topics: repo.topics,
+  openIssues: repo.open_issues,
+});