From bb65e2f4a6b0049336112ee3274258e7b0223ab6 Mon Sep 17 00:00:00 2001 From: Dan Livings Date: Mon, 7 Oct 2024 14:53:12 +0100 Subject: [PATCH] Implement GitHub webhook handlers To update dependency or repository information in an efficient way, Towtruck should listen to GitHub events rather than polling for updates. Towtruck needs to listen to the following events: - `pull_request.opened` and `pull_request.closed` to update the list of open pull requests - `issues.opened` and `issues.closed` to update the list of open issues - `push` and `issues.edited` to update the list of dependencies - `dependabot_alert` to update the list of vulnerabilities - `repository` to update basic information about repositories --- webhooks/alerts.js | 31 ++++++ webhooks/alerts.test.js | 87 +++++++++++++++++ webhooks/dependencies.js | 64 +++++++++++++ webhooks/dependencies.test.js | 172 ++++++++++++++++++++++++++++++++++ webhooks/index.js | 38 ++++++++ webhooks/issues.js | 42 +++++++++ webhooks/issues.test.js | 75 +++++++++++++++ webhooks/pullRequests.js | 42 +++++++++ webhooks/pullRequests.test.js | 77 +++++++++++++++ webhooks/repository.js | 30 ++++++ webhooks/repository.test.js | 66 +++++++++++++ 11 files changed, 724 insertions(+) create mode 100644 webhooks/alerts.js create mode 100644 webhooks/alerts.test.js create mode 100644 webhooks/dependencies.js create mode 100644 webhooks/dependencies.test.js create mode 100644 webhooks/index.js create mode 100644 webhooks/issues.js create mode 100644 webhooks/issues.test.js create mode 100644 webhooks/pullRequests.js create mode 100644 webhooks/pullRequests.test.js create mode 100644 webhooks/repository.js create mode 100644 webhooks/repository.test.js diff --git a/webhooks/alerts.js b/webhooks/alerts.js new file mode 100644 index 0000000..1d7b92d --- /dev/null +++ b/webhooks/alerts.js @@ -0,0 +1,31 @@ +import { TowtruckDatabase } from "../db/index.js"; +import { getDependabotAlertsForRepo } from "../utils/githubApi/fetchDependabotAlerts.js"; + +/** + * @typedef {import("./index.js").Event} Event + * @template {string} T + */ + +/** + * @param {Event<"dependabot_alert">} event + * @param {TowtruckDatabase} db + */ +export const handleEvent = async ({ payload, octokit }, db) => { + const alerts = await getDependabotAlertsForRepo({ + octokit, + repository: payload.repository, + }); + + db.saveToRepository(payload.repository.name, "dependabotAlerts", alerts); +}; + +/** + * Handles the `"dependabot_alert"` webhook event. + * @param {Event<"dependabot_alert">} event + */ +export const onDependabotAlert = async (event) => { + console.log( + `Dependabot alert #${event.payload.alert.number} in ${event.payload.repository.name} updated: ${event.payload.action}`, + ); + handleEvent(event, new TowtruckDatabase()); +}; diff --git a/webhooks/alerts.test.js b/webhooks/alerts.test.js new file mode 100644 index 0000000..d05b508 --- /dev/null +++ b/webhooks/alerts.test.js @@ -0,0 +1,87 @@ +import { describe, it } from "node:test"; +import expect from "node:assert"; +import { handleEvent } from "./alerts.js"; + +const db = { + saveToRepository: () => {}, +}; + +const octokit = { + request: async () => { + return Promise.resolve({}); + }, +}; + +describe("handleEvent", () => { + it("should update the Dependabot alert information in the database for the repository", async (t) => { + const responses = { + "/repos/{owner}/{repo}/dependabot/alerts": { + data: [ + { + state: "open", + security_vulnerability: { + severity: "critical", + }, + }, + { + state: "open", + security_vulnerability: { + severity: "high", + }, + }, + { + state: "open", + security_vulnerability: { + severity: "medium", + }, + }, + { + state: "open", + security_vulnerability: { + severity: "low", + }, + }, + ], + }, + }; + + t.mock.method(octokit, "request", async (url) => + Promise.resolve(responses[url]), + ); + t.mock.method(db, "saveToRepository"); + + const payload = { + repository: { + name: "repo", + owner: { + login: "dxw", + }, + }, + }; + + const event = { + payload, + octokit, + }; + + const expected = { + criticalSeverityAlerts: 1, + highSeverityAlerts: 1, + mediumSeverityAlerts: 1, + lowSeverityAlerts: 1, + totalOpenAlerts: 4, + }; + + await handleEvent(event, db); + + expect.strictEqual(octokit.request.mock.callCount(), 1); + + expect.strictEqual(db.saveToRepository.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToRepository.mock.calls[0].arguments, [ + payload.repository.name, + "dependabotAlerts", + expected, + ]); + }); +}); diff --git a/webhooks/dependencies.js b/webhooks/dependencies.js new file mode 100644 index 0000000..9cd08d8 --- /dev/null +++ b/webhooks/dependencies.js @@ -0,0 +1,64 @@ +import { TowtruckDatabase } from "../db/index.js"; +import { + fetchAllDependencyLifetimes, + saveAllDependencyLifetimes, +} from "../utils/endOfLifeDateApi/fetchAllDependencyEolInfo.js"; +import { EndOfLifeDateApiClient } from "../utils/endOfLifeDateApi/index.js"; +import { getDependenciesForRepo } from "../utils/renovate/dependencyDashboard.js"; + +/** + * @typedef {import("./index.js").Event} Event + * @template {string} T + */ + +/** + * @param {Event<"push" | "issues.edited">} event + * @param {TowtruckDatabase} db + * @param {EndOfLifeDateApiClient} apiClient + */ +export const handleEvent = async ({ payload, octokit }, db, apiClient) => { + const dependencies = await getDependenciesForRepo({ + octokit, + repository: payload.repository, + }); + + db.saveToRepository(payload.repository.name, "dependencies", dependencies); + + const allLifetimes = await fetchAllDependencyLifetimes(db, apiClient); + await saveAllDependencyLifetimes(allLifetimes, db); +}; + +/** + * Handles the `"push"` webhook event. + * @param {Event<"push">} event + */ +export const onPush = async (event) => { + console.log(`Push to ${event.payload.repository.name}: ${event.payload.ref}`); + if (event.payload.ref === "refs/heads/main") { + return await handleEvent( + event, + new TowtruckDatabase(), + new EndOfLifeDateApiClient(), + ); + } +}; + +/** + * Handles the `"issues.edited"` webhook event. + * @param {Event<"issues.edited">} event + */ +export const onIssueEdited = async (event) => { + console.log( + `Issue #${event.payload.issue.number} updated in ${event.payload.repository.name}: ${event.payload.issue.title}`, + ); + if ( + event.payload.issue.user.login === "renovate[bot]" && + event.payload.issue.pull_request === undefined + ) { + return await handleEvent( + event, + new TowtruckDatabase(), + new EndOfLifeDateApiClient(), + ); + } +}; diff --git a/webhooks/dependencies.test.js b/webhooks/dependencies.test.js new file mode 100644 index 0000000..7fc4015 --- /dev/null +++ b/webhooks/dependencies.test.js @@ -0,0 +1,172 @@ +import { describe, it } from "node:test"; +import expect from "node:assert"; +import { handleEvent } from "./dependencies.js"; +import { Dependency } from "../model/Dependency.js"; + +const db = { + transaction: (fn) => { + return (arg) => fn(arg); + }, + saveToRepository: () => {}, + saveToDependency: () => {}, + getAllRepositories: () => {}, +}; + +const octokit = { + request: async () => { + return Promise.resolve({}); + }, +}; + +const apiClient = { + getAllCycles: (dependency) => ({ dependency, cycles: [] }), +}; + +describe("handleEvent", () => { + it("should update the dependency information in the database for the repository", async (t) => { + const responses = { + "https://some.api/issues": { + data: [ + { + user: { + login: "renovate[bot]", + }, + body: "## Detected dependencies\n\n- `foobar 1.2.3`\n- `libquux 0.1.1-alpha` \n\n---", + }, + ], + }, + }; + + const repository = { + name: "repo", + owner: { + login: "dxw", + }, + issues_url: "https://some.api/issues", + }; + + const repositories = { + repo: { + repository, + dependencies: [], + }, + }; + + t.mock.method(octokit, "request", async (url) => + Promise.resolve(responses[url]), + ); + t.mock.method(db, "saveToRepository"); + t.mock.method(db, "getAllRepositories", () => repositories); + + const payload = { + repository, + }; + + const event = { + payload, + octokit, + }; + + const expected = [ + new Dependency("foobar", "1.2.3"), + new Dependency("libquux", "0.1.1-alpha"), + ]; + + await handleEvent(event, db, apiClient); + + expect.strictEqual(octokit.request.mock.callCount(), 1); + + expect.strictEqual(db.saveToRepository.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToRepository.mock.calls[0].arguments, [ + payload.repository.name, + "dependencies", + expected, + ]); + }); + + it("should update the lifetime information stored in the database", async (t) => { + const responses = { + "https://some.api/issues": { + data: [ + { + user: { + login: "renovate[bot]", + }, + body: "## Detected dependencies\n\n- `foobar 1.2.3`\n- `libquux 0.1.1-alpha` \n\n---", + }, + ], + }, + }; + + const repository = { + name: "repo", + owner: { + login: "dxw", + }, + issues_url: "https://some.api/issues", + }; + + const repositories = { + repo: { + repository, + dependencies: [new Dependency("foobar", "1.2.3")], + }, + }; + + const cycles = { + foobar: [ + { + cycle: "1.2", + latestVersion: "1.2.3", + releaseDate: "2024-01-01", + }, + ], + }; + + t.mock.method(octokit, "request", async (url) => + Promise.resolve(responses[url]), + ); + t.mock.method(db, "transaction"); + t.mock.method(db, "saveToDependency"); + t.mock.method(db, "getAllRepositories", () => repositories); + t.mock.method(apiClient, "getAllCycles", (dependency) => { + if (cycles[dependency]) { + return { dependency, cycles: cycles[dependency] }; + } else { + return { message: "Product not found" }; + } + }); + + const expected = [ + { + dependency: "foobar", + lifetimes: { + dependency: "foobar", + cycles: cycles.foobar, + }, + }, + ]; + + const payload = { + repository, + }; + + const event = { + payload, + octokit, + }; + + await handleEvent(event, db, apiClient); + + expect.strictEqual(apiClient.getAllCycles.mock.callCount(), 1); + + expect.strictEqual(db.saveToDependency.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToDependency.mock.calls[0].arguments, [ + expected[0].dependency, + "lifetimes", + expected[0].lifetimes, + ]); + }); +}); diff --git a/webhooks/index.js b/webhooks/index.js new file mode 100644 index 0000000..ae10859 --- /dev/null +++ b/webhooks/index.js @@ -0,0 +1,38 @@ +import { createNodeMiddleware } from "@octokit/app"; +import { OctokitApp } from "../octokitApp.js"; +import { onPullRequestClosed, onPullRequestOpened } from "./pullRequests.js"; +import { onIssueClosed, onIssueOpened } from "./issues.js"; +import { onIssueEdited, onPush } from "./dependencies.js"; +import { onDependabotAlert } from "./alerts.js"; +import { onRepository } from "./repository.js"; + +/** + * @typedef {import("@octokit/webhooks/dist-types/index").EmitterWebhookEvent} EmitterWebhookEvent + * @template {string} T + */ + +/** + * @typedef {import("@octokit/core/dist-types/index").Octokit} Octokit + */ + +/** + * @typedef {EmitterWebhookEvent & { octokit: Octokit; }} Event + * @template {string} T + */ + +const registerWebhook = OctokitApp.app.webhooks.on; + +registerWebhook("pull_request.opened", onPullRequestOpened); +registerWebhook("pull_request.closed", onPullRequestClosed); + +registerWebhook("issues.opened", onIssueOpened); +registerWebhook("issues.closed", onIssueClosed); + +registerWebhook("push", onPush); +registerWebhook("issues.edited", onIssueEdited); + +registerWebhook("dependabot_alert", onDependabotAlert); + +registerWebhook("repository", onRepository); + +export const handleWebhooks = createNodeMiddleware(OctokitApp.app); diff --git a/webhooks/issues.js b/webhooks/issues.js new file mode 100644 index 0000000..d9f8343 --- /dev/null +++ b/webhooks/issues.js @@ -0,0 +1,42 @@ +import { TowtruckDatabase } from "../db/index.js"; +import { getOpenIssuesForRepo } from "../utils/githubApi/fetchOpenIssues.js"; + +/** + * @typedef {import("./index.js").Event} Event + * @template {string} T + */ + +/** + * @param {Event<"issues">} event + * @param {TowtruckDatabase} db + */ +export const handleEvent = async ({ payload, octokit }, db) => { + const issueInfo = await getOpenIssuesForRepo({ + octokit, + repository: payload.repository, + }); + + db.saveToRepository(payload.repository.name, "issues", issueInfo); +}; + +/** + * Handles the `"issues.opened"` webhook event. + * @param {Event<"issues.opened">} event + */ +export const onIssueOpened = async (event) => { + console.log( + `New issue #${event.payload.issue.number} opened in ${event.payload.repository.name}: ${event.payload.issue.title}`, + ); + return await handleEvent(event, new TowtruckDatabase()); +}; + +/** + * Handles the `"issues.closed"` webhook event. + * @param {Event<"issues.closed">} event + */ +export const onIssueClosed = async (event) => { + console.log( + `Issue #${event.payload.issue.number} closed in ${event.payload.repository.name}: ${event.payload.issue.title}`, + ); + return await handleEvent(event, new TowtruckDatabase()); +}; diff --git a/webhooks/issues.test.js b/webhooks/issues.test.js new file mode 100644 index 0000000..0921907 --- /dev/null +++ b/webhooks/issues.test.js @@ -0,0 +1,75 @@ +import { describe, it } from "node:test"; +import expect from "node:assert"; +import { handleEvent } from "./issues.js"; + +const db = { + saveToRepository: () => {}, +}; + +const octokit = { + request: async () => { + return Promise.resolve({}); + }, +}; + +describe("handleEvent", () => { + it("should update the open issue information in the database for the repository", async (t) => { + const responses = { + "https://some.api/issues": { + data: [ + { + user: { + login: "foo", + }, + created_at: "2024-01-01T12:34:56.789Z", + state: "open", + }, + { + user: { + login: "bar", + }, + created_at: "2024-10-10T11:22:33.444Z", + state: "open", + }, + ], + }, + }; + + t.mock.method(octokit, "request", async (url) => + Promise.resolve(responses[url]), + ); + t.mock.method(db, "saveToRepository"); + + const payload = { + repository: { + name: "repo", + owner: { + login: "dxw", + }, + issues_url: "https://some.api/issues", + }, + }; + + const event = { + payload, + octokit, + }; + + const expected = { + oldestOpenIssueOpenedAt: new Date("2024-01-01T12:34:56.789Z"), + mostRecentIssueOpenedAt: new Date("2024-10-10T11:22:33.444Z"), + }; + + await handleEvent(event, db); + + expect.strictEqual(octokit.request.mock.callCount(), 1); + + expect.strictEqual(db.saveToRepository.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToRepository.mock.calls[0].arguments, [ + payload.repository.name, + "issues", + expected, + ]); + }); +}); diff --git a/webhooks/pullRequests.js b/webhooks/pullRequests.js new file mode 100644 index 0000000..9120039 --- /dev/null +++ b/webhooks/pullRequests.js @@ -0,0 +1,42 @@ +import { TowtruckDatabase } from "../db/index.js"; +import { getOpenPRsForRepo } from "../utils/githubApi/fetchOpenPrs.js"; + +/** + * @typedef {import("./index.js").Event} Event + * @template {string} T + */ + +/** + * @param {Event<"pull_request">} event + * @param {TowtruckDatabase} db + */ +export const handleEvent = async ({ payload, octokit }, db) => { + const prInfo = await getOpenPRsForRepo({ + octokit, + repository: payload.repository, + }); + + db.saveToRepository(payload.repository.name, "pullRequests", prInfo); +}; + +/** + * Handles the `"pull_request.opened"` webhook event. + * @param {Event<"pull_request.opened">} event + */ +export const onPullRequestOpened = async (event) => { + console.log( + `New PR #${event.payload.number} opened in ${event.payload.repository.name}: ${event.payload.pull_request.title}`, + ); + return await handleEvent(event, new TowtruckDatabase()); +}; + +/** + * Handles the `"pull_request.closed"` webhook event. + * @param {Event<"pull_request.closed">} event + */ +export const onPullRequestClosed = async (event) => { + console.log( + `PR #${event.payload.number} closed in ${event.payload.repository.name}: ${event.payload.pull_request.title}`, + ); + return await handleEvent(event, new TowtruckDatabase()); +}; diff --git a/webhooks/pullRequests.test.js b/webhooks/pullRequests.test.js new file mode 100644 index 0000000..747263f --- /dev/null +++ b/webhooks/pullRequests.test.js @@ -0,0 +1,77 @@ +import { describe, it } from "node:test"; +import expect from "node:assert"; +import { handleEvent } from "./pullRequests.js"; + +const db = { + saveToRepository: () => {}, +}; + +const octokit = { + request: async () => { + return Promise.resolve({}); + }, +}; + +describe("handleEvent", () => { + it("should update the open pull request information in the database for the repository", async (t) => { + const responses = { + "https://some.api/pulls": { + data: [ + { + user: { + login: "renovate[bot]", + }, + created_at: "2024-10-10T11:22:33.444Z", + state: "open", + }, + { + user: { + login: "foo", + }, + created_at: "2024-01-01T12:34:56.789Z", + state: "open", + }, + ], + }, + }; + + t.mock.method(octokit, "request", async (url) => + Promise.resolve(responses[url]), + ); + t.mock.method(db, "saveToRepository"); + + const payload = { + repository: { + name: "repo", + owner: { + login: "dxw", + }, + pulls_url: "https://some.api/pulls", + }, + }; + + const event = { + payload, + octokit, + }; + + const expected = { + openPrCount: 2, + openBotPrCount: 1, + oldestOpenPrOpenedAt: new Date("2024-01-01T12:34:56.789Z"), + mostRecentPrOpenedAt: new Date("2024-10-10T11:22:33.444Z"), + }; + + await handleEvent(event, db); + + expect.strictEqual(octokit.request.mock.callCount(), 1); + + expect.strictEqual(db.saveToRepository.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToRepository.mock.calls[0].arguments, [ + payload.repository.name, + "pullRequests", + expected, + ]); + }); +}); diff --git a/webhooks/repository.js b/webhooks/repository.js new file mode 100644 index 0000000..45dfd6b --- /dev/null +++ b/webhooks/repository.js @@ -0,0 +1,30 @@ +import { TowtruckDatabase } from "../db/index.js"; +import { mapRepoFromApiForStorage } from "../utils/index.js"; + +/** + * @typedef {import("./index.js").Event} Event + * @template {string} T + */ + +/** + * @param {Event<"repository">} event + * @param {TowtruckDatabase} db + */ +export const handleEvent = async ({ payload }, db) => { + if (payload.repository.archived) return; + + let repo = mapRepoFromApiForStorage(payload.repository); + + db.saveToRepository(payload.repository.name, "main", repo); +}; + +/** + * Handles the `"repository"` webhook event. + * @param {Event<"repository">} event + */ +export const onRepository = async (event) => { + console.log( + `${event.payload.repository.name} updated: ${event.payload.action}`, + ); + handleEvent(event, new TowtruckDatabase()); +}; diff --git a/webhooks/repository.test.js b/webhooks/repository.test.js new file mode 100644 index 0000000..c3d5d6a --- /dev/null +++ b/webhooks/repository.test.js @@ -0,0 +1,66 @@ +import { describe, it } from "node:test"; +import expect from "node:assert"; +import { handleEvent } from "./repository.js"; + +const db = { + saveToRepository: () => {}, +}; + +const octokit = { + request: async () => { + return Promise.resolve({}); + }, +}; + +describe("handleEvent", () => { + it("should update the core information in the database for the repository", async (t) => { + t.mock.method(db, "saveToRepository"); + + const payload = { + repository: { + name: "repo", + description: "Some repo", + owner: { + login: "dxw", + }, + url: "https://some.api/dxw/repo", + html_url: "https://some.site/dxw/repo", + issues_url: "https://some.api/issues", + pulls_url: "https://some.api/pulls", + updated_at: "2024-01-01T12:34:56.789Z", + language: "Ruby", + topics: ["foo", "bar"], + open_issues: 4, + }, + }; + + const event = { + payload, + octokit, + }; + + const expected = { + name: "repo", + owner: "dxw", + description: "Some repo", + htmlUrl: "https://some.site/dxw/repo", + apiUrl: "https://some.api/dxw/repo", + pullsUrl: "https://some.api/pulls", + issuesUrl: "https://some.api/issues", + updatedAt: "2024-01-01T12:34:56.789Z", + language: "Ruby", + topics: ["foo", "bar"], + openIssues: 4, + }; + + await handleEvent(event, db); + + expect.strictEqual(db.saveToRepository.mock.callCount(), 1); + + expect.deepStrictEqual(db.saveToRepository.mock.calls[0].arguments, [ + payload.repository.name, + "main", + expected, + ]); + }); +});