feat: fetch all organization contributors for home page

.github/workflows/ci.yml

@@ -10,6 +10,8 @@ jobs:
   verify_files:
     name: Verify Files
     runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
     steps:
     - name: Checkout
       uses: actions/checkout@v3
@@ -22,6 +24,8 @@ jobs:
       run: npm ci
     - name: Shape libs-data.json
       run: npm run fetch-data
+    - name: Shape contributors.json
+      run: npm run fetch-contributors
     - name: Shape packages-versions.json
       run: npm run get-packages-versions
     - name: Lint Files

.gitignore

@@ -26,7 +26,6 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
-/src/libs-data.json
-/src/packages-versions.json
+/src/data/*.json
 
 .env

package.json

@@ -76,8 +76,9 @@
     "start": "next dev",
     "build": "next build",
     "fetch-data": "node ./scripts/prefetch-data.mjs",
+    "fetch-contributors": "node ./scripts/prefetch-contributors.mjs",
     "get-packages-versions": "node ./scripts/get-packages-versions.mjs",
-    "prepare-metadata": "npm run fetch-data && npm run get-packages-versions",
+    "prepare-metadata": "npm run fetch-data && npm run fetch-contributors && npm run get-packages-versions",
     "prestart": "npm run prepare-metadata",
     "prebuild": "npm run prepare-metadata",
     "pretest": "npm run prepare-metadata",

scripts/get-packages-versions.mjs

@@ -11,7 +11,7 @@ const packagesMap = {
 const packageJsonPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '../package.json');
 const packagesVersionsPath = path.join(
     path.dirname(fileURLToPath(import.meta.url)),
-    '../src/packages-versions.json',
+    '../src/data/packages-versions.json',
 );
 
 const getPackagesVersions = () => {

scripts/github.mjs

@@ -2,7 +2,9 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 import {Octokit} from '@octokit/rest';
 
-const octokit = new Octokit();
+const octokit = new Octokit({
+    auth: process.env.GITHUB_TOKEN,
+});
 
 const CONTRIBUTOR_IGNORE_LIST = ['dependabot', 'dependabot[bot]', 'gravity-ui-bot', 'yc-ui-bot'];
 
@@ -29,6 +31,15 @@ export async function getRepositoryContributors(repoOwner, repo) {
     return contributors;
 }
 
+export async function getOrganizationRepositories(org) {
+    return await octokit.paginate('GET /orgs/{org}/repos', {
+        org,
+        headers: {
+            'X-GitHub-Api-Version': '2022-11-28',
+        },
+    });
+}
+
 export async function fetchRepositoryCodeOwners(repoOwner, repo) {
     const url = `https://raw.githubusercontent.com/${repoOwner}/${repo}/main/CODEOWNERS`;
     const res = await fetch(url);

scripts/prefetch-contributors.mjs

@@ -0,0 +1,83 @@
+/**
+ * This script is used to prefetch all organization contributors.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+import * as dotenv from 'dotenv';
+
+import {getOrganizationRepositories, getRepositoryContributors} from './github.mjs';
+
+dotenv.config();
+
+const LIBS_DATA_FILENAME = 'libs-data.json';
+const CONTRIB_FILENAME = 'contributors.json';
+
+const contribDataPath = path.resolve(
+    path.dirname(fileURLToPath(import.meta.url)),
+    '../src/data',
+    CONTRIB_FILENAME,
+);
+
+const readLibsData = async () => {
+    const libsDataPath = path.resolve(
+        path.dirname(fileURLToPath(import.meta.url)),
+        '../src/data',
+        LIBS_DATA_FILENAME,
+    );
+
+    const libsDataContent = await fs.promises.readFile(libsDataPath, 'utf-8');
+    return JSON.parse(libsDataContent);
+};
+
+const start = async () => {
+    if (fs.existsSync(contribDataPath) && !process.env.GITHUB_TOKEN) {
+        console.error(
+            `The ${CONTRIB_FILENAME} file exists. You can delete it manually for refetch.
+            Learn more about the limitations of the GitHub API...
+            https://docs.github.com/ru/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#rate-limiting`,
+        );
+
+        return;
+    }
+
+    const libsData = await readLibsData();
+    const repos = await getOrganizationRepositories('gravity-ui');
+
+    const rawContributors = await Promise.all(
+        repos.map(async (repo) => {
+            if (libsData[repo.name]) {
+                return libsData[repo.name].contributors;
+            }
+
+            return await getRepositoryContributors(repo.owner.login, repo.name);
+        }),
+    );
+
+    const contributors = {};
+
+    for (const list of rawContributors) {
+        for (const contributor of list) {
+            const {login, contributions} = contributor;
+
+            if (contributors[login]) {
+                contributors[login].contributions += contributions;
+            } else {
+                contributors[login] = contributor;
+            }
+        }
+    }
+
+    const sortedContributors = Object.values(contributors).sort(
+        (a, b) => b.contributions - a.contributions,
+    );
+
+    fs.writeFileSync(contribDataPath, JSON.stringify(sortedContributors, null, 4), 'utf8');
+};
+
+start().catch((err) => {
+    console.error(err.message);
+    process.exit(1);
+});

scripts/prefetch-data.mjs

@@ -13,7 +13,7 @@ dotenv.config();
 
 const libsFetchedData = path.join(
     path.dirname(fileURLToPath(import.meta.url)),
-    '../src/libs-data.json',
+    '../src/data/libs-data.json',
 );
 
 const fetchNpmInfo = () => {

src/content/components/utils.ts

@@ -1,5 +1,5 @@
 import {TARGET_PROFILE} from '../../constants';
-import packagesVersions from '../../packages-versions.json';
+import packagesVersions from '../../data/packages-versions.json';
 import {Repos} from '../../types/common';
 
 const githubTargetProfile = process.env.GITHUB_PROFILE || TARGET_PROFILE;

src/utils/lib.ts

@@ -1,9 +1,10 @@
 import _ from 'lodash';
 import mm from 'micromatch';
 
-import libsData from '../libs-data.json';
+import allContributors from '../data/contributors.json';
+import libsData from '../data/libs-data.json';
+import packagesVersions from '../data/packages-versions.json';
 import {libs} from '../libs.mjs';
-import packagesVersions from '../packages-versions.json';
 
 type LibConfig = {
     id: string;
@@ -20,7 +21,7 @@
 };
 
 export type Contributor = {
     [x: string]: any;
     login: string;
     url: string;
     avatarUrl: string;
@@ -53,7 +54,7 @@
     const result: Lib[] = [];
 
     libs.forEach((config) => {
         const data = (libsData as any)[config.id];
 
         if (!data) {
             throw new Error(`Can't find fetched data for lib with id – ${config.id}`);
@@ -75,7 +76,7 @@
         throw new Error(`Can't find config for lib with id – ${id}`);
     }
 
     const data = (libsData as any)[id];
 
     if (!data) {
         throw new Error(`Can't find fetched data for lib with id – ${id}`);
@@ -95,7 +96,7 @@
     }
 
     try {
         libraryVersion = (packagesVersions as any)[id];
     } catch {
         libraryVersion = undefined;
     }
@@ -106,25 +107,7 @@
 export const getLibraryGithubUrl = (library: Lib) =>
     `https://github.com/${library.config.githubId}`;
 
-export const getAllContributors = () => {
-    const libraries = getLibsList();
-
-    const contributors: Record<string, Contributor> = {};
-
-    for (const {data} of libraries) {
-        for (const contributor of data.contributors) {
-            const {login, contributions} = contributor;
-
-            if (contributors[login]) {
-                contributors[login].contributions += contributions;
-            } else {
-                contributors[login] = contributor;
-            }
-        }
-    }
-
-    return Object.values(contributors).sort((a, b) => b.contributions - a.contributions);
-};
+export const getAllContributors = () => allContributors;
 
 export const getMaintainers = (lib: Lib, path = '/'): Contributor[] => {
     const {contributors, codeOwners} = lib.data;
@@ -144,7 +127,7 @@
             const maintainer = contributors.find((contributor) => contributor.login === owner);
 
             if (!maintainer) {
                 console.warn(`code owner ${owner} is not a contributor`);
 
                 return {
                     login: owner,