Add table of all repos on github

fetchRepos.js

@@ -0,0 +1,60 @@
+const fs = require('fs');
+const fetch = require('node-fetch');
+
+process.on('unhandledRejection', error => {
+  console.log('unhandledRejection', error.message);
+});
+
+const readme = 'README.md';
+const API = 'https://api.github.com/';
+const options = {
+  method: 'GET',
+  headers: {
+    'User-Agent': 'awesome-docker script listing',
+    'Content-Type': 'application/json',
+    Authorization: `token ${process.env.TOKEN}`,
+  },
+};
+
+function get(path, opt) {
+  return fetch(`${API}repos/${path}`, {
+    ...options,
+    ...opt,
+  })
+    .catch(err => console.error(err))
+    .then(r => r.json());
+}
+
+function delay(ms) {
+  return new Promise(resolve => {
+    setTimeout(() => resolve(), ms);
+  });
+}
+
+function extractAllRepos() {
+  const re = /https:\/\/github\.com\/([a-zA-Z0-9-._]+)\/([a-zA-Z0-9-._]+)/g;
+  const markdown = fs.readFileSync(readme, 'utf-8');
+  const md = markdown.match(re);
+  const uniq = [...new Set(md)];
+  const repos = uniq.map(x => x.slice('https://github.com/'.length, x.length));
+  fs.writeFileSync('data/list_repos.json', JSON.stringify(uniq, null, 2));
+  return repos;
+}
+
+async function main() {
+  const repos = extractAllRepos();
+  let data = [];
+  const batchSize = 10;
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < repos.length; i += batchSize) {
+    const batch = repos.slice(i, i + batchSize);
+    console.log({ batch });
+    const res = await Promise.all(batch.map(async path => get(path)));
+    data = data.concat(res);
+    await delay(3000);
+  }
+  if (process.env.DEBUG) console.log({ data });
+  fs.writeFileSync('data/fetched_repo_data.txt', JSON.stringify(data, null, 2));
+}
+
+main();