mirrors/awesome-docker
1
0
Fork 0
mirror of https://github.com/veggiemonk/awesome-docker.git synced 2025-04-29 07:37:58 +02:00
Code Activity

Improve update table script

Browse source
This commit is contained in:
Julien Bisconti 2018-05-29 00:40:36 +02:00 • committed by Andreas Gebhardt
parent 0c24343c82
commit d22672dffa
5 changed files with 34222 additions and 1494 deletions
Download patch file Download diff file Expand all files Collapse all files

51
fetchRepos.js
View file

@ -1,6 +1,8 @@
const fs = require('fs');
const fetch = require('node-fetch');
require('draftlog').into(console);
process.on('unhandledRejection', error => {
console.log('unhandledRejection', error.message);
});
@ -8,7 +10,8 @@ process.on('unhandledRejection', error => {
if (!process.env.TOKEN) {
throw new Error('no github token found');
}
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 10;
const DELAY = parseInt(process.env.DELAY, 10) || 3000;
const readme = 'README.md';
const API = 'https://api.github.com/';
const options = {
@ -26,7 +29,11 @@ function get(path, opt) {
...opt,
})
.catch(err => console.error(err))
.then(r => r.json());
.then(r => {
if (r.ok) return r.json();
throw new Error('Network response was not ok.');
})
.catch(err => console.error(err));
}
function delay(ms) {
@ -35,28 +42,44 @@ function delay(ms) {
});
}
function extractAllRepos() {
function extractAllRepos(markdown) {
const re = /https:\/\/github\.com\/([a-zA-Z0-9-._]+)\/([a-zA-Z0-9-._]+)/g;
const markdown = fs.readFileSync(readme, 'utf-8');
const md = markdown.match(re);
const uniq = [...new Set(md)];
const repos = uniq.map(x => x.slice('https://github.com/'.length, x.length));
fs.writeFileSync('data/list_repos.json', JSON.stringify(uniq, null, 2));
return repos;
return [...new Set(md)];
}
const barLine = console.draft('Starting batch...');
function ProgressBar(i, batchSize, total) {
const progress = Math.round(i / total * 100);
const units = Math.round(progress / 2);
return barLine(
`[${'='.repeat(units)}${' '.repeat(50 - units)}] ${progress}% - # ${i}`,
);
}
async function main() {
const repos = extractAllRepos();
const markdown = fs.readFileSync(readme, 'utf-8');
const githubRepos = extractAllRepos(markdown);
const repos = githubRepos.map(x =>
x.slice('https://github.com/'.length, x.length),
);
fs.writeFileSync(
'data/list_repos.json',
JSON.stringify(githubRepos, null, 2),
);
let data = [];
const batchSize = 10;
/* eslint-disable no-await-in-loop */
for (let i = 0; i < repos.length; i += batchSize) {
const batch = repos.slice(i, i + batchSize);
console.log({ batch });
for (let i = 0; i < repos.length; i += BATCH_SIZE) {
const batch = repos.slice(i, i + BATCH_SIZE);
const res = await Promise.all(batch.map(async path => get(path)));
data = data.concat(res);
await delay(3000);
if (process.env.DEBUG) console.log({ batch });
ProgressBar(i, BATCH_SIZE, repos.length);
await delay(DELAY);
}
ProgressBar(repos.length, BATCH_SIZE, repos.length);
if (process.env.DEBUG) console.log({ data });
fs.writeFileSync(
'data/fetched_repo_data.json',