diff --git a/package.json b/package.json index b002fdd..6ab37bd 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "rimraf ./dist/ && node build.js", "test:test": "jest", - "test": "node pull_request.js" + "test": "node pr_check_gql.js" }, "repository": { "type": "git", diff --git a/pr_check_gql.js b/pr_check_gql.js new file mode 100644 index 0000000..c9941a6 --- /dev/null +++ b/pr_check_gql.js @@ -0,0 +1,218 @@ +const fs = require('fs-extra'); +const fetch = require('node-fetch'); +const exclude = require('./exclude_in_test.json'); + +function envvar_undefined(variable_name) { + throw new Error(`${variable_name} must be defined`); +} +console.log({ + DEBUG: process.env.DEBUG || false, +}); + +const README = 'README.md'; +const GITHUB_GQL_API = 'https://api.github.com/graphql'; +const TOKEN = process.env.GITHUB_TOKEN || envvar_undefined('GITHUB_TOKEN'); + +const LINKS_OPTIONS = { + redirect: 'error', + headers: { + 'Content-Type': 'application/json', + 'user-agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }, +}; + +const Authorization = `token ${TOKEN}`; + +const make_GQL_options = query => ({ + method: 'POST', + headers: { + Authorization, + 'Content-Type': 'application/json', + 'user-agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36', + }, + body: JSON.stringify({ query }), +}); + +const LOG = { + error: (...args) => console.error('❌ ERROR', args), + error_string: (...args) => + console.error('❌ ERROR', JSON.stringify({ ...args }, null, ' ')), + debug: (...args) => { + if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args }); + }, + debug_string: (...args) => { + if (process.env.DEBUG) + console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, ' ')); + }, +}; + +const handleFailure = error => { + console.error(`${error.message}: ${error.stack}`, { error }); + process.exit(1); +}; + +process.on('unhandledRejection', handleFailure); + +const extract_all_links = markdown => { + // if you have a problem and you try to solve it with a regex, + // now you have two problems + // TODO: replace this mess with a mardown parser ? + const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g; + return markdown.match(re); +}; + +const find_duplicates = arr => { + const hm = {}; + const dup = []; + arr.forEach(e => { + if (hm[e]) dup.push(e); + else hm[e] = true; + }); + return dup; +}; + +const partition = (arr, func) => { + const ap = [[], []]; + arr.forEach(e => (func(e) ? ap[0].push(e) : ap[1].push(e))); + return ap; +}; + +async function fetch_link(url) { + try { + const { ok, statusText, redirected } = await fetch(url, LINKS_OPTIONS); + return [url, { ok, status: statusText, redirected }]; + } catch (error) { + return [url, { ok: false, status: error.message }]; + } +} + +async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) { + const result = []; + /* eslint-disable no-await-in-loop */ + for (let i = 0; i < arr.length; i += BATCH_SIZE) { + const batch = arr.slice(i, i + BATCH_SIZE); + LOG.debug_string({ batch }); + let res = await Promise.all(batch.map(get)); + console.log(`batch fetched...${i + BATCH_SIZE}`); + res = post_filter_func ? res.filter(post_filter_func) : res; + LOG.debug_string({ res }); + result.push(...res); + } + return result; +} + +const extract_repos = arr => + arr + .map(e => e.substr('https://github.com/'.length).split('/')) + .filter(r => r.length === 2 && r[1] !== ''); + +const generate_GQL_query = arr => + `query AWESOME_REPOS{ ${arr + .map( + ([owner, name]) => + `repo_${owner.replace(/(-|\.)/g, '_')}_${name.replace( + /(-|\.)/g, + '_', + )}: repository(owner: "${owner}", name:"${name}"){ nameWithOwner } `, + ) + .join('')} }`; + +// ============================================================= +// const batch_github_repos = async (github_links) => { +// const BATCH_SIZE = 50; +// const repos = extract_repos(github_links); +// for (let i = 0; i < repos.length; i += BATCH_SIZE) { +// const batch = repos.slice(i, i + BATCH_SIZE); +// const query = generate_GQL_query(batch); +// LOG.debug({ query }); +// const gql_response = await fetch( +// 'https://api.github.com/graphql', +// make_GQL_options(query), +// ).then((r) => r.json()); +// LOG.debug({ gql_response }); +// } +// }; +// ============================================================= + +const exclude_length = exclude.length; +const exclude_from_list = link => { + let is_excluded = false; + for (let i = 0; i < exclude_length; i += 1) { + if (link.startsWith(exclude[i])) { + is_excluded = true; + break; + } + } + return is_excluded; +}; + +async function main() { + const has_error = { + show: false, + duplicates: '', + other_links_error: '', + github_repos: '', + }; + const markdown = await fs.readFile(README, 'utf8'); + let links = extract_all_links(markdown); + links = links.filter(l => !exclude_from_list(l)); // exclude websites + LOG.debug_string({ links }); + + console.log(`total links to check ${links.length}`); + + console.log('checking for duplicates links...'); + + const duplicates = find_duplicates(links); + if (duplicates.length > 0) { + has_error.show = true; + has_error.duplicates = duplicates; + } + LOG.debug_string({ duplicates }); + const [github_links, external_links] = partition(links, link => + link.startsWith('https://github.com'), + ); + + console.log(`checking ${external_links.length} external links...`); + + const external_links_error = await batch_fetch({ + arr: external_links, + get: fetch_link, + post_filter_func: x => !x[1].ok, + BATCH_SIZE: 8, + }); + if (external_links_error.length > 0) { + has_error.show = true; + has_error.other_links_error = external_links_error; + } + + console.log(`checking ${github_links.length} GitHub repositories...`); + + const repos = extract_repos(github_links); + const query = generate_GQL_query(repos); + const options = make_GQL_options(query); + LOG.debug_string(query); + const gql_response = await fetch(GITHUB_GQL_API, options) + .then(r => r.json()) + .catch(err => console.error({ err })); + LOG.debug(gql_response); + if (gql_response.errors) { + has_error.show = true; + has_error.github_repos = gql_response.errors; + } + + console.log({ + TEST_PASSED: !has_error.show, + GITHUB_REPOSITORY: github_links.length, + EXTERNAL_LINKS: external_links.length, + }); + + if (has_error.show) { + LOG.error_string(has_error); + process.exit(1); + } +} + +console.log('starting...'); +main();