From cca943ea92f5168976b551e9f18daff0bdea2a1e Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 01:06:36 -0500 Subject: [PATCH 001/773] fix: minor stuff --- .github/workflows/branches.yml | 2 ++ tests/presets.test.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/branches.yml b/.github/workflows/branches.yml index 09fdcf04788..bd0bf5f7a8d 100644 --- a/.github/workflows/branches.yml +++ b/.github/workflows/branches.yml @@ -28,3 +28,5 @@ jobs: "", "Thanks for your cooperation 🦾!" ].join("\n")}) + - name: Change base branch + run: exit 1 diff --git a/tests/presets.test.js b/tests/presets.test.js index 8f88e9850e0..d661081555d 100644 --- a/tests/presets.test.js +++ b/tests/presets.test.js @@ -21,7 +21,7 @@ catch { if (!/^[-\w\d]+\/[-\w\d]+$/.test(repo)) throw new Error(`invalid repo: ${repo}`) console.log(`cloning: ${repo}@${branch}`) - processes.execSync(`git clone https://github.com/${repo}.git ${__presets} --branch ${branch} --single-branch`) + processes.execFileSync("git", ["clone", `https://github.com/${repo}.git`, __presets, "--branch", branch, "--single-branch"]) } //Generate presets examples From ac995db988be462cc3b9ca89fd011cab20500a6e Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 01:11:22 -0500 Subject: [PATCH 002/773] chore: replace nosettings and mock by sandbox --- .github/scripts/preview.mjs | 2 +- source/app/action/index.mjs | 4 ++-- source/app/metrics/setup.mjs | 6 +++--- source/app/web/index.mjs | 2 +- source/app/web/instance.mjs | 15 ++++++++++++--- source/plugins/languages/analyzers.mjs | 2 +- tests/metrics.test.js | 2 +- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/scripts/preview.mjs b/.github/scripts/preview.mjs index 0d631596e42..c2d61a56469 100644 --- a/.github/scripts/preview.mjs +++ b/.github/scripts/preview.mjs @@ -19,7 +19,7 @@ const __preview_templates_ = paths.join(__preview, ".templates_") const __preview_about = paths.join(__preview, "about/.statics") //Extract from web server -const { conf, Templates } = await setup({ nosettings: true, log: false }) +const { conf, Templates } = await setup({ log: false }) const templates = Object.entries(Templates).map(([name]) => ({ name, enabled: true })) const metadata = Object.fromEntries( Object.entries(conf.metadata.plugins) diff --git a/source/app/action/index.mjs b/source/app/action/index.mjs index d2423bca60d..c639b1d5102 100644 --- a/source/app/action/index.mjs +++ b/source/app/action/index.mjs @@ -88,7 +88,7 @@ async function retry(func, {retries = 1, delay = 0} = {}) { } //Load configuration - const {conf, Plugins, Templates} = await setup({log:false, nosettings:true, community:{templates:core.getInput("setup_community_templates")}}) + const {conf, Plugins, Templates} = await setup({log:false, community:{templates:core.getInput("setup_community_templates")}}) const {metadata} = conf conf.settings.extras = {default:true} info("Setup", "complete") @@ -288,7 +288,7 @@ async function retry(func, {retries = 1, delay = 0} = {}) { await new Promise(async (solve, reject) => { let stdout = "" setTimeout(() => reject("Timeout while waiting for Insights webserver"), 5 * 60 * 1000) - const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env:{...process.env, NO_SETTINGS:true}}) + const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env:{...process.env}}) web.stdout.on("data", data => (console.debug(`web > ${data}`), stdout += data, /Server ready !/.test(stdout) ? solve() : null)) web.stderr.on("data", data => console.debug(`web > ${data}`)) }) diff --git a/source/app/metrics/setup.mjs b/source/app/metrics/setup.mjs index 417ee693621..aa942dea9e0 100644 --- a/source/app/metrics/setup.mjs +++ b/source/app/metrics/setup.mjs @@ -13,7 +13,7 @@ const Templates = {} const Plugins = {} /**Setup */ -export default async function({log = true, nosettings = false, community = {}} = {}) { +export default async function({log = true, sandbox = false, community = {}} = {}) { //Paths const __metrics = path.join(path.dirname(url.fileURLToPath(import.meta.url)), "../../..") const __statics = path.join(__metrics, "source/app/web/statics") @@ -42,8 +42,8 @@ export default async function({log = true, nosettings = false, community = {}} = //Load settings logger("metrics/setup > load settings.json") if (fs.existsSync(__settings)) { - if (nosettings) - logger("metrics/setup > load settings.json > skipped because no settings is enabled") + if (sandbox) + logger("metrics/setup > load settings.json > skipped because in sandbox is enabled") else { conf.settings = JSON.parse(`${await fs.promises.readFile(__settings)}`) logger("metrics/setup > load settings.json > success") diff --git a/source/app/web/index.mjs b/source/app/web/index.mjs index 6b386eb738a..6d18442d335 100644 --- a/source/app/web/index.mjs +++ b/source/app/web/index.mjs @@ -1,4 +1,4 @@ import app from "./instance.mjs" ;(async function() { - await app({mock:process.env.USE_MOCKED_DATA, nosettings:process.env.NO_SETTINGS}) + await app({sandbox:process.env.SANDBOX}) })() diff --git a/source/app/web/instance.mjs b/source/app/web/instance.mjs index 01b18c40ecc..8df66cd250a 100644 --- a/source/app/web/instance.mjs +++ b/source/app/web/instance.mjs @@ -12,11 +12,20 @@ import presets from "../metrics/presets.mjs" import setup from "../metrics/setup.mjs" /**App */ -export default async function({mock, nosettings} = {}) { +export default async function({sandbox} = {}) { //Load configuration settings - const {conf, Plugins, Templates} = await setup({nosettings}) + const {conf, Plugins, Templates} = await setup({sandbox}) const {token, maxusers = 0, restricted = [], debug = false, cached = 30 * 60 * 1000, port = 3000, ratelimiter = null, plugins = null} = conf.settings - mock = mock || conf.settings.mocked + const mock = sandbox || conf.settings.mocked + + //Sandbox mode + if (sandbox) { + console.debug("metrics/app > sandbox mode is specified, enabling advanced features") + conf.settings.extras = conf.settings.extras ?? {} + conf.settings.extras.default = true + conf.settings["plugins.default"] = true + conf.settings.optimize = true + } //Process mocking and default plugin state for (const plugin of Object.keys(Plugins).filter(x => !["base", "core"].includes(x))) { diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index c280532a6e7..e20f95741c8 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -229,7 +229,7 @@ if (/languages.analyzers.mjs$/.test(process.argv[1])) { process.exit(1) } const {default:setup} = await import("../../app/metrics/setup.mjs") - const {conf:{metadata}} = await setup({log:false, nosettings:true}) + const {conf:{metadata}} = await setup({log:false}) const {"commits.authoring":authoring} = await metadata.plugins.base.inputs({q:{"commits.authoring":_authoring}, account:"bypass"}) const data = {shared:{"commits.authoring":authoring}} diff --git a/tests/metrics.test.js b/tests/metrics.test.js index cd667e642c0..00c9f705799 100644 --- a/tests/metrics.test.js +++ b/tests/metrics.test.js @@ -33,7 +33,7 @@ web.run = async vars => (await axios(`http://localhost:3000/lowlighter?${new url web.start = async () => new Promise(solve => { let stdout = "" - web.instance = processes.spawn("node", ["source/app/web/index.mjs"], { env: { ...process.env, USE_MOCKED_DATA: true, NO_SETTINGS: true } }) + web.instance = processes.spawn("node", ["source/app/web/index.mjs"], { env: { ...process.env, SANDBOX: true } }) web.instance.stdout.on("data", data => (stdout += data, /Server ready !/.test(stdout) ? solve() : null)) web.instance.stderr.on("data", data => console.error(`${data}`)) }) From 11b000b1023c2e205bb44bec33202f6f981b8eb6 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 01:12:37 -0500 Subject: [PATCH 003/773] ci: update presets_examples --- .github/scripts/presets_examples.mjs | 32 ++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/scripts/presets_examples.mjs b/.github/scripts/presets_examples.mjs index 23ee8f26032..9d1d9f7bc8d 100644 --- a/.github/scripts/presets_examples.mjs +++ b/.github/scripts/presets_examples.mjs @@ -5,6 +5,7 @@ import yaml from "js-yaml" import paths from "path" import sgit from "simple-git" import url from "url" +import fetch from "node-fetch" //Mode const [mode = "dryrun"] = process.argv.slice(2) @@ -20,26 +21,20 @@ const git = sgit(__presets) await git.pull() const staged = new Set() -//Github action -const action = yaml.load(await fs.readFile(paths.join(__metrics, "action.yml"), "utf8")) -action.defaults = Object.fromEntries(Object.entries(action.inputs).map(([key, { default: value }]) => [key, value])) -action.input = vars => Object.fromEntries([...Object.entries(action.defaults), ...Object.entries(vars)].map(([key, value]) => [`INPUT_${key.toLocaleUpperCase()}`, value])) -action.run = async vars => - await new Promise((solve, reject) => { - let [stdout, stderr] = ["", ""] - const env = { ...process.env, ...action.input(vars), GITHUB_REPOSITORY: "lowlighter/metrics" } - const child = processes.spawn("node", ["source/app/action/index.mjs"], { env }) - child.stdout.on("data", data => stdout += data) - child.stderr.on("data", data => stderr += data) - child.on("close", code => { - if (code === 0) - return solve(stdout.match(/(?)/)?.groups?.svg ?? ``) - console.log(stdout, stderr) - reject(stdout) - }) +//Web instance +const web = {} +web.run = async vars => await fetch(`http://localhost:3000/lowlighter?${new url.URLSearchParams(Object.fromEntries(Object.entries(vars).map(([key, value]) => [key.replace(/^plugin_/, "").replace(/_/g, "."), value])))}`).then(response => response.text()) +web.start = async () => + new Promise(solve => { + let stdout = "" + web.instance = processes.spawn("node", ["source/app/web/index.mjs"], { env: { ...process.env, SANDBOX: true } }) + web.instance.stdout.on("data", data => (stdout += data, /Server ready !/.test(stdout) ? solve() : null)) + web.instance.stderr.on("data", data => console.error(`${data}`)) }) +web.stop = async () => await web.instance.kill("SIGKILL") //Generate presets examples +await web.start() for (const path of await fs.readdir(__presets)) { if (/^[.@]/.test(path)) continue @@ -49,7 +44,7 @@ for (const path of await fs.readdir(__presets)) { //Example console.log(`generating: ${preset}/example.svg`) - const svg = await action.run({ config_presets: `@${preset}`, debug_print: true, plugins_errors_fatal: true, dryrun: true, use_mocked_data: true, verify: true, token: "MOCKED_TOKEN" }) + const svg = await web.run({ config_presets: `@${preset}`, plugins_errors_fatal: true, verify: true, optimize: "css, xml" }) await fs.writeFile(paths.join(__presets, path, "example.svg"), svg) staged.add(paths.join(__presets, path, "example.svg")) @@ -71,6 +66,7 @@ for (const path of await fs.readdir(__presets)) { ) staged.add(paths.join(__presets, path, "README.md")) } +await web.stop() //Commit and push if (mode === "publish") { From 1d6ede7d410738b87992019b184f5fb5232bc1a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 27 Jan 2022 06:33:53 +0000 Subject: [PATCH 004/773] chore: code formatting --- .github/scripts/presets_examples.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/presets_examples.mjs b/.github/scripts/presets_examples.mjs index 9d1d9f7bc8d..dc9fd208310 100644 --- a/.github/scripts/presets_examples.mjs +++ b/.github/scripts/presets_examples.mjs @@ -2,10 +2,10 @@ import fs from "fs/promises" import processes from "child_process" import yaml from "js-yaml" +import fetch from "node-fetch" import paths from "path" import sgit from "simple-git" import url from "url" -import fetch from "node-fetch" //Mode const [mode = "dryrun"] = process.argv.slice(2) From b1137b397371266a62634479a524f4b210849d7b Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 01:41:48 -0500 Subject: [PATCH 005/773] docs(plugins/community): fix path --- .github/readme/partials/templated/plugins.community.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/readme/partials/templated/plugins.community.md b/.github/readme/partials/templated/plugins.community.md index cca86aa22db..eb0282b1624 100644 --- a/.github/readme/partials/templated/plugins.community.md +++ b/.github/readme/partials/templated/plugins.community.md @@ -10,7 +10,7 @@ const cells = [["even", elements[i]], ["odd", elements[i+1]]] for (const [cell, [plugin, {name, readme}]] of cells) { if (cell === "even") { %>
<% } %> - <% if (plugin) { %><%= name -%><% } %><% + <% if (plugin) { %><%= name -%><% } %><% if (cell === "odd") { %>
<% }} From 12ec9778aac22c3e446bfe91b6064d737c69f8bb Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 01:42:53 -0500 Subject: [PATCH 006/773] build: fix presets caching --- .github/scripts/presets_examples.mjs | 2 +- source/app/web/instance.mjs | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/scripts/presets_examples.mjs b/.github/scripts/presets_examples.mjs index 9d1d9f7bc8d..914d8923038 100644 --- a/.github/scripts/presets_examples.mjs +++ b/.github/scripts/presets_examples.mjs @@ -44,7 +44,7 @@ for (const path of await fs.readdir(__presets)) { //Example console.log(`generating: ${preset}/example.svg`) - const svg = await web.run({ config_presets: `@${preset}`, plugins_errors_fatal: true, verify: true, optimize: "css, xml" }) + const svg = await web.run({ config_presets: `@${preset}`, plugins_errors_fatal: true }) await fs.writeFile(paths.join(__presets, path, "example.svg"), svg) staged.add(paths.join(__presets, path, "example.svg")) diff --git a/source/app/web/instance.mjs b/source/app/web/instance.mjs index 8df66cd250a..872b1523205 100644 --- a/source/app/web/instance.mjs +++ b/source/app/web/instance.mjs @@ -15,17 +15,13 @@ import setup from "../metrics/setup.mjs" export default async function({sandbox} = {}) { //Load configuration settings const {conf, Plugins, Templates} = await setup({sandbox}) - const {token, maxusers = 0, restricted = [], debug = false, cached = 30 * 60 * 1000, port = 3000, ratelimiter = null, plugins = null} = conf.settings - const mock = sandbox || conf.settings.mocked - //Sandbox mode if (sandbox) { console.debug("metrics/app > sandbox mode is specified, enabling advanced features") - conf.settings.extras = conf.settings.extras ?? {} - conf.settings.extras.default = true - conf.settings["plugins.default"] = true - conf.settings.optimize = true + Object.assign(conf.settings, {optimize: true, cached:0, "plugins.default":true, extras:{default:true}}) } + const {token, maxusers = 0, restricted = [], debug = false, cached = 30 * 60 * 1000, port = 3000, ratelimiter = null, plugins = null} = conf.settings + const mock = sandbox || conf.settings.mocked //Process mocking and default plugin state for (const plugin of Object.keys(Plugins).filter(x => !["base", "core"].includes(x))) { From e53361de9e9eaa12ecde67bbc8c4a761fdb81c56 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 12:32:00 -0500 Subject: [PATCH 007/773] fix: test commands --- .github/workflows/test.yml | 4 ++-- package.json | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34015f88962..38dd6e204dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - name: Setup metrics run: npm ci - name: Check contributions requirements - run: npm test -- ci.test.js --noStackTrace + run: npm run test-contrib env: PR_AUTHOR: ${{ github.event.pull_request.user.login }} - name: Run linter @@ -47,7 +47,7 @@ jobs: - name: Build lowlighter/metrics:${{ github.head_ref || 'master' }} run: docker build -t lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') . - name: Run tests - run: docker run --entrypoint="" lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') npm test -- metrics.test.js + run: docker run --entrypoint="" lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') npm test-metrics # Run CodeQL on branch analyze: diff --git a/package.json b/package.json index 2f0725995c5..cd343d8375f 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,10 @@ "main": "index.mjs", "scripts": { "start": "node source/app/web/index.mjs", - "test": "jest --runInBand --testPathIgnorePatterns presets.test.js", - "test-presets": "jest --runInBand presets.test.js", + "test": "jest --runInBand", + "test-contrib": "jest --runInBand ci.test.js --noStackTrace", + "test-presets": "jest --runInBand presets.test.js --noStackTrace", + "test-metrics": "jest --runInBand metrics.test.js", "build": "node .github/scripts/build.mjs", "presets": "node .github/scripts/presets_examples.mjs", "quickstart": "node .github/scripts/quickstart/index.mjs", From f3c938466172c5d2c6b294fb1b9d972209078d83 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 12:45:51 -0500 Subject: [PATCH 008/773] fix(plugins/stargazers): add missing default --- source/plugins/stargazers/metadata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/stargazers/metadata.yml b/source/plugins/stargazers/metadata.yml index e7ec57b2ded..67aae895ebb 100644 --- a/source/plugins/stargazers/metadata.yml +++ b/source/plugins/stargazers/metadata.yml @@ -24,7 +24,7 @@ inputs: - `classic`: `
` based charts, simple and lightweight - `chartist`: `` based charts, smooth type: string - default: + default: classic values: - classic - chartist \ No newline at end of file From d13677280d8ba258e3987580640047258d44923f Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 12:46:05 -0500 Subject: [PATCH 009/773] fix: test commands --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38dd6e204dc..b0a00829ef6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: - name: Build lowlighter/metrics:${{ github.head_ref || 'master' }} run: docker build -t lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') . - name: Run tests - run: docker run --entrypoint="" lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') npm test-metrics + run: docker run --entrypoint="" lowlighter/metrics:$(echo ${{ github.head_ref || 'master' }} | sed 's/\//-/g') npm run test-metrics # Run CodeQL on branch analyze: From 65aeac29549c66d6ec59545b8a06e63cd9568429 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 20:08:09 -0500 Subject: [PATCH 010/773] fix(tests): web broken tests --- tests/cases/nightscout.plugin.yml | 2 +- tests/cases/poopmap.plugin.yml | 3 ++- tests/cases/posts.plugin.yml | 4 ++-- tests/cases/screenshot.plugin.yml | 2 +- tests/cases/stackoverflow.plugin.yml | 2 +- tests/cases/starlists.plugin.yml | 2 +- tests/cases/topics.plugin.yml | 4 ++-- tests/mocks/api/axios/get/poopmap.mjs | 2 +- tests/secrets.json | 3 ++- 9 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/cases/nightscout.plugin.yml b/tests/cases/nightscout.plugin.yml index b88fbab9d9b..8323918e207 100644 --- a/tests/cases/nightscout.plugin.yml +++ b/tests/cases/nightscout.plugin.yml @@ -1,6 +1,6 @@ - uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_nightscout: 'yes' plugin_nightscout_url: https://testapp.herokuapp.com/ use_mocked_data: 'yes' diff --git a/tests/cases/poopmap.plugin.yml b/tests/cases/poopmap.plugin.yml index 321e5f1fc8a..c4c3df0fc76 100644 --- a/tests/cases/poopmap.plugin.yml +++ b/tests/cases/poopmap.plugin.yml @@ -1,6 +1,7 @@ - uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED + plugin_poopmap_token: MOCKED_TOKEN plugin_poopmap: 'yes' use_mocked_data: 'yes' verify: 'yes' diff --git a/tests/cases/posts.plugin.yml b/tests/cases/posts.plugin.yml index 9c9fc02735f..9f4bf94b3ab 100644 --- a/tests/cases/posts.plugin.yml +++ b/tests/cases/posts.plugin.yml @@ -1,7 +1,7 @@ - name: ✒️ Recent posts - Recent posts uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_posts: 'yes' plugin_posts_source: dev.to use_mocked_data: 'yes' @@ -9,7 +9,7 @@ - name: ✒️ Recent posts - Recent posts with descriptions and cover images uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_posts: 'yes' plugin_posts_source: dev.to plugin_posts_limit: 2 diff --git a/tests/cases/screenshot.plugin.yml b/tests/cases/screenshot.plugin.yml index c840701a87f..bc884b13c02 100644 --- a/tests/cases/screenshot.plugin.yml +++ b/tests/cases/screenshot.plugin.yml @@ -1,7 +1,7 @@ - name: 📸 Website screenshot - XKCD of the day uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_screenshot: 'yes' plugin_screenshot_title: XKCD of the day plugin_screenshot_url: https://xkcd.com diff --git a/tests/cases/stackoverflow.plugin.yml b/tests/cases/stackoverflow.plugin.yml index 138c119886a..c397e8d31ec 100644 --- a/tests/cases/stackoverflow.plugin.yml +++ b/tests/cases/stackoverflow.plugin.yml @@ -1,7 +1,7 @@ - name: 🗨️ StackOverflow plugin - Top answers from stackoverflow uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_stackoverflow: 'yes' plugin_stackoverflow_user: 1 plugin_stackoverflow_sections: answers-top diff --git a/tests/cases/starlists.plugin.yml b/tests/cases/starlists.plugin.yml index 1476a38f2c4..7e8ac2a462a 100644 --- a/tests/cases/starlists.plugin.yml +++ b/tests/cases/starlists.plugin.yml @@ -1,7 +1,7 @@ - name: 💫 Starlists - Featured star list uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_starlists: 'yes' plugin_starlists_limit_repositories: 2 plugin_starlists_only: TC39 diff --git a/tests/cases/topics.plugin.yml b/tests/cases/topics.plugin.yml index bc6cf181a67..1f8b9bbb4c5 100644 --- a/tests/cases/topics.plugin.yml +++ b/tests/cases/topics.plugin.yml @@ -1,7 +1,7 @@ - name: 📌 Starred topics - Labels uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_topics: 'yes' plugin_topics_limit: 12 use_mocked_data: 'yes' @@ -9,7 +9,7 @@ - name: 📌 Starred topics - Icons uses: lowlighter/metrics@latest with: - token: MOCKED_TOKEN + token: NOT_NEEDED plugin_topics: 'yes' plugin_topics_limit: 0 plugin_topics_mode: icons diff --git a/tests/mocks/api/axios/get/poopmap.mjs b/tests/mocks/api/axios/get/poopmap.mjs index 5b313deb95d..82a4fe60248 100644 --- a/tests/mocks/api/axios/get/poopmap.mjs +++ b/tests/mocks/api/axios/get/poopmap.mjs @@ -1,7 +1,7 @@ /**Mocked data */ export default function({ faker, url, options, login = faker.internet.userName() }) { //Wakatime api - if (/^https:..api.poopmap.net$/.test(url)) { + if (/^https:..api.poopmap.net/.test(url)) { //Get user profile if (/public_links\/MOCKED_TOKEN/.test(url)) { console.debug(`metrics/compute/mocks > mocking poopmap api result > ${url}`) diff --git a/tests/secrets.json b/tests/secrets.json index d98417893da..f28d02d45cc 100644 --- a/tests/secrets.json +++ b/tests/secrets.json @@ -8,5 +8,6 @@ "NIGHTSCOUT_URL":"https://testapp.herokuapp.com/", "WAKATIME_TOKEN":"MOCKED_TOKEN", "TWITTER_TOKEN":"MOCKED_TOKEN", - "STOCK_TOKEN":"MOCKED_TOKEN" + "STOCK_TOKEN":"MOCKED_TOKEN", + "POOPMAP_TOKEN":"MOCKED_TOKEN" } \ No newline at end of file From b1906e3de81fa7df7df809695b93e0fa41eb41af Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 20:09:26 -0500 Subject: [PATCH 011/773] docs: update metadata.yml where token isn't needed --- .github/workflows/examples.yml | 14 +++++++------- source/plugins/community/README.md | 10 +++++----- source/plugins/community/nightscout/README.md | 2 +- source/plugins/community/nightscout/examples.yml | 2 +- source/plugins/community/poopmap/README.md | 3 ++- source/plugins/community/poopmap/examples.yml | 3 ++- source/plugins/community/screenshot/README.md | 2 +- source/plugins/community/screenshot/examples.yml | 2 +- source/plugins/core/README.md | 4 +++- source/plugins/core/metadata.yml | 4 +++- source/plugins/posts/README.md | 4 ++-- source/plugins/posts/examples.yml | 4 ++-- source/plugins/stackoverflow/README.md | 2 +- source/plugins/stackoverflow/examples.yml | 2 +- source/plugins/stargazers/README.md | 6 +++--- source/plugins/stargazers/metadata.yml | 4 ++-- source/plugins/starlists/README.md | 2 +- source/plugins/starlists/examples.yml | 2 +- source/plugins/topics/README.md | 4 ++-- source/plugins/topics/examples.yml | 4 ++-- 20 files changed, 43 insertions(+), 37 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 34786238c07..3037d6d6f16 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -203,7 +203,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.topics.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 12 @@ -216,7 +216,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.topics.icons.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 0 @@ -673,7 +673,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.starlists.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_starlists: yes plugin_starlists_limit_repositories: 2 @@ -834,7 +834,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.stackoverflow.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_stackoverflow: yes plugin_stackoverflow_user: 1 @@ -927,7 +927,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.posts.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to @@ -940,7 +940,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.posts.full.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to @@ -997,7 +997,7 @@ jobs: uses: lowlighter/metrics@master with: filename: metrics.plugin.screenshot.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_screenshot: yes plugin_screenshot_title: XKCD of the day diff --git a/source/plugins/community/README.md b/source/plugins/community/README.md index 9ab1e8b2590..4b1c0401b44 100644 --- a/source/plugins/community/README.md +++ b/source/plugins/community/README.md @@ -2,8 +2,8 @@

🎲 Community plugins

Additional plugins maintained by community for even more features! - 🥠 Fortune - 💉 Nightscout + 🥠 Fortune + 💉 Nightscout @@ -15,8 +15,8 @@ - 💩 PoopMap plugin - 📸 Website screenshot + 💩 PoopMap plugin + 📸 Website screenshot @@ -28,7 +28,7 @@ - 💹 Stock prices + 💹 Stock prices diff --git a/source/plugins/community/nightscout/README.md b/source/plugins/community/nightscout/README.md index ddca1aba52f..53303f8ea23 100644 --- a/source/plugins/community/nightscout/README.md +++ b/source/plugins/community/nightscout/README.md @@ -125,7 +125,7 @@ Check out the instructions there. ```yaml uses: lowlighter/metrics@latest with: - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED plugin_nightscout: yes plugin_nightscout_url: ${{ secrets.NIGHTSCOUT_URL }} diff --git a/source/plugins/community/nightscout/examples.yml b/source/plugins/community/nightscout/examples.yml index f6fd61e7d05..aaedd35413d 100644 --- a/source/plugins/community/nightscout/examples.yml +++ b/source/plugins/community/nightscout/examples.yml @@ -1,6 +1,6 @@ - uses: lowlighter/metrics@latest with: - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED plugin_nightscout: yes plugin_nightscout_url: ${{ secrets.NIGHTSCOUT_URL }} prod: diff --git a/source/plugins/community/poopmap/README.md b/source/plugins/community/poopmap/README.md index 9cdebd56d14..77df53ff866 100644 --- a/source/plugins/community/poopmap/README.md +++ b/source/plugins/community/poopmap/README.md @@ -97,7 +97,8 @@ This token will not expire and it will be able to access only public details. ```yaml uses: lowlighter/metrics@latest with: - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED + plugin_poopmap_token: ${{ secrets.POOPMAP_TOKEN }} plugin_poopmap: yes ``` diff --git a/source/plugins/community/poopmap/examples.yml b/source/plugins/community/poopmap/examples.yml index 2821c08850b..ef7c6988893 100644 --- a/source/plugins/community/poopmap/examples.yml +++ b/source/plugins/community/poopmap/examples.yml @@ -1,6 +1,7 @@ - uses: lowlighter/metrics@latest with: - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED + plugin_poopmap_token: ${{ secrets.POOPMAP_TOKEN }} plugin_poopmap: yes prod: skip: true \ No newline at end of file diff --git a/source/plugins/community/screenshot/README.md b/source/plugins/community/screenshot/README.md index cda82999ed6..0b6a36052c1 100644 --- a/source/plugins/community/screenshot/README.md +++ b/source/plugins/community/screenshot/README.md @@ -91,7 +91,7 @@ name: XKCD of the day uses: lowlighter/metrics@latest with: filename: metrics.plugin.screenshot.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_screenshot: yes plugin_screenshot_title: XKCD of the day diff --git a/source/plugins/community/screenshot/examples.yml b/source/plugins/community/screenshot/examples.yml index 6b146aa085f..202608e22e4 100644 --- a/source/plugins/community/screenshot/examples.yml +++ b/source/plugins/community/screenshot/examples.yml @@ -2,7 +2,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.screenshot.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_screenshot: yes plugin_screenshot_title: XKCD of the day diff --git a/source/plugins/core/README.md b/source/plugins/core/README.md index a8e72148930..48bbf1790cc 100644 --- a/source/plugins/core/README.md +++ b/source/plugins/core/README.md @@ -382,7 +382,9 @@ On forks, this feature is disable to take into account any changes you made on i token

GitHub Personal Access Token

No scopes are required by default, though some plugins and features may require additional scopes

-

When using a configuration which does not requires a GitHub PAT, you may pass NOT_NEEDED instead

+

When using a configuration which does not requires a GitHub PAT, you may pass NOT_NEEDED instead. +Note that when doing so, all defaults values using .user.* will not be applicable meaning that they need to be filled manually. +Most of the time user option must also be set.

diff --git a/source/plugins/core/metadata.yml b/source/plugins/core/metadata.yml index 0d3f91c2256..d998f596365 100644 --- a/source/plugins/core/metadata.yml +++ b/source/plugins/core/metadata.yml @@ -14,7 +14,9 @@ inputs: No scopes are required by default, though some plugins and features may require additional scopes - When using a configuration which does not requires a GitHub PAT, you may pass `NOT_NEEDED` instead + When using a configuration which does not requires a GitHub PAT, you may pass `NOT_NEEDED` instead. + Note that when doing so, all defaults values using `.user.*` will not be applicable meaning that they need to be filled manually. + Most of the time `user` option must also be set. type: token required: true diff --git a/source/plugins/posts/README.md b/source/plugins/posts/README.md index 71f4831ed61..8e590c3791a 100644 --- a/source/plugins/posts/README.md +++ b/source/plugins/posts/README.md @@ -109,7 +109,7 @@ name: Recent posts uses: lowlighter/metrics@latest with: filename: metrics.plugin.posts.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to @@ -120,7 +120,7 @@ name: Recent posts with descriptions and cover images uses: lowlighter/metrics@latest with: filename: metrics.plugin.posts.full.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to diff --git a/source/plugins/posts/examples.yml b/source/plugins/posts/examples.yml index f952b4f8060..dc53c9ba763 100644 --- a/source/plugins/posts/examples.yml +++ b/source/plugins/posts/examples.yml @@ -2,7 +2,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.posts.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to @@ -11,7 +11,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.posts.full.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_posts: yes plugin_posts_source: dev.to diff --git a/source/plugins/stackoverflow/README.md b/source/plugins/stackoverflow/README.md index edca9540799..d7195aa35b0 100644 --- a/source/plugins/stackoverflow/README.md +++ b/source/plugins/stackoverflow/README.md @@ -119,7 +119,7 @@ name: Top answers from stackoverflow uses: lowlighter/metrics@latest with: filename: metrics.plugin.stackoverflow.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_stackoverflow: yes plugin_stackoverflow_user: 1 diff --git a/source/plugins/stackoverflow/examples.yml b/source/plugins/stackoverflow/examples.yml index 4203287ecb8..5946ab16ce4 100644 --- a/source/plugins/stackoverflow/examples.yml +++ b/source/plugins/stackoverflow/examples.yml @@ -2,7 +2,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.stackoverflow.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_stackoverflow: yes plugin_stackoverflow_user: 1 diff --git a/source/plugins/stargazers/README.md b/source/plugins/stargazers/README.md index 17b005a0ac3..cde173fab48 100644 --- a/source/plugins/stargazers/README.md +++ b/source/plugins/stargazers/README.md @@ -15,8 +15,8 @@ -
Chartist charts
-
Classic charts
+
Classic charts
+
Chartist charts
@@ -53,7 +53,7 @@ ✨ On master/main
type: string
-default: null
+default: classic
allowed values: diff --git a/source/plugins/stargazers/metadata.yml b/source/plugins/stargazers/metadata.yml index 67aae895ebb..12d261f9708 100644 --- a/source/plugins/stargazers/metadata.yml +++ b/source/plugins/stargazers/metadata.yml @@ -2,8 +2,8 @@ name: "✨ Stargazers over last weeks" category: github description: This plugin displays your stargazers evolution across all of your repositories over the last two weeks. examples: - +chartist charts: https://github.com/lowlighter/metrics/blob/examples/metrics.plugin.stargazers.chartist.svg - classic charts: https://github.com/lowlighter/metrics/blob/examples/metrics.plugin.stargazers.svg + +classic charts: https://github.com/lowlighter/metrics/blob/examples/metrics.plugin.stargazers.svg + chartist charts: https://github.com/lowlighter/metrics/blob/examples/metrics.plugin.stargazers.chartist.svg index: 10 supports: - user diff --git a/source/plugins/starlists/README.md b/source/plugins/starlists/README.md index 0f4b7870e03..e017adc751f 100644 --- a/source/plugins/starlists/README.md +++ b/source/plugins/starlists/README.md @@ -109,7 +109,7 @@ name: Featured star list uses: lowlighter/metrics@latest with: filename: metrics.plugin.starlists.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_starlists: yes plugin_starlists_limit_repositories: 2 diff --git a/source/plugins/starlists/examples.yml b/source/plugins/starlists/examples.yml index 088d2c8ad48..331d55591d3 100644 --- a/source/plugins/starlists/examples.yml +++ b/source/plugins/starlists/examples.yml @@ -2,7 +2,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.starlists.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_starlists: yes plugin_starlists_limit_repositories: 2 diff --git a/source/plugins/topics/README.md b/source/plugins/topics/README.md index eafbaaa4225..3e12d791d51 100644 --- a/source/plugins/topics/README.md +++ b/source/plugins/topics/README.md @@ -101,7 +101,7 @@ name: Labels uses: lowlighter/metrics@latest with: filename: metrics.plugin.topics.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 12 @@ -112,7 +112,7 @@ name: Icons uses: lowlighter/metrics@latest with: filename: metrics.plugin.topics.icons.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 0 diff --git a/source/plugins/topics/examples.yml b/source/plugins/topics/examples.yml index 309c2840bf4..9b663a1e047 100644 --- a/source/plugins/topics/examples.yml +++ b/source/plugins/topics/examples.yml @@ -2,7 +2,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.topics.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 12 @@ -11,7 +11,7 @@ uses: lowlighter/metrics@latest with: filename: metrics.plugin.topics.icons.svg - token: ${{ secrets.METRICS_TOKEN }} + token: NOT_NEEDED base: "" plugin_topics: yes plugin_topics_limit: 0 From f1b4a6aad96107e2abad6fe3051f2e2bb810a418 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 20:10:12 -0500 Subject: [PATCH 012/773] tests: change nosttings/mock to sandbox setting --- source/plugins/core/index.mjs | 2 +- source/plugins/music/index.mjs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/plugins/core/index.mjs b/source/plugins/core/index.mjs index 4a857ea4924..e22d15beacf 100644 --- a/source/plugins/core/index.mjs +++ b/source/plugins/core/index.mjs @@ -60,7 +60,7 @@ export default async function({login, q}, {conf, data, rest, graphql, plugins, q pending.push((async () => { try { console.debug(`metrics/compute/${login}/plugins > ${name} > started`) - data.plugins[name] = await imports.plugins[name]({login, q, imports, data, computed, rest, graphql, queries, account}, {extras:conf.settings?.extras?.features ?? conf.settings?.extras?.default ?? false, ...plugins[name]}) + data.plugins[name] = await imports.plugins[name]({login, q, imports, data, computed, rest, graphql, queries, account}, {extras:conf.settings?.extras?.features ?? conf.settings?.extras?.default ?? false, sandbox:conf.settings?.sandbox ?? false, ...plugins[name]}) console.debug(`metrics/compute/${login}/plugins > ${name} > completed`) } catch (error) { diff --git a/source/plugins/music/index.mjs b/source/plugins/music/index.mjs index b7f8b3baf18..42c721bfd0d 100644 --- a/source/plugins/music/index.mjs +++ b/source/plugins/music/index.mjs @@ -28,7 +28,7 @@ const modes = { } //Setup -export default async function({login, imports, data, q, account}, {enabled = false, token = ""} = {}) { +export default async function({login, imports, data, q, account}, {enabled = false, token = "", sandbox = false} = {}) { //Plugin execution try { //Check if plugin is enabled and requirements are met @@ -47,7 +47,12 @@ export default async function({login, imports, data, q, account}, {enabled = fal let tracks = null //Load inputs - let {provider, mode, playlist, limit, user, "played.at":played_at, "time.range":time_range, "top.type":top_type} = imports.metadata.plugins.music.inputs({data, account, q}) + let {provider, mode, playlist, limit, user, "played.at":played_at, "time.range":time_range, "top.type":top_type, token:_token} = imports.metadata.plugins.music.inputs({data, account, q}) + if ((sandbox)&&(_token)) { + token = _token + console.debug(`metrics/compute/${login}/plugins > music > overriden token value through user inputs as sandbox mode is enabled`) + } + //Auto-guess parameters if (!mode) { if (playlist) { @@ -119,7 +124,7 @@ export default async function({login, imports, data, q, account}, {enabled = fal name:tr.querySelector("td:nth-child(2) div div:nth-child(1)").innerText, artist:tr.querySelector("td:nth-child(2) div div:nth-child(2)").innerText, //Spotify doesn't provide artworks so we fallback on playlist artwork instead - artwork:window.getComputedStyle(document.querySelector("button[title=Play]").parentNode, null).backgroundImage.match(/^url\("(?https:...+)"\)$/)?.groups?.url ?? null, + artwork:window.getComputedStyle(document.querySelector("button[title=Play]")?.parentNode ?? document.querySelector("button").parentNode, null).backgroundImage.match(/^url\("(?https:...+)"\)$/)?.groups?.url ?? null, })) ), ] From 6118a0cc222308c5964e0b15dacdf3eed697f5a4 Mon Sep 17 00:00:00 2001 From: lowlighter <22963968+lowlighter@users.noreply.github.com> Date: Thu, 27 Jan 2022 20:12:49 -0500 Subject: [PATCH 013/773] fix(app/metrics): force dimensions to be at least one --- source/app/metrics/utils.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index 97aa6e078fd..94815614e45 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -457,8 +457,8 @@ export const svg = { //Get bounds and resize let {y:height, width} = document.querySelector("svg #metrics-end").getBoundingClientRect() console.debug(`bounds width=${width}, height=${height}`) - height = Math.ceil(height * padding.height + padding.absolute.height) - width = Math.ceil(width * padding.width + padding.absolute.width) + height = Math.max(1, Math.ceil(height * padding.height + padding.absolute.height)) + width = Math.max(1, Math.ceil(width * padding.width + padding.absolute.width)) console.debug(`bounds after applying padding width=${width} (*${padding.width}+${padding.absolute.width}), height=${height} (*${padding.height}+${padding.absolute.height})`) //Resize svg if (document.querySelector("svg").getAttribute("height") === "auto") From 9f4e255c85896db50004e14f2960dfe43d99b247 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:22:06 +0100 Subject: [PATCH 014/773] feat(plugins/base): add bulk query and fallback to unit (#825) --- source/plugins/base/index.mjs | 95 +++++++++++-------- .../base/queries/organization.x.graphql | 20 ++++ source/plugins/base/queries/user.x.graphql | 55 +++++++++++ 3 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 source/plugins/base/queries/organization.x.graphql create mode 100644 source/plugins/base/queries/user.x.graphql diff --git a/source/plugins/base/index.mjs b/source/plugins/base/index.mjs index 94664bf3f68..17773249278 100644 --- a/source/plugins/base/index.mjs +++ b/source/plugins/base/index.mjs @@ -30,53 +30,60 @@ export default async function({login, graphql, rest, data, q, queries, imports}, const queried = await graphql(queries.base[account]({login})) Object.assign(data, {user:queried[account]}) postprocess?.[account]({login, data}) - //Query basic fields - const fields = { - user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"], - organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"], - }[account] ?? [] - for (const field of fields) { - try { - Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account]) - } - catch { - console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`) - data.user[field] = {totalCount:NaN} - } + try { + Object.assign(data.user, (await graphql(queries.base[`${account}.x`]({login, account, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account]) + console.debug(`metrics/compute/${login}/base > successfully loaded bulk query`) } - //Query repositories fields - for (const field of ["totalCount", "totalDiskUsage"]) { - try { - Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories) + catch { + console.debug(`metrics/compute/${login}/base > failed to load bulk query, falling back to unit queries`) + //Query basic fields + const fields = { + user:["packages", "starredRepositories", "watching", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "followers", "following", "issueComments", "organizations", "repositoriesContributedTo(includeUserRepositories: true)"], + organization:["packages", "sponsorshipsAsSponsor", "sponsorshipsAsMaintainer", "membersWithRole"], + }[account] ?? [] + for (const field of fields) { + try { + Object.assign(data.user, (await graphql(queries.base.field({login, account, field})))[account]) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve ${field}`) + data.user[field] = {totalCount:NaN} + } } - catch (error) { - console.log(error) - console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`) - data.user.repositories[field] = NaN + //Query repositories fields + for (const field of ["totalCount", "totalDiskUsage"]) { + try { + Object.assign(data.user.repositories, (await graphql(queries.base["field.repositories"]({login, account, field})))[account].repositories) + } + catch (error) { + console.log(error) + console.debug(`metrics/compute/${login}/base > failed to retrieve repositories.${field}`) + data.user.repositories[field] = NaN + } } - } - //Query user account fields - if (account === "user") { - //Query contributions collection - { - const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"] - for (const field of fields) { - try { - Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection) - } - catch { - console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`) - data.user.contributionsCollection[field] = NaN + //Query user account fields + if (account === "user") { + //Query contributions collection + { + const fields = ["totalRepositoriesWithContributedCommits", "totalCommitContributions", "restrictedContributionsCount", "totalIssueContributions", "totalPullRequestContributions", "totalPullRequestReviewContributions"] + for (const field of fields) { + try { + Object.assign(data.user.contributionsCollection, (await graphql(queries.base.contributions({login, account, field})))[account].contributionsCollection) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve contributionsCollection.${field}`) + data.user.contributionsCollection[field] = NaN + } } } - } - //Query calendar - try { - Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account]) - } - catch { - console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`) - data.user.calendar = {contributionCalendar:{weeks:[]}} + //Query calendar + try { + Object.assign(data.user, (await graphql(queries.base.calendar({login, "calendar.from":new Date(Date.now() - 14 * 24 * 60 * 60 * 1000).toISOString(), "calendar.to":(new Date()).toISOString()})))[account]) + } + catch { + console.debug(`metrics/compute/${login}/base > failed to retrieve contributions calendar`) + data.user.calendar = {contributionCalendar:{weeks:[]}} + } } } //Query repositories from GitHub API @@ -107,6 +114,10 @@ export default async function({login, graphql, rest, data, q, queries, imports}, data.user[type].nodes.push(...nodes) pushed = nodes.length console.debug(`metrics/compute/${login}/base > retrieved ${pushed} ${type} after ${cursor}`) + if (pushed < repositories) { + console.debug(`metrics/compute/${login}/base > retrieved less repositories than expected, probably no more to fetch`) + break + } } while ((pushed) && (cursor) && ((data.user.repositories?.nodes?.length ?? 0) + (data.user.repositoriesContributedTo?.nodes?.length ?? 0) < repositories)) //Limit repositories console.debug(`metrics/compute/${login}/base > keeping only ${repositories} ${type}`) diff --git a/source/plugins/base/queries/organization.x.graphql b/source/plugins/base/queries/organization.x.graphql new file mode 100644 index 00000000000..86c7f92e079 --- /dev/null +++ b/source/plugins/base/queries/organization.x.graphql @@ -0,0 +1,20 @@ +query BaseOrganizationX { + organization(login: "$login") { + packages { + totalCount + } + sponsorshipsAsSponsor { + totalCount + } + sponsorshipsAsMaintainer { + totalCount + } + membersWithRole { + totalCount + } + repositories(last: 0) { + totalCount + totalDiskUsage + } + } +} diff --git a/source/plugins/base/queries/user.x.graphql b/source/plugins/base/queries/user.x.graphql new file mode 100644 index 00000000000..39b27231803 --- /dev/null +++ b/source/plugins/base/queries/user.x.graphql @@ -0,0 +1,55 @@ +query BaseUserX { + user(login: "$login") { + packages { + totalCount + } + starredRepositories { + totalCount + } + watching { + totalCount + } + sponsorshipsAsSponsor { + totalCount + } + sponsorshipsAsMaintainer { + totalCount + } + followers { + totalCount + } + following { + totalCount + } + issueComments { + totalCount + } + organizations { + totalCount + } + repositoriesContributedTo(includeUserRepositories: true) { + totalCount + } + repositories(last: 0) { + totalCount + totalDiskUsage + } + contributionsCollection { + totalRepositoriesWithContributedCommits + totalCommitContributions + restrictedContributionsCount + totalIssueContributions + totalPullRequestContributions + totalPullRequestReviewContributions + } + calendar:contributionsCollection(from: "$calendar.from", to: "$calendar.to") { + contributionCalendar { + weeks { + contributionDays { + color + } + } + } + } + } +} From 3945af02d1be39ee499e40ebe728df7be895f281 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Fri, 28 Jan 2022 02:22:42 +0100 Subject: [PATCH 015/773] feat(app/web): update ratelimit and placeholders (#826) --- source/app/web/instance.mjs | 32 +- source/app/web/statics/about/index.html | 11 +- source/app/web/statics/about/script.js | 11 +- source/app/web/statics/app.js | 25 +- source/app/web/statics/app.placeholder.js | 47 +- source/app/web/statics/index.html | 25 +- .../placeholders/isocalendar.full-year.svg | 1964 +++++++++++++++++ .../placeholders/isocalendar.half-year.svg | 1002 +++++++++ .../web/statics/placeholders/screenshot.png | Bin 0 -> 101296 bytes .../app/web/statics/placeholders/skyline.png | Bin 0 -> 88912 bytes source/app/web/statics/placeholders/stock.svg | 34 + source/app/web/statics/style.css | 1 + 12 files changed, 3124 insertions(+), 28 deletions(-) create mode 100644 source/app/web/statics/placeholders/isocalendar.full-year.svg create mode 100644 source/app/web/statics/placeholders/isocalendar.half-year.svg create mode 100644 source/app/web/statics/placeholders/screenshot.png create mode 100644 source/app/web/statics/placeholders/skyline.png create mode 100644 source/app/web/statics/placeholders/stock.svg diff --git a/source/app/web/instance.mjs b/source/app/web/instance.mjs index 872b1523205..0af6cf1b625 100644 --- a/source/app/web/instance.mjs +++ b/source/app/web/instance.mjs @@ -12,13 +12,13 @@ import presets from "../metrics/presets.mjs" import setup from "../metrics/setup.mjs" /**App */ -export default async function({sandbox} = {}) { +export default async function({sandbox = false} = {}) { //Load configuration settings const {conf, Plugins, Templates} = await setup({sandbox}) //Sandbox mode if (sandbox) { console.debug("metrics/app > sandbox mode is specified, enabling advanced features") - Object.assign(conf.settings, {optimize: true, cached:0, "plugins.default":true, extras:{default:true}}) + Object.assign(conf.settings, {sandbox:true, optimize: true, cached:0, "plugins.default":true, extras:{default:true}}) } const {token, maxusers = 0, restricted = [], debug = false, cached = 30 * 60 * 1000, port = 3000, ratelimiter = null, plugins = null} = conf.settings const mock = sandbox || conf.settings.mocked @@ -93,17 +93,28 @@ export default async function({sandbox} = {}) { const enabled = Object.entries(metadata).filter(([_name, {category}]) => category !== "core").map(([name]) => ({name, category:metadata[name]?.category ?? "community", enabled:plugins[name]?.enabled ?? false})) const templates = Object.entries(Templates).map(([name]) => ({name, enabled:(conf.settings.templates.enabled.length ? conf.settings.templates.enabled.includes(name) : true) ?? false})) const actions = {flush:new Map()} - let requests = {limit:0, used:0, remaining:0, reset:NaN} + const requests = {rest:{limit:0, used:0, remaining:0, reset:NaN}, graphql:{limit:0, used:0, remaining:0, reset:NaN}} + let _requests_refresh = false if (!conf.settings.notoken) { - requests = (await rest.rateLimit.get()).data.rate - setInterval(async () => { + const refresh = async () => { try { - requests = (await rest.rateLimit.get()).data.rate + const {limit} = await graphql("{ limit:rateLimit {limit remaining reset:resetAt used} }") + Object.assign(requests, { + rest:(await rest.rateLimit.get()).data.rate, + graphql:{...limit, reset:new Date(limit.reset).getTime()} + }) } catch { console.debug("metrics/app > failed to update remaining requests") } - }, 5 * 60 * 1000) + } + await refresh() + setInterval(refresh, 15 * 60 * 1000) + setInterval(() => { + if (_requests_refresh) + refresh() + _requests_refresh = false + }, 15 * 1000) } //Web app.get("/", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/index.html`)) @@ -119,6 +130,8 @@ export default async function({sandbox} = {}) { app.get("/.templates/:template", limiter, (req, res) => req.params.template in conf.templates ? res.status(200).json(conf.templates[req.params.template]) : res.sendStatus(404)) for (const template in conf.templates) app.use(`/.templates/${template}/partials`, express.static(`${conf.paths.templates}/${template}/partials`)) + //Placeholders + app.use("/.placeholders", express.static(`${conf.paths.statics}/placeholders`)) //Styles app.get("/.css/style.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.css`)) app.get("/.css/style.vars.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.vars.css`)) @@ -205,6 +218,9 @@ export default async function({sandbox} = {}) { console.error(error) return res.status(500).send("Internal Server Error: failed to process metrics correctly") } + finally { + _requests_refresh = true + } }) //Metrics @@ -309,8 +325,8 @@ export default async function({sandbox} = {}) { } finally { //After rendering - solve?.() + _requests_refresh = true } }) diff --git a/source/app/web/statics/about/index.html b/source/app/web/statics/about/index.html index 61783dbc469..035a3a9e901 100644 --- a/source/app/web/statics/about/index.html +++ b/source/app/web/statics/about/index.html @@ -1,3 +1,4 @@ + @@ -28,7 +29,7 @@

Search a GitHub user

- {{ requests.remaining }} GitHub requests remaining + Remaining GitHub requests: {{ requests.rest.remaining }} REST / {{ requests.graphql.remaining }} GraphQL Send feedback on GitHub discussions!
@@ -42,6 +43,10 @@

+
+ This web instance has run out of GitHub API requests. + Please wait until {{ rlreset }} to generate metrics again. +
Display rankings, contributions, highlights, commits calendar, used languages and recent activity from any user account! @@ -237,7 +242,7 @@

{{ habits[h] }} -
+
{{ `${h}`.padStart(2, 0) }}
@@ -369,6 +374,6 @@

- + diff --git a/source/app/web/statics/about/script.js b/source/app/web/statics/about/script.js index 8895b238acb..7ab67d38126 100644 --- a/source/app/web/statics/about/script.js +++ b/source/app/web/statics/about/script.js @@ -93,6 +93,10 @@ } finally { this.pending = false + try { + const { data: requests } = await axios.get("/.requests") + this.requests = requests + } catch {} } }, }, @@ -111,6 +115,7 @@ return this.metrics?.rendered.plugins.followup ?? null }, habits() { + console.log(this.metrics?.rendered.plugins.habits.commits.hours) return this.metrics?.rendered.plugins.habits.commits.hours ?? null }, isocalendar() { @@ -142,6 +147,10 @@ preview() { return /-preview$/.test(this.version) }, + rlreset() { + const reset = new Date(Math.max(this.requests.graphql.reset, this.requests.rest.reset)) + return `${reset.getHours()}:${reset.getMinutes()}` + } }, //Data initialization data: { @@ -151,7 +160,7 @@ embed: false, localstorage: false, searchable: false, - requests: { limit: 0, used: 0, remaining: 0, reset: 0 }, + requests: {rest:{limit:0, used:0, remaining:0, reset:NaN}, graphql:{limit:0, used:0, remaining:0, reset:NaN}}, palette: "light", metrics: null, pending: false, diff --git a/source/app/web/statics/app.js b/source/app/web/statics/app.js index d685ba94626..e8737217ce5 100644 --- a/source/app/web/statics/app.js +++ b/source/app/web/statics/app.js @@ -90,11 +90,25 @@ tab: "overview", palette: "light", clipboard: null, - requests: { limit: 0, used: 0, remaining: 0, reset: 0 }, + requests: {rest:{limit:0, used:0, remaining:0, reset:NaN}, graphql:{limit:0, used:0, remaining:0, reset:NaN}}, cached: new Map(), config: Object.fromEntries(Object.entries(metadata.core.web).map(([key, { defaulted }]) => [key, defaulted])), metadata: Object.fromEntries(Object.entries(metadata).map(([key, { web }]) => [key, web])), hosted: null, + docs:{ + overview:{ + link:"https://github.com/lowlighter/metrics#-documentation", + name:"Complete documentation", + }, + markdown:{ + link:"https://github.com/lowlighter/metrics/blob/master/.github/readme/partials/documentation/setup/shared.md", + name:"Setup using the shared instance", + }, + action:{ + link:"https://github.com/lowlighter/metrics/blob/master/.github/readme/partials/documentation/setup/action.md", + name:"Setup using GitHub Action on a profile repository", + } + }, plugins: { base: {}, list: [], @@ -251,6 +265,11 @@ preview() { return /-preview$/.test(this.version) }, + //Rate limit reset + rlreset() { + const reset = new Date(Math.max(this.requests.graphql.reset, this.requests.rest.reset)) + return `${reset.getHours()}:${reset.getMinutes()}` + } }, //Methods methods: { @@ -299,6 +318,10 @@ } finally { this.generated.pending = false + try { + const { data: requests } = await axios.get("/.requests") + this.requests = requests + } catch {} } }, }, diff --git a/source/app/web/statics/app.placeholder.js b/source/app/web/statics/app.placeholder.js index afd922842c6..abb05efca2a 100644 --- a/source/app/web/statics/app.placeholder.js +++ b/source/app/web/statics/app.placeholder.js @@ -18,6 +18,12 @@ values.push(probability) return values.sort((a, b) => b - a) } + //Static complex placeholder + async function staticPlaceholder(condition, name) { + if (!condition) + return "" + return await fetch(`/.placeholders/${name}`).then(response => response.text()).catch(() => "(could not render placeholder)") + } //Placeholder function globalThis.placeholder = async function(set) { //Load templates informations @@ -241,8 +247,21 @@ ? ({ notable: { contributions: new Array(2 + faker.datatype.number(2)).fill(null).map(_ => ({ - name: `${options["notable.repositories"] ? `${faker.lorem.slug()}/` : ""}${faker.lorem.slug()}`, + get name() { return options["notable.repositories"] ? this.handle : this.handle.split("/")[0] }, + handle: `${faker.lorem.slug()}/${faker.lorem.slug()}`, avatar: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg==", + organization: faker.datatype.boolean(), + stars: faker.datatype.number(1000), + aggregated: faker.datatype.number(100), + history: faker.datatype.number(1000), + ...(options["notable.indepth"] ? { + user:{ + commits: faker.datatype.number(100), + percentage: faker.datatype.float({ max: 1 }), + maintainer: false, + stars: faker.datatype.number(100), + } + } : null) })), }, }) @@ -322,7 +341,7 @@ }, }, comments: options["reactions.limit"], - details: options["reactions.details"], + details: options["reactions.details"].split(",").map(x => x.trim()), days: options["reactions.days"], }, }) @@ -451,7 +470,7 @@ ...(set.plugins.enabled.stock ? ({ stock: { - chart: "(stock chart is not displayed in placeholder)", + chart: await staticPlaceholder(set.plugins.enabled.stock, "stock.svg"), currency: "USD", price: faker.datatype.number(10000) / 100, previous: faker.datatype.number(10000) / 100, @@ -553,10 +572,12 @@ music: { provider: "(music provider)", mode: "Suggested tracks", + played_at: options["music.played.at"], tracks: new Array(Number(options["music.limit"])).fill(null).map(_ => ({ name: faker.random.words(5), artist: faker.random.words(), artwork: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg==", + played_at: options["music.played.at"] ? faker.date.recent() : null, })), }, }) @@ -578,6 +599,16 @@ }, }) : null), + //Fortune + ...(set.plugins.enabled.fortune + ? ({ + fortune: faker.random.arrayElement([ + {chance:.06, color:"#43FD3B", text:"Good news will come to you by mail"}, + {chance:.06, color:"#00CBB0", text:"ｷﾀ━━━━━━(ﾟ∀ﾟ)━━━━━━ !!!!"}, + {chance: 0.03, color: "#FD4D32", text: "Excellent Luck"} + ]), + }) + : null), //Pagespeed ...(set.plugins.enabled.pagespeed ? ({ @@ -666,6 +697,7 @@ started: faker.datatype.number(1000), comments: faker.datatype.number(1000), answers: faker.datatype.number(1000), + display: { categories: options["discussions.categories"] ? { limit: options["discussions.categories.limit"] || Infinity } : null }, }, }) : null), @@ -690,6 +722,7 @@ ? ({ topics: { mode: options["topics.mode"], + type: {starred:"labels", labels:"labels", mastered:"icons", icons:"icons"}[options["topics.mode"]] || "labels", list: new Array(Number(options["topics.limit"]) || 20).fill(null).map(_ => ({ name: faker.lorem.words(2), description: faker.lorem.sentence(), @@ -770,7 +803,7 @@ ...(set.plugins.enabled.repositories ? ({ repositories: { - list: new Array(Number(options["repositories.featured"].split(",").length) - 1).fill(null).map((_, i) => ({ + list: new Array(Number(options["repositories.featured"].split(",").map(x => x.trim()).length)).fill(null).map((_, i) => ({ created: faker.date.past(), description: faker.lorem.sentence(), forkCount: faker.datatype.number(100), @@ -1058,7 +1091,7 @@ streak: { max: 30 + faker.datatype.number(20), current: faker.datatype.number(30) }, max: 10 + faker.datatype.number(40), average: faker.datatype.float(10), - svg: "(isometric calendar is not displayed in placeholder)", + svg: await staticPlaceholder(set.plugins.enabled.isocalendar, `isocalendar.${options["isocalendar.duration"]}.svg`), duration: options["isocalendar.duration"], }, }) @@ -1076,7 +1109,7 @@ ...(set.plugins.enabled.screenshot ? ({ screenshot: { - image: "data:image/jpg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg==", + image: "/.placeholders/screenshot.png", title: options["screenshot.title"], height: 440, width: 454, @@ -1087,7 +1120,7 @@ ...(set.plugins.enabled.skyline ? ({ skyline: { - animation: "data:image/jpg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mOcOnfpfwAGfgLYttYINwAAAABJRU5ErkJggg==", + animation: "/.placeholders/skyline.png", width: 454, height: 284, compatibility: false, diff --git a/source/app/web/statics/index.html b/source/app/web/statics/index.html index c310cbcd893..ae5da4d1c85 100644 --- a/source/app/web/statics/index.html +++ b/source/app/web/statics/index.html @@ -1,3 +1,4 @@ + @@ -49,8 +50,8 @@
- - - {{ requests.remaining }} GitHub requests remaining + Remaining GitHub requests: + {{ requests.rest.remaining }} REST / {{ requests.graphql.remaining }} GraphQL Metrics are rendered by metrics.lecoq.io in preview mode. Any backend editions won't be reflected but client-side rendering can still be tested.
- Metrics cannot be generated because the following plugins are not available on this web instance: {{ unusable.join(", ") }} + The following plugins are not available on this web instance: {{ unusable.join(", ") }} +
+
+ This web instance has run out of GitHub API requests. + Please wait until {{ rlreset }} to generate metrics again.
@@ -101,7 +107,7 @@