+ This service is intended for emergency data recovery from a defunct Diigo + service. By using it, you trust this service with your Diigo credentials. +
+ + + +Idle.+ +
Data is appended as each page arrives. You can copy it at any point.
+ + ++ Feedback / View Source: + PR #245 +
+ + + + diff --git a/projects/diigo-bak/rescue-site/netlify.toml b/projects/diigo-bak/rescue-site/netlify.toml new file mode 100644 index 0000000..3602e27 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify.toml @@ -0,0 +1,7 @@ +[build] + publish = "." + command = "echo rescue-site" + +[functions] + directory = "netlify/functions" + node_bundler = "esbuild" diff --git a/projects/diigo-bak/rescue-site/netlify/functions/_config.js b/projects/diigo-bak/rescue-site/netlify/functions/_config.js new file mode 100644 index 0000000..8b24ec1 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/_config.js @@ -0,0 +1,68 @@ +function parseTs(name) { + const raw = process.env[name]; + if (!raw) { + return null; + } + const ms = Date.parse(raw); + if (Number.isNaN(ms)) { + return null; + } + return ms; +} + +function utcDay(ms) { + return new Date(ms).toISOString().slice(0, 10); +} + +function getConfig(now = Date.now()) { + return { + enabled: process.env.RESCUE_ENABLED !== "false", + startAt: parseTs("RESCUE_START_AT"), + sunsetAt: parseTs("RESCUE_SUNSET_AT"), + diigoApiKey: process.env.DIIGO_API_KEY || "", + knownIp: process.env.DIIGO_KNOWN_IP || "54.148.192.94", + maxActiveJobs: Number(process.env.MAX_ACTIVE_JOBS || "3"), + maxGlobalStartsPerDay: Number(process.env.MAX_GLOBAL_STARTS_PER_DAY || "50"), + maxUserStartsPerDay: Number(process.env.MAX_USER_STARTS_PER_DAY || "5"), + maxDownloadAgeMs: Number( + process.env.MAX_DOWNLOAD_AGE_MS || String(60 * 60 * 1000), + ), + now, + dayKey: utcDay(now), + }; +} + +function checkWindow(cfg) { + if (!cfg.enabled) { + return { ok: false, status: 503, message: "service disabled" }; + } + if (!cfg.diigoApiKey) { + return { ok: false, status: 503, message: "server is not configured" }; + } + if (cfg.startAt === null || cfg.sunsetAt === null) { + return { + ok: false, + status: 503, + message: "start/sunset timestamps are not configured", + }; + } + if (cfg.sunsetAt - cfg.startAt > 72 * 60 * 60 * 1000) { + return { + ok: false, + status: 503, + message: "invalid window: must be 72h or less", + }; + } + if (cfg.now < cfg.startAt) { + return { ok: false, status: 403, message: "service not open yet" }; + } + if (cfg.now >= cfg.sunsetAt) { + return { ok: false, status: 410, message: "service window has ended" }; + } + return { ok: true }; +} + +module.exports = { + getConfig, + checkWindow, +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/_diigo-client.js b/projects/diigo-bak/rescue-site/netlify/functions/_diigo-client.js new file mode 100644 index 0000000..a937b25 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/_diigo-client.js @@ -0,0 +1,142 @@ +const https = require("https"); +const { URLSearchParams } = require("url"); + +const API_HOST = "secure.diigo.com"; +const API_BASE = "https://secure.diigo.com/api/v2"; + +function authHeader(username, password) { + const token = Buffer.from(username + ":" + password, "utf8").toString("base64"); + return "Basic " + token; +} + +async function requestPrimary(path, headers) { + const res = await fetch(API_BASE + path, { method: "GET", headers }); + const text = await res.text(); + return { + status: res.status, + body: text, + }; +} + +function requestKnownIp(path, headers, knownIp) { + return new Promise((resolve, reject) => { + const req = https.request( + { + host: knownIp, + port: 443, + method: "GET", + path: "/api/v2" + path, + servername: API_HOST, + headers: Object.assign({}, headers, { Host: API_HOST }), + }, + (res) => { + let data = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { + data += chunk; + }); + res.on("end", () => { + resolve({ status: res.statusCode || 0, body: data }); + }); + }, + ); + req.on("error", reject); + req.end(); + }); +} + +function parseArray(body) { + let parsed; + try { + parsed = JSON.parse(body); + } catch { + if (typeof body === "string" && body.trim().startsWith("<")) { + throw new Error("Diigo returned HTML instead of JSON"); + } + throw new Error("Invalid JSON from Diigo API"); + } + if (!Array.isArray(parsed)) { + throw new Error("unexpected Diigo response shape"); + } + return parsed; +} + +async function requestPage({ + username, + password, + apiKey, + start, + count, + knownIp, +}) { + const params = new URLSearchParams({ + user: username, + sort: "2", + filter: "all", + start: String(start), + count: String(count), + key: apiKey, + }); + const path = "/bookmarks?" + params.toString(); + const headers = { + Authorization: authHeader(username, password), + Accept: "application/json", + }; + + try { + const primary = await requestPrimary(path, headers); + if (primary.status === 200) { + return parseArray(primary.body); + } + if (primary.status === 401) { + throw new Error("authentication failed"); + } + } catch (err) { + // Fall through to known-ip strategy. + if (!knownIp) { + throw err; + } + } + + if (!knownIp) { + throw new Error("Diigo API unavailable"); + } + const fallback = await requestKnownIp(path, headers, knownIp); + if (fallback.status === 200) { + return parseArray(fallback.body); + } + if (fallback.status === 401) { + throw new Error("authentication failed"); + } + throw new Error("Diigo API unavailable: HTTP " + String(fallback.status)); +} + +async function* fetchAllBookmarks({ + username, + password, + apiKey, + knownIp, + pageSize = 100, +}) { + let start = 0; + for (;;) { + const page = await requestPage({ + username, + password, + apiKey, + start, + count: pageSize, + knownIp, + }); + if (page.length === 0) { + return; + } + yield page; + start += pageSize; + } +} + +module.exports = { + requestPage, + fetchAllBookmarks, +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/_export-runner.js b/projects/diigo-bak/rescue-site/netlify/functions/_export-runner.js new file mode 100644 index 0000000..c6c1292 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/_export-runner.js @@ -0,0 +1,40 @@ +const { fetchAllBookmarks } = require("./_diigo-client"); +const { updateJob } = require("./_jobs"); + +async function runExportJob({ jobId, username, password, apiKey, knownIp }) { + updateJob(jobId, { state: "running", error: null }); + const lines = []; + let fetched = 0; + try { + for await (const page of fetchAllBookmarks({ + username, + password, + apiKey, + knownIp, + })) { + for (const item of page) { + lines.push(JSON.stringify(item)); + } + fetched += page.length; + updateJob(jobId, { fetched }); + } + updateJob(jobId, { + state: "done", + fetched, + ndjson: lines.join("\n") + (lines.length ? "\n" : ""), + }); + } catch (err) { + updateJob(jobId, { + state: "error", + error: String(err && err.message ? err.message : err), + }); + } finally { + // Scrub local references. + username = ""; + password = ""; + } +} + +module.exports = { + runExportJob, +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/_jobs.js b/projects/diigo-bak/rescue-site/netlify/functions/_jobs.js new file mode 100644 index 0000000..5b2f55b --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/_jobs.js @@ -0,0 +1,68 @@ +const crypto = require("crypto"); + +const jobs = new Map(); + +function makeJobId() { + return "j_" + crypto.randomBytes(16).toString("hex"); +} + +function usernameHash(username) { + return crypto.createHash("sha256").update(username).digest("hex").slice(0, 16); +} + +function createJob({ usernameHashValue, now }) { + const id = makeJobId(); + const job = { + id, + state: "queued", + fetched: 0, + createdAt: now, + updatedAt: now, + usernameHash: usernameHashValue, + ndjson: "", + error: null, + }; + jobs.set(id, job); + return job; +} + +function getJob(id) { + return jobs.get(id) || null; +} + +function updateJob(id, patch) { + const job = jobs.get(id); + if (!job) { + return null; + } + Object.assign(job, patch, { updatedAt: Date.now() }); + jobs.set(id, job); + return job; +} + +function countActiveJobs() { + let n = 0; + for (const job of jobs.values()) { + if (job.state === "queued" || job.state === "running") { + n += 1; + } + } + return n; +} + +function purgeOldJobs(maxAgeMs, now = Date.now()) { + for (const [id, job] of jobs.entries()) { + if (now - job.createdAt > maxAgeMs) { + jobs.delete(id); + } + } +} + +module.exports = { + usernameHash, + createJob, + getJob, + updateJob, + countActiveJobs, + purgeOldJobs, +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/_rate-limit.js b/projects/diigo-bak/rescue-site/netlify/functions/_rate-limit.js new file mode 100644 index 0000000..4e47d41 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/_rate-limit.js @@ -0,0 +1,49 @@ +const perUserByDay = new Map(); +const globalByDay = new Map(); + +function _key(dayKey, username) { + return dayKey + "|" + username.toLowerCase(); +} + +function _bump(map, key) { + const n = (map.get(key) || 0) + 1; + map.set(key, n); + return n; +} + +function _read(map, key) { + return map.get(key) || 0; +} + +function checkAndConsume({ dayKey, username, maxUserStartsPerDay, maxGlobalStartsPerDay }) { + const userKey = _key(dayKey, username); + const gKey = dayKey; + + const userCount = _read(perUserByDay, userKey); + if (userCount >= maxUserStartsPerDay) { + return { + ok: false, + status: 429, + message: "daily per-user export limit reached", + userCount, + }; + } + + const globalCount = _read(globalByDay, gKey); + if (globalCount >= maxGlobalStartsPerDay) { + return { + ok: false, + status: 429, + message: "daily service cap reached", + globalCount, + }; + } + + _bump(perUserByDay, userKey); + _bump(globalByDay, gKey); + return { ok: true }; +} + +module.exports = { + checkAndConsume, +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-download.js b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-download.js new file mode 100644 index 0000000..5519a30 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-download.js @@ -0,0 +1,84 @@ +const { getConfig, checkWindow } = require("./_config"); +const { checkAndConsume } = require("./_rate-limit"); +const { fetchAllBookmarks } = require("./_diigo-client"); + +exports.handler = async (event) => { + const cfg = getConfig(); + if (event.httpMethod !== "POST") { + return { + statusCode: 405, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ error: "method not allowed" }), + }; + } + const windowCheck = checkWindow(cfg); + if (!windowCheck.ok) { + return { + statusCode: windowCheck.status, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ error: windowCheck.message }), + }; + } + + let body; + try { + body = JSON.parse(event.body || "{}"); + } catch { + body = {}; + } + const username = String(body.username || "").trim(); + const password = String(body.password || ""); + if (!username || !password) { + return { + statusCode: 400, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ error: "username and password are required" }), + }; + } + + const limit = checkAndConsume({ + dayKey: cfg.dayKey, + username, + maxUserStartsPerDay: cfg.maxUserStartsPerDay, + maxGlobalStartsPerDay: cfg.maxGlobalStartsPerDay, + }); + if (!limit.ok) { + return { + statusCode: limit.status, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ error: limit.message }), + }; + } + + const lines = []; + try { + for await (const page of fetchAllBookmarks({ + username, + password, + apiKey: cfg.diigoApiKey, + knownIp: cfg.knownIp, + })) { + for (const item of page) { + lines.push(JSON.stringify(item)); + } + } + } catch (err) { + return { + statusCode: 502, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ + error: String(err && err.message ? err.message : err), + }), + }; + } + + return { + statusCode: 200, + headers: { + "content-type": "application/x-ndjson; charset=utf-8", + "content-disposition": 'attachment; filename="diigo-bookmarks.ndjson"', + "cache-control": "no-store", + }, + body: lines.join("\n") + (lines.length ? "\n" : ""), + }; +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-start.js b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-start.js new file mode 100644 index 0000000..94cfc4f --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-start.js @@ -0,0 +1,75 @@ +const { getConfig, checkWindow } = require("./_config"); +const { + createJob, + countActiveJobs, + purgeOldJobs, + usernameHash, +} = require("./_jobs"); +const { checkAndConsume } = require("./_rate-limit"); +const { runExportJob } = require("./_export-runner"); + +function json(statusCode, body) { + return { + statusCode, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify(body), + }; +} + +exports.handler = async (event) => { + if (event.httpMethod !== "POST") { + return json(405, { error: "method not allowed" }); + } + const cfg = getConfig(); + const windowCheck = checkWindow(cfg); + if (!windowCheck.ok) { + return json(windowCheck.status, { error: windowCheck.message }); + } + + purgeOldJobs(cfg.maxDownloadAgeMs); + if (countActiveJobs() >= cfg.maxActiveJobs) { + return json(429, { error: "too many active exports; try later" }); + } + + let body; + try { + body = JSON.parse(event.body || "{}"); + } catch { + return json(400, { error: "invalid JSON body" }); + } + const username = String(body.username || "").trim(); + const password = String(body.password || ""); + if (!username || !password) { + return json(400, { error: "username and password are required" }); + } + + const limit = checkAndConsume({ + dayKey: cfg.dayKey, + username, + maxUserStartsPerDay: cfg.maxUserStartsPerDay, + maxGlobalStartsPerDay: cfg.maxGlobalStartsPerDay, + }); + if (!limit.ok) { + return json(limit.status, { error: limit.message }); + } + + const job = createJob({ + usernameHashValue: usernameHash(username), + now: cfg.now, + }); + + runExportJob({ + jobId: job.id, + username, + password, + apiKey: cfg.diigoApiKey, + knownIp: cfg.knownIp, + }); + + return json(202, { + jobId: job.id, + state: job.state, + statusUrl: "/.netlify/functions/diigo-export-status?id=" + job.id, + downloadUrl: "/.netlify/functions/diigo-export-download?id=" + job.id, + }); +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-status.js b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-status.js new file mode 100644 index 0000000..f4a3d5d --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-status.js @@ -0,0 +1,43 @@ +const { getConfig, checkWindow } = require("./_config"); +const { getJob, purgeOldJobs } = require("./_jobs"); + +function json(statusCode, body) { + return { + statusCode, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify(body), + }; +} + +exports.handler = async (event) => { + const cfg = getConfig(); + purgeOldJobs(cfg.maxDownloadAgeMs); + + const id = event.queryStringParameters && event.queryStringParameters.id; + if (!id) { + return json(400, { error: "missing id" }); + } + const job = getJob(id); + if (!job) { + return json(404, { error: "job not found or expired" }); + } + + const windowCheck = checkWindow(cfg); + if (!windowCheck.ok && job.state !== "done") { + return json(windowCheck.status, { + id: job.id, + state: "expired", + fetched: job.fetched, + error: windowCheck.message, + }); + } + + return json(200, { + id: job.id, + state: job.state, + fetched: job.fetched, + error: job.error, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + }); +}; diff --git a/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-worker.js b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-worker.js new file mode 100644 index 0000000..338a382 --- /dev/null +++ b/projects/diigo-bak/rescue-site/netlify/functions/diigo-export-worker.js @@ -0,0 +1,10 @@ +exports.handler = async () => { + return { + statusCode: 501, + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ + error: "not implemented", + note: "Current build runs export work directly from export-start.", + }), + }; +}; diff --git a/projects/diigo-bak/rescue-site/static/js/diigo-rescue.js b/projects/diigo-bak/rescue-site/static/js/diigo-rescue.js new file mode 100644 index 0000000..82d5711 --- /dev/null +++ b/projects/diigo-bak/rescue-site/static/js/diigo-rescue.js @@ -0,0 +1,53 @@ +(function () { + const form = document.getElementById("export-form"); + const statusEl = document.getElementById("status"); + const outputEl = document.getElementById("output"); + + const setStatus = (obj) => { + statusEl.textContent = + typeof obj === "string" ? obj : JSON.stringify(obj, null, 2); + }; + + form.addEventListener("submit", async (ev) => { + ev.preventDefault(); + outputEl.value = ""; + + const username = document.getElementById("username").value.trim(); + const password = document.getElementById("password").value; + const count = 100; + let start = 0; + let pageNo = 0; + let total = 0; + setStatus("Starting export..."); + + try { + for (;;) { + pageNo += 1; + setStatus( + "Fetching page " + pageNo + " starting from record " + (start + 1), + ); + const res = await fetch("/.netlify/functions/diigo-export-page", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username, password, start, count }), + }); + const body = await res.json(); + if (!res.ok) { + setStatus(body); + return; + } + if (body.ndjson) { + outputEl.value += body.ndjson; + } + total += body.fetched || 0; + if (body.done) { + setStatus("Export complete. Total records: " + total); + return; + } + start = body.nextStart; + } + } catch (err) { + setStatus({ error: String(err) }); + } + }); +})();