From d81b33b9d7b1604bd253d2174c77028a209d35d2 Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Mon, 12 May 2025 17:17:07 -0700 Subject: [PATCH 1/7] add image identity - fingerprinting --- tools/asset-identity/index.html | 41 ++++++ tools/asset-identity/scripts.js | 53 ++++++++ tools/asset-identity/styles.css | 188 ++++++++++++++++++++++++++ tools/image-audit/index.html | 2 +- tools/image-audit/scripts.js | 9 +- utils/identity.js | 225 ++++++++++++++++++++++++++++++++ 6 files changed, 516 insertions(+), 2 deletions(-) create mode 100644 tools/asset-identity/index.html create mode 100644 tools/asset-identity/scripts.js create mode 100644 tools/asset-identity/styles.css create mode 100644 utils/identity.js diff --git a/tools/asset-identity/index.html b/tools/asset-identity/index.html new file mode 100644 index 00000000..d2e08065 --- /dev/null +++ b/tools/asset-identity/index.html @@ -0,0 +1,41 @@ + + + + + + + Asset Identity + + + + + + + + +
+
+
+

Asset Identity

+

+ Playground for bringing asset identity AI into the browser. +

+
+ +
+
+
+ + +
+
+
+
+ + + + + \ No newline at end of file diff --git a/tools/asset-identity/scripts.js b/tools/asset-identity/scripts.js new file mode 100644 index 00000000..4def6159 --- /dev/null +++ b/tools/asset-identity/scripts.js @@ -0,0 +1,53 @@ +import { initIdentity, getImageFingerprint } from '../../utils/identity.js'; + +function disableForm() { + const viewbox = document.querySelector('.viewbox'); + if (viewbox) { + viewbox.classList.remove('hover'); + viewbox.dataset.status = 'preview'; + } + const label = document.querySelector('label[for="upload"]'); + if (label) { + label.setAttribute('aria-hidden', true); + } + const upload = document.querySelector('input[name="upload"]'); + if (upload) { + upload.disabled = true; + } +} + +async function initUpload() { + // get input element with name "upload" + const upload = document.querySelector('input[name="upload"]'); + const viewbox = document.querySelector('.viewbox'); + upload.addEventListener('change', () => { + const file = upload.files[0]; + if (file) { + const reader = new FileReader(); + reader.addEventListener('load', async (e) => { + // show image + const imgWrapper = document.createElement('div'); + imgWrapper.classList.add('preview'); + imgWrapper.innerHTML = `${file.name}`; + viewbox.appendChild(imgWrapper); + disableForm(); + + const img = imgWrapper.querySelector('img'); + const fingerprint = await getImageFingerprint(img); + console.debug('Fingerprint:', fingerprint); + }); + reader.readAsDataURL(file); + } + }); +} + +async function init() { + initUpload(); + + // await initIdentity(); + // load test image for debugging + // const fingerprint = await getImageFingerprint('/tools/asset-identity/samples/earth.png'); + // console.debug('Fingerprint:', fingerprint); +} + +init(); diff --git a/tools/asset-identity/styles.css b/tools/asset-identity/styles.css new file mode 100644 index 00000000..6e6c3834 --- /dev/null +++ b/tools/asset-identity/styles.css @@ -0,0 +1,188 @@ +/* form */ +.asset-identity { + --color-checkerboard-bg-fill: #fefefe; + --color-checkerboard-bg-border: rgb(214 213 213 / 40%); + --checkerboard-background: linear-gradient( var(--color-checkerboard-bg-border) 1px, transparent 1px ), linear-gradient( 90deg, var(--color-checkerboard-bg-border) 1px, transparent 1px ), linear-gradient( 45deg, var(--color-checkerboard-bg-fill) 50%, var(--color-checkerboard-bg-fill) 51% ); + --checkerboard-background-size: clamp(15px, 3vw, 25px) clamp(15px, 3vw, 25px); +} + +.asset-identity.block { + text-align: left; +} + +.asset-identity form { + position: relative; +} + +.asset-identity form input[name='upload'] { + display: block; + cursor: pointer; + position: absolute; + inset: 0; + opacity: 0; +} + +.asset-identity form input[name='upload']:disabled, +.asset-identity form label[for='upload'][aria-hidden="true"] { + display: none; + visibility: hidden; +} + +.asset-identity form label { + transition: color .3s; +} + +.asset-identity form .viewbox[data-mode='dark'] label { + color: white; +} + +.asset-identity form label p { + color: inherit; + text-align: center; +} + +.asset-identity form label span + p { + margin-top: .5em; +} + +.asset-identity .form .form-number-wrapper, +.asset-identity .form label[for="width"], .asset-identity .form input#width, +.asset-identity .form label[for="height"], .asset-identity .form input#height { + visibility: hidden; + display: none; +} + +/* viewbox */ +.asset-identity .viewbox { + display: flex; + align-items: center; + justify-content: center; + + /* width: 100%; */ + height: 100vw; + outline: 2px solid transparent; + border: 1px solid var(--color-checkerboard-bg-border); + padding: 1rem; + background-image: var(--checkerboard-background); + background-size: var(--checkerboard-background-size); + background-position: center; +} + +.asset-identity .viewbox, +.asset-identity .viewbox [data-mode='light'] { + --color-checkerboard-bg-fill: #fefefe; + --color-checkerboard-bg-border: rgb(214 213 213 / 40%); + --checkerboard-background: linear-gradient( + var(--color-checkerboard-bg-border) 1px, + transparent 1px + ), + linear-gradient( + 90deg, + var(--color-checkerboard-bg-border) 1px, + transparent 1px + ), + linear-gradient( + 45deg, + var(--color-checkerboard-bg-fill) 50%, + var(--color-checkerboard-bg-fill) 51% + ); +} + +.asset-identity .viewbox[data-mode='dark'] { + --color-checkerboard-bg-border: rgb(41 42 42 / 40%); + --color-checkerboard-bg-fill: #010101; +} + +.asset-identity .viewbox[data-status='upload'] { + transition: background-color .3s, outline .3s; +} + +.asset-identity .viewbox[data-status='upload'].hover, +.asset-identity .viewbox[data-status='upload']:hover { + --color-checkerboard-bg-fill: var(--gray-100); + + outline: 2px solid var(--link-hover-color); +} + +.asset-identity .viewbox[data-status='upload'][data-mode='dark'].hover, +.asset-identity .viewbox[data-status='upload'][data-mode='dark']:hover { + --color-checkerboard-bg-fill: #070707; +} + +.asset-identity .viewbox svg { + width: 100%; + height: auto; + max-height: calc(100vw - 3rem); +} + +@media (width >= 900px) { + .asset-identity .viewbox { + height: 860px; + } + + .asset-identity .viewbox svg { + max-height: calc(860px - 3rem); + } +} + +/* icons/glyphs */ +.asset-identity .glyph, +.asset-identity .glyph::before, +.asset-identity .glyph::after { + box-sizing: border-box; + display: block; + position: relative; + color: var(--color-text); +} + +.asset-identity .glyph::before, +.asset-identity .glyph::after { + content: ''; + position: absolute; +} + +.asset-identity .glyph-upload { + width: 20px; + height: 10px; + margin: auto; + margin-top: 12px; + border: 2px solid; + border-top: 0; + border-bottom-left-radius: 2px; + border-bottom-right-radius: 2px; +} + +.asset-identity .glyph-upload::after { + left: 3px; + bottom: 7px; + width: 10px; + height: 10px; + border-left: 2px solid; + border-top: 2px solid; + transform: rotate(45deg); +} + +.asset-identity .glyph-upload::before { + left: 7px; + bottom: 5px; + width: 2px; + height: 12px; + background: currentcolor; +} + +/* preview */ +.asset-identity .viewbox .preview { + width: 100%; + height: 100%; + color: black; +} + +.asset-identity .viewbox .preview img { + width: 100%; + height: 100%; + object-fit: contain; +} + +.asset-identity .viewbox[data-mode='dark'] .preview { + color: white; +} diff --git a/tools/image-audit/index.html b/tools/image-audit/index.html index 87068965..01098d3e 100644 --- a/tools/image-audit/index.html +++ b/tools/image-audit/index.html @@ -4,7 +4,7 @@ diff --git a/tools/image-audit/scripts.js b/tools/image-audit/scripts.js index 4a168379..1f129e92 100644 --- a/tools/image-audit/scripts.js +++ b/tools/image-audit/scripts.js @@ -3,6 +3,8 @@ import { buildModal } from '../../scripts/scripts.js'; import { decorateIcons } from '../../scripts/aem.js'; import { initConfigField } from '../../utils/config/config.js'; +import { getImageFingerprint } from '../../utils/identity.js'; + /* reporting utilities */ /** * Generates sorted array of audit report rows. @@ -213,10 +215,15 @@ function displayImages(images) { // build image const { href } = new URL(data.src, data.origin); const img = document.createElement('img'); - img.dataset.src = href; + img.crossOrigin = 'Anonymous'; + img.dataset.src = `https://little-forest-58aa.david8603.workers.dev/?url=${encodeURIComponent(href)}`; img.width = data.width; img.height = data.height; img.loading = 'lazy'; + img.onload = async () => { + const fingerprint = await getImageFingerprint(img); + // console.debug('Fingerprint:', fingerprint); + }; figure.append(img); // load the image when it comes into view const observer = new IntersectionObserver((entries) => { diff --git a/utils/identity.js b/utils/identity.js new file mode 100644 index 00000000..33b4b986 --- /dev/null +++ b/utils/identity.js @@ -0,0 +1,225 @@ +let jimp; +const sessions = {}; + +/** + * A simple sequential task queue implementation that will ensure that each `run()` will + * wait for the previous `run()` to complete first. Provide the execution function as + * a constructor argument. + * + * Example usage: + * ``` + * // this fetches URLs one at a time + * const taskQueue = new TaskQueue(async (url) => { + * return fetch(url); + * }); + * + * await taskQueue.run('https://example.com/1'); + * await taskQueue.run('https://example.com/2'); + * await taskQueue.run('https://example.com/3'); + * ``` + */ +class TaskQueue { + /** + * Create a new task queue. + * @param {Function} fn - The task execution function. + */ + constructor(fn) { + this.pending = Promise.resolve(); + this.fn = fn; + } + + async #process(...args) { + try { + await this.pending; + } catch (_e) { + // ignore, should be handled by other Promise branch returned by run() + } + return this.fn(...args); + } + + /** + * Run the task. Will wait for the previous `run()` to complete first. + * @param {...any} args - The arguments to pass to the task execution function. + * @returns {Promise} A promise that resolves to the result of the task execution. + */ + async run(...args) { + this.pending = this.#process(...args); + return this.pending; + } +} + +/** + * Optional initialization function. Usually, getImageFingerprint() will + * initialize the identity automatically (load libraries & models). + * Use this function to pre-initialize the identity model manually, if needed. + * + * @returns {Promise} + */ +export async function initIdentity() { + if (!window.ort) { + console.debug('Loading ONNX runtime...'); + + // eslint-disable-next-line import/no-unresolved + jimp = await import('https://cdn.jsdelivr.net/npm/jimp@1.6.0/+esm'); + + const start = Date.now(); + /* global ort */ + // for some reason the ESM import below is not working + // import onnxruntimeWeb from 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/+esm' + // hence we load the web distribution the old school way + const onnxScript = document.createElement('script'); + + // all providers (184 kb + 16 kb for wasm dep) + // onnxScript.src = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.1/dist/ort.all.min.js'; + // only wasm (93 kb + 16 kb for wasm dep) + // onnxScript.src = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.1/dist/ort.min.js'; + // only webgpu (93 kb + 16 kb for wasm dep) + onnxScript.src = 'https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.1/dist/ort.webgpu.min.js'; + + document.head.appendChild(onnxScript); + await new Promise((resolve) => { + onnxScript.onload = resolve; + }); + console.debug('ONNX runtime loaded in', Date.now() - start, 'ms'); + } + + if (!sessions.fingerprint) { + const EXECUTION_PROVIDERS = [/* 'webnn', */'webgpu', 'wasm']; + console.debug('Loading fingerprint model with execution provider preference:', EXECUTION_PROVIDERS.join(', ')); + const start = Date.now(); + const FINGERPRINTER_MODEL_URL = '/tools/asset-identity/models/fingerprinter_behance_c5_grad_v2.onnx'; + // ort.env.debug = true; + // ort.env.logLevel = 'verbose'; + sessions.fingerprint = await ort.InferenceSession.create(FINGERPRINTER_MODEL_URL, { + executionProviders: EXECUTION_PROVIDERS, + }); + console.debug('Loaded fingerprint model in', Date.now() - start, 'ms'); + } +} + +// function tensorValueNCHW(tensor, n, c, h, w) { +// const W = tensor.dims[3]; +// const HW = tensor.dims[2] * W; +// const CHW = tensor.dims[1] * HW; +// return tensor.data[n * CHW + c * HW + h * W + w]; +// } + +// function logPixel(tensor, x, y) { +// // for NCHW and RGB format +// const R = tensorValueNCHW(tensor, 0, 0, y, x); +// const G = tensorValueNCHW(tensor, 0, 1, y, x); +// const B = tensorValueNCHW(tensor, 0, 2, y, x); +// console.debug(`${R.toFixed(6)}\t${G.toFixed(6)}\t${B.toFixed(6)}\t\t[${x},${y}]`); +// } + +function getImageData(imageElement) { + const canvas = document.createElement('canvas'); + canvas.width = imageElement.width; + canvas.height = imageElement.height; + const ctx = canvas.getContext('2d'); + ctx.drawImage(imageElement, 0, 0); + return ctx.getImageData(0, 0, canvas.width, canvas.height); +} + +function jimpImageToOnnxTensorRGB(image, dims, normalize) { + const [R, G, B] = [[], [], []]; + + // Loop through the image buffer and extract the R, G, and B channels + for (let y = 0; y < dims[2]; y += 1) { + for (let x = 0; x < dims[3]; x += 1) { + const rgba = jimp.intToRGBA(image.getPixelColor(x, y)); + // convert RGB 0...255 to float 0...1.0 + // and normalize if requested + if (normalize) { + R.push(((rgba.r / 255.0) - normalize.mean[0]) / normalize.std[0]); + G.push(((rgba.g / 255.0) - normalize.mean[1]) / normalize.std[1]); + B.push(((rgba.b / 255.0) - normalize.mean[2]) / normalize.std[2]); + } else { + R.push(rgba.r / 255.0); + G.push(rgba.g / 255.0); + B.push(rgba.b / 255.0); + } + } + } + // Concatenate RGB to transpose [H, W, 3] -> [3, H, W] to a number array + const rgbData = R.concat(G).concat(B); + + // create the Float32Array size 3 * H * W for these dimensions output + const float32Data = new Float32Array(dims[1] * dims[2] * dims[3]); + for (let i = 0; i < rgbData.length; i += 1) { + float32Data[i] = rgbData[i]; + } + + return new ort.Tensor('float32', float32Data, dims); +} + +async function runImageFingerprint(img) { + const INPUT_SIZE = 384; + const NORMALIZE = { + mean: [0.485, 0.456, 0.406], + std: [0.229, 0.224, 0.225], + }; + + await initIdentity(); + + const { Jimp } = jimp; + + const start = Date.now(); + + let image; + if (typeof img === 'string') { + image = await Jimp.read(img); + } else if (img instanceof ArrayBuffer) { + image = await Jimp.fromBuffer(img); + } else if (img instanceof ImageData) { + image = await Jimp.fromBitmap(img); + } else if (img instanceof HTMLImageElement) { + image = await Jimp.fromBitmap(getImageData(img)); + } else { + throw new Error('Invalid type of img argument in getImageFingerprint()'); + } + + console.debug('Fingerprinting image:', image.width, image.height, image); + + image.resize({ w: INPUT_SIZE, h: INPUT_SIZE, mode: Jimp.RESIZE_BILINEAR }); + + const shape = [1, 3, INPUT_SIZE, INPUT_SIZE]; + const inputTensor = jimpImageToOnnxTensorRGB(image, shape, NORMALIZE); + + // console.debug('Image input tensor:', imgInput.dims, imgInput); + // logPixel(imgInput, 0, 0); + // logPixel(imgInput, 200, 100); + // logPixel(imgInput, 300, 200); + // logPixel(imgInput, 382, 382); + // logPixel(imgInput, 383, 383); + // logPixel(imgInput, 3000, 1000); + // logPixel(imgInput, image.width - 1, image.height - 1); + + console.debug('Image loaded in', Date.now() - start, 'ms'); + + const startInference = Date.now(); + + const result = await sessions.fingerprint.run({ image: inputTensor }); + + const features = await result.embedding.getData(); + + console.debug('Feature vector calculated in', Date.now() - startInference, 'ms'); + + return features; +} + +const queues = { + imageFingerprint: new TaskQueue(runImageFingerprint), +}; + +/** + * Get a 256-dimension float32 vector fingerprint of an image. + * + * @param {string|HTMLImageElement|ImageData|Buffer|ArrayBuffer} img - The image to fingerprint. + * Can be a URL, an HTMLImageElement, an ImageData object, + * a Buffer, or an ArrayBuffer. + * @returns {Promise} A 256-dimension float32 vector fingerprint of the image. + */ +export async function getImageFingerprint(img) { + return queues.imageFingerprint.run(img); +} From 82335589dd2c5ef357e4e165f45ce0196190b842 Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Tue, 24 Jun 2025 11:59:05 +0200 Subject: [PATCH 2/7] fix: make identity an optional checkbox --- tools/asset-identity/.gitignore | 3 + tools/asset-identity/scripts.js | 10 ++- tools/image-audit/index.html | 11 ++- tools/image-audit/scripts.js | 70 +++++++++------- utils/identity.js | 137 ++++++++++++++++++++++++++++---- 5 files changed, 184 insertions(+), 47 deletions(-) create mode 100644 tools/asset-identity/.gitignore diff --git a/tools/asset-identity/.gitignore b/tools/asset-identity/.gitignore new file mode 100644 index 00000000..7e7e1583 --- /dev/null +++ b/tools/asset-identity/.gitignore @@ -0,0 +1,3 @@ +# we must ensure the models are NOT published on the internet as is +# hence do not check them into a helix git repo +models/ \ No newline at end of file diff --git a/tools/asset-identity/scripts.js b/tools/asset-identity/scripts.js index 4def6159..a8ad765b 100644 --- a/tools/asset-identity/scripts.js +++ b/tools/asset-identity/scripts.js @@ -1,4 +1,4 @@ -import { initIdentity, getImageFingerprint } from '../../utils/identity.js'; +import { getImageFingerprint } from '../../utils/identity.js'; function disableForm() { const viewbox = document.querySelector('.viewbox'); @@ -33,8 +33,12 @@ async function initUpload() { disableForm(); const img = imgWrapper.querySelector('img'); - const fingerprint = await getImageFingerprint(img); - console.debug('Fingerprint:', fingerprint); + try { + const fingerprint = await getImageFingerprint(img, file.name); + console.debug('Fingerprint:', fingerprint); + } catch (error) { + console.error('Error getting fingerprint:', error); + } }); reader.readAsDataURL(file); } diff --git a/tools/image-audit/index.html b/tools/image-audit/index.html index 01098d3e..80a7692b 100644 --- a/tools/image-audit/index.html +++ b/tools/image-audit/index.html @@ -37,13 +37,22 @@

Image Audit

- +
+
+
+ +
+
+

diff --git a/tools/image-audit/scripts.js b/tools/image-audit/scripts.js index 1f129e92..6043948b 100644 --- a/tools/image-audit/scripts.js +++ b/tools/image-audit/scripts.js @@ -201,8 +201,9 @@ function findUniqueImages(data) { /** * Displays a collection of images in the gallery. * @param {Object[]} images - Array of image data objects to be displayed. + * @param {boolean} [identity=false] - Enable advanced image identity features. */ -function displayImages(images) { +function displayImages(images, identity = false) { const gallery = document.getElementById('image-gallery'); images.forEach((data) => { // create a new figure to hold the image and its metadata @@ -215,31 +216,39 @@ function displayImages(images) { // build image const { href } = new URL(data.src, data.origin); const img = document.createElement('img'); - img.crossOrigin = 'Anonymous'; - img.dataset.src = `https://little-forest-58aa.david8603.workers.dev/?url=${encodeURIComponent(href)}`; + img.dataset.src = href; + + if (identity) { + // load immediately so identity can run in the background + // need to proxy so we can access image data from HTMLImageElement for fingerprinting + img.src = `https://little-forest-58aa.david8603.workers.dev/?url=${encodeURIComponent(href)}`; + img.onload = async () => { + // load from HTMLImageElement so that we get support for SVG + const fingerprint = await getImageFingerprint(img, img.dataset.src); + // console.debug('Fingerprint:', fingerprint); + // TODO: what to do with the fingerprint? + }; + } else { + // load the image when it comes into view + img.loading = 'lazy'; + const observer = new IntersectionObserver((entries) => { + entries.forEach((entry) => { + if (entry.isIntersecting) { + entry.target.timeoutId = setTimeout(() => { + img.src = img.dataset.src; + observer.disconnect(); + }, 500); // delay image loading + } else { + // cancel loading delay if image is scrolled out of view + clearTimeout(entry.target.timeoutId); + } + }); + }, { threshold: 0 }); + observer.observe(figure); + } img.width = data.width; img.height = data.height; - img.loading = 'lazy'; - img.onload = async () => { - const fingerprint = await getImageFingerprint(img); - // console.debug('Fingerprint:', fingerprint); - }; figure.append(img); - // load the image when it comes into view - const observer = new IntersectionObserver((entries) => { - entries.forEach((entry) => { - if (entry.isIntersecting) { - entry.target.timeoutId = setTimeout(() => { - img.src = img.dataset.src; - observer.disconnect(); - }, 500); // delay image loading - } else { - // cancel loading delay if image is scrolled out of view - clearTimeout(entry.target.timeoutId); - } - }); - }, { threshold: 0 }); - observer.observe(figure); // build info button const info = document.createElement('button'); info.setAttribute('aria-label', 'More information'); @@ -324,7 +333,7 @@ function updateCounter(counter, increment, float = false) { const targetValue = increment ? value + increment : 0; counter.textContent = float ? targetValue.toFixed(1) : Math.floor(targetValue); } -async function fetchAndDisplayImages(url) { +async function fetchAndDisplayImages(url, identity = false) { const data = []; const main = document.querySelector('main'); const results = document.getElementById('audit-results'); @@ -348,7 +357,7 @@ async function fetchAndDisplayImages(url) { window.audit.push(...uniqueBatchData); const imagesCounter = document.getElementById('images-counter'); imagesCounter.textContent = parseInt(imagesCounter.textContent, 10) + uniqueBatchData.length; - displayImages(uniqueBatchData); + displayImages(uniqueBatchData, identity); decorateIcons(gallery); data.length = 0; download.disabled = false; @@ -573,10 +582,14 @@ function registerListeners(doc) { // eslint-disable-next-line no-return-assign [...sortActions, ...filterActions].forEach((action) => action.checked = false); const { - url, path, + url, path, identity, } = getFormData(form); - window.history.pushState({}, '', `${window.location.pathname}?url=${encodeURIComponent(url)}&path=${encodeURIComponent(path)}`); + const urlParams = new URLSearchParams(); + urlParams.set('url', url); + urlParams.set('path', path); + if (identity) urlParams.set('identity', '1'); + window.history.pushState({}, '', `${window.location.pathname}?${urlParams.toString()}`); try { const sitemapUrls = url.endsWith('.json') ? fetchQueryIndex(url) : fetchSitemap(url); @@ -586,7 +599,7 @@ function registerListeners(doc) { window.audit = []; for await (const sitemapUrl of sitemapUrls) { if (sitemapUrl.pathname.startsWith(path)) { - await fetchAndDisplayImages(sitemapUrl); + await fetchAndDisplayImages(sitemapUrl, identity); } } clearInterval(timer); @@ -705,6 +718,7 @@ async function init() { const params = new URLSearchParams(window.location.search); if (params.has('url')) document.getElementById('url').value = decodeURIComponent(params.get('url')); if (params.has('path')) document.getElementById('path').value = decodeURIComponent(params.get('path')); + if (params.has('identity')) document.getElementById('identity').checked = ['1', 'true'].includes(params.get('identity')); registerListeners(document); } diff --git a/utils/identity.js b/utils/identity.js index 33b4b986..07f18945 100644 --- a/utils/identity.js +++ b/utils/identity.js @@ -1,5 +1,9 @@ +/* eslint-disable no-return-assign */ +/* eslint-disable no-param-reassign */ +// eslint-disable-next-line max-classes-per-file let jimp; const sessions = {}; +let identityDB; /** * A simple sequential task queue implementation that will ensure that each `run()` will @@ -48,6 +52,75 @@ class TaskQueue { } } +class AsyncIndexedDB { + constructor(db) { + this.db = db; + + // TODO: might need to be configurable? + this.db.onversionchange = (event) => { + console.debug('Received a versionchange event for AsyncIndexedDB, closing connnection:', event); + this.db.close(); + }; + } + + static async open(name, schema, version) { + return new Promise((resolve, reject) => { + const request = indexedDB.open(name, version); + + request.onupgradeneeded = (event) => { + if (event.oldVersion === 0) { + console.debug('Initializing IndexedDB', name, `(version ${event.newVersion})`); + } else { + console.debug('Migrating IndexedDB', name, `from version ${event.oldVersion} to ${event.newVersion}`); + } + schema(event); + }; + request.onsuccess = () => resolve(new AsyncIndexedDB(request.result)); + request.onerror = () => reject(request.error); + }); + } + + async add(store, doc) { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(store, 'readwrite'); + const request = tx.objectStore(store).add(doc); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + } + + async get(store, index, key) { + if (!key) { + key = index; + index = undefined; + } + return new Promise((resolve, reject) => { + const tx = this.db.transaction(store, 'readonly'); + const objStore = tx.objectStore(store); + const request = index ? objStore.index(index).get(key) : objStore.get(key); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + } +} + +async function openIdentityDB() { + const schema = (event) => { + const { transaction, result: db } = event.target; + if (event.oldVersion === 0) { + // initialize new DB with current schema + db.createObjectStore('images', { keyPath: 'url' }); + } else if (event.oldVersion < 4) { + // v1 => v2 upgrade + const store = transaction.objectStore('images'); + if (store.indexNames.includes('url')) { + store.deleteIndex('url'); + } + } + }; + return AsyncIndexedDB.open('identity', schema, 4); +} + /** * Optional initialization function. Usually, getImageFingerprint() will * initialize the identity automatically (load libraries & models). @@ -153,7 +226,40 @@ function jimpImageToOnnxTensorRGB(image, dims, normalize) { return new ort.Tensor('float32', float32Data, dims); } -async function runImageFingerprint(img) { +async function runImageFingerprint(img, url) { + if (typeof img === 'string') { + url = url || img; + } else if (img instanceof HTMLImageElement) { + url = url || img.src; + } + + // ignore data URLs for now + if (url && url.startsWith('data:')) { + url = undefined; + } + + if (!identityDB) { + try { + identityDB = await openIdentityDB(); + } catch (error) { + console.warn('Error opening identity cache DB. Ignoring.', error); + } + } + + if (identityDB && url) { + try { + const image = await identityDB.get('images', url); + if (image && image.fingerprint) { + console.debug(`Found fingerprint in IndexedDB cache: ${url}`); + return image.fingerprint; + } + } catch (error) { + console.warn('Error getting image from identity cache DB. Ignoring.', error); + } + } + + // calculcate using identity + const INPUT_SIZE = 384; const NORMALIZE = { mean: [0.485, 0.456, 0.406], @@ -179,23 +285,14 @@ async function runImageFingerprint(img) { throw new Error('Invalid type of img argument in getImageFingerprint()'); } - console.debug('Fingerprinting image:', image.width, image.height, image); + // console.debug('Fingerprinting image:', image.width, image.height, url); image.resize({ w: INPUT_SIZE, h: INPUT_SIZE, mode: Jimp.RESIZE_BILINEAR }); const shape = [1, 3, INPUT_SIZE, INPUT_SIZE]; const inputTensor = jimpImageToOnnxTensorRGB(image, shape, NORMALIZE); - // console.debug('Image input tensor:', imgInput.dims, imgInput); - // logPixel(imgInput, 0, 0); - // logPixel(imgInput, 200, 100); - // logPixel(imgInput, 300, 200); - // logPixel(imgInput, 382, 382); - // logPixel(imgInput, 383, 383); - // logPixel(imgInput, 3000, 1000); - // logPixel(imgInput, image.width - 1, image.height - 1); - - console.debug('Image loaded in', Date.now() - start, 'ms'); + const loadTime = Date.now() - start; const startInference = Date.now(); @@ -203,7 +300,15 @@ async function runImageFingerprint(img) { const features = await result.embedding.getData(); - console.debug('Feature vector calculated in', Date.now() - startInference, 'ms'); + console.debug(`Image loaded in ${loadTime} ms, feature vector calculated in ${Date.now() - startInference} ms: ${url || ''}`); + + if (url) { + try { + await identityDB.add('images', { url, fingerprint: features }); + } catch (error) { + console.warn('Error adding image to identityDB. Ignoring.', error); + } + } return features; } @@ -218,8 +323,10 @@ const queues = { * @param {string|HTMLImageElement|ImageData|Buffer|ArrayBuffer} img - The image to fingerprint. * Can be a URL, an HTMLImageElement, an ImageData object, * a Buffer, or an ArrayBuffer. + * @param {string} [url] - Optional URL of the image (identifier) if 'img' object is one without + * a URL or to set a different URL. * @returns {Promise} A 256-dimension float32 vector fingerprint of the image. */ -export async function getImageFingerprint(img) { - return queues.imageFingerprint.run(img); +export async function getImageFingerprint(img, url) { + return queues.imageFingerprint.run(img, url); } From 1ac660bc234561fe6ebbe9d223891d46d3128906 Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Tue, 1 Jul 2025 16:51:09 +0200 Subject: [PATCH 3/7] feat(image-audit): show image clusters --- tools/image-audit/index.html | 3 + tools/image-audit/scripts.js | 277 ++++++++++++++++++++++++++++++++++- tools/image-audit/styles.css | 26 ++++ utils/identity.js | 30 +++- 4 files changed, 329 insertions(+), 7 deletions(-) diff --git a/tools/image-audit/index.html b/tools/image-audit/index.html index 80a7692b..3daa25e5 100644 --- a/tools/image-audit/index.html +++ b/tools/image-audit/index.html @@ -66,6 +66,9 @@

Image Audit

+
diff --git a/tools/image-audit/scripts.js b/tools/image-audit/scripts.js index 6043948b..c9ebc71d 100644 --- a/tools/image-audit/scripts.js +++ b/tools/image-audit/scripts.js @@ -1,9 +1,9 @@ /* eslint-disable no-restricted-syntax */ /* eslint-disable class-methods-use-this */ import { buildModal } from '../../scripts/scripts.js'; -import { decorateIcons } from '../../scripts/aem.js'; +import { decorateIcons, loadScript } from '../../scripts/aem.js'; import { initConfigField } from '../../utils/config/config.js'; -import { getImageFingerprint } from '../../utils/identity.js'; +import { getImageFingerprint, stringToFloat64Array } from '../../utils/identity.js'; /* reporting utilities */ /** @@ -149,6 +149,259 @@ function displayModal(figure) { } modal.showModal(); } + +async function renderImageCluster(canvas, figures) { + if (!window.Chart) { + await loadScript('https://cdn.jsdelivr.net/npm/chart.js'); + await loadScript('https://cdn.jsdelivr.net/npm/chartjs-plugin-zoom'); + } + if (!window.druid) { + await loadScript('https://cdn.jsdelivr.net/npm/@saehrimnir/druidjs'); + } + + // collect all fingerprints + const fingerprints = []; + + for (const figure of figures) { + if (figure.dataset.fingerprint) { + fingerprints.push(stringToFloat64Array(figure.dataset.fingerprint)); + } else { + fingerprints.push(new Float64Array(256).fill(NaN)); + } + } + + // reduce dimensionality to 2D + + /* global druid */ + const matrix = druid.Matrix.from(fingerprints, 'col'); + + // candidates: PCA, T-SNE or UMAP + + // FASTMAP: all become 0,0 + // PCA: all are NaN + // UMAP: well distributed but not much patterns + // TSNE: all are NaN + const algorithm = 'UMAP'; + const parameters = {}; + if (algorithm === 'UMAP' && fingerprints.length <= 15) { + parameters.n_neighbors = fingerprints.length - 1; + } + // if (algorithm === 'TSNE') { + // parameters.epsilon = 100; + // } + const dimReduction = new druid[algorithm](matrix, parameters); + console.debug(`running ${algorithm} dimension reduction`); + const projection = dimReduction.transform(); + const fingerprints2D = projection.to2dArray; + console.debug('fingerprints2D[0]', fingerprints2D[0]); + + // prepare data and images for chart + // chartjs data + const data = []; + // store original images for tooltips + const images = []; + // images to be rendered as points + const pointStyle = []; + + const IMAGE_POINT_SIZE = 32; + + for (let i = 0; i < fingerprints2D.length; i += 1) { + const fingerprint = fingerprints2D[i]; + if (fingerprint) { + data.push({ x: fingerprint[0], y: fingerprint[1] }); + + const img = figures[i].querySelector('img'); + if (img && img.complete && img.naturalWidth > 0 && img.naturalHeight > 0) { + images.push(img); + + const ratio = img.width / img.height; + if (ratio > 1) { + // set custom draw size - see below + img.chartDrawWidth = IMAGE_POINT_SIZE; + img.chartDrawHeight = IMAGE_POINT_SIZE / ratio; + } else { + img.chartDrawHeight = IMAGE_POINT_SIZE; + img.chartDrawWidth = IMAGE_POINT_SIZE * ratio; + } + pointStyle.push(img); + } else { + // fingerprint but no image + images.push(null); + pointStyle.push('rect'); + } + } else { + // no fingerprint, no image + data.push({}); + images.push(null); + } + } + + // HACK to change chart.js code that is hardcoded to the original image width and height + // but we want to reuse the original html image but draw it at a smaller size + // https://github.com/chartjs/Chart.js/blob/b5ee134effb0d1b28d48bf8c0146eff13f2fa3e5/src/helpers/helpers.canvas.ts#L186 + const ctx = canvas.getContext('2d'); + if (!ctx.originalDrawImage) { + ctx.originalDrawImage = ctx.drawImage; + ctx.drawImage = (img, x, y, width, height, dx, dy, dWidth, dHeight) => { + if (img instanceof HTMLImageElement && img.chartDrawWidth && img.chartDrawHeight) { + const w = img.chartDrawWidth; + const h = img.chartDrawHeight; + ctx.originalDrawImage(img, -w / 2, -h / 2, w, h); + } else { + ctx.originalDrawImage(img, x, y, width, height, dx, dy, dWidth, dHeight); + } + }; + } + + // wait for cloned point images (in pointStyle[]) to be resized + setTimeout(() => { + // render cluster using chart.js + + if (canvas.chart) { + canvas.chart.destroy(); + } + + /* global Chart */ + // eslint-disable-next-line no-new + canvas.chart = new Chart(canvas, { + type: 'scatter', + data: { + datasets: [{ + data, + radius: IMAGE_POINT_SIZE, + pointStyle, + }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + scales: { + x: { + grid: { + display: false, + }, + display: false, + }, + y: { + grid: { + display: false, + }, + display: false, + }, + }, + plugins: { + legend: { + display: false, + }, + zoom: { + pan: { + enabled: true, + mode: 'xy', + modifierKey: 'alt', + threshold: 0.01, + }, + zoom: { + wheel: { + enabled: true, + speed: 0.05, + }, + pinch: { + enabled: true, + }, + }, + }, + tooltip: { + enabled: false, + external(context) { + // custom tooltip that shows the image in full size + const { chart, tooltip } = context; + let tooltipEl = chart.canvas.parentNode.querySelector('div'); + + // create tooltip container element initially + if (!tooltipEl) { + tooltipEl = document.createElement('div'); + tooltipEl.style.background = 'rgba(0, 0, 0, 0.7)'; + tooltipEl.style.borderRadius = '3px'; + tooltipEl.style.color = 'white'; + tooltipEl.style.opacity = 1; + tooltipEl.style.pointerEvents = 'none'; + tooltipEl.style.position = 'absolute'; + tooltipEl.style.transform = 'translate(-50%, 0)'; + tooltipEl.style.transition = 'all .1s ease'; + + chart.canvas.parentNode.appendChild(tooltipEl); + } + + // Hide if no tooltip + if (tooltip.opacity === 0) { + tooltipEl.style.opacity = 0; + return; + } + + // Set tooltip content + if (tooltip.body) { + // replace tooltip content with image + tooltipEl.innerHTML = ''; + + const dataIndex = tooltip.dataPoints[0]?.dataIndex; + if (dataIndex !== undefined) { + const img = images[dataIndex]; + if (img) { + tooltipEl.appendChild(img); + } + } + } + + // Display, position, and set styles for font + const { offsetLeft: positionX, offsetTop: positionY } = chart.canvas; + tooltipEl.style.opacity = 1; + tooltipEl.style.left = `${positionX + tooltip.caretX}px`; + tooltipEl.style.top = `${positionY + tooltip.caretY}px`; + tooltipEl.style.padding = `${tooltip.options.padding}px ${tooltip.options.padding}px`; + }, + }, + }, + }, + plugins: [{ + beforeDatasetsUpdate(chart) { + // some transparency so that multiple images on top of each are visible + chart.ctx.globalAlpha = 0.8; + }, + }], + }); + }, 0); +} + +async function showImageClusterModal(gallery) { + const figures = [...gallery.querySelectorAll('figure')]; + const id = 'image-cluster'; + + // check if a modal with this ID already exists + let modal = document.getElementById(id); + if (!modal) { + // build new modal + const [newModal, body] = buildModal(); + newModal.id = id; + newModal.classList.add('fullsize'); + modal = newModal; + // define and populate modal content + const h4 = document.createElement('h4'); + h4.classList.add('title'); + h4.innerText = 'Image Clusters'; + // insert h4 before the first child of modal + modal.insertBefore(h4, modal.firstChild); + const canvas = document.createElement('canvas'); + body.append(canvas); + document.body.append(modal); + } + + // render fresh each time the modal is shown + const canvas = modal.querySelector('canvas'); + await renderImageCluster(canvas, figures); + + modal.showModal(); +} + /* image processing and display */ /** * Validates that every image in an array has alt text. @@ -221,12 +474,14 @@ function displayImages(images, identity = false) { if (identity) { // load immediately so identity can run in the background // need to proxy so we can access image data from HTMLImageElement for fingerprinting + img.crossOrigin = 'Anonymous'; img.src = `https://little-forest-58aa.david8603.workers.dev/?url=${encodeURIComponent(href)}`; img.onload = async () => { // load from HTMLImageElement so that we get support for SVG const fingerprint = await getImageFingerprint(img, img.dataset.src); - // console.debug('Fingerprint:', fingerprint); - // TODO: what to do with the fingerprint? + + // adding to dataset turns it into a string, comma-separated floating point decimals + figure.dataset.fingerprint = fingerprint; }; } else { // load the image when it comes into view @@ -337,8 +592,10 @@ async function fetchAndDisplayImages(url, identity = false) { const data = []; const main = document.querySelector('main'); const results = document.getElementById('audit-results'); - const download = results.querySelector('button'); + const download = results.querySelector('#download-report'); download.disabled = true; + const visualizeClusters = results.querySelector('#visualize-image-clusters'); + visualizeClusters.parentElement.hidden = !identity; const gallery = document.getElementById('image-gallery'); // reset counters @@ -556,6 +813,7 @@ function registerListeners(doc) { const canvas = doc.getElementById('canvas'); const gallery = canvas.querySelector('.gallery'); const downloadReport = doc.getElementById('download-report'); + const visualizeClusters = doc.getElementById('visualize-image-clusters'); const actionbar = canvas.querySelector('.action-bar'); const sortActions = actionbar.querySelectorAll('input[name="sort"]'); const filterActions = actionbar.querySelectorAll('input[name="filter"]'); @@ -622,6 +880,7 @@ function registerListeners(doc) { form.addEventListener('reset', () => { errorWrapper.setAttribute('aria-hidden', 'true'); + visualizeClusters.parentElement.hidden = true; }); // handle gallery clicks to display modals gallery.addEventListener('click', (e) => { @@ -647,6 +906,10 @@ function registerListeners(doc) { } }); + visualizeClusters.addEventListener('click', () => { + showImageClusterModal(gallery); + }); + sortActions.forEach((action) => { action.addEventListener('click', (e) => { const { target } = e; @@ -720,6 +983,10 @@ async function init() { if (params.has('path')) document.getElementById('path').value = decodeURIComponent(params.get('path')); if (params.has('identity')) document.getElementById('identity').checked = ['1', 'true'].includes(params.get('identity')); registerListeners(document); + + const visualizeClusters = document.getElementById('visualize-image-clusters'); + const identity = params.has('identity') && ['1', 'true'].includes(params.get('identity')); + visualizeClusters.parentElement.hidden = !identity; } init(); diff --git a/tools/image-audit/styles.css b/tools/image-audit/styles.css index aa87405e..ae9f7238 100644 --- a/tools/image-audit/styles.css +++ b/tools/image-audit/styles.css @@ -2,6 +2,10 @@ display: none; } +.image-audit [hidden] { + display: none; +} + .image-audit .field-group { display: flex; flex-wrap: wrap; @@ -156,6 +160,28 @@ label, legend, .field-help-text, .form-error { border-bottom: var(--border-s) solid var(--gray-300); } +.image-audit dialog > .title { + position: absolute; + top: var(--border-l); + margin: 0; + padding-top: 5px; + padding-left: var(--horizontal-spacing); + width: 100%; + height: 44px; + text-align: center; +} + +.image-audit dialog.fullsize { + width: 100%; + max-width: calc(100% - var(--horizontal-spacing)); + height: calc(100vh - 2 * var(--spacing-xl)); +} + +.image-audit #image-cluster > div { + position: relative; + width: 100%; + height: 100%; +} .image-audit i.symbol-square::after { inset: 0; diff --git a/utils/identity.js b/utils/identity.js index 07f18945..ce7ecdbd 100644 --- a/utils/identity.js +++ b/utils/identity.js @@ -191,7 +191,11 @@ function getImageData(imageElement) { canvas.height = imageElement.height; const ctx = canvas.getContext('2d'); ctx.drawImage(imageElement, 0, 0); - return ctx.getImageData(0, 0, canvas.width, canvas.height); + try { + return ctx.getImageData(0, 0, canvas.width, canvas.height); + } catch (e) { + throw new Error(`Failed to get image data for ${imageElement.src}: ${e.message}`); + } } function jimpImageToOnnxTensorRGB(image, dims, normalize) { @@ -250,7 +254,7 @@ async function runImageFingerprint(img, url) { try { const image = await identityDB.get('images', url); if (image && image.fingerprint) { - console.debug(`Found fingerprint in IndexedDB cache: ${url}`); + // console.debug(`Found fingerprint in IndexedDB cache: ${url}`); return image.fingerprint; } } catch (error) { @@ -330,3 +334,25 @@ const queues = { export async function getImageFingerprint(img, url) { return queues.imageFingerprint.run(img, url); } + +/** + * Convert a string of comma-separated floating point decimals to a Float32Array. + * For example, '1.0,-2.4,3.9' -> [1.0, -2.4, 3.9] + * + * @param {string} str - The string to convert. + * @returns {Float32Array} The Float32Array. + */ +export function stringToFloat32Array(str) { + return new Float32Array(str.split(',').map(Number)); +} + +/** + * Convert a string of comma-separated floating point decimals to a Float64Array. + * For example, '1.0,-2.4,3.9' -> [1.0, -2.4, 3.9] + * + * @param {string} str - The string to convert. + * @returns {Float64Array} The Float64Array. + */ +export function stringToFloat64Array(str) { + return new Float64Array(str.split(',').map(Number)); +} From e01bb61bbb410df72feed985571c11862b1e68b1 Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Tue, 1 Jul 2025 19:02:31 +0200 Subject: [PATCH 4/7] fix: document how to install identity models --- .gitignore | 3 +++ README.md | 14 ++++++++++++++ tools/asset-identity/.gitignore | 3 --- utils/identity.js | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) delete mode 100644 tools/asset-identity/.gitignore diff --git a/.gitignore b/.gitignore index 2eb44f6e..9ddf5848 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ helix-importer-ui .DS_Store *.bak .idea + +# for now ensure ML models are not accidentally published +models/ \ No newline at end of file diff --git a/README.md b/README.md index fd423cd2..f1b45f3e 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,17 @@ npm run lint 1. Install the [AEM CLI](https://github.com/adobe/helix-cli): `npm install -g @adobe/aem-cli` 1. Start AEM Proxy: `aem up` (opens your browser at `http://localhost:3000`) 1. Open the `helix-labs-website` directory in your favorite IDE and start coding :) + +### Image Identity + +`Image identity` features require ML models that are not part of this repository as they cannot be published on the internet (i.e. commited to an Helix repo). To demo one has to run helix-labs locally with the models present on the local machine. + + +1. Check out this repo +1. Create a `models/` directory in the root of the project +1. Get the model files in onnx format. These are Adobe-internal and distributed outside this repo. +1. Copy the model files into the `models/` directory +1. Run `aem up` +1. Go to + - + - (enable `Advanced Image Identity`) diff --git a/tools/asset-identity/.gitignore b/tools/asset-identity/.gitignore deleted file mode 100644 index 7e7e1583..00000000 --- a/tools/asset-identity/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# we must ensure the models are NOT published on the internet as is -# hence do not check them into a helix git repo -models/ \ No newline at end of file diff --git a/utils/identity.js b/utils/identity.js index ce7ecdbd..5f324d24 100644 --- a/utils/identity.js +++ b/utils/identity.js @@ -160,7 +160,7 @@ export async function initIdentity() { const EXECUTION_PROVIDERS = [/* 'webnn', */'webgpu', 'wasm']; console.debug('Loading fingerprint model with execution provider preference:', EXECUTION_PROVIDERS.join(', ')); const start = Date.now(); - const FINGERPRINTER_MODEL_URL = '/tools/asset-identity/models/fingerprinter_behance_c5_grad_v2.onnx'; + const FINGERPRINTER_MODEL_URL = '/models/fingerprinter_behance_c5_grad_v2.onnx'; // ort.env.debug = true; // ort.env.logLevel = 'verbose'; sessions.fingerprint = await ort.InferenceSession.create(FINGERPRINTER_MODEL_URL, { From d0e3a2ca9a99d79b3f9de3275514e7d310a50828 Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Wed, 2 Jul 2025 16:31:17 +0200 Subject: [PATCH 5/7] feat(image-audit): support different image cluster algorithms --- scripts/scripts.js | 1 + tools/image-audit/scripts.js | 55 ++++++++++++++++++++++-------------- tools/image-audit/styles.css | 19 ++++++++++++- 3 files changed, 53 insertions(+), 22 deletions(-) diff --git a/scripts/scripts.js b/scripts/scripts.js index 82fbbe74..be408a5c 100644 --- a/scripts/scripts.js +++ b/scripts/scripts.js @@ -126,6 +126,7 @@ export function buildModal() { close.innerHTML = ''; close.addEventListener('click', closeModal); const body = document.createElement('div'); + body.classList.add('modal-body'); dialog.append(close, body); return [dialog, body]; } diff --git a/tools/image-audit/scripts.js b/tools/image-audit/scripts.js index c9ebc71d..99850db7 100644 --- a/tools/image-audit/scripts.js +++ b/tools/image-audit/scripts.js @@ -166,7 +166,11 @@ async function renderImageCluster(canvas, figures) { if (figure.dataset.fingerprint) { fingerprints.push(stringToFloat64Array(figure.dataset.fingerprint)); } else { - fingerprints.push(new Float64Array(256).fill(NaN)); + // using NaN will give only NaN results for many DruidJS algorithms + // using 0 is a simple stopgap. ideally we should skip them but then + // we'd also need to make sure the data, images and pointStyle arrays + // below are in sync + fingerprints.push(new Float64Array(256).fill(0)); } } @@ -175,25 +179,16 @@ async function renderImageCluster(canvas, figures) { /* global druid */ const matrix = druid.Matrix.from(fingerprints, 'col'); - // candidates: PCA, T-SNE or UMAP + const algorithm = document.getElementById('dr-image-cluster').value; - // FASTMAP: all become 0,0 - // PCA: all are NaN - // UMAP: well distributed but not much patterns - // TSNE: all are NaN - const algorithm = 'UMAP'; const parameters = {}; if (algorithm === 'UMAP' && fingerprints.length <= 15) { parameters.n_neighbors = fingerprints.length - 1; } - // if (algorithm === 'TSNE') { - // parameters.epsilon = 100; - // } const dimReduction = new druid[algorithm](matrix, parameters); console.debug(`running ${algorithm} dimension reduction`); const projection = dimReduction.transform(); const fingerprints2D = projection.to2dArray; - console.debug('fingerprints2D[0]', fingerprints2D[0]); // prepare data and images for chart // chartjs data @@ -315,20 +310,12 @@ async function renderImageCluster(canvas, figures) { external(context) { // custom tooltip that shows the image in full size const { chart, tooltip } = context; - let tooltipEl = chart.canvas.parentNode.querySelector('div'); + let tooltipEl = chart.canvas.parentNode.querySelector('.image-cluster-tooltip'); // create tooltip container element initially if (!tooltipEl) { tooltipEl = document.createElement('div'); - tooltipEl.style.background = 'rgba(0, 0, 0, 0.7)'; - tooltipEl.style.borderRadius = '3px'; - tooltipEl.style.color = 'white'; - tooltipEl.style.opacity = 1; - tooltipEl.style.pointerEvents = 'none'; - tooltipEl.style.position = 'absolute'; - tooltipEl.style.transform = 'translate(-50%, 0)'; - tooltipEl.style.transition = 'all .1s ease'; - + tooltipEl.classList.add('image-cluster-tooltip'); chart.canvas.parentNode.appendChild(tooltipEl); } @@ -390,8 +377,34 @@ async function showImageClusterModal(gallery) { h4.innerText = 'Image Clusters'; // insert h4 before the first child of modal modal.insertBefore(h4, modal.firstChild); + + const drSelectWrapper = document.createElement('div'); + drSelectWrapper.classList.add('dr-dropdown-wrapper'); + drSelectWrapper.innerHTML = ` + + `; + const canvas = document.createElement('canvas'); + + const drDropdown = drSelectWrapper.querySelector('select'); + drDropdown.addEventListener('change', () => { + renderImageCluster(canvas, figures); + }); + body.append(canvas); + body.append(drSelectWrapper); document.body.append(modal); } diff --git a/tools/image-audit/styles.css b/tools/image-audit/styles.css index ae9f7238..710593ef 100644 --- a/tools/image-audit/styles.css +++ b/tools/image-audit/styles.css @@ -177,10 +177,27 @@ label, legend, .field-help-text, .form-error { height: calc(100vh - 2 * var(--spacing-xl)); } -.image-audit #image-cluster > div { +.image-audit #image-cluster > .modal-body { position: relative; width: 100%; height: 100%; + padding-bottom: 56px; +} + +.image-audit .image-cluster-tooltip { + position: absolute; + background: rgba(0, 0, 0, 0.7); + border-radius: 3px; + color: white; + opacity: 1; + pointer-events: none; + position: absolute; + transform: translate(-50%, 0); + transition: all .1s ease; +} + +.image-audit .dr-dropdown-wrapper select { + width: auto; } .image-audit i.symbol-square::after { From b48f8e8f7eba8a3cfdf8929aedfb23ef1cc2608e Mon Sep 17 00:00:00 2001 From: Alexander Klimetschek Date: Tue, 22 Jul 2025 11:53:53 +0200 Subject: [PATCH 6/7] fix(image-audit): update cluster everytime dialog is opened --- tools/image-audit/scripts.js | 308 ++++++++++++++++++++--------------- tools/image-audit/styles.css | 10 ++ 2 files changed, 183 insertions(+), 135 deletions(-) diff --git a/tools/image-audit/scripts.js b/tools/image-audit/scripts.js index 99850db7..8964b357 100644 --- a/tools/image-audit/scripts.js +++ b/tools/image-audit/scripts.js @@ -150,7 +150,134 @@ function displayModal(figure) { modal.showModal(); } -async function renderImageCluster(canvas, figures) { +function renderImageClusterChart(canvas, chartData) { + canvas.classList.remove('loading-spinner'); + + // render cluster using chart.js + + /* global Chart */ + canvas.chart = new Chart(canvas, { + type: 'scatter', + data: { + datasets: [{ + data: chartData.data, + radius: chartData.radius, + pointStyle: chartData.pointStyle, + }], + }, + options: { + responsive: true, + maintainAspectRatio: false, + scales: { + x: { + grid: { + display: false, + }, + display: false, + }, + y: { + grid: { + display: false, + }, + display: false, + }, + }, + plugins: { + legend: { + display: false, + }, + zoom: { + pan: { + enabled: true, + mode: 'xy', + modifierKey: 'alt', + threshold: 0.01, + }, + zoom: { + wheel: { + enabled: true, + speed: 0.05, + }, + pinch: { + enabled: true, + }, + }, + }, + tooltip: { + enabled: false, + external(context) { + // custom tooltip that shows the image in full size + const { chart, tooltip } = context; + let tooltipEl = chart.canvas.parentNode.querySelector('.image-cluster-tooltip'); + + // create tooltip container element initially + if (!tooltipEl) { + tooltipEl = document.createElement('div'); + tooltipEl.classList.add('image-cluster-tooltip'); + chart.canvas.parentNode.appendChild(tooltipEl); + } + + // Hide if no tooltip + if (tooltip.opacity === 0) { + tooltipEl.style.opacity = 0; + return; + } + + // Set tooltip content + if (tooltip.body) { + // replace tooltip content with image + tooltipEl.innerHTML = ''; + + const dataIndex = tooltip.dataPoints[0]?.dataIndex; + if (dataIndex !== undefined) { + let img = chartData.clonedImages[dataIndex]; + if (!img) { + img = chartData.images[dataIndex]; + if (img) { + img = img.cloneNode(true); + chartData.clonedImages[dataIndex] = img; + } else { + chartData.clonedImages[dataIndex] = null; + } + } + if (img) { + tooltipEl.appendChild(img); + } + } + } + + // Display, position, and set styles for font + const { offsetLeft: positionX, offsetTop: positionY } = chart.canvas; + tooltipEl.style.opacity = 1; + tooltipEl.style.left = `${positionX + tooltip.caretX}px`; + tooltipEl.style.top = `${positionY + tooltip.caretY}px`; + tooltipEl.style.padding = `${tooltip.options.padding}px ${tooltip.options.padding}px`; + }, + }, + }, + onClick: (event, elements) => { + if (event.type === 'click' && elements[0]) { + const dataIndex = elements[0].index; + const currentFigures = document.querySelectorAll('#canvas .gallery figure'); + if (currentFigures[dataIndex]) { + displayModal(currentFigures[dataIndex]); + } + } + }, + onHover: (event, chartElement) => { + event.native.target.style.cursor = chartElement[0] ? 'pointer' : 'default'; + }, + }, + plugins: [{ + beforeDatasetsUpdate(chart) { + // some transparency so that multiple images on top of each are visible + chart.ctx.globalAlpha = 0.8; + }, + }], + }); +} + +async function renderImageCluster(canvas, gallery) { if (!window.Chart) { await loadScript('https://cdn.jsdelivr.net/npm/chart.js'); await loadScript('https://cdn.jsdelivr.net/npm/chartjs-plugin-zoom'); @@ -162,6 +289,7 @@ async function renderImageCluster(canvas, figures) { // collect all fingerprints const fingerprints = []; + const figures = [...gallery.querySelectorAll('figure')]; for (const figure of figures) { if (figure.dataset.fingerprint) { fingerprints.push(stringToFloat64Array(figure.dataset.fingerprint)); @@ -177,57 +305,60 @@ async function renderImageCluster(canvas, figures) { // reduce dimensionality to 2D /* global druid */ + // prep data const matrix = druid.Matrix.from(fingerprints, 'col'); + // set algorithm const algorithm = document.getElementById('dr-image-cluster').value; - const parameters = {}; if (algorithm === 'UMAP' && fingerprints.length <= 15) { parameters.n_neighbors = fingerprints.length - 1; } const dimReduction = new druid[algorithm](matrix, parameters); - console.debug(`running ${algorithm} dimension reduction`); - const projection = dimReduction.transform(); - const fingerprints2D = projection.to2dArray; - // prepare data and images for chart - // chartjs data - const data = []; - // store original images for tooltips - const images = []; - // images to be rendered as points - const pointStyle = []; + // reduce dimensionality (can take some time) + const fingerprints2D = dimReduction.transform().to2dArray; - const IMAGE_POINT_SIZE = 32; + // prepare data and images for chart + const chartData = { + // chartjs data + data: [], + // store original images for tooltips + images: [], + clonedImages: [], + // images to be rendered as points + pointStyle: [], + radius: 32, + }; for (let i = 0; i < fingerprints2D.length; i += 1) { const fingerprint = fingerprints2D[i]; if (fingerprint) { - data.push({ x: fingerprint[0], y: fingerprint[1] }); + chartData.data.push({ x: fingerprint[0], y: fingerprint[1] }); const img = figures[i].querySelector('img'); if (img && img.complete && img.naturalWidth > 0 && img.naturalHeight > 0) { - images.push(img); + chartData.images.push(img); const ratio = img.width / img.height; if (ratio > 1) { - // set custom draw size - see below - img.chartDrawWidth = IMAGE_POINT_SIZE; - img.chartDrawHeight = IMAGE_POINT_SIZE / ratio; + // set custom draw size - see below + img.chartDrawWidth = chartData.radius; + img.chartDrawHeight = chartData.radius / ratio; } else { - img.chartDrawHeight = IMAGE_POINT_SIZE; - img.chartDrawWidth = IMAGE_POINT_SIZE * ratio; + img.chartDrawHeight = chartData.radius; + img.chartDrawWidth = chartData.radius * ratio; } - pointStyle.push(img); + chartData.pointStyle.push(img); } else { // fingerprint but no image - images.push(null); - pointStyle.push('rect'); + chartData.images.push(null); + chartData.pointStyle.push('rect'); } } else { // no fingerprint, no image - data.push({}); - images.push(null); + chartData.data.push({}); + chartData.images.push(null); } } @@ -250,117 +381,24 @@ async function renderImageCluster(canvas, figures) { // wait for cloned point images (in pointStyle[]) to be resized setTimeout(() => { - // render cluster using chart.js - - if (canvas.chart) { - canvas.chart.destroy(); - } - - /* global Chart */ - // eslint-disable-next-line no-new - canvas.chart = new Chart(canvas, { - type: 'scatter', - data: { - datasets: [{ - data, - radius: IMAGE_POINT_SIZE, - pointStyle, - }], - }, - options: { - responsive: true, - maintainAspectRatio: false, - scales: { - x: { - grid: { - display: false, - }, - display: false, - }, - y: { - grid: { - display: false, - }, - display: false, - }, - }, - plugins: { - legend: { - display: false, - }, - zoom: { - pan: { - enabled: true, - mode: 'xy', - modifierKey: 'alt', - threshold: 0.01, - }, - zoom: { - wheel: { - enabled: true, - speed: 0.05, - }, - pinch: { - enabled: true, - }, - }, - }, - tooltip: { - enabled: false, - external(context) { - // custom tooltip that shows the image in full size - const { chart, tooltip } = context; - let tooltipEl = chart.canvas.parentNode.querySelector('.image-cluster-tooltip'); - - // create tooltip container element initially - if (!tooltipEl) { - tooltipEl = document.createElement('div'); - tooltipEl.classList.add('image-cluster-tooltip'); - chart.canvas.parentNode.appendChild(tooltipEl); - } - - // Hide if no tooltip - if (tooltip.opacity === 0) { - tooltipEl.style.opacity = 0; - return; - } + renderImageClusterChart(canvas, chartData); + }, 0); +} - // Set tooltip content - if (tooltip.body) { - // replace tooltip content with image - tooltipEl.innerHTML = ''; +async function updateImageCluster(canvas, gallery) { + if (canvas.chart) { + canvas.chart.destroy(); + } - const dataIndex = tooltip.dataPoints[0]?.dataIndex; - if (dataIndex !== undefined) { - const img = images[dataIndex]; - if (img) { - tooltipEl.appendChild(img); - } - } - } + // show loading animation while clusters are being calculated + canvas.classList.add('loading-spinner'); - // Display, position, and set styles for font - const { offsetLeft: positionX, offsetTop: positionY } = chart.canvas; - tooltipEl.style.opacity = 1; - tooltipEl.style.left = `${positionX + tooltip.caretX}px`; - tooltipEl.style.top = `${positionY + tooltip.caretY}px`; - tooltipEl.style.padding = `${tooltip.options.padding}px ${tooltip.options.padding}px`; - }, - }, - }, - }, - plugins: [{ - beforeDatasetsUpdate(chart) { - // some transparency so that multiple images on top of each are visible - chart.ctx.globalAlpha = 0.8; - }, - }], - }); + setTimeout(() => { + renderImageCluster(canvas, gallery); }, 0); } -async function showImageClusterModal(gallery) { - const figures = [...gallery.querySelectorAll('figure')]; +function showImageClusterModal(gallery) { const id = 'image-cluster'; // check if a modal with this ID already exists @@ -384,9 +422,9 @@ async function showImageClusterModal(gallery) {