From bbcd0b7f45dad0570766c544fed176d431fc155c Mon Sep 17 00:00:00 2001 From: Danesh Kuruppu Date: Wed, 17 Jun 2026 10:59:36 +0530 Subject: [PATCH 1/2] update the document discovery logic to rely on sitemap --- src/lib/connector-utils/connector-utils.ts | 90 ++++++++++++++++------ src/lib/connector-utils/index.ts | 2 +- src/pages/ConnectorDetailPage.tsx | 34 ++++++-- 3 files changed, 92 insertions(+), 34 deletions(-) diff --git a/src/lib/connector-utils/connector-utils.ts b/src/lib/connector-utils/connector-utils.ts index 3db4f0b..0f0ac2a 100644 --- a/src/lib/connector-utils/connector-utils.ts +++ b/src/lib/connector-utils/connector-utils.ts @@ -356,6 +356,7 @@ const CONNECTOR_DOCS: Record = { 'aws.s3': `${DOCS_BASE}/storage-file/aws.s3/aws-s3-connector-overview`, azure_storage_service: `${DOCS_BASE}/storage-file/azure_storage_service/overview`, 'microsoft.onedrive': `${DOCS_BASE}/storage-file/microsoft.onedrive/microsoft-onedrive-connector-overview`, + 'microsoft.sharepoint.pages': `${DOCS_BASE}/storage-file/microsoft.sharepoint.pages/connector-overview`, }; /** @@ -598,37 +599,76 @@ export function isConnectorDocHardcoded(packageName: string): boolean { return packageName in CONNECTOR_DOCS; } -const DOCS_CHECK_CACHE_PREFIX = 'connector_docs_check_'; +const SITEMAP_CACHE_KEY = 'connector_docs_sitemap'; +const SITEMAP_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours +const DOCS_SITEMAP_URL = `${DOCS_BASE.replace('/connectors/catalog', '')}/sitemap.xml`; + +interface SitemapCache { + packageNames: string[]; + timestamp: number; +} + +// Module-level promise deduplicates concurrent calls during the same page session +let sitemapPromise: Promise> | null = null; /** - * Verifies that a derived docs URL resolves (HTTP 200) via a HEAD request. - * Results are cached in sessionStorage for the duration of the browser session. - * Returns false on network failure or CORS error (cross-origin dev environments). + * Returns the set of connector package names that have published docs pages, + * determined by parsing the docs sitemap. Result is cached in localStorage for + * 6 hours so subsequent calls within that window are instant. + * Falls back to an empty set on network failure or CORS error. */ -export async function checkDerivedDocsUrl(packageName: string, url: string): Promise { - const cacheKey = `${DOCS_CHECK_CACHE_PREFIX}${packageName}`; - try { - const cached = sessionStorage.getItem(cacheKey); - if (cached !== null) return cached === 'true'; - } catch { - /* sessionStorage unavailable */ - } +export function getDocumentedConnectors(): Promise> { + if (sitemapPromise) return sitemapPromise; + + sitemapPromise = (async (): Promise> => { + try { + const cached = localStorage.getItem(SITEMAP_CACHE_KEY); + if (cached) { + const { packageNames, timestamp }: SitemapCache = JSON.parse(cached); + if (Date.now() - timestamp < SITEMAP_CACHE_TTL) { + return new Set(packageNames); + } + } + } catch { + /* localStorage unavailable */ + } - let valid = false; - try { - const response = await fetch(url, { method: 'HEAD' }); - valid = response.ok; - } catch { - /* network failure or CORS error — treat as invalid */ - } + try { + const response = await fetch(DOCS_SITEMAP_URL); + const xml = await response.text(); + + // Extract package names from URLs like: + // .../connectors/catalog/{category}/{packageName}/connector-overview + const packageNames: string[] = []; + const regex = /connectors\/catalog\/[^/<]+\/([^/<]+)\/connector-overview/g; + let match; + while ((match = regex.exec(xml)) !== null) { + packageNames.push(match[1]); + } - try { - sessionStorage.setItem(cacheKey, String(valid)); - } catch { - /* sessionStorage unavailable or full */ - } + try { + localStorage.setItem( + SITEMAP_CACHE_KEY, + JSON.stringify({ packageNames, timestamp: Date.now() }) + ); + } catch { + /* localStorage unavailable or full */ + } + + return new Set(packageNames); + } catch { + /* network failure or CORS error */ + } + + return new Set(); + })(); + + // Reset on failure so the next call retries + sitemapPromise.catch(() => { + sitemapPromise = null; + }); - return valid; + return sitemapPromise; } /** diff --git a/src/lib/connector-utils/index.ts b/src/lib/connector-utils/index.ts index 90691b5..ce32c1b 100644 --- a/src/lib/connector-utils/index.ts +++ b/src/lib/connector-utils/index.ts @@ -12,6 +12,6 @@ export { HIDDEN_PACKAGES, getConnectorDocsUrl, isConnectorDocHardcoded, - checkDerivedDocsUrl, + getDocumentedConnectors, } from './connector-utils'; export type { SortOption } from './connector-utils'; diff --git a/src/pages/ConnectorDetailPage.tsx b/src/pages/ConnectorDetailPage.tsx index bf5337a..52f7e4d 100644 --- a/src/pages/ConnectorDetailPage.tsx +++ b/src/pages/ConnectorDetailPage.tsx @@ -48,7 +48,7 @@ import { getDisplayName, getConnectorDocsUrl, isConnectorDocHardcoded, - checkDerivedDocsUrl, + getDocumentedConnectors, } from '@/lib/connector-utils'; import MarkdownContent from '@/components/MarkdownContent'; import Footer from '@/components/Footer'; @@ -125,7 +125,8 @@ export default function ConnectorDetailPage() { name: string; documentationUrl?: string; } | null>(null); - const [docsUrl, setDocsUrl] = useState(undefined); + // null = sitemap check in progress, string = confirmed URL, undefined = no docs + const [docsUrl, setDocsUrl] = useState(null); const effectiveMode = useMemo(() => { if (mode === 'system') { @@ -169,9 +170,15 @@ export default function ConnectorDetailPage() { loadPackageDetails(); }, [org, name, version]); + // Prefetch sitemap in parallel with fetchPackageDetails so it's ready (or cached) + // by the time package details finish loading. + useEffect(() => { + getDocumentedConnectors().catch(() => {}); + }, []); + useEffect(() => { if (!packageDetails) { - setDocsUrl(undefined); + setDocsUrl(null); return; } @@ -181,16 +188,22 @@ export default function ConnectorDetailPage() { return; } + // Hardcoded entries are manually verified — no sitemap check needed if (isConnectorDocHardcoded(packageDetails.name)) { setDocsUrl(tentative); return; } - // Derived URL: verify it resolves before showing the Documentation button + // Derived URL: confirm it's in the published sitemap before showing the button + setDocsUrl(null); let cancelled = false; - checkDerivedDocsUrl(packageDetails.name, tentative).then((valid) => { - if (!cancelled) setDocsUrl(valid ? tentative : undefined); - }); + getDocumentedConnectors() + .then((documented) => { + if (!cancelled) setDocsUrl(documented.has(packageDetails.name) ? tentative : undefined); + }) + .catch(() => { + if (!cancelled) setDocsUrl(undefined); + }); return () => { cancelled = true; }; @@ -452,7 +465,12 @@ export default function ConnectorDetailPage() { )} - {docsUrl ? ( + {docsUrl === null ? ( + + ) : docsUrl ? (