Auto-discover connector documentation links via docs sitemap

src/lib/connector-utils/connector-utils.ts

@@ -356,6 +356,7 @@ const CONNECTOR_DOCS: Record<string, string> = {
   'aws.s3': `${DOCS_BASE}/storage-file/aws.s3/aws-s3-connector-overview`,
   azure_storage_service: `${DOCS_BASE}/storage-file/azure_storage_service/overview`,
   'microsoft.onedrive': `${DOCS_BASE}/storage-file/microsoft.onedrive/microsoft-onedrive-connector-overview`,
+  'microsoft.sharepoint.pages': `${DOCS_BASE}/storage-file/microsoft.sharepoint.pages/connector-overview`,
 };
 
 /**
@@ -598,37 +599,71 @@ export function isConnectorDocHardcoded(packageName: string): boolean {
   return packageName in CONNECTOR_DOCS;
 }
 
-const DOCS_CHECK_CACHE_PREFIX = 'connector_docs_check_';
+const SITEMAP_CACHE_KEY = 'connector_docs_sitemap';
+const SITEMAP_CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours
+const DOCS_SITEMAP_URL = `${DOCS_BASE.replace('/connectors/catalog', '')}/sitemap.xml`;
+
+interface SitemapCache {
+  packageNames: string[];
+  timestamp: number;
+}
+
+// Module-level promise deduplicates concurrent calls during the same page session
+let sitemapPromise: Promise<Set<string>> | null = null;
 
 /**
- * Verifies that a derived docs URL resolves (HTTP 200) via a HEAD request.
- * Results are cached in sessionStorage for the duration of the browser session.
- * Returns false on network failure or CORS error (cross-origin dev environments).
+ * Returns the set of connector package names that have published docs pages,
+ * determined by parsing the docs sitemap. Result is cached in localStorage for
+ * 6 hours so subsequent calls within that window are instant.
+ * Falls back to an empty set on network failure or CORS error.
  */
-export async function checkDerivedDocsUrl(packageName: string, url: string): Promise<boolean> {
-  const cacheKey = `${DOCS_CHECK_CACHE_PREFIX}${packageName}`;
-  try {
-    const cached = sessionStorage.getItem(cacheKey);
-    if (cached !== null) return cached === 'true';
-  } catch {
-    /* sessionStorage unavailable */
-  }
+export function getDocumentedConnectors(): Promise<Set<string>> {
+  if (sitemapPromise) return sitemapPromise;
+
+  sitemapPromise = (async (): Promise<Set<string>> => {
+    try {
+      const cached = localStorage.getItem(SITEMAP_CACHE_KEY);
+      if (cached) {
+        const { packageNames, timestamp }: SitemapCache = JSON.parse(cached);
+        if (Date.now() - timestamp < SITEMAP_CACHE_TTL) {
+          return new Set(packageNames);
+        }
+      }
+    } catch {
+      /* localStorage unavailable */
+    }
 
-  let valid = false;
-  try {
-    const response = await fetch(url, { method: 'HEAD' });
-    valid = response.ok;
-  } catch {
-    /* network failure or CORS error — treat as invalid */
-  }
+    // Let network/parse errors propagate so the outer .catch() can reset
+    // sitemapPromise and allow a retry on the next call.
+    const response = await fetch(DOCS_SITEMAP_URL);
+    const xml = await response.text();
+
+    // Extract package names from URLs like:
+    // .../connectors/catalog/{category}/{packageName}/connector-overview
+    const packageNames: string[] = [];
+    const regex = /connectors\/catalog\/[^/<]+\/([^/<]+)\/connector-overview/g;
+    let match;
+    while ((match = regex.exec(xml)) !== null) {
+      packageNames.push(match[1]);
+    }
 
-  try {
-    sessionStorage.setItem(cacheKey, String(valid));
-  } catch {
-    /* sessionStorage unavailable or full */
-  }
+    try {
+      localStorage.setItem(
+        SITEMAP_CACHE_KEY,
+        JSON.stringify({ packageNames, timestamp: Date.now() })
+      );
+    } catch {
+      /* localStorage unavailable or full */
+    }
+
+    return new Set(packageNames);
+  })();
 
-  return valid;
+  // Reset on failure so the next call retries, then return an empty set to callers
+  return sitemapPromise.catch((_e) => {
+    sitemapPromise = null;
+    return new Set<string>();
+  });
 }
 
 /**

src/lib/connector-utils/index.ts

@@ -12,6 +12,6 @@ export {
   HIDDEN_PACKAGES,
   getConnectorDocsUrl,
   isConnectorDocHardcoded,
-  checkDerivedDocsUrl,
+  getDocumentedConnectors,
 } from './connector-utils';
 export type { SortOption } from './connector-utils';

src/pages/ConnectorDetailPage.tsx

@@ -48,7 +48,7 @@ import {
   getDisplayName,
   getConnectorDocsUrl,
   isConnectorDocHardcoded,
-  checkDerivedDocsUrl,
+  getDocumentedConnectors,
 } from '@/lib/connector-utils';
 import MarkdownContent from '@/components/MarkdownContent';
 import Footer from '@/components/Footer';
@@ -125,7 +125,8 @@ export default function ConnectorDetailPage() {
     name: string;
     documentationUrl?: string;
   } | null>(null);
-  const [docsUrl, setDocsUrl] = useState<string | undefined>(undefined);
+  // null = sitemap check in progress, string = confirmed URL, undefined = no docs
+  const [docsUrl, setDocsUrl] = useState<string | null | undefined>(null);
 
   const effectiveMode = useMemo(() => {
     if (mode === 'system') {
@@ -169,9 +170,15 @@ export default function ConnectorDetailPage() {
     loadPackageDetails();
   }, [org, name, version]);
 
+  // Prefetch sitemap in parallel with fetchPackageDetails so it's ready (or cached)
+  // by the time package details finish loading.
+  useEffect(() => {
+    getDocumentedConnectors().catch(() => {});
+  }, []);
+
   useEffect(() => {
     if (!packageDetails) {
-      setDocsUrl(undefined);
+      setDocsUrl(null);
       return;
     }
 
@@ -181,16 +188,22 @@ export default function ConnectorDetailPage() {
       return;
     }
 
+    // Hardcoded entries are manually verified — no sitemap check needed
     if (isConnectorDocHardcoded(packageDetails.name)) {
       setDocsUrl(tentative);
       return;
     }
 
-    // Derived URL: verify it resolves before showing the Documentation button
+    // Derived URL: confirm it's in the published sitemap before showing the button
+    setDocsUrl(null);
     let cancelled = false;
-    checkDerivedDocsUrl(packageDetails.name, tentative).then((valid) => {
-      if (!cancelled) setDocsUrl(valid ? tentative : undefined);
-    });
+    getDocumentedConnectors()
+      .then((documented) => {
+        if (!cancelled) setDocsUrl(documented.has(packageDetails.name) ? tentative : undefined);
+      })
+      .catch(() => {
+        if (!cancelled) setDocsUrl(undefined);
+      });
     return () => {
       cancelled = true;
     };
@@ -452,7 +465,12 @@ export default function ConnectorDetailPage() {
           <Divider sx={{ my: 2 }} />
         </>
       )}
-      {docsUrl ? (
+      {docsUrl === null ? (
+        <Button fullWidth variant="contained" color="primary" disabled>
+          <CircularProgress size={14} sx={{ mr: 1 }} />
+          Documentation
+        </Button>
+      ) : docsUrl ? (
         <Button
           fullWidth
           variant="contained"