Improve sitemap settings so crawling is more effective

docs/tools/vscode-extension/cursor/index.md

@@ -17,7 +17,6 @@ Adding Flow docs lets you interact with our docs directly and get the most accur
 ![Cursor Settings](./images//use-cursor-1.png)
 2. Set Flow Docs:
     - Enter the URL of the Flow docs: `https://developers.flow.com/tools` and press Enter.
-      - Note:  This **will index all** the docs.  We're investigating why you need `/tools`
     - Cursor will automatically detect the Flow docs and index them for you.
     - Ensure the name is `Flow`, and click "Confirm" to add the docs.
 ![Cursor Settings](./images//use-cursor-2.png)

docsearch.config.json

@@ -1,29 +1,65 @@
 {
   "index_name": "flow_docs",
   "start_urls": [
-    "https://developers.flow.com/"
-  ],
-  "sitemap_urls": [
-    "https://developers.flow.com/sitemap.xml"
+    {
+      "url": "https://developers.flow.com/",
+      "selectors_key": "homepage",
+      "tags": ["homepage"],
+      "page_rank": 1
+    },
+    {
+      "url": "https://developers.flow.com/build/",
+      "selectors_key": "default",
+      "tags": ["build"],
+      "page_rank": 2
+    },
+    {
+      "url": "https://developers.flow.com/tools/",
+      "selectors_key": "default",
+      "tags": ["tools"],
+      "page_rank": 2
+    },
+    {
+      "url": "https://developers.flow.com/ecosystem/",
+      "selectors_key": "default",
+      "tags": ["ecosystem"],
+      "page_rank": 2
+    }
   ],
+  "sitemap_urls": ["https://developers.flow.com/sitemap.xml"],
   "sitemap_alternate_links": true,
-  "stop_urls": [
-    "/tests"
-  ],
+  "stop_urls": ["/tests"],
   "selectors": {
-    "lvl0": {
-      "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]",
-      "type": "xpath",
-      "global": true,
-      "default_value": "Documentation"
+    "default": {
+      "lvl0": {
+        "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]",
+        "type": "xpath",
+        "global": true,
+        "default_value": "Documentation"
+      },
+      "lvl1": "article h1, header h1",
+      "lvl2": "article h2",
+      "lvl3": "article h3",
+      "lvl4": "article h4",
+      "lvl5": "article h5, article td:first-child",
+      "lvl6": "article h6",
+      "text": "article p, article li, article td:last-child"
     },
-    "lvl1": "article h1, header h1",
-    "lvl2": "article h2",
-    "lvl3": "article h3",
-    "lvl4": "article h4",
-    "lvl5": "article h5, article td:first-child",
-    "lvl6": "article h6",
-    "text": "article p, article li, article td:last-child"
+    "homepage": {
+      "lvl0": {
+        "selector": "nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]",
+        "type": "xpath",
+        "global": true,
+        "default_value": "Flow Developer Portal"
+      },
+      "lvl1": "h1, header h1, .hero__title, main h1",
+      "lvl2": "h2, .hero__subtitle, .cards h3, main h2",
+      "lvl3": "h3, .cards h4, main h3",
+      "lvl4": "h4, main h4",
+      "lvl5": "h5",
+      "lvl6": "h6",
+      "text": "p, li, td, .hero__subtitle, .cards p, main p"
+    }
   },
   "strip_chars": " .,;:#",
   "custom_settings": {
@@ -32,7 +68,8 @@
       "language",
       "version",
       "type",
-      "docusaurus_tag"
+      "docusaurus_tag",
+      "tags"
     ],
     "attributesToRetrieve": [
       "hierarchy",
@@ -41,6 +78,14 @@
       "url",
       "url_without_anchor",
       "type"
-    ]
-  }
-}
+    ],
+    "distinct": true,
+    "attributeForDistinct": "url"
+  },
+  "js_render": true,
+  "js_wait": 2,
+  "use_anchors": true,
+  "scrape_start_urls": true,
+  "min_indexed_level": 0,
+  "only_content_level": false
+}

docusaurus.config.js

@@ -153,7 +153,7 @@ const config = {
   url: getUrl(),
   // Set the /<baseUrl>/ pathname under which your site is served
   // For GitHub pages deployment, it is often '/<projectName>/'
-  baseUrl,
+  baseUrl: baseUrl,
 
   // GitHub pages deployment config.
   // If you aren't using GitHub pages, you don't need these.
@@ -203,13 +203,18 @@ const config = {
             require.resolve('./src/css/custom.css'),
           ],
         },
+        sitemap: {
+          changefreq: 'daily',
+          priority: 0.5,
+          ignorePatterns: ['/tags/**'],
+        },
         ...(process.env.GTAG
           ? {
-            gtag: {
-              trackingID: process.env.GTAG,
-              anonymizeIP: true,
-            },
-          }
+              gtag: {
+                trackingID: process.env.GTAG,
+                anonymizeIP: true,
+              },
+            }
           : {}),
       }),
     ],
@@ -244,14 +249,27 @@ const config = {
       image: 'img/flow-docs-og-1200-630.png',
       metadata: [
         { name: 'twitter:card', content: 'summary_large_image' },
-        { name: 'twitter:image', content: getUrl() + '/img/flow-docs-og-1200-630.png' },
-        { property: 'og:image', content: getUrl() + '/img/flow-docs-og-1200-630.png' },
+        {
+          name: 'twitter:image',
+          content: getUrl() + '/img/flow-docs-og-1200-630.png',
+        },
+        {
+          property: 'og:image',
+          content: getUrl() + '/img/flow-docs-og-1200-630.png',
+        },
         { property: 'og:image:type', content: 'image/png' },
         { property: 'og:image:width', content: '1200' },
         { property: 'og:image:height', content: '630' },
         { property: 'og:type', content: 'website' },
-        { property: 'og:description', content: 'Flow Developer Documentation - The future of culture and digital assets is built on Flow' },
-        { property: 'og:logo', content: getUrl() + '/img/flow-docs-logo-light.png' },
+        {
+          property: 'og:description',
+          content:
+            'Flow Developer Documentation - The future of culture and digital assets is built on Flow',
+        },
+        {
+          property: 'og:logo',
+          content: getUrl() + '/img/flow-docs-logo-light.png',
+        },
       ],
       docs: {
         sidebar: {