Add llms.txt file generation (#30827)

* Add llms.txt generation package

* Add llms.txt build command to Makefile

* Tweak log message

* Fix llms.txt build

* Add llms.txt build to dependencies

Author: jhgilbert

.gitignore

@@ -299,3 +299,6 @@ content/en/**/*.ast.json
 
 # Cursor editor configuration
 .cursor/
+
+# Generated file for LLMs
+static/llms.txt

Makefile

@@ -72,6 +72,12 @@ watch-cdocs:
 	@echo "Compiling .mdoc files to HTML";
 	@node ./local/bin/js/cdocs-build.js --watch;
 
+# compile .mdoc.md files to HTML
+# so Hugo can include them in the site
+build-llms-txt:
+	@echo "Launching llms.txt build ...";
+	@node ./local/bin/js/llms-txt-build.js;
+
 start:
 	@make setup-build-scripts ## Build and run docs including external content.
 	@make dependencies
@@ -116,7 +122,7 @@ node_modules: package.json yarn.lock
 
 # All the requirements for a full build
 dependencies: clean
-	make hugpython all-examples update_pre_build node_modules build-cdocs
+	make hugpython all-examples update_pre_build node_modules build-cdocs build-llms-txt
 
 integrations_data/extracted/vector:
 	$(call source_repo,vector,https://github.com/vectordotdev/vector.git,master,true,website/)

package.json

@@ -52,6 +52,7 @@
         "bootstrap": "^5.2",
         "cdocs-hugo-integration": "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/cdocs-hugo-integration-v1.4.0.tgz",
         "del": "4.1.1",
+        "docs-llms-txt": "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/docs-llms-txt-v1.0.0.tgz",
         "fancy-log": "^1.3.3",
         "geo-locate": "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/geo-locate-v1.0.2.tgz",
         "hugo-bin": "0.123.0",

yarn.lock

@@ -7364,6 +7364,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"debug@npm:^4.4.0":
+  version: 4.4.1
+  resolution: "debug@npm:4.4.1"
+  dependencies:
+    ms: ^2.1.3
+  peerDependenciesMeta:
+    supports-color:
+      optional: true
+  checksum: a43826a01cda685ee4cec00fb2d3322eaa90ccadbef60d9287debc2a886be3e835d9199c80070ede75a409ee57828c4c6cd80e4b154f2843f0dc95a570dc0729
+  languageName: node
+  linkType: hard
+
 "decamelize@npm:^1.2.0":
   version: 1.2.0
   resolution: "decamelize@npm:1.2.0"
@@ -7588,6 +7600,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"docs-llms-txt@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/docs-llms-txt-v1.0.0.tgz":
+  version: 1.0.0
+  resolution: "docs-llms-txt@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/docs-llms-txt-v1.0.0.tgz"
+  dependencies:
+    js-yaml: ^4.1.0
+    rimraf: ^5.0.5
+    simple-git: ^3.28.0
+    uuid: ^11.1.0
+    zod: ^3.22.4
+  checksum: 92d15672e53188cb68d3f1aa32c93f556c3177a56ee23c8eaeaac04cd9b0a0d273a4edc0fb3fccf8c921f3f6b1b3d26a5b0287fa14426a4e49ddc8f55a4c29dc
+  languageName: node
+  linkType: hard
+
 "docs@workspace:.":
   version: 0.0.0-use.local
   resolution: "docs@workspace:."
@@ -7608,6 +7633,7 @@ __metadata:
     cdocs-hugo-integration: "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/cdocs-hugo-integration-v1.4.0.tgz"
     cross-env: ^5.2.1
     del: 4.1.1
+    docs-llms-txt: "https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/docs-llms-txt-v1.0.0.tgz"
     eslint: ^6.8.0
     eslint-config-airbnb: ^18.2.1
     eslint-config-prettier: ^6.15.0
@@ -15516,6 +15542,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"simple-git@npm:^3.28.0":
+  version: 3.28.0
+  resolution: "simple-git@npm:3.28.0"
+  dependencies:
+    "@kwsites/file-exists": ^1.1.1
+    "@kwsites/promise-deferred": ^1.1.1
+    debug: ^4.4.0
+  checksum: d4f4a56f0eaf916e3c4b801342f6e6c32e73a3eb8c5497f61f71ff21f30462c64537ba76f2cc835d6cd1ffe6198bd52b3e242b0a70444c8cd21faa9f61f75634
+  languageName: node
+  linkType: hard
+
 "simple-swizzle@npm:^0.2.2":
   version: 0.2.2
   resolution: "simple-swizzle@npm:0.2.2"
@@ -17102,7 +17139,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"uuid@npm:^11.0.5":
+"uuid@npm:^11.0.5, uuid@npm:^11.1.0":
   version: 11.1.0
   resolution: "uuid@npm:11.1.0"
   bin: