Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 35b77d3

Browse files
cometkimcometkim
authoredMar 11, 2025··
replace gray-matter and deprecated remark plugins (#984)
* replace gray-matter and deprecated remark plugins * fix * fix scripts * fix compability issues * fix headers * use interface file
1 parent f233d14 commit 35b77d3

File tree

10 files changed

+18706
-11756
lines changed

10 files changed

+18706
-11756
lines changed
 

‎package-lock.json

Lines changed: 18520 additions & 11598 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
"@babel/traverse": "^7.24.7",
2323
"@docsearch/react": "^3.5.2",
2424
"@headlessui/react": "^1.2.0",
25-
"@mdx-js/loader": "^2.3.0",
25+
"@mdx-js/loader": "^3.1.0",
2626
"@rescript/core": "^1.4.0",
2727
"@rescript/react": "^0.12.0-alpha.3",
2828
"@rescript/tools": "^0.5.0",
@@ -33,28 +33,29 @@
3333
"fuse.js": "^6.4.3",
3434
"gentype": "^3.44.0",
3535
"glob": "^7.1.4",
36-
"gray-matter": "^4.0.3",
3736
"highlight.js": "^11.9.0",
3837
"highlightjs-rescript": "^0.2.2",
3938
"lz-string": "^1.4.4",
4039
"next": "^14.2.21",
41-
"next-mdx-remote": "^4.4.1",
40+
"next-mdx-remote": "^5.0.0",
4241
"prettier": "^1.18.2",
4342
"react": "^18.2.0",
4443
"react-dom": "^18.2.0",
45-
"react-markdown": "^9.0.1",
46-
"rehype-slug": "^5.1.0",
44+
"react-markdown": "^10.1.0",
45+
"rehype-slug": "^6.0.0",
46+
"rehype-stringify": "^10.0.1",
4747
"remark-comment": "^1.0.0",
48-
"remark-frontmatter": "^4.0.1",
49-
"remark-gfm": "^3.0.1",
50-
"remark-mdx-frontmatter": "^3.0.0",
51-
"remark-parse": "^10.0.2",
52-
"remark-slug": "^5.1.2",
53-
"remark-stringify": "^7.0.3",
48+
"remark-frontmatter": "^5.0.0",
49+
"remark-gfm": "^4.0.1",
50+
"remark-mdx-frontmatter": "^5.0.0",
51+
"remark-parse": "^11.0.0",
52+
"remark-rehype": "^11.1.1",
53+
"remark-stringify": "^11.0.0",
5454
"request": "^2.88.0",
5555
"rescript": "^11.1.0",
5656
"stringify-object": "^3.3.0",
57-
"unified": "^8.4.0"
57+
"unified": "^11.0.5",
58+
"vfile-matter": "^5.0.0"
5859
},
5960
"scripts": {
6061
"dev": "next",

‎scripts/extract-indices.mjs

Lines changed: 3 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -5,77 +5,19 @@
55
* - Module names (h1)
66
* - function names (```res sig)
77
*/
8-
import unified from "unified";
9-
import markdown from "remark-parse";
10-
import stringify from "remark-stringify";
11-
import slug from "remark-slug";
128
import glob from "glob";
139
import path from "path";
1410
import fs from "fs";
1511
import { URL } from 'url';
1612

13+
import { defaultProcessor } from "./markdown.js";
14+
1715
const pathname = new URL('.', import.meta.url).pathname;
1816
const __dirname = process.platform !== 'win32' ? pathname : pathname.substring(1)
1917

20-
const headers = options => (tree, file) => {
21-
const headers = [];
22-
let mainHeader;
23-
tree.children.forEach(child => {
24-
if (child.type === "heading" && child.depth === 1) {
25-
if (child.children.length > 0) {
26-
mainHeader = child.children.map(element => element.value).join("");
27-
}
28-
}
29-
if (child.type === "heading" && child.depth === 2) {
30-
if (child.children.length > 0) {
31-
const id = child.data.id || "";
32-
const name = child.children.map(element => element.value).join("");
33-
headers.push({ name, href: id });
34-
}
35-
}
36-
});
37-
38-
file.data = Object.assign({}, file.data, { headers, mainHeader });
39-
};
40-
41-
const codeblocks = options => (tree, file) => {
42-
const { children } = tree;
43-
const codeblocks = {};
44-
45-
const formatter = value => {
46-
// Strip newlines and weird spacing
47-
return value
48-
.replace(/\n/g, " ")
49-
.replace(/\s+/g, " ")
50-
.replace(/\(\s+/g, "(")
51-
.replace(/\s+\)/g, ")");
52-
};
53-
54-
children.forEach(child => {
55-
if (child.type === "code" && child.value) {
56-
const { meta, lang } = child;
57-
if (meta === "sig" && lang === "re") {
58-
if (codeblocks[lang] == null) {
59-
codeblocks[lang] = [];
60-
}
61-
codeblocks[lang].push(formatter(child.value));
62-
}
63-
}
64-
});
65-
66-
file.data = Object.assign({}, file.data, { codeblocks });
67-
};
68-
69-
const processor = unified()
70-
.use(markdown, { gfm: true })
71-
.use(slug)
72-
.use(stringify)
73-
.use(headers)
74-
.use(codeblocks);
75-
7618
const processFile = filepath => {
7719
const content = fs.readFileSync(filepath, "utf8");
78-
const result = processor.processSync(content);
20+
const result = defaultProcessor.processSync(content);
7921

8022
const pagesPath = path.resolve("./pages");
8123
const relFilepath = path.relative(pagesPath, filepath);

‎scripts/extract-syntax.mjs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,29 @@
1-
import matter from "gray-matter";
21
import glob from "glob";
32
import path from "path";
43
import fs from "fs";
54
import { URL } from 'url';
65

6+
import { defaultProcessor } from "./markdown.js";
7+
78
const pathname = new URL('.', import.meta.url).pathname;
89
const __dirname = process.platform !== 'win32' ? pathname : pathname.substring(1)
910

1011
const processFile = filepath => {
11-
const raw = fs.readFileSync(filepath, "utf8");
12-
const { data } = matter(raw);
12+
const content = fs.readFileSync(filepath, "utf8");
13+
const { data: { matter } } = defaultProcessor.processSync(content);
1314

1415
const syntaxPath = path.resolve("./misc_docs/syntax");
1516
const relFilePath = path.relative(syntaxPath, filepath);
1617
const parsedPath = path.parse(relFilePath);
1718

18-
if (data.id && data.keywords && data.name && data.summary && data.category) {
19+
if (matter.id && matter.keywords && matter.name && matter.summary && matter.category) {
1920
return {
2021
file: parsedPath.name,
21-
id: data.id,
22-
keywords: data.keywords,
23-
name: data.name,
24-
summary: data.summary,
25-
category: data.category
22+
id: matter.id,
23+
keywords: matter.keywords,
24+
name: matter.name,
25+
summary: matter.summary,
26+
category: matter.category
2627
}
2728
}
2829

‎scripts/extract-tocs.mjs

Lines changed: 6 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,14 @@
22
* This script is used for generating the table of contents for prose
33
* text documents
44
*/
5-
import unified from "unified";
6-
import markdown from "remark-parse";
7-
import matter from "gray-matter";
8-
import stringify from "remark-stringify";
9-
import slug from "remark-slug";
5+
import { unified } from "unified";
106
import glob from "glob";
117
import path from "path";
128
import fs from "fs";
139
import { URL } from "url";
1410

11+
import { defaultProcessor } from "./markdown.js";
12+
1513
const pathname = new URL(".", import.meta.url).pathname;
1614
const __dirname =
1715
process.platform !== "win32" ? pathname : pathname.substring(1);
@@ -56,39 +54,11 @@ const collapseHeaderChildren = (children) => {
5654
}, "");
5755
};
5856

59-
const headers = (options) => (tree, file) => {
60-
const headers = [];
61-
let mainHeader;
62-
tree.children.forEach((child) => {
63-
if (child.type === "heading" && child.depth === 1) {
64-
if (child.children.length > 0) {
65-
mainHeader = collapseHeaderChildren(child.children);
66-
}
67-
}
68-
if (child.type === "heading" && child.depth === 2) {
69-
if (child.children.length > 0) {
70-
// Take the id generated from remark-slug
71-
const headerValue = collapseHeaderChildren(child.children);
72-
const id = child.data.id || "";
73-
headers.push({ name: headerValue, href: id });
74-
}
75-
}
76-
});
77-
78-
file.data = Object.assign({}, file.data, { headers, mainHeader });
79-
};
80-
81-
const processor = unified()
82-
.use(markdown, { gfm: true })
83-
.use(slug)
84-
.use(stringify)
85-
.use(headers);
86-
8757
// sidebarJson: { [category: string]: array<plain_filename_without_ext> }
8858
const processFile = (filepath, sidebarJson = {}) => {
89-
const raw = fs.readFileSync(filepath, "utf8");
90-
const { content, data } = matter(raw);
91-
const result = processor.processSync(content);
59+
const content = fs.readFileSync(filepath, "utf8");
60+
const result = defaultProcessor.processSync(content);
61+
const data = result.data.matter;
9262

9363
const pagesPath = path.resolve("./pages");
9464
const relFilepath = path.relative(pagesPath, filepath);

‎scripts/markdown.js

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { unified } from "unified";
2+
import remarkParse from "remark-parse";
3+
import remarkGfm from "remark-gfm";
4+
import remarkFrontmatter from "remark-frontmatter";
5+
import remarkRehype from "remark-rehype";
6+
import rehypeSlug from "rehype-slug";
7+
import rehypeStringify from "rehype-stringify";
8+
import { matter } from "vfile-matter";
9+
10+
const remarkVfileMatter = options => (tree, file) => {
11+
matter(file);
12+
};
13+
14+
const remarkCodeblocks = options => (tree, file) => {
15+
const { children } = tree;
16+
const codeblocks = {};
17+
18+
const formatter = value => {
19+
// Strip newlines and weird spacing
20+
return value
21+
.replace(/\n/g, " ")
22+
.replace(/\s+/g, " ")
23+
.replace(/\(\s+/g, "(")
24+
.replace(/\s+\)/g, ")");
25+
};
26+
27+
children.forEach(child => {
28+
if (child.type === "code" && child.value) {
29+
const { meta, lang } = child;
30+
if (meta === "sig" && lang === "re") {
31+
if (codeblocks[lang] == null) {
32+
codeblocks[lang] = [];
33+
}
34+
codeblocks[lang].push(formatter(child.value));
35+
}
36+
}
37+
});
38+
39+
Object.assign(file.data, { codeblocks });
40+
};
41+
42+
const rehypeHeaders = options => (tree, file) => {
43+
const headers = [];
44+
let mainHeader;
45+
tree.children.forEach(child => {
46+
if (child.tagName === "h1") {
47+
if (child.children.length > 0) {
48+
mainHeader = child.children.map(element => element.value).join("");
49+
}
50+
}
51+
if (child.tagName === "h2") {
52+
if (child.children.length > 0) {
53+
const id = child.properties.id || "";
54+
const name = child.children.map(element => element.value).join("");
55+
headers.push({ name, href: id });
56+
}
57+
}
58+
});
59+
60+
Object.assign(file.data, { headers, mainHeader });
61+
};
62+
63+
export const defaultProcessor = unified()
64+
.use(remarkParse)
65+
.use(remarkGfm)
66+
.use(remarkFrontmatter, [{ type: 'yaml', marker: '-' }])
67+
.use(remarkVfileMatter)
68+
.use(remarkCodeblocks)
69+
.use(remarkRehype)
70+
.use(rehypeSlug)
71+
.use(rehypeHeaders)
72+
.use(rehypeStringify);

‎scripts/test-hrefs.mjs

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,15 @@
66
* the website.
77
*/
88

9-
import unified from "unified";
10-
import markdown from "remark-parse";
11-
import stringify from "remark-stringify";
12-
import {config} from "dotenv"
9+
import { config } from "dotenv"
1310
import glob from "glob";
1411
import path from "path";
1512
import fs from "fs";
1613
import urlModule from "url";
1714
import { URL } from 'url';
18-
import {getAllPosts, blogPathToSlug} from '../src/common/BlogApi.mjs'
15+
import { getAllPosts, blogPathToSlug } from '../src/common/BlogApi.mjs'
16+
17+
import { defaultProcessor } from "./markdown.js";
1918

2019
config()
2120

@@ -30,7 +29,7 @@ const mapBlogFilePath = path => {
3029

3130
if (match) {
3231
let relPath = match[1];
33-
let data = getAllPosts().find(({path}) => path === relPath);
32+
let data = getAllPosts().find(({ path }) => path === relPath);
3433
if (data != null) {
3534
return `./pages/blog/${blogPathToSlug(data.path)}`;
3635
}
@@ -54,10 +53,10 @@ const createPageIndex = files => {
5453
// We need to consider all the different file formats used in pages
5554
// Calculate the website url by stripping .re, .bs.js, .md(x), etc.
5655
let url;
57-
if(path.startsWith("./_blogposts")) {
56+
if (path.startsWith("./_blogposts")) {
5857
url = mapBlogFilePath(path)
5958
}
60-
else if(path.startsWith("./public/static")) {
59+
else if (path.startsWith("./public/static")) {
6160
url = mapStaticFilePath(path);
6261
}
6362
else {
@@ -95,10 +94,7 @@ const hrefs = options => (tree, file) => {
9594
file.data = Object.assign({}, file.data, { links });
9695
};
9796

98-
const processor = unified()
99-
.use(markdown, { gfm: true })
100-
.use(stringify)
101-
.use(hrefs);
97+
const processor = defaultProcessor.use(hrefs);
10298

10399
const processFile = filepath => {
104100
const content = fs.readFileSync(filepath, "utf8");
@@ -133,7 +129,7 @@ const apiIndexModules = [...createApiIndexModules(latestVersion), ...createApiIn
133129

134130
const testFile = (pageMap, test) => {
135131
const filepath = test.filepath;
136-
132+
137133
// Used for storing failed / ok hrefs
138134
const results = [];
139135

@@ -142,7 +138,7 @@ const testFile = (pageMap, test) => {
142138
if (link.url.includes("/manual/latest/")) {
143139
link.url = link.url.replace("/latest/", `/${latestVersion}/`);
144140
}
145-
141+
146142
if (link.url.includes("/manual/next/")) {
147143
link.url = link.url.replace("/next/", `/${nextVersion}/`);
148144
}
@@ -182,7 +178,7 @@ const testFile = (pageMap, test) => {
182178
resolved = path.join("/", path.dirname(filepath), parsed.pathname);
183179
}
184180
else {
185-
if(parsed.pathname.startsWith("/static")) {
181+
if (parsed.pathname.startsWith("/static")) {
186182
console.log("Static");
187183
resolved = path.join(parsed.pathname);
188184
}
@@ -192,10 +188,10 @@ const testFile = (pageMap, test) => {
192188
}
193189
}
194190

195-
191+
196192

197193
if (
198-
resolved.startsWith(`/pages/docs/manual/${latestVersion}/api`) ||
194+
resolved.startsWith(`/pages/docs/manual/${latestVersion}/api`) ||
199195
resolved.startsWith(`/pages/docs/manual/${nextVersion}/api`)
200196
) {
201197
const pathToModule = resolved.replace("/pages/docs/manual/", "");
@@ -279,9 +275,9 @@ const main = () => {
279275
const allFiles = pageMapFiles.concat(staticFiles);
280276

281277
const pageMap = createPageIndex(allFiles);
282-
278+
283279
const processedFiles = files.map(processFile);
284-
280+
285281
const allTested = processedFiles.map(file => testFile(pageMap, file));
286282

287283
const failed = allTested.reduce((acc, test) => {

‎src/common/BlogApi.res

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,6 @@
2323
See `pages/blog.re` for more context on why we need this API.
2424
*/
2525

26-
module GrayMatter = {
27-
type output = {
28-
data: JSON.t,
29-
content: string,
30-
}
31-
32-
@module("gray-matter") external matter: string => output = "default"
33-
}
34-
3526
type post = {
3627
path: string,
3728
archived: bool,
@@ -51,9 +42,9 @@ let getAllPosts = () => {
5142
let archivedPostsDirectory = Node.Path.join2(postsDirectory, "archive")
5243

5344
let nonArchivedPosts = mdxFiles(postsDirectory)->Array.map(path => {
54-
let {GrayMatter.data: data} =
55-
Node.Path.join2(postsDirectory, path)->Node.Fs.readFileSync->GrayMatter.matter
56-
switch BlogFrontmatter.decode(data) {
45+
let {frontmatter} =
46+
Node.Path.join2(postsDirectory, path)->Node.Fs.readFileSync->MarkdownParser.parseSync
47+
switch BlogFrontmatter.decode(frontmatter) {
5748
| Error(msg) => Exn.raiseError(msg)
5849
| Ok(d) => {
5950
path,
@@ -64,9 +55,9 @@ let getAllPosts = () => {
6455
})
6556

6657
let archivedPosts = mdxFiles(archivedPostsDirectory)->Array.map(path => {
67-
let {GrayMatter.data: data} =
68-
Node.Path.join2(archivedPostsDirectory, path)->Node.Fs.readFileSync->GrayMatter.matter
69-
switch BlogFrontmatter.decode(data) {
58+
let {frontmatter} =
59+
Node.Path.join2(archivedPostsDirectory, path)->Node.Fs.readFileSync->MarkdownParser.parseSync
60+
switch BlogFrontmatter.decode(frontmatter) {
7061
| Error(msg) => Exn.raiseError(msg)
7162
| Ok(d) => {
7263
path: Node.Path.join2("archive", path),
@@ -85,9 +76,9 @@ let getLivePosts = () => {
8576
let postsDirectory = Node.Path.join2(Node.Process.cwd(), "_blogposts")
8677

8778
let livePosts = mdxFiles(postsDirectory)->Array.map(path => {
88-
let {GrayMatter.data: data} =
89-
Node.Path.join2(postsDirectory, path)->Node.Fs.readFileSync->GrayMatter.matter
90-
switch BlogFrontmatter.decode(data) {
79+
let {frontmatter} =
80+
Node.Path.join2(postsDirectory, path)->Node.Fs.readFileSync->MarkdownParser.parseSync
81+
switch BlogFrontmatter.decode(frontmatter) {
9182
| Error(msg) => Exn.raiseError(msg)
9283
| Ok(d) => {
9384
path,
@@ -107,9 +98,9 @@ let getArchivedPosts = () => {
10798
let archivedPostsDirectory = Node.Path.join2(postsDirectory, "archive")
10899

109100
let archivedPosts = mdxFiles(archivedPostsDirectory)->Array.map(path => {
110-
let {GrayMatter.data: data} =
111-
Node.Path.join2(archivedPostsDirectory, path)->Node.Fs.readFileSync->GrayMatter.matter
112-
switch BlogFrontmatter.decode(data) {
101+
let {frontmatter} =
102+
Node.Path.join2(archivedPostsDirectory, path)->Node.Fs.readFileSync->MarkdownParser.parseSync
103+
switch BlogFrontmatter.decode(frontmatter) {
113104
| Error(msg) => Exn.raiseError(msg)
114105
| Ok(d) => {
115106
path: Node.Path.join2("archive", path),

‎src/common/MarkdownParser.res

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
type t
2+
type plugin
3+
type vfile<'a> = {..} as 'a
4+
5+
external makePlugin: 'a => plugin = "%identity"
6+
7+
@module("unified") external make: unit => t = "unified"
8+
9+
@module("remark-parse") external remarkParse: plugin = "default"
10+
@module("remark-gfm") external remarkGfm: plugin = "default"
11+
@module("remark-comment") external remarkComment: plugin = "default"
12+
@module("remark-frontmatter") external remarkFrontmatter: plugin = "default"
13+
@module("remark-stringify") external remarkStringify: plugin = "default"
14+
15+
@send external use: (t, plugin) => t = "use"
16+
@send external useOptions: (t, plugin, array<{..}>) => t = "use"
17+
18+
@send external processSync: (t, string) => vfile<'a> = "processSync"
19+
@send external toString: vfile<'a> => string = "toString"
20+
21+
@module("vfile-matter") external vfileMatter: vfile<'a> => unit = "matter"
22+
23+
type result = {
24+
frontmatter: JSON.t,
25+
content: string,
26+
}
27+
28+
let vfileMatterPlugin = makePlugin(_options => (_tree, vfile) => vfileMatter(vfile))
29+
30+
let parser =
31+
make()
32+
->use(remarkParse)
33+
->use(remarkStringify)
34+
->use(remarkGfm)
35+
->use(remarkComment)
36+
->useOptions(remarkFrontmatter, [{"type": "yaml", "marker": "-"}])
37+
->use(vfileMatterPlugin)
38+
39+
let parseSync = content => {
40+
let vfile = parser->processSync(content)
41+
let frontmatter = (vfile["data"]["matter"] :> option<JSON.t>)
42+
let frontmatter = frontmatter->Option.getOr(JSON.Object(Dict.make()))
43+
let content = vfile->toString
44+
{frontmatter, content}
45+
}

‎src/common/MarkdownParser.resi

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
type t
2+
type plugin
3+
type vfile<'a> = {..} as 'a
4+
5+
type result = {
6+
frontmatter: JSON.t,
7+
content: string,
8+
}
9+
10+
let parseSync: string => result

0 commit comments

Comments
 (0)
Please sign in to comment.