Skip to content

fix(wikidoc): sanitize generated MDX #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ss-o merged 1 commit into from
Jul 26, 2026
Merged

fix(wikidoc): sanitize generated MDX #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 136 additions & 32 deletions internal/wikidoc/wikidoc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,24 @@
// Package wikidoc transforms gomarkdoc-generated Markdown into MDX-safe content
// and injects it into a marked region of a Docusaurus .mdx page. It is dev/CI
// tooling, not product code.
//
// # Known limitations
//
// Step 4 (bare-character escaping) operates line-by-line and does NOT exempt
// inline code spans (single-backtick runs) on prose lines. Characters inside
// `backtick spans` on prose lines will be escaped along with the surrounding
// text. This is acceptable for the current gomarkdoc output, which uses only
// indented blocks and fenced blocks for code samples, not inline spans
// containing angle brackets or braces.
package wikidoc

import (
"fmt"
"html"
"regexp"
"strings"
)

var (
reHTMLComment = regexp.MustCompile(`(?s)<!--.*?-->`)
reHTMLAnchor = regexp.MustCompile(`</?a\b[^>]*>`)
reAngleLink = regexp.MustCompile(`\]\(<([^>\s]*)>\)`)
reHeading = regexp.MustCompile(`(?m)^(#{1,6})([ \t]+)`)
reAPIHeading = regexp.MustCompile(`(?m)^#{2,} (func|type|var|const) ([A-Za-z_][A-Za-z0-9_]*)\b.*$`)
reHTMLComment = regexp.MustCompile(`(?s)<!--.*?-->`)
reHTMLAnchor = regexp.MustCompile(`</?a\b[^>]*>`)
reAngleLink = regexp.MustCompile(`\]\(<([^>\s]*)>\)`)
reHeading = regexp.MustCompile(`(?m)^(#{1,6})([ \t]+)`)
reAPIHeading = regexp.MustCompile(`(?m)^#{2,} (func|type|var|const) ([A-Za-z_][A-Za-z0-9_]*)\b.*$`)
reAPIMethodHeading = regexp.MustCompile(
`(?m)^#{2,} func \\\(([A-Za-z_][A-Za-z0-9_]*)\\\) ([A-Za-z_][A-Za-z0-9_]*)\b.*$`,
)
)

// Sanitize transforms a gomarkdoc Markdown string into MDX-safe content by
Expand All @@ -35,11 +30,14 @@ var (
// and closing <a> tags are stripped (inner text, if any, is preserved).
// 3. Unwrap angle-bracketed link destinations (](<#Run>) → ](#Run)).
// 4. Normalize leading tabs to four spaces for Markdown indentation.
// 5. Escape bare <, >, {, } on prose lines (not inside fenced or indented code).
// 6. Demote generated headings by three levels so they nest under the wiki
// 5. Convert indented code blocks to fenced blocks that MDX parses as code.
// 6. Escape bare <, >, {, } on prose lines (not inside fenced code).
// 7. Restore gomarkdoc's escaped inline-code spans, removing Markdown
// punctuation escapes and decoding entities only inside paired spans.
// 8. Demote generated headings by three levels so they nest under the wiki
// page's "### Reference" section.
// 7. Rewrite top-level Go declaration fragment links to Docusaurus slugs, so
// gomarkdoc fragment links like #Run resolve as #func-run.
// 9. Rewrite Go declaration fragment links to Docusaurus slugs, so gomarkdoc
// fragments like #Run and #Backquotes.Analyze resolve to their headings.
func Sanitize(md string) string {
// Step 1: remove HTML comments.
out := reHTMLComment.ReplaceAllString(md, "")
Expand All @@ -53,13 +51,21 @@ func Sanitize(md string) string {
// Step 4: use spaces for Markdown indentation and repository whitespace.
out = normalizeIndent(out)

// Step 5: escape bare MDX special chars on prose lines only.
// Step 5: MDX does not recognize Markdown-indented code blocks. Fence them
// so braces in generated examples are not parsed as JSX expressions.
out = fenceIndentedCodeBlocks(out)

// Step 6: escape bare MDX special chars on prose lines only.
out = escapeProse(out)

// Step 6: nest generated headings under the page's Reference section.
// Step 7: restore inline code after prose escaping so code contents remain
// literal, including angle brackets and braces.
out = normalizeEscapedInlineCode(out)

// Step 8: nest generated headings under the page's Reference section.
out = demoteHeadings(out)

// Step 7: target the slugs Docusaurus derives from declaration headings.
// Step 9: target the slugs Docusaurus derives from declaration headings.
out = rewriteAPIFragmentLinks(out)

return out
Expand All @@ -78,6 +84,49 @@ func normalizeIndent(s string) string {
return strings.Join(lines, "\n")
}

// fenceIndentedCodeBlocks converts Markdown-indented code into fenced blocks.
// MDX v3 treats four-space indentation as ordinary text, so raw braces in such
// blocks are otherwise parsed as expressions instead of rendered as code.
func fenceIndentedCodeBlocks(s string) string {
lines := strings.Split(s, "\n")
out := make([]string, 0, len(lines))
inFenced := false

for i := 0; i < len(lines); {
trimmed := strings.TrimSpace(lines[i])
if strings.HasPrefix(trimmed, "```") {
inFenced = !inFenced
out = append(out, lines[i])
i++
continue
}
if inFenced || !strings.HasPrefix(lines[i], " ") {
out = append(out, lines[i])
i++
continue
}

block := make([]string, 0, 1)
for i < len(lines) && strings.HasPrefix(lines[i], " ") {
block = append(block, strings.TrimPrefix(lines[i], " "))
i++
}
for len(block) > 0 && strings.TrimSpace(block[len(block)-1]) == "" {
block = block[:len(block)-1]
}
if len(block) == 0 {
out = append(out, "")
continue
}

out = append(out, "```")
out = append(out, block...)
out = append(out, "```")
}

return strings.Join(out, "\n")
}

// demoteHeadings shifts generated Markdown headings down three levels. Markdown
// has only six heading levels, so deeper input is clamped at h6.
func demoteHeadings(s string) string {
Expand All @@ -88,22 +137,81 @@ func demoteHeadings(s string) string {
})
}

// rewriteAPIFragmentLinks rewrites gomarkdoc's top-level declaration links to
// the slugs Docusaurus derives from headings such as "## func Run".
// rewriteAPIFragmentLinks rewrites gomarkdoc declaration links to the slugs
// Docusaurus derives from headings such as "## func Run" and
// "### func \(Backquotes\) Analyze".
func rewriteAPIFragmentLinks(s string) string {
for _, match := range reAPIHeading.FindAllStringSubmatch(s, -1) {
name := match[2]
slug := strings.ToLower(match[1] + "-" + name)
s = strings.ReplaceAll(s, "](#"+name+")", "](#"+slug+")")
}
for _, match := range reAPIMethodHeading.FindAllStringSubmatch(s, -1) {
receiver := match[1]
method := match[2]
slug := strings.ToLower("func-" + receiver + "-" + method)
s = strings.ReplaceAll(s, "](#"+receiver+"."+method+")", "](#"+slug+")")
}
return s
}

// escapeProse escapes bare <, >, {, } on non-code lines.
// Code lines are:
// - lines inside a fenced block (toggled by lines whose trimmed text starts
// with three backticks), or
// - indented lines (start with a tab or 4+ spaces).
// normalizeEscapedInlineCode restores paired inline-code spans emitted by
// gomarkdoc's plain formatter. Unmatched delimiters are left unchanged.
func normalizeEscapedInlineCode(s string) string {
lines := strings.Split(s, "\n")
for i, line := range lines {
lines[i] = normalizeEscapedInlineCodeLine(line)
}
return strings.Join(lines, "\n")
}

func normalizeEscapedInlineCodeLine(line string) string {
const delimiter = "\\`"

var out strings.Builder
for {
start := strings.Index(line, delimiter)
if start < 0 {
out.WriteString(line)
break
}
afterStart := start + len(delimiter)
endOffset := strings.Index(line[afterStart:], delimiter)
if endOffset < 0 {
out.WriteString(line)
break
}
end := afterStart + endOffset

out.WriteString(line[:start])
out.WriteByte('`')
out.WriteString(html.UnescapeString(unescapeMarkdownPunctuation(line[afterStart:end])))
out.WriteByte('`')
line = line[end+len(delimiter):]
}
return out.String()
}

func unescapeMarkdownPunctuation(s string) string {
var out strings.Builder
for i := 0; i < len(s); i++ {
if s[i] == '\\' && i+1 < len(s) && isASCIIPunctuation(s[i+1]) {
i++
}
out.WriteByte(s[i])
}
return out.String()
}

func isASCIIPunctuation(b byte) bool {
return (b >= '!' && b <= '/') ||
(b >= ':' && b <= '@') ||
(b >= '[' && b <= '`') ||
(b >= '{' && b <= '~')
}

// escapeProse escapes bare <, >, {, } on non-code lines. Fenced code content
// is left verbatim.
func escapeProse(s string) string {
lines := strings.Split(s, "\n")
inFenced := false
Expand All @@ -118,10 +226,6 @@ func escapeProse(s string) string {
if inFenced {
continue
}
// Indented code line: starts with a tab or 4+ spaces.
if strings.HasPrefix(line, "\t") || strings.HasPrefix(line, " ") {
continue
}
// Prose line — escape MDX special characters.
lines[i] = escapeLine(line)
}
Expand Down
91 changes: 77 additions & 14 deletions internal/wikidoc/wikidoc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,56 @@ func TestSanitize_RewriteDocusaurusHeadingID(t *testing.T) {
}
}

// TestSanitize_RewriteDocusaurusMethodHeadingID verifies receiver-method links
// target the slug Docusaurus derives from the corresponding generated heading.
func TestSanitize_RewriteDocusaurusMethodHeadingID(t *testing.T) {
input := "[func \\(r Backquotes\\) Analyze\\(ctx \\*analyzer.Context\\)](#Backquotes.Analyze)\n\n" +
"### func \\(Backquotes\\) Analyze"
got := wikidoc.Sanitize(input)
want := "[func \\(r Backquotes\\) Analyze\\(ctx \\*analyzer.Context\\)](#func-backquotes-analyze)\n\n" +
"###### func \\(Backquotes\\) Analyze"
if got != want {
t.Errorf("Sanitize(%q) = %q; want %q", input, got, want)
}
}

// TestSanitize_EscapedInlineCode verifies gomarkdoc's plain-format escapes are
// removed only inside complete inline-code spans.
func TestSanitize_EscapedInlineCode(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "punctuation and entities",
input: "ID: \\`style/backquotes\\`\n" +
"Suppression: \\`\\# zsh\\-lint disable=style/backquotes \\-\\- \\&lt;reason\\&gt;\\`",
want: "ID: `style/backquotes`\n" +
"Suppression: `# zsh-lint disable=style/backquotes -- <reason>`",
},
{
name: "unmatched delimiter",
input: "Prefix \\`unterminated",
want: "Prefix \\`unterminated",
},
{
name: "non punctuation backslashes",
input: "Path: \\`C:\\temp\\name\\`",
want: "Path: `C:\\temp\\name`",
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := wikidoc.Sanitize(tc.input)
if got != tc.want {
t.Errorf("Sanitize(%q) = %q; want %q", tc.input, got, tc.want)
}
})
}
}

// TestSanitize_DemoteHeadings verifies generated headings nest beneath the
// wiki page's h3 Reference section without producing invalid h7 headings.
func TestSanitize_DemoteHeadings(t *testing.T) {
Expand All @@ -211,12 +261,23 @@ func TestSanitize_DemoteHeadings(t *testing.T) {
}
}

// TestSanitize_NormalizeIndent verifies generated Markdown code indentation
// uses spaces so wiki whitespace checks accept newly generated lines.
func TestSanitize_NormalizeIndent(t *testing.T) {
input := "\timport \"example.test/pkg\"\n\t\tdeep"
// TestSanitize_FenceIndentedCode verifies gomarkdoc's tab-indented code is
// converted to fenced code that MDX both parses and renders as code.
func TestSanitize_FenceIndentedCode(t *testing.T) {
input := "\tfunc render() { print ok; }\n\t print done\n\t\n\nProse"
got := wikidoc.Sanitize(input)
want := " import \"example.test/pkg\"\n deep"
want := "```\nfunc render() { print ok; }\n print done\n```\n\nProse"
if got != want {
t.Errorf("Sanitize(%q) = %q; want %q", input, got, want)
}
}

// TestSanitize_WhitespaceOnlyIndent verifies a tab-only separator is cleaned
// without producing an empty fenced block.
func TestSanitize_WhitespaceOnlyIndent(t *testing.T) {
input := "\t\n\nProse"
got := wikidoc.Sanitize(input)
want := "\n\nProse"
if got != want {
t.Errorf("Sanitize(%q) = %q; want %q", input, got, want)
}
Expand All @@ -232,14 +293,14 @@ func TestSanitize_EscapeProseChars(t *testing.T) {
}
}

// TestSanitize_IndentedCodeNotEscaped verifies tab-indented code lines are
// normalized to spaces without escaping their code contents.
func TestSanitize_IndentedCodeNotEscaped(t *testing.T) {
// TestSanitize_IndentedCodePreservesContent verifies fenced conversion leaves
// code contents untouched.
func TestSanitize_IndentedCodePreservesContent(t *testing.T) {
input := "\tfunc F(a <T>) {}"
got := wikidoc.Sanitize(input)
want := " func F(a <T>) {}"
want := "```\nfunc F(a <T>) {}\n```"
if got != want {
t.Errorf("Sanitize should normalize but not escape tab-indented code; got: %q, want: %q", got, want)
t.Errorf("Sanitize should fence tab-indented code without escaping it; got: %q, want: %q", got, want)
}
}

Expand All @@ -252,11 +313,13 @@ func TestSanitize_FencedCodeNotEscaped(t *testing.T) {
}
}

// TestSanitize_FourSpaceIndentNotEscaped verifies 4-space-indented code lines are left verbatim.
func TestSanitize_FourSpaceIndentNotEscaped(t *testing.T) {
// TestSanitize_FourSpaceIndentFenced verifies existing Markdown-indented code
// is converted to MDX-compatible fenced code.
func TestSanitize_FourSpaceIndentFenced(t *testing.T) {
input := " func F(a <T>) {}"
got := wikidoc.Sanitize(input)
if got != input {
t.Errorf("Sanitize should not escape 4-space-indented code; got: %q, want: %q", got, input)
want := "```\nfunc F(a <T>) {}\n```"
if got != want {
t.Errorf("Sanitize should fence 4-space-indented code; got: %q, want: %q", got, want)
}
}