diff --git a/CLI_REFERENCE.adoc b/CLI_REFERENCE.adoc index 737c6eec..cb6643cf 100644 --- a/CLI_REFERENCE.adoc +++ b/CLI_REFERENCE.adoc @@ -490,6 +490,7 @@ display help for command * `migrate-rpcn-metadata` - One-time migration of inline connector == Metadata blocks into regenerated partials. Dry run unless --write is given. * `property-docs` - Generate JSON and consolidated AsciiDoc partials for Redpanda configuration properties. Defaults to branch "dev" if neither --tag nor --branch is specified. * `rpk-docs` - Generate rpk CLI documentation from source. Builds rpk and parses source for platform detection. +* `rpk-plugin-stubs` - Reconcile single-source stub pages and nav against the docs repo's rpk plugin partials. Run from the consumer repo root. * `rpk-overrides` - Validate rpk-overrides.json against schema and check for common issues * `helm-spec` - Generate AsciiDoc documentation for Helm charts. Requires either --tag or --branch for GitHub URLs. * `cloud-regions` - Generate Markdown table of cloud regions and tiers from GitHub YAML file @@ -693,6 +694,18 @@ Path to local rpk source (src/go/rpk directory) `--from-json `:: Regenerate docs from an existing versioned JSON file (skips building) +`--plugin `:: +Refresh a single rpk plugin's docs (ai, connect, k8s, check). Requires --from-json. Installs the plugin, splices its fresh subtree into the snapshot, and re-renders. + +`--plugin-version `:: +Plugin version to install and record (for example, 4.102.0). Defaults to the latest published version. + +`--plugin-pin `:: +Pin a plugin version for the installs during full generation (repeatable, for example --plugin-pin k8s=26.3.1-beta.1). Required for pre-GA plugins with no promoted latest version. (default: {}) + +`--rpk-bin `:: +Path to an existing rpk binary for the plugin refresh (skips download/build) + `--overrides `:: Path to overrides JSON file (default: "docs-data/rpk-overrides.json") diff --git a/__tests__/tools/rpk-docs/generate-rpk-docs.test.js b/__tests__/tools/rpk-docs/generate-rpk-docs.test.js index 2e50c085..dbc2cab0 100644 --- a/__tests__/tools/rpk-docs/generate-rpk-docs.test.js +++ b/__tests__/tools/rpk-docs/generate-rpk-docs.test.js @@ -18,7 +18,8 @@ const { shouldUsePartialDir, updateNavFile, getOutputPath, - findTopLevelWithSubcommands + findTopLevelWithSubcommands, + generateRpkDocs } = require('../../../tools/rpk-docs/generate-rpk-docs.js') describe('rpk Docs Generation', () => { @@ -238,6 +239,22 @@ describe('rpk Docs Generation', () => { expect(formatDescription(undefined)).toBe('') }) + test('strips a dangling e.g. with nothing after it', () => { + // Upstream rpai help ends some flag descriptions mid-example; the + // e.g. transform would otherwise render "for example,." + const input = 'fields optionally suffixed with " desc" (default ascending), e.g.' + expect(formatDescription(input)).toBe('fields optionally suffixed with " desc" (default ascending)') + }) + + test('keeps e.g. transform when an example follows', () => { + expect(formatDescription('durations, e.g. 30s or 1.5m')).toBe('durations, for example, 30s or 1.5m') + }) + + test('removes stray space before a closing parenthesis', () => { + const input = 'topic:partition_id (repeatable; e.g. -t foo:0,1,2 )' + expect(formatDescription(input)).toBe('topic:partition_id (repeatable; for example, `-t` foo:0,1,2)') + }) + test('converts markdown-style dash lists to AsciiDoc asterisk lists', () => { const input = 'States:\n - Active: The item is active.\n - Inactive: The item is inactive.' const result = formatDescription(input) @@ -786,6 +803,7 @@ Specify a time range.` const nav = [ '* xref:get-started:index.adoc[]', '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk.adoc[]', '*** xref:reference:rpk/rpk-commands.adoc[]', '*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]', '*** xref:reference:rpk/rpk-topic/rpk-topic.adoc[]', @@ -803,9 +821,12 @@ Specify a time range.` expect(written).toContain('** xref:reference:rpk/index.adoc[rpk Commands]') expect(written).toContain('*** xref:reference:rpk/rpk-commands.adoc[]') expect(written).toContain('*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]') - // The root rpk command is represented by the hand-written index.adoc - // landing page; it must not get its own generated rpk.adoc nav entry. - expect(written).not.toContain('xref:reference:rpk/rpk.adoc[]') + // The generated root rpk.adoc page is listed ahead of the hand-written + // entries; without it Antora reports the page as unlisted. + const lines = written.split('\n') + const rootIdx = lines.indexOf('*** xref:reference:rpk/rpk.adoc[]') + expect(rootIdx).toBeGreaterThan(-1) + expect(rootIdx).toBeLessThan(lines.indexOf('*** xref:reference:rpk/rpk-commands.adoc[]')) }) test('generates entries at correct nesting depths', () => { @@ -947,5 +968,380 @@ Specify a time range.` expect(result.navUpdated).toBe(true) expect(result.navEntriesGenerated).toBe(3) }) + + test('preserves nav entries for protected plugins absent from the tree', () => { + const nav = [ + '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk-connect/rpk-connect.adoc[]', + '**** xref:reference:rpk/rpk-connect/rpk-connect-run.adoc[]', + '*** xref:reference:rpk/rpk-topic/rpk-topic.adoc[]', + '** xref:reference:glossary.adoc[]', + ].join('\n') + + const navPath = makeNav(nav) + // connect is entirely absent from this run's tree + const tree = makeTree(['rpk topic', 'rpk topic create']) + const commands = flattenCommands(tree) + const topLevel = findTopLevelWithSubcommands(tree) + updateNavFile(navPath, commands, {}, topLevel, ['connect']) + + const written = fs.readFileSync(navPath, 'utf8') + expect(written).toContain('*** xref:reference:rpk/rpk-connect/rpk-connect.adoc[]') + expect(written).toContain('**** xref:reference:rpk/rpk-connect/rpk-connect-run.adoc[]') + expect(written).toContain('*** xref:reference:rpk/rpk-topic/rpk-topic.adoc[]') + }) + + test('does not duplicate preserved entries that also exist in regenerated output', () => { + const nav = [ + '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk-connect/rpk-connect.adoc[]', + '**** xref:reference:rpk/rpk-connect/rpk-connect-run.adoc[]', + '** xref:reference:glossary.adoc[]', + ].join('\n') + + const navPath = makeNav(nav) + // connect IS in this run's tree, but protection wins: no connect + // entries are generated, and the previous nav block is preserved in + // place exactly once — never regenerated AND preserved. + const tree = makeTree(['rpk connect', 'rpk connect run', 'rpk topic', 'rpk topic create']) + const commands = flattenCommands(tree) + const topLevel = findTopLevelWithSubcommands(tree) + updateNavFile(navPath, commands, {}, topLevel, ['connect']) + + const written = fs.readFileSync(navPath, 'utf8') + const count = (needle) => written.split('\n').filter(l => l.includes(needle)).length + expect(count('rpk-connect/rpk-connect.adoc')).toBe(1) + expect(count('rpk-connect/rpk-connect-run.adoc')).toBe(1) + expect(count('rpk-topic/rpk-topic.adoc')).toBe(1) + }) + + test('protected-plugin preservation is idempotent across runs', () => { + const nav = [ + '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk-connect/rpk-connect.adoc[]', + '** xref:reference:glossary.adoc[]', + ].join('\n') + + const navPath = makeNav(nav) + const tree = makeTree(['rpk topic', 'rpk topic create']) + const commands = flattenCommands(tree) + const topLevel = findTopLevelWithSubcommands(tree) + + updateNavFile(navPath, commands, {}, topLevel, ['connect']) + const firstRun = fs.readFileSync(navPath, 'utf8') + updateNavFile(navPath, commands, {}, topLevel, ['connect']) + const secondRun = fs.readFileSync(navPath, 'utf8') + + expect(firstRun).toBe(secondRun) + expect(firstRun.split('\n').filter(l => l.includes('rpk-connect/rpk-connect.adoc')).length).toBe(1) + }) + + test('splices preserved plugin entries in place under the plugin parent, not at the end', () => { + // Realistic pre-GA shim scenario: the previous nav has the full k8s + // block; this run's tree has only the k8s shim. + const nav = [ + '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk.adoc[]', + '*** xref:reference:rpk/rpk-commands.adoc[]', + '*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]', + '*** xref:reference:rpk/rpk-iotune.adoc[]', + '*** xref:reference:rpk/rpk-k8s/rpk-k8s.adoc[]', + '**** xref:reference:rpk/rpk-k8s/rpk-k8s-install.adoc[]', + '**** xref:reference:rpk/rpk-k8s/rpk-k8s-multicluster.adoc[]', + '***** xref:reference:rpk/rpk-k8s/rpk-k8s-multicluster-bootstrap.adoc[]', + '**** xref:reference:rpk/rpk-k8s/rpk-k8s-uninstall.adoc[]', + '*** xref:reference:rpk/rpk-version.adoc[]', + '** xref:reference:glossary.adoc[]', + ].join('\n') + + const navPath = makeNav(nav) + const tree = makeTree([ + 'rpk iotune', + 'rpk k8s', 'rpk k8s install', 'rpk k8s uninstall', 'rpk k8s upgrade', + 'rpk version' + ]) + const commands = flattenCommands(tree) + const topLevel = findTopLevelWithSubcommands(tree) + updateNavFile(navPath, commands, {}, topLevel, ['k8s']) + + const written = fs.readFileSync(navPath, 'utf8') + const outLines = written.split('\n') + const idx = (needle) => outLines.findIndex(l => l.includes(needle)) + + // The whole k8s block stays together, in its original position: + // directly after rpk-iotune, parent first, children nested under it, + // and rpk-version still follows the block — nothing lands at the end. + const parentIdx = idx('rpk-k8s/rpk-k8s.adoc') + expect(parentIdx).toBe(idx('rpk-iotune.adoc') + 1) + expect(idx('rpk-k8s-install.adoc')).toBe(parentIdx + 1) + expect(idx('rpk-k8s-multicluster.adoc')).toBe(parentIdx + 2) + expect(idx('rpk-k8s-multicluster-bootstrap.adoc')).toBe(parentIdx + 3) + expect(idx('rpk-k8s-uninstall.adoc')).toBe(parentIdx + 4) + expect(idx('rpk-version.adoc')).toBe(parentIdx + 5) + + // In this scenario the rebuilt nav is byte-identical to the input. + expect(written).toBe(nav) + }) + + test('keeps a fully absent plugin block in its original position', () => { + const nav = [ + '** xref:reference:rpk/index.adoc[rpk Commands]', + '*** xref:reference:rpk/rpk.adoc[]', + '*** xref:reference:rpk/rpk-commands.adoc[]', + '*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]', + '*** xref:reference:rpk/rpk-connect/rpk-connect.adoc[]', + '**** xref:reference:rpk/rpk-connect/rpk-connect-run.adoc[]', + '*** xref:reference:rpk/rpk-topic/rpk-topic.adoc[]', + '**** xref:reference:rpk/rpk-topic/rpk-topic-create.adoc[]', + '** xref:reference:glossary.adoc[]', + ].join('\n') + + const navPath = makeNav(nav) + // connect is entirely absent from this run's tree + const tree = makeTree(['rpk topic', 'rpk topic create']) + const commands = flattenCommands(tree) + const topLevel = findTopLevelWithSubcommands(tree) + updateNavFile(navPath, commands, {}, topLevel, ['connect']) + + const written = fs.readFileSync(navPath, 'utf8') + expect(written).toBe(nav) + }) + }) + + describe('stale file cleanup with protected plugins', () => { + let tmpDir + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rpk-cleanup-test-')) + }) + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }) + }) + + const writePage = (...segments) => { + const filePath = path.join(tmpDir, ...segments) + fs.mkdirSync(path.dirname(filePath), { recursive: true }) + fs.writeFileSync(filePath, '= Existing page\n', 'utf8') + return filePath + } + + const cmd = (name, extra = {}) => ({ + name, + description: `${name} command.`, + usage: `rpk ${name} [flags]`, + flags: [], + ...extra + }) + + test('pages under a protected plugin dir survive the sweep while a stale non-plugin page is deleted', async () => { + // Pre-existing pages from an earlier run where the connect plugin + // installed successfully and rpk topic had a "describe" subcommand. + const connectRoot = writePage('rpk-connect', 'rpk-connect.adoc') + const connectRun = writePage('rpk-connect', 'rpk-connect-run.adoc') + const staleTopic = writePage('rpk-topic', 'rpk-topic-describe.adoc') + + // This run's tree: connect is entirely absent (failed plugin install), + // and rpk topic no longer has "describe". + const tree = { + name: 'rpk', + description: 'Root command', + commands: [ + cmd('topic', { commands: [cmd('create')] }) + ], + global_flags: [] + } + + const result = await generateRpkDocs({ + tree, + outputDir: tmpDir, + rpkVersion: 'test', + pluginVersions: {} + }) + + // Protected plugin pages must survive: connect is a known plugin whose + // subtree is absent from this run's tree. + expect(fs.existsSync(connectRoot)).toBe(true) + expect(fs.existsSync(connectRun)).toBe(true) + + // The genuinely stale non-plugin page must be swept. + expect(fs.existsSync(staleTopic)).toBe(false) + expect(result.filesDeleted).toBe(1) + + // Regenerated pages exist as usual. + expect(fs.existsSync(path.join(tmpDir, 'rpk-topic', 'rpk-topic.adoc'))).toBe(true) + expect(fs.existsSync(path.join(tmpDir, 'rpk-topic', 'rpk-topic-create.adoc'))).toBe(true) + }, 30000) + + test('shim-only plugin subtree still protects the plugin dir', async () => { + // Page from an earlier run where the plugin binary was installed. + const connectRun = writePage('rpk-connect', 'rpk-connect-run.adoc') + + // This run's tree has the connect shim (install/uninstall/upgrade) + // but none of the real plugin commands: the plugin failed to install. + const tree = { + name: 'rpk', + description: 'Root command', + commands: [ + cmd('connect', { commands: [cmd('install'), cmd('uninstall'), cmd('upgrade')] }), + cmd('topic', { commands: [cmd('create')] }) + ], + global_flags: [] + } + + const result = await generateRpkDocs({ + tree, + outputDir: tmpDir, + rpkVersion: 'test', + pluginVersions: {} + }) + + // The shim-only subtree marks connect as auto-protected, so the page + // written by the earlier (successful) run is not treated as stale. + expect(fs.existsSync(connectRun)).toBe(true) + expect(result.filesDeleted).toBe(0) + }, 30000) + + test('explicitly protected plugins are honored alongside auto-protection', async () => { + const aiPage = writePage('rpk-ai', 'rpk-ai-agent.adoc') + const stale = writePage('rpk-cluster', 'rpk-cluster-old.adoc') + + const tree = { + name: 'rpk', + description: 'Root command', + commands: [ + cmd('cluster', { commands: [cmd('health')] }) + ], + global_flags: [] + } + + const result = await generateRpkDocs({ + tree, + outputDir: tmpDir, + rpkVersion: 'test', + pluginVersions: {}, + protectedPlugins: ['ai'] + }) + + expect(fs.existsSync(aiPage)).toBe(true) + expect(fs.existsSync(stale)).toBe(false) + expect(result.filesDeleted).toBe(1) + }, 30000) + + test('shim-only plugin pages are not rewritten — parent page content stays untouched', async () => { + // Pages from an earlier run where the connect plugin was installed: + // the parent page documents the full plugin (including `run` in its + // Subcommands table) and a child page exists for the real command. + const parentContent = '= rpk connect\n\nFull plugin page with run subcommand.\n' + const parent = path.join(tmpDir, 'rpk-connect', 'rpk-connect.adoc') + fs.mkdirSync(path.dirname(parent), { recursive: true }) + fs.writeFileSync(parent, parentContent, 'utf8') + const childRun = writePage('rpk-connect', 'rpk-connect-run.adoc') + const childContent = fs.readFileSync(childRun, 'utf8') + + // This run's tree has only the connect shim. + const tree = { + name: 'rpk', + description: 'Root command', + commands: [ + cmd('connect', { commands: [cmd('install'), cmd('uninstall'), cmd('upgrade')] }), + cmd('topic', { commands: [cmd('create')] }) + ], + global_flags: [] + } + + await generateRpkDocs({ + tree, + outputDir: tmpDir, + rpkVersion: 'test', + pluginVersions: {} + }) + + // The parent page must not be regenerated with shim-only content, and + // the child page must not become an orphan or change. + expect(fs.readFileSync(parent, 'utf8')).toBe(parentContent) + expect(fs.readFileSync(childRun, 'utf8')).toBe(childContent) + + // No shim pages are written under the protected subtree. + expect(fs.existsSync(path.join(tmpDir, 'rpk-connect', 'rpk-connect-install.adoc'))).toBe(false) + expect(fs.existsSync(path.join(tmpDir, 'rpk-connect', 'rpk-connect-uninstall.adoc'))).toBe(false) + expect(fs.existsSync(path.join(tmpDir, 'rpk-connect', 'rpk-connect-upgrade.adoc'))).toBe(false) + + // Non-plugin pages regenerate as usual. + expect(fs.existsSync(path.join(tmpDir, 'rpk-topic', 'rpk-topic.adoc'))).toBe(true) + expect(fs.existsSync(path.join(tmpDir, 'rpk-topic', 'rpk-topic-create.adoc'))).toBe(true) + }, 30000) + + test('fully installed plugin regenerates pages and sweeps its stale files as usual', async () => { + // Previous run left a page for a subcommand that no longer exists. + const stale = writePage('rpk-connect', 'rpk-connect-removed.adoc') + const parent = path.join(tmpDir, 'rpk-connect', 'rpk-connect.adoc') + fs.writeFileSync(parent, '= Old parent page\n', 'utf8') + + // This run's tree has the real plugin commands (beyond the shim), so + // connect is NOT protected and generation proceeds normally. + const tree = { + name: 'rpk', + description: 'Root command', + commands: [ + cmd('connect', { + commands: [cmd('install'), cmd('uninstall'), cmd('upgrade'), cmd('run')] + }) + ], + global_flags: [] + } + + const result = await generateRpkDocs({ + tree, + outputDir: tmpDir, + rpkVersion: 'test', + pluginVersions: {} + }) + + // Parent page is regenerated (content replaced) and the real + // subcommand page is written. + expect(fs.readFileSync(parent, 'utf8')).not.toBe('= Old parent page\n') + expect(fs.readFileSync(parent, 'utf8')).toContain('rpk connect') + expect(fs.existsSync(path.join(tmpDir, 'rpk-connect', 'rpk-connect-run.adoc'))).toBe(true) + + // The genuinely stale page is swept. + expect(fs.existsSync(stale)).toBe(false) + expect(result.filesDeleted).toBe(1) + }, 30000) + }) +}) + +describe('capToTwoSentences with code blocks', () => { + const { capToTwoSentences } = require('../../../tools/rpk-docs/generate-rpk-docs.js') + + test('ends the summary at a colon that introduces a code block', () => { + const input = 'Prints a config according to an expression. The expression takes three lists, divided by slashes:\n\n[,text]\n----\nredpanda-connect create stdin/bloblang,awk/nats\n----\n\nIf omitted a default config is created.' + // The block and its dangling colon introducer are both dropped + expect(capToTwoSentences(input)).toBe('Prints a config according to an expression.') + }) + + test('drops a mid-prose block and keeps surrounding sentences', () => { + const input = 'Reports progress.\n\n[,bash]\n----\nrpk thing status 4\n----\n\nUse --detailed for more.' + expect(capToTwoSentences(input)).toBe('Reports progress. Use --detailed for more.') + }) +}) + +describe('placeholder brace escaping', () => { + const { formatDescription } = require('../../../tools/rpk-docs/generate-rpk-docs.js') + + test('escapes template placeholders so Asciidoctor keeps them', () => { + const input = 'Enable code mode: adds {name}_search and {name}_execute tools.' + const out = formatDescription(input) + expect(out).toContain('\\{name}_search') + expect(out).toContain('\\{name}_execute') + }) + + test('leaves a bare vbar attribute in prose alone', () => { + // {vbar} in prose is a real attribute (pipe escaping); placeholders in + // backtick spans were already escaped by the span-protection step + const input = 'Format: `table{vbar}json` uses {vbar} in prose.' + const out = formatDescription(input) + expect(out).toContain('uses {vbar} in prose') + expect(out).toContain('`table\\{vbar}json`') }) }) diff --git a/__tests__/tools/rpk-docs/plugin-flags.test.js b/__tests__/tools/rpk-docs/plugin-flags.test.js new file mode 100644 index 00000000..50d5eba8 --- /dev/null +++ b/__tests__/tools/rpk-docs/plugin-flags.test.js @@ -0,0 +1,180 @@ +'use strict' + +const { parseCobraFlags, enrichPluginTreeWithFlags } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + +describe('parseCobraFlags', () => { + const HELP = [ + 'Reconcile LLM providers from one or more YAML manifests.', + '', + 'Usage:', + ' rpk ai llm-provider apply [flags]', + '', + 'Flags:', + ' --allow-empty Allow applying zero manifests', + ' -f, --file strings Manifest paths, - for stdin (default [])', + ' -h, --help help for apply', + ' --timeout duration How long to wait for the reconcile to', + ' complete before giving up (default 30s)', + ' -o, --format string Output format (default "text")', + '', + 'Global Flags:', + ' -c, --config string rpk config file', + ' -v, --verbose enable verbose logging', + '', + 'Use "rpk ai llm-provider apply [command] --help" for more information.' + ].join('\n') + + test('parses local flags with shorthand, type, and default', () => { + const flags = parseCobraFlags(HELP) + const byName = Object.fromEntries(flags.map(f => [f.name, f])) + + expect(byName['allow-empty']).toMatchObject({ type: 'bool' }) + expect(byName['file']).toMatchObject({ shorthand: 'f', type: 'strings', default: '[]' }) + expect(byName['format']).toMatchObject({ shorthand: 'o', type: 'string', default: '"text"' }) + }) + + test('joins wrapped descriptions and extracts trailing defaults', () => { + const flags = parseCobraFlags(HELP) + const timeout = flags.find(f => f.name === 'timeout') + expect(timeout.description).toBe('How long to wait for the reconcile to complete before giving up') + expect(timeout.default).toBe('30s') + }) + + test('skips --help and the Global Flags section', () => { + const flags = parseCobraFlags(HELP) + expect(flags.map(f => f.name)).not.toContain('help') + expect(flags.map(f => f.name)).not.toContain('config') + expect(flags.map(f => f.name)).not.toContain('verbose') + }) + + test('returns empty for help without a Flags section', () => { + expect(parseCobraFlags('Usage:\n rpk ai\n\nUse "rpk ai --help".')).toEqual([]) + expect(parseCobraFlags('')).toEqual([]) + }) +}) + +describe('enrichPluginTreeWithFlags', () => { + test('fills flagless commands and leaves shim flags alone', () => { + const node = { + name: 'ai', + commands: [ + { name: 'install', flags: [{ name: 'ai-version', type: 'string' }], commands: [] }, + { name: 'auth', commands: [{ name: 'login', commands: [] }] } + ] + } + const calls = [] + const enriched = enrichPluginTreeWithFlags(node, (argPath) => { + calls.push(argPath.join(' ')) + return 'Flags:\n --no-browser Print the URL instead of opening it\n' + }) + + // install already has flags: not queried + expect(calls).not.toContain('ai install') + expect(calls).toContain('ai auth login') + const login = node.commands[1].commands[0] + expect(login.flags).toHaveLength(1) + expect(login.flags[0]).toMatchObject({ name: 'no-browser', type: 'bool' }) + expect(node.commands[0].flags[0].name).toBe('ai-version') + expect(enriched).toBeGreaterThanOrEqual(2) // root + auth + login minus empties + }) + + test('help failures are non-fatal', () => { + const node = { name: 'ai', commands: [{ name: 'run', commands: [] }] } + const enriched = enrichPluginTreeWithFlags(node, () => null) + expect(enriched).toBe(0) + expect(node.commands[0].flags).toBeUndefined() + }) +}) + +describe('mergeVisibleDeprecationsIntoOverrides', () => { + const fs = require('fs') + const path = require('path') + const os = require('os') + const { mergeVisibleDeprecationsIntoOverrides } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + + const tree = { + name: 'rpk', + commands: [ + { name: 'oldcmd', commands: [] }, + { name: 'topic', commands: [] } + ] + } + + let dir, overridesPath + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dep-merge-')) + overridesPath = path.join(dir, 'overrides.json') + }) + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) + + test('annotates visible deprecated commands, skips hidden ones', () => { + fs.writeFileSync(overridesPath, JSON.stringify({ commands: {} })) + mergeVisibleDeprecationsIntoOverrides({ + 'rpk oldcmd': { deprecated: true, deprecatedMessage: 'use rpk newcmd', replacement: 'See `rpk newcmd`.' }, + 'rpk hiddencmd': { deprecated: true, _note: 'Hidden: true' } + }, tree, overridesPath) + + const result = JSON.parse(fs.readFileSync(overridesPath, 'utf8')) + expect(result.commands['rpk oldcmd']).toMatchObject({ + deprecated: true, + deprecatedMessage: 'use rpk newcmd', + replacement: 'See `rpk newcmd`.' + }) + // hiddencmd is not in the tree: no page to annotate + expect(result.commands['rpk hiddencmd']).toBeUndefined() + }) + + test('never overwrites curated deprecation overrides', () => { + fs.writeFileSync(overridesPath, JSON.stringify({ + commands: { 'rpk oldcmd': { deprecated: false, deprecatedMessage: 'curated text' } } + })) + mergeVisibleDeprecationsIntoOverrides({ + 'rpk oldcmd': { deprecated: true, deprecatedMessage: 'scanner text' } + }, tree, overridesPath) + + const result = JSON.parse(fs.readFileSync(overridesPath, 'utf8')) + expect(result.commands['rpk oldcmd'].deprecated).toBe(false) + expect(result.commands['rpk oldcmd'].deprecatedMessage).toBe('curated text') + }) + + test('no-op when nothing to annotate', () => { + fs.writeFileSync(overridesPath, JSON.stringify({ commands: {} })) + const before = fs.readFileSync(overridesPath, 'utf8') + mergeVisibleDeprecationsIntoOverrides({}, tree, overridesPath) + expect(fs.readFileSync(overridesPath, 'utf8')).toBe(before) + }) +}) + +describe('parseUrfaveFlags (Redpanda Connect help format)', () => { + const { parseUrfaveFlags, parseHelpFlags } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + + const HELP = [ + 'NAME:', + ' redpanda-connect run - Run', + '', + 'OPTIONS:', + ' --log.level value override the log level', + ' --set value, -s value [ --set value, -s value ] set a field', + ' --chilled continue on lint errors (default: false)', + ' --watcher, -w watch config files (default: false)', + '', + 'GLOBAL OPTIONS:', + ' --verbose noisy' + ].join('\n') + + test('parses names, shorthands, types, and defaults', () => { + const flags = parseUrfaveFlags(HELP) + const byName = Object.fromEntries(flags.map(f => [f.name, f])) + expect(byName['log.level']).toMatchObject({ type: 'string' }) + expect(byName['set']).toMatchObject({ shorthand: 's', type: 'strings' }) + expect(byName['chilled']).toMatchObject({ type: 'bool', default: 'false' }) + expect(byName['watcher']).toMatchObject({ shorthand: 'w', type: 'bool' }) + expect(flags.map(f => f.name)).not.toContain('verbose') + }) + + test('parseHelpFlags dispatches by section header', () => { + expect(parseHelpFlags(HELP).length).toBe(4) + expect(parseHelpFlags('Flags:\n --no-browser print URL\n').length).toBe(1) + expect(parseHelpFlags('Usage: nothing here')).toEqual([]) + }) +}) diff --git a/__tests__/tools/rpk-docs/plugin-refresh.test.js b/__tests__/tools/rpk-docs/plugin-refresh.test.js new file mode 100644 index 00000000..dd0328b3 --- /dev/null +++ b/__tests__/tools/rpk-docs/plugin-refresh.test.js @@ -0,0 +1,154 @@ +'use strict' + +const { + splicePluginNode, + preserveLinuxOnlyCommands, + pluginNodeHasRealCommands, + REFRESHABLE_PLUGINS, + PLUGIN_INSTALL_VERSION_FLAGS, + PLUGIN_MANIFEST_SLUGS +} = require('../../../tools/rpk-docs/rpk-docs-handler.js') + +describe('Plugin refresh (--plugin mode)', () => { + const shimOnlyNode = { + name: 'k8s', + description: 'Kubernetes plugin', + commands: [ + { name: 'install' }, + { name: 'uninstall' }, + { name: 'upgrade' } + ] + } + + const installedNode = { + name: 'connect', + description: 'Redpanda Connect plugin', + commands: [ + { name: 'install' }, + { name: 'uninstall' }, + { name: 'upgrade' }, + { name: 'run', description: 'Run a pipeline' }, + { name: 'lint', description: 'Lint a config' } + ] + } + + const baseTree = { + name: 'rpk', + global_flags: [{ name: '--config' }], + commands: [ + { name: 'topic', commands: [{ name: 'create' }] }, + { name: 'connect', commands: [{ name: 'install' }, { name: 'run', description: 'old run' }] }, + { name: 'cluster', commands: [{ name: 'health' }] } + ] + } + + describe('pluginNodeHasRealCommands', () => { + test('false for a shim-only node (install/uninstall/upgrade)', () => { + expect(pluginNodeHasRealCommands(shimOnlyNode)).toBe(false) + }) + + test('true when real plugin commands are present', () => { + expect(pluginNodeHasRealCommands(installedNode)).toBe(true) + }) + + test('false for a node with no subcommands', () => { + expect(pluginNodeHasRealCommands({ name: 'ai' })).toBe(false) + }) + }) + + describe('splicePluginNode', () => { + test('replaces the plugin node and keeps everything else', () => { + const result = splicePluginNode(baseTree, 'connect', installedNode) + + expect(result.commands.map(c => c.name)).toEqual(['topic', 'connect', 'cluster']) + const connect = result.commands.find(c => c.name === 'connect') + expect(connect.commands.map(c => c.name)).toContain('lint') + expect(result.commands.find(c => c.name === 'topic')).toBe(baseTree.commands[0]) + expect(result.global_flags).toEqual(baseTree.global_flags) + }) + + test('does not mutate the input tree', () => { + splicePluginNode(baseTree, 'connect', installedNode) + const connect = baseTree.commands.find(c => c.name === 'connect') + expect(connect.commands.find(c => c.name === 'run').description).toBe('old run') + }) + + test('throws when the plugin is not in the base tree', () => { + expect(() => splicePluginNode(baseTree, 'k8s', shimOnlyNode)) + .toThrow(/not present in the base tree/) + }) + + test('keeps the tree linux_only_commands list through a splice', () => { + const treeWithMarkers = { ...baseTree, linux_only_commands: ['rpk debug bundle', 'rpk iotune'] } + const result = splicePluginNode(treeWithMarkers, 'connect', installedNode) + expect(result.linux_only_commands).toEqual(['rpk debug bundle', 'rpk iotune']) + }) + }) + + describe('preserveLinuxOnlyCommands', () => { + const linuxOnly = ['rpk debug bundle', 'rpk iotune'] + + test('inherits the snapshot list when the working tree lacks it', () => { + const result = preserveLinuxOnlyCommands(baseTree, { ...baseTree, linux_only_commands: linuxOnly }) + expect(result.linux_only_commands).toEqual(linuxOnly) + // Copies, not aliases: mutating the result must not touch the snapshot + expect(result.linux_only_commands).not.toBe(linuxOnly) + expect(result.commands).toBe(baseTree.commands) + }) + + test('keeps the working tree list when it already has one', () => { + const tree = { ...baseTree, linux_only_commands: linuxOnly } + const result = preserveLinuxOnlyCommands(tree, { ...baseTree, linux_only_commands: ['rpk other'] }) + expect(result).toBe(tree) + expect(result.linux_only_commands).toEqual(linuxOnly) + }) + + test('is a no-op when neither tree carries the list', () => { + expect(preserveLinuxOnlyCommands(baseTree, baseTree)).toBe(baseTree) + expect(preserveLinuxOnlyCommands(baseTree, undefined)).toBe(baseTree) + expect(preserveLinuxOnlyCommands(null, baseTree)).toBe(null) + }) + + test('refresh persistence chain keeps markers for the saved snapshot', () => { + // Mirrors the --plugin save path: derive the working tree from the + // snapshot, splice the fresh subtree, then preserve before persisting. + const snapshot = { + raw_tree: { ...baseTree, linux_only_commands: linuxOnly }, + tree: { ...baseTree, linux_only_commands: linuxOnly } + } + let tree = snapshot.raw_tree || snapshot.tree + tree = preserveLinuxOnlyCommands(tree, snapshot.tree || snapshot.raw_tree) + tree = splicePluginNode(tree, 'connect', installedNode) + tree = preserveLinuxOnlyCommands(tree, snapshot.raw_tree || snapshot.tree) + expect(tree.linux_only_commands).toEqual(linuxOnly) + }) + + test('refresh persistence chain restores markers when only the enhanced tree has them', () => { + // Older snapshots may carry the field on only one stored tree; the + // derivation step must inherit it so the re-saved snapshot keeps it. + const snapshot = { + raw_tree: { ...baseTree }, + tree: { ...baseTree, linux_only_commands: linuxOnly } + } + let tree = snapshot.raw_tree || snapshot.tree + tree = preserveLinuxOnlyCommands(tree, snapshot.tree || snapshot.raw_tree) + expect(tree.linux_only_commands).toEqual(linuxOnly) + }) + }) + + describe('plugin constants', () => { + test('every refreshable plugin has a version pin flag', () => { + for (const plugin of REFRESHABLE_PLUGINS) { + expect(PLUGIN_INSTALL_VERSION_FLAGS[plugin]).toMatch(/^--[a-z-]+$/) + } + }) + + test('oxla is not refreshable (stub with no installable binary)', () => { + expect(REFRESHABLE_PLUGINS).not.toContain('oxla') + }) + + test('ai maps to the rpai manifest slug', () => { + expect(PLUGIN_MANIFEST_SLUGS.ai).toBe('rpai') + }) + }) +}) diff --git a/__tests__/tools/rpk-docs/plugin-stubs.test.js b/__tests__/tools/rpk-docs/plugin-stubs.test.js new file mode 100644 index 00000000..88b2c14f --- /dev/null +++ b/__tests__/tools/rpk-docs/plugin-stubs.test.js @@ -0,0 +1,190 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const os = require('os') + +const { + readPartialTitles, + inferIncludePrefix, + renderStub, + reconcileStubs +} = require('../../../tools/rpk-docs/generate-plugin-stubs.js') + +describe('plugin stub reconciler', () => { + let dir, partialsDir, stubDir, navFile + + const writePartial = (file, title) => { + fs.writeFileSync(path.join(partialsDir, file), `= ${title}\n:description: x\n\n// tag::single-source[]\nBody.\n// end::single-source[]\n`) + } + + const writeStub = (file, title, partialFile) => { + fs.writeFileSync(path.join(stubDir, file), renderStub({ + title, + file: partialFile || file, + includePrefix: 'streaming:reference:partial$rpk-ai/', + attributes: [':page-preview: true'] + })) + } + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'stub-recon-')) + partialsDir = path.join(dir, 'partials') + stubDir = path.join(dir, 'stubs') + navFile = path.join(dir, 'nav.adoc') + fs.mkdirSync(partialsDir) + fs.mkdirSync(stubDir) + fs.writeFileSync(navFile, [ + '** xref:reference:rpk/index.adoc[rpk Command Reference]', + '*** xref:reference:rpk/rpk-ai/rpk-ai.adoc[rpk ai]', + '**** xref:reference:rpk/rpk-ai/rpk-ai-old.adoc[]', + '*** xref:reference:rpk-install.adoc[Install rpk]', + '' + ].join('\n')) + }) + + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })) + + const run = (overrides = {}) => reconcileStubs({ + partials: readPartialTitles(partialsDir), + stubDir, + navFile, + plugin: 'ai', + includePrefix: 'streaming:reference:partial$rpk-ai/', + ...overrides + }) + + test('creates stubs for new partials and deletes orphaned managed stubs', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + writePartial('rpk-ai-auth.adoc', 'rpk ai auth') + writePartial('rpk-ai-auth-login.adoc', 'rpk ai auth login') + writeStub('rpk-ai-old.adoc', 'rpk ai old') // partial gone + + const result = run() + + expect(result.created.sort()).toEqual(['rpk-ai-auth-login.adoc', 'rpk-ai-auth.adoc', 'rpk-ai.adoc']) + expect(result.deleted).toEqual(['rpk-ai-old.adoc']) + const stub = fs.readFileSync(path.join(stubDir, 'rpk-ai-auth-login.adoc'), 'utf8') + expect(stub).toContain('= rpk ai auth login') + expect(stub).toContain(':page-preview: true') + expect(stub).toContain('include::streaming:reference:partial$rpk-ai/rpk-ai-auth-login.adoc[tag=single-source]') + }) + + test('rebuilds the nav block hierarchically and preserves surrounding nav', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + writePartial('rpk-ai-auth.adoc', 'rpk ai auth') + writePartial('rpk-ai-auth-login.adoc', 'rpk ai auth login') + + const result = run() + + expect(result.navUpdated).toBe(true) + const nav = fs.readFileSync(navFile, 'utf8').split('\n') + expect(nav[1]).toBe('*** xref:reference:rpk/rpk-ai/rpk-ai.adoc[rpk ai]') + expect(nav[2]).toBe('**** xref:reference:rpk/rpk-ai/rpk-ai-auth.adoc[]') + expect(nav[3]).toBe('***** xref:reference:rpk/rpk-ai/rpk-ai-auth-login.adoc[]') + expect(nav[4]).toBe('*** xref:reference:rpk-install.adoc[Install rpk]') + expect(nav.join('\n')).not.toContain('rpk-ai-old.adoc') + }) + + test('never deletes pages that are not managed stubs', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + fs.writeFileSync(path.join(stubDir, 'hand-written.adoc'), '= Concepts\n\nReal prose, no include.\n') + + const result = run() + + expect(result.keptNonStub).toEqual(['hand-written.adoc']) + expect(fs.existsSync(path.join(stubDir, 'hand-written.adoc'))).toBe(true) + }) + + test('flags likely renames for reviewer alias decisions', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + writePartial('rpk-ai-llm-provider.adoc', 'rpk ai llm-provider') + writeStub('rpk-ai-llm.adoc', 'rpk ai llm') + + const result = run() + + expect(result.renameCandidates).toEqual([ + { deleted: 'rpk-ai-llm.adoc', created: 'rpk-ai-llm-provider.adoc' } + ]) + }) + + test('is idempotent', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + writePartial('rpk-ai-run.adoc', 'rpk ai run') + run() + const second = run() + expect(second.created).toEqual([]) + expect(second.deleted).toEqual([]) + expect(second.navUpdated).toBe(false) + }) + + test('infers the include prefix from an existing stub', () => { + writeStub('rpk-ai-run.adoc', 'rpk ai run') + expect(inferIncludePrefix(stubDir, 'ai')).toBe('streaming:reference:partial$rpk-ai/') + expect(inferIncludePrefix(path.join(dir, 'missing'), 'ai')).toBe(null) + }) + + test('supports page-family includes (rp-connect-docs pattern)', () => { + writePartial('rpk-connect-run.adoc', 'rpk connect run') + const result = reconcileStubs({ + partials: readPartialTitles(partialsDir), + stubDir, + navFile: null, + plugin: 'connect', + includePrefix: 'streaming:reference:page$rpk/rpk-connect/' + }) + expect(result.created).toEqual(['rpk-connect-run.adoc']) + const stub = fs.readFileSync(path.join(stubDir, 'rpk-connect-run.adoc'), 'utf8') + expect(stub).toContain('include::streaming:reference:page$rpk/rpk-connect/rpk-connect-run.adoc[tag=single-source]') + // A second run recognizes the page-family stub as managed + const second = reconcileStubs({ + partials: [], + stubDir, + navFile: null, + plugin: 'connect', + includePrefix: 'streaming:reference:page$rpk/rpk-connect/' + }) + expect(second.deleted).toEqual(['rpk-connect-run.adoc']) + }) + + test('dry run reports without writing', () => { + writePartial('rpk-ai.adoc', 'rpk ai') + const result = run({ dryRun: true }) + expect(result.created).toEqual(['rpk-ai.adoc']) + expect(fs.existsSync(path.join(stubDir, 'rpk-ai.adoc'))).toBe(false) + }) +}) + +describe('alias-collision guard', () => { + const fs2 = require('fs') + const path2 = require('path') + const os2 = require('os') + const { reconcileStubs: recon, readPartialTitles: readTitles, renderStub: render } = require('../../../tools/rpk-docs/generate-plugin-stubs.js') + + test('never creates a stub whose name is claimed as a page alias', () => { + const dir = fs2.mkdtempSync(path2.join(os2.tmpdir(), 'alias-guard-')) + const partialsDir = path2.join(dir, 'partials'); fs2.mkdirSync(partialsDir) + const stubDir = path2.join(dir, 'stubs'); fs2.mkdirSync(stubDir) + + // Upstream still has the old-name partial; this repo's renamed page claims it as alias + fs2.writeFileSync(path2.join(partialsDir, 'rpk-ai-llm.adoc'), '= rpk ai llm\n') + fs2.writeFileSync(path2.join(partialsDir, 'rpk-ai-llm-provider.adoc'), '= rpk ai llm-provider\n') + fs2.writeFileSync(path2.join(stubDir, 'rpk-ai-llm-provider.adoc'), + '= rpk ai llm-provider\n:page-aliases: reference:rpk/rpk-ai/rpk-ai-llm.adoc\n\ninclude::streaming:reference:partial$rpk-ai/rpk-ai-llm-provider.adoc[tag=single-source]\n') + + const result = recon({ + partials: readTitles(partialsDir), + stubDir, + navFile: null, + plugin: 'ai', + includePrefix: 'streaming:reference:partial$rpk-ai/' + }) + + expect(result.skippedAliasTargets).toEqual([ + { file: 'rpk-ai-llm.adoc', claimedBy: 'rpk-ai-llm-provider.adoc' } + ]) + expect(result.created).toEqual([]) + expect(fs2.existsSync(path2.join(stubDir, 'rpk-ai-llm.adoc'))).toBe(false) + fs2.rmSync(dir, { recursive: true, force: true }) + }) +}) diff --git a/__tests__/tools/rpk-docs/pr-summary.test.js b/__tests__/tools/rpk-docs/pr-summary.test.js new file mode 100644 index 00000000..6b412f36 --- /dev/null +++ b/__tests__/tools/rpk-docs/pr-summary.test.js @@ -0,0 +1,151 @@ +'use strict' + +const { generatePRSummary } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + +describe('generatePRSummary change reporting', () => { + const baseOptions = { + rpkVersion: 'v2.0.0', + commandCount: 100, + filesGenerated: 100, + outputDir: 'modules/reference/pages/rpk' + } + + const diffWith = (summary, details) => ({ + comparison: { oldVersion: 'v1.0.0', newVersion: 'v2.0.0' }, + summary: { + newCommands: 0, + removedCommands: 0, + newFlags: 0, + removedFlags: 0, + changedDefaults: 0, + changedFlagTypes: 0, + changedFlagRequirements: 0, + changedFlagDescriptions: 0, + descriptionChanges: 0, + ...summary + }, + details: { + newCommands: [], + removedCommands: [], + newFlags: [], + removedFlags: [], + changedDefaults: [], + changedFlagTypes: [], + changedFlagRequirements: [], + changedFlagDescriptions: [], + descriptionChanges: [], + ...details + } + }) + + test('renders changed flag defaults with values', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { changedDefaults: 1 }, + { changedDefaults: [{ commandPath: 'rpk topic create', flagName: 'timeout', oldDefault: '5s', newDefault: '30s' }] } + ) + }) + expect(summary).toContain('| Changed flag defaults | 1 |') + expect(summary).toContain('`--timeout` default `5s` → `30s`') + }) + + test('renders array defaults as JSON', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { changedDefaults: 1 }, + { changedDefaults: [{ commandPath: 'rpk x', flagName: 'brokers', oldDefault: ['a'], newDefault: ['b'] }] } + ) + }) + expect(summary).toContain('["b"]') + expect(summary).not.toContain('[object Object]') + }) + + test('renders command description changes using the correct field name', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { descriptionChanges: 2 }, + { descriptionChanges: [{ path: 'rpk topic' }, { path: 'rpk cluster' }] } + ) + }) + expect(summary).toContain('| Changed command descriptions | 2 |') + expect(summary).toContain('`rpk topic`') + }) + + test('defaults-only changes do not report "no changes"', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { changedDefaults: 1 }, + { changedDefaults: [{ commandPath: 'rpk x', flagName: 'y', oldDefault: 1, newDefault: 2 }] } + ) + }) + expect(summary).not.toContain('No command, flag, or default changes detected.') + }) + + test('reports no changes when the diff is empty', () => { + const summary = generatePRSummary({ ...baseOptions, diffData: diffWith({}, {}) }) + expect(summary).toContain('No command, flag, or default changes detected.') + }) + + test('lists removed flags and flag type changes', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { removedFlags: 1, changedFlagTypes: 1 }, + { + removedFlags: [{ commandPath: 'rpk topic create', flagName: 'legacy' }], + changedFlagTypes: [{ commandPath: 'rpk topic create', flagName: 'partitions', oldType: 'int', newType: 'int32' }] + } + ) + }) + expect(summary).toContain('Removed Flags') + expect(summary).toContain('`--legacy`') + expect(summary).toContain('type `int` → `int32`') + }) + + test('renders deprecated commands when present', () => { + const summary = generatePRSummary({ + ...baseOptions, + diffData: diffWith( + { newlyDeprecatedCommands: 1 }, + { newlyDeprecatedCommands: [{ path: 'rpk redpanda admin', message: 'use rpk cluster instead', hidden: true }] } + ) + }) + expect(summary).toContain('| Deprecated commands | 1 |') + expect(summary).toContain('use rpk cluster instead') + expect(summary).toContain('_(hidden from help output)_') + }) +}) + +describe('computeDescriptionCoverage', () => { + const { computeDescriptionCoverage } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + + const tree = { + name: 'rpk', + commands: [ + { name: 'group', description: 'x'.repeat(2000), commands: [] }, + { name: 'version', description: 'Prints the version.', commands: [] } + ] + } + + test('flags overrides that hide substantially longer source help', () => { + const overrides = { commands: { + 'rpk group': { description: 'Manage groups.' }, + 'rpk version': { description: 'Print version info.' } + } } + const result = computeDescriptionCoverage(tree, overrides) + expect(result).toHaveLength(1) + expect(result[0]).toMatchObject({ commandPath: 'rpk group', sourceChars: 2000 }) + }) + + test('ignores overrides without descriptions and unknown commands', () => { + const overrides = { commands: { + 'rpk group': { flags: {} }, + 'rpk nonexistent': { description: 'x' } + } } + expect(computeDescriptionCoverage(tree, overrides)).toEqual([]) + }) +}) diff --git a/__tests__/tools/rpk-docs/report-delta.test.js b/__tests__/tools/rpk-docs/report-delta.test.js index be21f80b..8ee0f89c 100644 --- a/__tests__/tools/rpk-docs/report-delta.test.js +++ b/__tests__/tools/rpk-docs/report-delta.test.js @@ -197,6 +197,198 @@ describe('rpk Docs Diff Generation', () => { expect(diff.comparison.newVersion).toBe('v2.0.0') expect(diff.comparison.timestamp).toBeDefined() }) + + test('detects flag type changes', () => { + const diff = generateRpkDiff(oldTree, newTree, { oldVersion: 'v1', newVersion: 'v2' }) + expect(diff.summary.changedFlagTypes).toBe(1) + expect(diff.details.changedFlagTypes[0]).toMatchObject({ + commandPath: 'rpk topic create', + flagName: 'partitions', + oldType: 'int', + newType: 'int32' + }) + }) + + test('detects flag default, requirement, and description changes', () => { + const before = { + name: 'rpk', + commands: [{ + name: 'topic', + flags: [ + { name: 'timeout', type: 'duration', default: '5s', description: 'Wait time', required: false } + ], + commands: [] + }] + } + const after = { + name: 'rpk', + commands: [{ + name: 'topic', + flags: [ + { name: 'timeout', type: 'duration', default: '30s', description: 'Maximum wait time', required: true } + ], + commands: [] + }] + } + const diff = generateRpkDiff(before, after) + expect(diff.summary.changedDefaults).toBe(1) + expect(diff.details.changedDefaults[0]).toMatchObject({ oldDefault: '5s', newDefault: '30s' }) + expect(diff.summary.changedFlagRequirements).toBe(1) + expect(diff.details.changedFlagRequirements[0]).toMatchObject({ oldRequired: false, newRequired: true }) + expect(diff.summary.changedFlagDescriptions).toBe(1) + }) + }) + + describe('deprecation detection', () => { + const oldTreeDep = { + name: 'rpk', + commands: [ + { name: 'redpanda', commands: [{ name: 'admin', commands: [{ name: 'brokers', commands: [{ name: 'list', commands: [] }] }] }] }, + { name: 'gone', commands: [] } + ] + } + const newTreeDep = { + name: 'rpk', + commands: [ + { name: 'redpanda', commands: [] } + ] + } + const newDeprecated = { + 'rpk redpanda admin': { + deprecated: true, + _note: 'Hidden: true, found by scanning Go source', + deprecatedMessage: 'use `rpk cluster` subcommands', + replacement: 'Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.' + } + } + + test('reports newly deprecated commands with metadata', () => { + const diff = generateRpkDiff(oldTreeDep, newTreeDep, { + newVersion: 'v2', + oldDeprecatedCommands: {}, + newDeprecatedCommands: newDeprecated + }) + expect(diff.summary.newlyDeprecatedCommands).toBe(1) + const dep = diff.details.newlyDeprecatedCommands[0] + expect(dep.path).toBe('rpk redpanda admin') + expect(dep.hidden).toBe(true) + expect(dep.message).toContain('rpk cluster') + expect(dep.replacement).toContain('xref:') + }) + + test('reclassifies hidden-deprecated subtree removals as deprecations', () => { + const diff = generateRpkDiff(oldTreeDep, newTreeDep, { + newVersion: 'v2', + oldDeprecatedCommands: {}, + newDeprecatedCommands: newDeprecated + }) + // rpk gone is a genuine removal; the admin family is not + expect(diff.details.removedCommands.map(c => c.path)).toEqual(['rpk gone']) + const dep = diff.details.newlyDeprecatedCommands[0] + expect(dep.affectedSubcommands).toEqual( + expect.arrayContaining(['rpk redpanda admin brokers', 'rpk redpanda admin brokers list']) + ) + }) + + test('already-deprecated commands do not re-report', () => { + const diff = generateRpkDiff(oldTreeDep, newTreeDep, { + newVersion: 'v2', + oldDeprecatedCommands: newDeprecated, + newDeprecatedCommands: newDeprecated + }) + expect(diff.summary.newlyDeprecatedCommands).toBe(0) + // Without a NEW deprecation, the subtree disappearance counts as removal + expect(diff.summary.removedCommands).toBe(4) + }) + }) + + describe('generateWhatsNewSection change coverage', () => { + const { generateWhatsNewSection } = require('../../../tools/rpk-docs/report-delta.js') + + const baseDiff = (details) => ({ + comparison: { oldVersion: 'v1', newVersion: 'v2' }, + summary: {}, + details: { + newCommands: [], + removedCommands: [], + newFlags: [], + removedFlags: [], + changedDefaults: [], + changedFlagTypes: [], + descriptionChanges: [], + ...details + } + }) + + test('renders removed commands and flags', () => { + const section = generateWhatsNewSection(baseDiff({ + removedCommands: [{ path: 'rpk old command' }], + removedFlags: [{ commandPath: 'rpk topic create', flagName: 'legacy' }] + })) + expect(section).toContain('=== Removed commands') + expect(section).toContain('`rpk old command`') + expect(section).toContain('=== Removed flags') + expect(section).toContain('`--legacy`') + }) + + test('renders changed flag types', () => { + const section = generateWhatsNewSection(baseDiff({ + changedFlagTypes: [{ commandPath: 'rpk topic create', flagName: 'partitions', oldType: 'int', newType: 'int32' }] + })) + expect(section).toContain('=== Changed flag types') + expect(section).toContain('`int32`') + }) + + test('renders array defaults as JSON, not [object Object]', () => { + const section = generateWhatsNewSection(baseDiff({ + changedDefaults: [{ commandPath: 'rpk topic create', flagName: 'brokers', oldDefault: ['a'], newDefault: ['a', 'b'] }] + })) + expect(section).toContain('["a","b"]') + expect(section).not.toContain('[object Object]') + }) + + test('returns empty string when nothing changed', () => { + expect(generateWhatsNewSection(baseDiff({}))).toBe('') + }) + + test('routes command-group roots into their directory', () => { + const section = generateWhatsNewSection(baseDiff({ + newCommands: [ + { path: 'rpk check', description: 'Production readiness checks.' }, + { path: 'rpk check run', description: 'Run the checks.' }, + { path: 'rpk version', description: 'Prints the version.' } + ] + }), { hasSubcommands: (p) => p === 'rpk check' }) + expect(section).toContain('xref:reference:rpk/rpk-check/rpk-check.adoc[`rpk check`]') + expect(section).toContain('xref:reference:rpk/rpk-check/rpk-check-run.adoc[`rpk check run`]') + expect(section).toContain('xref:reference:rpk/rpk-version.adoc[`rpk version`]') + }) + + test('caps bullet descriptions at a sentence boundary', () => { + const section = generateWhatsNewSection(baseDiff({ + newCommands: [{ + path: 'rpk ai llm-provider diff', + description: 'Dry-run of apply for LLM providers. Prints, per manifest, whether apply would\ncreate, update, or leave the resource unchanged.' + }] + })) + expect(section).toContain(' - Dry-run of apply for LLM providers.') + expect(section).not.toContain('whether apply would') + }) + + test('renders deprecated commands with replacement and affected subcommands', () => { + const section = generateWhatsNewSection(baseDiff({ + newlyDeprecatedCommands: [{ + path: 'rpk redpanda admin', + message: 'use `rpk cluster` subcommands', + replacement: 'Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.', + hidden: true, + affectedSubcommands: ['rpk redpanda admin brokers'] + }] + })) + expect(section).toContain('=== Deprecated commands') + expect(section).toContain('xref:reference:rpk/rpk-cluster/rpk-cluster.adoc') + expect(section).toContain('rpk redpanda admin brokers') + }) }) describe('generateMarkdownSummary', () => { @@ -317,3 +509,138 @@ describe('rpk Docs Diff Generation', () => { }) }) }) + +describe('firstSentence', () => { + const { firstSentence } = require('../../../tools/rpk-docs/report-delta.js') + + test('cuts an unterminated summary at the paragraph break', () => { + // cobra descriptions often open with a periodless summary line + expect(firstSentence('Install Redpanda Check\n\nThis command installs the latest version by default.')) + .toBe('Install Redpanda Check') + }) + + test('joins hard-wrapped lines within the first paragraph', () => { + expect(firstSentence('Collects environment data that can help debug\nissues with a cluster. It then bundles the data.')) + .toBe('Collects environment data that can help debug issues with a cluster.') + }) + + test('keeps decimal numbers intact', () => { + expect(firstSentence('Installs version 4.32.0 of the plugin. More text.')) + .toBe('Installs version 4.32.0 of the plugin.') + }) +}) + +describe('flag data backfill guard', () => { + const { generateRpkDiff } = require('../../../tools/rpk-docs/report-delta.js') + + const tree = (cmds) => ({ name: 'rpk', commands: cmds }) + + test('does not report backfilled plugin flags as new', () => { + // v26.1.12-style baseline: plugin command exists but no flags captured + const oldTree = tree([{ name: 'connect', commands: [{ name: 'streams', flags: [] }] }]) + const newTree = tree([{ + name: 'connect', + commands: [{ + name: 'streams', + flags: [{ name: 'observability', type: 'string' }, { name: 'chilled', type: 'bool' }] + }] + }]) + + const diff = generateRpkDiff(oldTree, newTree, { oldVersion: 'v1', newVersion: 'v2' }) + expect(diff.summary.newFlags).toBe(0) + expect(diff.summary.flagDataBackfilled).toBe(1) + expect(diff.details.flagDataBackfilled).toEqual([ + { commandPath: 'rpk connect streams', flagCount: 2 } + ]) + }) + + test('still detects description changes on backfilled commands', () => { + const oldTree = tree([{ name: 'connect', commands: [{ name: 'streams', flags: [], description: 'old' }] }]) + const newTree = tree([{ name: 'connect', commands: [{ name: 'streams', flags: [{ name: 'x' }], description: 'new' }] }]) + + const diff = generateRpkDiff(oldTree, newTree) + expect(diff.summary.descriptionChanges).toBe(1) + }) + + test('reports genuinely new flags when the baseline has flag data', () => { + const oldTree = tree([{ name: 'cluster', commands: [{ name: 'info', flags: [{ name: 'brokers', type: 'string' }] }] }]) + const newTree = tree([{ + name: 'cluster', + commands: [{ name: 'info', flags: [{ name: 'brokers', type: 'string' }, { name: 'detailed', type: 'bool' }] }] + }]) + + const diff = generateRpkDiff(oldTree, newTree, { newVersion: 'v26.2.1' }) + expect(diff.summary.flagDataBackfilled).toBe(0) + expect(diff.summary.newFlags).toBe(1) + expect(diff.details.newFlags[0]).toMatchObject({ + commandPath: 'rpk cluster info', + flagName: 'detailed', + introducedInVersion: 'v26.2.1' + }) + }) + + test('new commands are unaffected by the guard', () => { + const oldTree = tree([]) + const newTree = tree([{ name: 'policy', flags: [{ name: 'format' }], commands: [] }]) + + const diff = generateRpkDiff(oldTree, newTree, { newVersion: 'v26.2.1' }) + expect(diff.summary.newCommands).toBe(1) + expect(diff.summary.flagDataBackfilled).toBe(0) + }) +}) + +describe('flag data backfill guard: group-level baseline detection', () => { + const { generateRpkDiff } = require('../../../tools/rpk-docs/report-delta.js') + const tree = (cmds) => ({ name: 'rpk', commands: cmds }) + + test('a zero-flag core command in a group with baseline data gets genuine new flags', () => { + // rpk cluster config status gained --format; the cluster group has + // baseline flag data elsewhere, so this is not backfill + const oldTree = tree([{ + name: 'cluster', + commands: [ + { name: 'health', flags: [{ name: 'watch', type: 'bool' }] }, + { name: 'config', commands: [{ name: 'status', flags: [] }] } + ] + }]) + const newTree = tree([{ + name: 'cluster', + commands: [ + { name: 'health', flags: [{ name: 'watch', type: 'bool' }] }, + { name: 'config', commands: [{ name: 'status', flags: [{ name: 'format', type: 'string' }] }] } + ] + }]) + + const diff = generateRpkDiff(oldTree, newTree, { newVersion: 'v26.2.1' }) + expect(diff.summary.flagDataBackfilled).toBe(0) + expect(diff.details.newFlags).toEqual([expect.objectContaining({ + commandPath: 'rpk cluster config status', + flagName: 'format' + })]) + }) + + test('shim flags do not count as plugin baseline data', () => { + // Old connect subtree only has the rpk-native install shim flag; the + // plugin commands themselves recorded no flags, so it is backfill + const oldTree = tree([{ + name: 'connect', + commands: [ + { name: 'install', flags: [{ name: 'connect-version', type: 'string' }] }, + { name: 'streams', flags: [] } + ] + }]) + const newTree = tree([{ + name: 'connect', + commands: [ + { name: 'install', flags: [{ name: 'connect-version', type: 'string' }] }, + { name: 'streams', flags: [{ name: 'observability', type: 'string' }] } + ] + }]) + + const diff = generateRpkDiff(oldTree, newTree) + expect(diff.summary.newFlags).toBe(0) + expect(diff.details.flagDataBackfilled).toEqual([ + { commandPath: 'rpk connect streams', flagCount: 1 } + ]) + }) +}) diff --git a/__tests__/tools/rpk-docs/rpk-docs-handler.test.js b/__tests__/tools/rpk-docs/rpk-docs-handler.test.js index 707f0a95..fcecc29e 100644 --- a/__tests__/tools/rpk-docs/rpk-docs-handler.test.js +++ b/__tests__/tools/rpk-docs/rpk-docs-handler.test.js @@ -76,6 +76,33 @@ describe('rpk Docs Handler', () => { const result = JSON.parse(fs.readFileSync(overridesPath, 'utf8')) expect(result.commands['rpk topic existing'].introducedInVersion).toBe('v26.1.0') }) + + test('stamps plugin commands with the plugin version, core commands with the rpk version', () => { + fs.writeFileSync(overridesPath, JSON.stringify({ commands: {} })) + + const diffData = { + summary: { newCommands: 2 }, + details: { + newCommands: [ + { path: 'rpk connect new-subcommand' }, + { path: 'rpk cluster new-command' } + ], + newFlags: [ + { commandPath: 'rpk connect run', flagName: 'new-flag' } + ], + removedCommands: [], + removedFlags: [], + changedDefaults: [] + } + } + + updateOverridesWithIntroducedVersions(diffData, overridesPath, 'v26.2.0', { connect: '4.103.0' }) + + const result = JSON.parse(fs.readFileSync(overridesPath, 'utf8')) + expect(result.commands['rpk connect new-subcommand'].introducedInVersion).toBe('4.103.0') + expect(result.commands['rpk cluster new-command'].introducedInVersion).toBe('v26.2.0') + expect(result.commands['rpk connect run'].flags['new-flag'].introducedInVersion).toBe('4.103.0') + }) }) describe('flag version tracking', () => { diff --git a/__tests__/tools/rpk-docs/table-conversion.test.js b/__tests__/tools/rpk-docs/table-conversion.test.js index 28c90b52..f08987a3 100644 --- a/__tests__/tools/rpk-docs/table-conversion.test.js +++ b/__tests__/tools/rpk-docs/table-conversion.test.js @@ -419,3 +419,79 @@ Without \`--node-id\`, the request is sent to any broker.` }) }) }) + +describe('unindented shell examples and colon-introduced code', () => { + const { + convertIndentedCodeBlocksToAsciiDoc, + parseDescriptionSections + } = require('../../../tools/rpk-docs/generate-rpk-docs.js') + + test('captures a column-0 $ invocation and its output as blocks', () => { + const store = [] + const input = [ + 'Progress is reported as follows.', + '', + '$ rpk cluster brokers decommission-status 4', + 'DECOMMISSION PROGRESS', + '=====================', + 'kafka/test/0 3 9 1699470920', + '', + 'Using --detailed prints granular reports.' + ].join('\n') + + const out = convertIndentedCodeBlocksToAsciiDoc(input, store) + expect(store).toHaveLength(1) + expect(store[0]).toContain('[,bash]\n----\nrpk cluster brokers decommission-status 4\n----') + expect(store[0]).toContain('[.no-copy]\n----\nDECOMMISSION PROGRESS\n=====================\nkafka/test/0 3 9 1699470920\n----') + expect(out).toContain('__EARLY_CODE_BLOCK_0__') + expect(out).toContain('Using --detailed prints granular reports.') + }) + + test('all-caps output title after a $ invocation is not a section header', () => { + const desc = [ + 'Reports progress.', + '', + '$ rpk thing status 4', + 'DECOMMISSION PROGRESS', + '=====================', + 'row 1', + '', + 'Trailing prose.' + ].join('\n') + + const { mainDescription, sections } = parseDescriptionSections(desc) + expect(Object.keys(sections)).toEqual([]) + expect(mainDescription).toContain('DECOMMISSION PROGRESS') + }) + + test('captures a colon-introduced indented code sample verbatim', () => { + const store = [] + const input = [ + 'Scope the resource to your MCP server, e.g.:', + '', + ' permit(principal, action == Action::"tools_call",', + ' resource == McpServer::"servicenow");', + '', + 'Data shaping is NOT configured here.' + ].join('\n') + + const out = convertIndentedCodeBlocksToAsciiDoc(input, store) + expect(store).toHaveLength(1) + expect(store[0]).toContain('permit(principal, action == Action::"tools_call",\n resource == McpServer::"servicenow");') + expect(out).toContain('Data shaping is NOT configured here.') + }) + + test('colon-introduced indented lists are not captured as code', () => { + const store = [] + const input = [ + 'The following conditions are met:', + '', + ' - All partitions have leaders', + ' - No brokers are down', + '' + ].join('\n') + + convertIndentedCodeBlocksToAsciiDoc(input, store) + expect(store).toHaveLength(0) + }) +}) diff --git a/__tests__/tools/rpk-docs/text-transformations.test.js b/__tests__/tools/rpk-docs/text-transformations.test.js index d0f2e964..76caba74 100644 --- a/__tests__/tools/rpk-docs/text-transformations.test.js +++ b/__tests__/tools/rpk-docs/text-transformations.test.js @@ -125,3 +125,47 @@ describe('Text Transformations', () => { }, 30000) }) }) + +describe('applyToCode rules in early code blocks', () => { + const { formatDescription } = require('../../../tools/rpk-docs/generate-rpk-docs.js') + + const transforms = { + replacements: [ + { pattern: '\\brpai\\b', replacement: 'rpk ai', flags: 'g', applyToCode: true }, + { pattern: '(^|\\n\\s*)Note:\\s', replacement: '$1NOTE: ', flags: 'g' } + ] + } + + test('applies only code-safe rules inside captured code blocks', () => { + const input = 'Run the agent, e.g.:\n\n rpai run claude -L anthropic\n Note: output follows\n\nDone.' + const out = formatDescription(input, transforms) + expect(out).toContain('rpk ai run claude -L anthropic') + // The admonition rule must NOT rewrite text inside the code block + expect(out).toContain('Note: output follows') + expect(out).not.toContain('NOTE: output follows') + }) +}) + +describe('applyTextTransformationsToExamples', () => { + const { applyTextTransformationsToExamples } = require('../../../tools/rpk-docs/generate-rpk-docs.js') + + const transforms = { + replacements: [ + { pattern: '"([a-z]{1,20})"', replacement: '`$1`', flags: 'g' }, + { pattern: '\\brpai\\b', replacement: 'rpk ai', flags: 'g', applyToCode: true } + ] + } + + test('caption rules never rewrite quoted strings inside command lines', () => { + const input = 'Import client "quotas" from a string:\n rpk cluster quotas import --from \'{"quotas":...}\'' + const out = applyTextTransformationsToExamples(input, transforms) + expect(out).toContain('`quotas` from a string') + expect(out).toContain(String.raw`'{"quotas":...}'`) + }) + + test('code-safe rules still apply to command lines', () => { + const input = 'Send a task:\n rpai agent a2a send hello' + const out = applyTextTransformationsToExamples(input, transforms) + expect(out).toContain(' rpk ai agent a2a send hello') + }) +}) diff --git a/__tests__/tools/rpk-docs/whats-new-merge.test.js b/__tests__/tools/rpk-docs/whats-new-merge.test.js new file mode 100644 index 00000000..02a11475 --- /dev/null +++ b/__tests__/tools/rpk-docs/whats-new-merge.test.js @@ -0,0 +1,162 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const os = require('os') + +const { updateWhatsNewFile } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + +describe('updateWhatsNewFile merge semantics', () => { + let tempDir + let whatsNewPath + + const diffWith = (details) => ({ + comparison: { oldVersion: 'v1', newVersion: 'v2' }, + summary: {}, + details: { + newCommands: [], + newlyDeprecatedCommands: [], + removedCommands: [], + newFlags: [], + removedFlags: [], + changedDefaults: [], + changedFlagTypes: [], + descriptionChanges: [], + ...details + } + }) + + const rcDiff = (cmdPath) => diffWith({ + newCommands: [{ path: cmdPath, description: 'Does things' }] + }) + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'whats-new-test-')) + whatsNewPath = path.join(tempDir, 'redpanda.adoc') + }) + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }) + }) + + test('creates a marked Redpanda CLI section when none exists', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n\n== Bug fixes\n\nStuff.\n') + updateWhatsNewFile(rcDiff('rpk topic new'), whatsNewPath, 'v2.0.1-rc1') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content).toContain('== Redpanda CLI') + expect(content).toContain('// AUTOGEN-RPK-CHANGES v2.0.1-rc1 START') + expect(content).toContain('`rpk topic new`') + // Inserted before Bug fixes + expect(content.indexOf('== Redpanda CLI')).toBeLessThan(content.indexOf('== Bug fixes')) + }) + + test('appends a second version block into the existing section', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n\n== Bug fixes\n\nStuff.\n') + updateWhatsNewFile(rcDiff('rpk topic new'), whatsNewPath, 'v2.0.1-rc1') + updateWhatsNewFile(rcDiff('rpk cluster newer'), whatsNewPath, 'v2.0.1-rc2') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content.match(/== Redpanda CLI/g)).toHaveLength(1) + expect(content).toContain('AUTOGEN-RPK-CHANGES v2.0.1-rc1 START') + expect(content).toContain('AUTOGEN-RPK-CHANGES v2.0.1-rc2 START') + expect(content).toContain('`rpk topic new`') + expect(content).toContain('`rpk cluster newer`') + // Both blocks stay inside the CLI section, before Bug fixes + expect(content.indexOf('rc2 START')).toBeLessThan(content.indexOf('== Bug fixes')) + }) + + test('re-running the same version replaces its block instead of duplicating', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n') + updateWhatsNewFile(rcDiff('rpk topic old-name'), whatsNewPath, 'v2.0.1-rc1') + updateWhatsNewFile(rcDiff('rpk topic renamed'), whatsNewPath, 'v2.0.1-rc1') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content.match(/AUTOGEN-RPK-CHANGES v2\.0\.1-rc1 START/g)).toHaveLength(1) + expect(content).toContain('`rpk topic renamed`') + expect(content).not.toContain('`rpk topic old-name`') + }) + + test('preserves a hand-written CLI section and appends after it', () => { + fs.writeFileSync(whatsNewPath, [ + '= What\'s New', + '', + '== Redpanda CLI (rpk)', + '', + '* *Hand-curated entry*: writers wrote this.', + '', + '== Bug fixes', + '', + 'Stuff.', + '' + ].join('\n')) + + updateWhatsNewFile(rcDiff('rpk topic new'), whatsNewPath, 'v2.0.1-rc3') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content).toContain('* *Hand-curated entry*: writers wrote this.') + expect(content).toContain('AUTOGEN-RPK-CHANGES v2.0.1-rc3 START') + // Appended inside the CLI section (before Bug fixes), after manual prose + expect(content.indexOf('Hand-curated entry')).toBeLessThan(content.indexOf('AUTOGEN-RPK-CHANGES')) + expect(content.indexOf('AUTOGEN-RPK-CHANGES v2.0.1-rc3 END')).toBeLessThan(content.indexOf('== Bug fixes')) + }) + + test('plugin runs write to a separate rpk plugins section without xrefs', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n\n== Redpanda CLI\n\n* Core entry.\n') + updateWhatsNewFile(rcDiff('rpk ai gateway'), whatsNewPath, 'ai plugin 0.3.0', { + xrefs: false, + sectionHeading: '== rpk plugins' + }) + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content).toContain('== rpk plugins') + expect(content).toContain('AUTOGEN-RPK-CHANGES ai plugin 0.3.0 START') + expect(content).toContain('=== ai plugin 0.3.0') + expect(content).toContain('`rpk ai gateway`') + expect(content).not.toContain('xref:') + // Core section untouched + expect(content).toContain('* Core entry.') + const cliIdx = content.indexOf('== Redpanda CLI') + expect(content.indexOf('AUTOGEN-RPK-CHANGES')).toBeGreaterThan(cliIdx) + }) + + test('block headings carry the version so accumulated blocks never collide', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n') + updateWhatsNewFile(rcDiff('rpk topic a'), whatsNewPath, 'v2.0.1-rc1') + updateWhatsNewFile(rcDiff('rpk topic b'), whatsNewPath, 'v2.0.1-rc2') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content).toContain('=== v2.0.1-rc1') + expect(content).toContain('=== v2.0.1-rc2') + // Category headings nest under the version heading + expect(content).toContain('==== New commands') + expect(content).not.toMatch(/^=== New commands$/m) + }) + + test('writes deprecations from the diff', () => { + fs.writeFileSync(whatsNewPath, '= What\'s New\n') + updateWhatsNewFile(diffWith({ + newlyDeprecatedCommands: [{ + path: 'rpk redpanda admin', + message: 'use `rpk cluster` subcommands', + replacement: 'Use xref:reference:rpk/rpk-cluster/rpk-cluster.adoc[`rpk cluster`] instead.', + hidden: true + }] + }), whatsNewPath, 'v2.0.0') + + const content = fs.readFileSync(whatsNewPath, 'utf8') + expect(content).toContain('==== Deprecated commands') + expect(content).toContain('rpk-cluster.adoc') + }) +}) + +describe('linkable predicate coverage', () => { + const { makeLinkablePredicate } = require('../../../tools/rpk-docs/rpk-docs-handler.js') + + test('cloud and security-secret commands are never linkable', () => { + const linkable = makeLinkablePredicate(null) + expect(linkable('rpk cloud auth list')).toBe(false) + expect(linkable('rpk security secret create')).toBe(false) + expect(linkable('rpk topic create')).toBe(true) + }) +}) diff --git a/bin/doc-tools.js b/bin/doc-tools.js index 1772f31b..b424260f 100755 --- a/bin/doc-tools.js +++ b/bin/doc-tools.js @@ -1109,6 +1109,17 @@ automation .option('-r, --ref ', 'Git branch or tag to document (e.g., dev, v26.2.0). Clones from GitHub.') .option('--from-source ', 'Path to local rpk source (src/go/rpk directory)') .option('--from-json ', 'Regenerate docs from an existing versioned JSON file (skips building)') + .option('--plugin ', 'Refresh a single rpk plugin\'s docs (ai, connect, k8s, check). Requires --from-json. Installs the plugin, splices its fresh subtree into the snapshot, and re-renders.') + .option('--plugin-version ', 'Plugin version to install and record (for example, 4.102.0). Defaults to the latest published version.') + .option('--plugin-pin ', 'Pin a plugin version for the installs during full generation (repeatable, for example --plugin-pin k8s=26.3.1-beta.1). Required for pre-GA plugins with no promoted latest version.', (value, pins) => { + const eq = value.indexOf('=') + if (eq < 1 || eq === value.length - 1) { + throw new Error(`Invalid --plugin-pin '${value}': expected =`) + } + pins[value.slice(0, eq)] = value.slice(eq + 1) + return pins + }, {}) + .option('--rpk-bin ', 'Path to an existing rpk binary for the plugin refresh (skips download/build)') .option('--overrides ', 'Path to overrides JSON file', 'docs-data/rpk-overrides.json') .option('--diff ', 'Generate diff against previous version') .option('--update-whats-new [path]', 'Update what\'s-new file with rpk changes from diff (default: modules/get-started/pages/release-notes/redpanda.adoc)') @@ -1124,6 +1135,12 @@ automation try { const { handleRpkDocsGeneration } = require('../tools/rpk-docs/rpk-docs-handler.js') + if (options.plugin && !options.fromJson) { + console.error('Error: --plugin requires --from-json ') + console.error('A plugin refresh splices the fresh subtree into an existing committed snapshot.') + process.exit(1) + } + // Handle --update-whats-new with optional path let whatsNewPath = null if (options.updateWhatsNew !== undefined) { @@ -1137,6 +1154,10 @@ automation ref: options.ref, fromSource: options.fromSource, fromJson: options.fromJson, + plugin: options.plugin, + pluginVersion: options.pluginVersion, + pluginPins: options.pluginPin, + rpkBin: options.rpkBin, overrides: options.overrides, diff: options.diff, updateWhatsNew: whatsNewPath, @@ -1150,6 +1171,10 @@ automation }) if (result.success) { + if (result.skipped) { + console.log(`\n✓ Skipped: ${result.reason}`) + process.exit(0) + } console.log('\n✓ rpk documentation generated successfully') // Write PR summary to file if requested (useful for GitHub Actions) @@ -1175,6 +1200,140 @@ automation } }) +/** + * generate rpk-plugin-stubs + * + * @description + * Reconciles a consumer repo's single-source stub pages and nav section + * against the rpk plugin partials generated in the docs repo. Run from the + * consumer repo root (for example, adp-docs for rpk ai). Creates stubs for + * new partials, deletes managed stubs whose partial is gone, rebuilds the + * plugin's nav block, and proposes page aliases for likely renames. + * Full reconcile, so it is idempotent and heals pre-existing drift. + */ +automation + .command('rpk-plugin-stubs') + .description('Reconcile single-source stub pages and nav against the docs repo\'s rpk plugin partials. Run from the consumer repo root.') + .option('--plugin ', 'rpk plugin command name', 'ai') + .option('--docs-repo ', 'Docs repo that owns the partials', 'redpanda-data/docs') + .option('--docs-ref ', 'Branch or tag to read partials from', 'main') + .option('--partials-dir ', 'Local partials directory (skips cloning the docs repo)') + .option('--source-path ', 'Path in the docs repo to read from (default: modules/reference/partials/rpk-; use modules/reference/pages/rpk/rpk-connect for page-family content)') + .option('--stub-dir ', 'Stub pages directory in the consumer repo (default: modules/reference/pages/rpk/rpk-)') + .option('--nav-file ', 'Nav file whose plugin block is rebuilt', 'modules/ROOT/nav.adoc') + .option('--include-prefix ', 'Antora resource prefix for stub includes. Default: inferred from an existing stub.') + .option('--attribute ', 'Page attribute line added to new stubs (repeatable)', (value, acc) => { acc.push(value); return acc }, []) + .option('--summary-file ', 'Write a markdown summary (for PR bodies)') + .option('--dry-run', 'Report what would change without writing') + .action(async (options) => { + try { + const { + readPartialTitles, fetchPartialsDir, inferIncludePrefix, reconcileStubs + } = require('../tools/rpk-docs/generate-plugin-stubs.js') + + const plugin = options.plugin + const stubDir = options.stubDir || `modules/reference/pages/rpk/rpk-${plugin}` + const partialsDir = options.partialsDir || fetchPartialsDir({ + docsRepo: options.docsRepo, + docsRef: options.docsRef, + plugin, + sourcePath: options.sourcePath + }) + + const includePrefix = options.includePrefix || inferIncludePrefix(stubDir, plugin) + if (!includePrefix) { + console.error('Error: could not infer the include prefix (no existing stubs). Pass --include-prefix, for example: streaming:reference:partial$rpk-ai/') + process.exit(1) + } + + const partials = readPartialTitles(partialsDir) + console.log(`Reconciling ${partials.length} partial(s) against ${stubDir}`) + + const result = reconcileStubs({ + partials, + stubDir, + navFile: options.navFile, + plugin, + includePrefix, + ...(options.attribute.length > 0 ? { attributes: options.attribute } : {}), + dryRun: options.dryRun + }) + + console.log(` Created: ${result.created.length}, deleted: ${result.deleted.length}, nav updated: ${result.navUpdated}`) + for (const f of result.created) console.log(` + ${f}`) + for (const f of result.deleted) console.log(` - ${f}`) + for (const f of result.keptNonStub) console.log(` ! kept (not a managed stub): ${f}`) + + const lines = [] + lines.push(`## rpk ${plugin} stub reconciliation`) + lines.push('') + lines.push(`Reconciled against \`${options.partialsDir ? partialsDir : `${options.docsRepo}@${options.docsRef}`}\`.`) + lines.push('') + if (result.created.length + result.deleted.length === 0 && !result.navUpdated) { + lines.push('No changes: stubs and nav already match the partials.') + } + if (result.created.length > 0) { + lines.push(`### New stubs (${result.created.length})`) + lines.push('') + result.created.forEach(f => lines.push(`- \`${f}\``)) + lines.push('') + } + if (result.deleted.length > 0) { + lines.push(`### Deleted stubs (${result.deleted.length})`) + lines.push('') + result.deleted.forEach(f => lines.push(`- \`${f}\``)) + lines.push('') + } + if ((result.skippedAliasTargets || []).length > 0) { + lines.push('### Skipped: names claimed as page aliases') + lines.push('') + lines.push('These partials exist upstream, but a page here already claims the name as a `:page-aliases:` target — creating the stub would make the Antora build fatal. Usually this means a rename alias exists while the upstream partial for the old name has not been cleaned up yet:') + lines.push('') + for (const t of result.skippedAliasTargets) { + lines.push(`- \`${t.file}\` (claimed by \`${t.claimedBy}\`)`) + } + lines.push('') + } + const straightDeletions = result.deleted.filter(d => !result.renameCandidates.some(rc => rc.deleted === d)) + if (straightDeletions.length > 0) { + lines.push('### Deletions with no rename partner') + lines.push('') + lines.push('These pages were removed with no successor detected. Their published URLs will 404 — consider adding a redirect or an alias on a related page:') + lines.push('') + straightDeletions.forEach(f => lines.push(`- \`${f}\``)) + lines.push('') + } + if (result.renameCandidates.length > 0) { + lines.push('### Possible renames — reviewer decision needed') + lines.push('') + lines.push('These deleted/created pairs look like renames. If so, add `:page-aliases:` for the old page name to the new stub so published URLs keep working:') + lines.push('') + for (const rc of result.renameCandidates) { + lines.push(`- \`${rc.deleted}\` → \`${rc.created}\`: add \`:page-aliases: reference:rpk/rpk-${plugin}/${rc.deleted}\` to the new stub`) + } + lines.push('') + } + if (result.keptNonStub.length > 0) { + lines.push('### Kept (not managed stubs)') + lines.push('') + lines.push('These pages do not match the managed stub shape, so they were not touched:') + lines.push('') + result.keptNonStub.forEach(f => lines.push(`- \`${f}\``)) + lines.push('') + } + const summary = lines.join('\n') + if (options.summaryFile) { + fs.writeFileSync(options.summaryFile, summary, 'utf8') + console.log(`Summary written to: ${options.summaryFile}`) + } + + process.exit(0) + } catch (err) { + console.error(`Error: ${err.message}`) + process.exit(1) + } + }) + /** * validate rpk-overrides * diff --git a/docs-data/rpk-overrides.schema.json b/docs-data/rpk-overrides.schema.json index be9cf77d..7bd6d734 100644 --- a/docs-data/rpk-overrides.schema.json +++ b/docs-data/rpk-overrides.schema.json @@ -36,6 +36,10 @@ "description": "Regular expression flags. Default: 'g' (global). Common: 'gi' (global, case-insensitive)", "default": "g", "pattern": "^[gimsuvy]*$" + }, + "applyToCode": { + "type": "boolean", + "description": "Also apply this replacement inside code blocks captured from help text (verbatim command examples). Default false: code blocks are protected from prose transformations. Use for binary-name rewrites like rpai to rpk ai." } }, "required": ["pattern", "replacement"], diff --git a/package-lock.json b/package-lock.json index f12ec71f..8109f504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "5.2.5", + "version": "5.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "5.2.5", + "version": "5.3.0", "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", diff --git a/package.json b/package.json index cf952d1e..cad0beaf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@redpanda-data/docs-extensions-and-macros", - "version": "5.2.5", + "version": "5.3.0", "description": "Antora extensions and macros developed for Redpanda documentation.", "keywords": [ "antora", diff --git a/tools/rpk-docs/generate-plugin-stubs.js b/tools/rpk-docs/generate-plugin-stubs.js new file mode 100644 index 00000000..b755b467 --- /dev/null +++ b/tools/rpk-docs/generate-plugin-stubs.js @@ -0,0 +1,291 @@ +'use strict' + +/** + * Stub reconciler for consumer repos that publish rpk plugin docs through + * single-source stubs (adp-docs for rpk ai). + * + * The docs repo owns the generated partials; consumer repos own one static + * stub page per command plus a nav entry. When a plugin release adds or + * removes commands, the stubs drift: a new partial has no stub (command + * invisible on the consumer site) and a deleted partial leaves a stub with an + * unresolved include (broken page). This module reconciles the stub set + * against the current partials rather than applying a diff, so it also heals + * pre-existing drift and is idempotent. + */ + +const fs = require('fs') +const path = require('path') +const os = require('os') +const { spawnSync } = require('child_process') + +/** + * Read command titles from generated partials. The title line is + * authoritative: dashified filenames cannot be reversed unambiguously + * (rpk-ai-llm-provider could be `llm provider` or `llm-provider`). + * @param {string} partialsDir - Directory of generated .adoc partials + * @returns {Array<{file: string, title: string}>} Sorted by command path + */ +function readPartialTitles(partialsDir) { + const partials = [] + for (const file of fs.readdirSync(partialsDir)) { + if (!file.endsWith('.adoc')) continue + const content = fs.readFileSync(path.join(partialsDir, file), 'utf8') + const match = content.match(/^= (.+)$/m) + if (!match) { + console.warn(`Warning: no title line in ${file}; skipping`) + continue + } + partials.push({ file, title: match[1].trim() }) + } + // Hierarchical order: sort by command words so parents precede children + partials.sort((a, b) => { + const aw = a.title.split(' ') + const bw = b.title.split(' ') + for (let i = 0; i < Math.max(aw.length, bw.length); i++) { + if (aw[i] === bw[i]) continue + if (aw[i] === undefined) return -1 + if (bw[i] === undefined) return 1 + return aw[i] < bw[i] ? -1 : 1 + } + return 0 + }) + return partials +} + +/** + * Sparse-clone the docs repo and return the path to a plugin's partials dir. + * @param {Object} params + * @param {string} params.docsRepo - owner/repo (e.g. redpanda-data/docs) + * @param {string} params.docsRef - Branch or tag to read (e.g. main) + * @param {string} params.plugin - Plugin command name (e.g. ai) + * @returns {string} Local path to the partials directory + */ +function fetchPartialsDir({ docsRepo, docsRef, plugin, sourcePath }) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-stubs-')) + const repoDir = path.join(tmpDir, 'docs') + // ai and cloud content renders as partials; connect renders as pages — + // both carry the single-source tag, so either family can be stubbed + const sparsePath = sourcePath || `modules/reference/partials/rpk-${plugin}` + + console.log(`Fetching ${sparsePath} from ${docsRepo}@${docsRef}...`) + const cloneResult = spawnSync('git', [ + 'clone', '--depth', '1', '--filter=blob:none', '--sparse', + '--branch', docsRef, + `https://github.com/${docsRepo}.git`, repoDir + ], { encoding: 'utf8', timeout: 180000 }) + if (cloneResult.status !== 0) { + throw new Error(`Failed to clone ${docsRepo}@${docsRef}: ${cloneResult.stderr}`) + } + + const sparseResult = spawnSync('git', ['sparse-checkout', 'set', sparsePath], { + cwd: repoDir, encoding: 'utf8', timeout: 60000 + }) + if (sparseResult.status !== 0) { + throw new Error(`Failed sparse checkout of ${sparsePath}: ${sparseResult.stderr}`) + } + + const partialsDir = path.join(repoDir, sparsePath) + if (!fs.existsSync(partialsDir)) { + throw new Error(`Partials directory not found in ${docsRepo}@${docsRef}: ${sparsePath}`) + } + return partialsDir +} + +/** + * Infer the include prefix from an existing managed stub, so the reconciler + * follows whatever component/module coordinates the consumer repo uses. + * @param {string} stubDir - Consumer repo stub directory + * @param {string} plugin - Plugin command name + * @returns {string|null} e.g. "streaming:reference:partial$rpk-ai/" + */ +function inferIncludePrefix(stubDir, plugin) { + if (!fs.existsSync(stubDir)) return null + for (const file of fs.readdirSync(stubDir)) { + if (!file.endsWith('.adoc')) continue + const content = fs.readFileSync(path.join(stubDir, file), 'utf8') + const match = content.match(new RegExp(`include::([^\\[]*(?:partial|page)\\$[^\\[]*rpk-${plugin}/)`)) + if (match) return match[1] + } + return null +} + +/** + * Render a stub page. + * @param {Object} params + * @param {string} params.title - Command path (e.g. "rpk ai auth login") + * @param {string} params.file - Partial filename + * @param {string} params.includePrefix - Antora resource prefix + * @param {Array} params.attributes - Page attribute lines + * @returns {string} + */ +function renderStub({ title, file, includePrefix, attributes }) { + const lines = [`= ${title}`] + for (const attr of attributes) lines.push(attr) + lines.push('') + lines.push(`include::${includePrefix}${file}[tag=single-source]`) + lines.push('') + return lines.join('\n') +} + +/** + * Reconcile a consumer repo's stub pages and nav section against the + * current set of generated partials. + * @param {Object} params + * @param {Array<{file: string, title: string}>} params.partials + * @param {string} params.stubDir - Consumer stub directory (created if missing) + * @param {string} params.navFile - Consumer nav.adoc path + * @param {string} params.plugin - Plugin command name (e.g. ai) + * @param {string} params.includePrefix - Antora resource prefix for includes + * @param {Array} [params.attributes] - Page attributes for new stubs + * @param {boolean} [params.dryRun] + * @returns {Object} { created, deleted, keptNonStub, navUpdated, renameCandidates } + */ +function reconcileStubs({ + partials, + stubDir, + navFile, + plugin, + includePrefix, + attributes = [':page-preview: true'], + dryRun = false +}) { + const managedStubRe = new RegExp(`include::[^\\[]*(?:partial|page)\\$[^\\[]*rpk-${plugin}/([\\w.-]+\\.adoc)\\[`) + const partialByFile = new Map(partials.map(p => [p.file, p])) + + fs.mkdirSync(stubDir, { recursive: true }) + const existingStubs = fs.readdirSync(stubDir).filter(f => f.endsWith('.adoc')) + + const created = [] + const deleted = [] + const keptNonStub = [] + const skippedAliasTargets = [] + + // Page names already claimed as aliases by other pages in this directory. + // Creating a page whose resource ID is an alias target makes the Antora + // build fatal ("Page alias cannot reference an existing page"), which + // happens when a rename alias exists here while the upstream partial for + // the old name still lingers. Skip those creations and surface them. + const aliasClaims = new Map() + for (const file of existingStubs) { + const content = fs.readFileSync(path.join(stubDir, file), 'utf8') + const aliasLine = content.match(/^:page-aliases:\s*(.+)$/m) + if (!aliasLine) continue + for (const target of aliasLine[1].split(',')) { + const base = target.trim().split('/').pop() + if (base) aliasClaims.set(base, file) + } + } + + // Delete managed stubs whose partial no longer exists. Pages that do not + // match the managed-stub shape are never deleted: they may be hand-written. + const deletedTitles = new Map() + for (const file of existingStubs) { + const stubPath = path.join(stubDir, file) + const content = fs.readFileSync(stubPath, 'utf8') + const match = content.match(managedStubRe) + if (!match) { + if (!partialByFile.has(file)) keptNonStub.push(file) + continue + } + if (!partialByFile.has(match[1])) { + const titleMatch = content.match(/^= (.+)$/m) + deletedTitles.set(file, titleMatch ? titleMatch[1].trim() : '') + if (!dryRun) fs.unlinkSync(stubPath) + deleted.push(file) + } + } + + // Create stubs for partials that have none + const remainingStubs = new Set( + fs.existsSync(stubDir) ? fs.readdirSync(stubDir).filter(f => f.endsWith('.adoc')) : [] + ) + for (const partial of partials) { + if (remainingStubs.has(partial.file)) continue + if (aliasClaims.has(partial.file)) { + skippedAliasTargets.push({ file: partial.file, claimedBy: aliasClaims.get(partial.file) }) + continue + } + if (!dryRun) { + fs.writeFileSync( + path.join(stubDir, partial.file), + renderStub({ ...partial, includePrefix, attributes }), + 'utf8' + ) + } + created.push(partial.file) + } + + // Rename candidates: same parent command, same depth, related last words + // (llm -> llm-provider). Proposed for the reviewer, who decides whether + // the new stub gets a page alias. + const renameCandidates = [] + for (const [dFile, dTitle] of deletedTitles) { + if (!dTitle) continue + const dWords = dTitle.split(' ') + for (const cFile of created) { + const cTitle = (partialByFile.get(cFile) || {}).title || '' + const cWords = cTitle.split(' ') + if (cWords.length !== dWords.length) continue + if (cWords.slice(0, -1).join(' ') !== dWords.slice(0, -1).join(' ')) continue + const dLast = dWords[dWords.length - 1] + const cLast = cWords[cWords.length - 1] + if (cLast.startsWith(dLast) || dLast.startsWith(cLast)) { + renameCandidates.push({ deleted: dFile, created: cFile }) + } + } + } + + // Rebuild the plugin's nav block: keep the labeled parent line, regenerate + // child entries from titles at star depth = parent depth + (words - 2) + let navUpdated = false + if (navFile && fs.existsSync(navFile)) { + const navLines = fs.readFileSync(navFile, 'utf8').split('\n') + const parentRe = new RegExp(`^(\\*+) xref:[^\\[]*rpk-${plugin}/rpk-${plugin}\\.adoc\\[`) + const parentIdx = navLines.findIndex(l => parentRe.test(l)) + if (parentIdx === -1) { + console.warn(`Warning: no rpk-${plugin} parent entry found in ${navFile}; nav not updated`) + } else { + const parentStars = navLines[parentIdx].match(parentRe)[1].length + const navPathPrefix = navLines[parentIdx].match(/xref:([^\[]*rpk-\w+\/)/)[1] + + // The block ends at the first line that is not a deeper entry + let end = parentIdx + 1 + while (end < navLines.length) { + const starMatch = navLines[end].match(/^(\*+) /) + if (!starMatch || starMatch[1].length <= parentStars) break + end++ + } + + const skippedFiles = new Set(skippedAliasTargets.map(t => t.file)) + const entries = [] + for (const partial of partials) { + if (skippedFiles.has(partial.file)) continue + const words = partial.title.split(' ').length + if (words <= 2) continue // the parent line represents the root command + const stars = '*'.repeat(parentStars + (words - 2)) + entries.push(`${stars} xref:${navPathPrefix}${partial.file}[]`) + } + + const rebuilt = [ + ...navLines.slice(0, parentIdx + 1), + ...entries, + ...navLines.slice(end) + ].join('\n') + + if (rebuilt !== navLines.join('\n')) { + if (!dryRun) fs.writeFileSync(navFile, rebuilt, 'utf8') + navUpdated = true + } + } + } + + return { created, deleted, keptNonStub, navUpdated, renameCandidates, skippedAliasTargets } +} + +module.exports = { + readPartialTitles, + fetchPartialsDir, + inferIncludePrefix, + renderStub, + reconcileStubs +} diff --git a/tools/rpk-docs/generate-rpk-docs.js b/tools/rpk-docs/generate-rpk-docs.js index e6ffb1c6..0fbc5840 100644 --- a/tools/rpk-docs/generate-rpk-docs.js +++ b/tools/rpk-docs/generate-rpk-docs.js @@ -778,6 +778,76 @@ function convertIndentedCodeBlocksToAsciiDoc(text, codeBlockStore = []) { while (i < lines.length) { const line = lines[i] + // Unindented shell example: "$ rpk ..." at column 0 followed by its + // sample output on the contiguous lines below. Rendered as a command + // block plus a no-copy output block so the output's =-underlined titles + // and aligned columns are never parsed as prose or headings. + if (/^\$\s/.test(line)) { + const command = line.replace(/^\$\s+/, '') + let j = i + 1 + const outputLines = [] + while (j < lines.length && lines[j].trim() !== '' && !/^\$\s/.test(lines[j])) { + outputLines.push(lines[j]) + j++ + } + let block = `\n[,bash]\n----\n${command}\n----\n` + if (outputLines.length > 0) { + block += `\n[.no-copy]\n----\n${outputLines.join('\n')}\n----\n` + } + const placeholder = `__EARLY_CODE_BLOCK_${codeBlockStore.length}__` + codeBlockStore.push(block) + result.push('') + result.push(placeholder) + result.push('') + i = j + continue + } + + // Colon-introduced code sample: prose ending with ":" followed by an + // indented block that is not a list (Cedar policies, config snippets). + // Deeply indented (4+ space) blocks are literals by help-text convention + // even without a colon introducer (path templates like + // " kafka/{topic}/{partition}_{revision}/" would otherwise render as + // prose whose braces Asciidoctor eats as attribute references). + // Captured verbatim (dedented) so inline-code transforms never touch it. + const prevNonBlank = [...result].reverse().find(l => l.trim() !== '') + // Only a chunk that starts after a blank line is a standalone literal; + // a deeply indented line mid-chunk is a wrapped continuation of a table + // row or list item and belongs to the converters below. + const atChunkStart = result.length === 0 || result[result.length - 1].trim() === '' + if ( + ( + (prevNonBlank && /:\s*$/.test(prevNonBlank) && /^[ ]{2,}\S/.test(line)) || + (atChunkStart && /^[ ]{4,}\S/.test(line)) + ) && + !/^[ ]{2,}(-|\*|\d+[.)])\s/.test(line) && + !/^[ ]{2,}(--|rpk\s|\$\s)/.test(line) + ) { + const blockLines = [] + let j = i + while (j < lines.length && (/^[ ]{2,}\S/.test(lines[j]) || lines[j].trim() === '')) { + if (lines[j].trim() === '' && (j + 1 >= lines.length || !/^[ ]{2,}\S/.test(lines[j + 1]))) break + blockLines.push(lines[j]) + j++ + } + // Column-aligned layouts (two or more lines with a run of spaces + // separating columns) are definition tables, not code: leave them for + // the indented-table/YAML converters below. + const alignedLines = blockLines.filter(l => /\S\s{2,}\S/.test(l.trim())).length + if (alignedLines < 2) { + const indent = Math.min(...blockLines.filter(l => l.trim() !== '').map(l => l.search(/\S/))) + const dedented = blockLines.map(l => l.slice(indent)).join('\n') + const codeBlock = `\n[,text]\n----\n${dedented}\n----\n` + const placeholder = `__EARLY_CODE_BLOCK_${codeBlockStore.length}__` + codeBlockStore.push(codeBlock) + result.push('') + result.push(placeholder) + result.push('') + i = j + continue + } + } + // Check for indented command example: 2+ spaces then --, rpk, or $ (shell prompt) // e.g. " --job-name test --labels ..." // e.g. " rpk cluster info" @@ -1240,10 +1310,13 @@ function convertIndentedTablesToAsciiDoc(text, options = {}) { * @param {Object|null} customTransformations * @returns {string} */ -function applyTextTransformations(text, customTransformations) { +function applyTextTransformations(text, customTransformations, options = {}) { if (!text || !customTransformations?.replacements) return text let result = text for (const rule of customTransformations.replacements) { + // Code blocks are verbatim: only rules explicitly marked applyToCode + // (like the rpai -> rpk ai binary-name rewrite) may touch them. + if (options.code && !rule.applyToCode) continue try { const flags = rule.flags || 'g' result = result.replace(new RegExp(rule.pattern, flags), rule.replacement) @@ -1254,6 +1327,24 @@ function applyTextTransformations(text, customTransformations) { return result } +/** + * Apply text transformations to examples content line by line. Indented + * lines are verbatim commands: only rules flagged applyToCode may touch + * them (a caption rule once rewrote '{"quotas":...}' inside a command to + * '{`quotas`:...}'). Caption lines get the full rule set. + * @param {string} text + * @param {Object|null} customTransformations + * @returns {string} + */ +function applyTextTransformationsToExamples(text, customTransformations) { + if (!text || !customTransformations?.replacements) return text + return text.split('\n').map(line => + /^[ ]{2,}\S/.test(line) + ? applyTextTransformations(line, customTransformations, { code: true }) + : applyTextTransformations(line, customTransformations) + ).join('\n') +} + /** * Format description by adding backticks around flags and code * @param {string} desc - Description text @@ -1470,9 +1561,19 @@ function formatDescription(desc, customTransformations = null, options = {}) { // Also handles '--flag/-f', '--flag help', and similar patterns .replace(/'(--[a-z][-a-z0-9]*(?:\/-[a-z])?(?:\s+\w+)?)'/gi, '$1') .replace(/'(-[a-z])'/gi, '$1') + // Drop a dangling example lead-in with nothing after it (upstream help + // strings sometimes end mid-example, which would render as "for example,.") + .replace(/[,;]?\s*\be\.g\.[\s,]*$/i, '') + // Stray space before a closing parenthesis (upstream help typo) + .replace(/ +\)/g, ')') // === STYLE GUIDE COMPLIANCE === - .replace(/\be\.g\.\s*/gi, 'for example, ') - .replace(/\bi\.e\.\s*/gi, 'that is, ') + // "e.g.:" introduces a block; keep the colon instead of rendering ", :". + // Horizontal whitespace only: crossing a newline would glue the next + // line (often a code-block placeholder) onto this sentence. + .replace(/,?[^\S\n]*\be\.g\.:[^\S\n]*/gi, ', for example: ') + .replace(/, for example: $/gm, ', for example:') + .replace(/\be\.g\.[^\S\n]*/gi, 'for example, ') + .replace(/\bi\.e\.[^\S\n]*/gi, 'that is, ') // === TYPO CORRECTIONS === // Fix "an" before consonant sounds (common source typos) .replace(/\ban\s+(prod|dev|test|local|remote|new|cluster|config|file)\b/gi, 'a $1') @@ -1592,12 +1693,23 @@ function formatDescription(desc, customTransformations = null, options = {}) { // e.g., "(see #2904)" → "(see https://github.com/redpanda-data/redpanda/issues/2904[#2904])" result = result.replace(/(? + idx % 2 === 1 ? segment : segment.replace(/(? + token === 'vbar' ? match : `\\{${token}}`) + ).join('') + // Restore early code blocks FIRST (from indented command/YAML detection) // This must happen before inline code restoration so that placeholders // inside the early code blocks (like __INLINE_CODE_X__) get resolved // Use function replacements to prevent $ special-pattern interpretation (e.g. `$` → before-match) earlyCodeBlocks.forEach((block, i) => { - result = result.replace(`__EARLY_CODE_BLOCK_${i}__`, () => block) + const transformed = applyTextTransformations(block, customTransformations, { code: true }) + result = result.replace(`__EARLY_CODE_BLOCK_${i}__`, () => transformed) }) // Restore inline code @@ -1807,11 +1919,37 @@ function parseDescriptionSections(desc) { let currentContent = [] const mainLines = [] - for (const line of lines) { + let skipNext = false + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + if (skipNext) { + skipNext = false + continue + } // Check for ALL CAPS header (at least 2 chars, possibly with spaces, hyphens, slashes, or ampersands) // Examples: "FIELDS", "BALANCER STATUS", "PRODUCER ID & EPOCH" - const headerMatch = line.match(/^([A-Z][A-Z\s\-\/&]{0,}[A-Z])$/) + // A line with runs of multiple spaces is a column-header row of aligned + // sample output (for example "PARTITION REASON"), not a section + // header, so leave it in the content. + let headerMatch = / /.test(line) ? null : line.match(/^([A-Z][A-Z\s\-\/&]{0,}[A-Z])$/) + // An all-caps line directly after a "$ command" invocation is the title + // of that command's sample output (for example "DECOMMISSION PROGRESS"), + // not a section header. Splitting there would strand the invocation in + // one section and its output table in another. + if (headerMatch) { + for (let p = i - 1; p >= 0; p--) { + if (lines[p].trim() === '') break + if (/^\$\s/.test(lines[p])) { headerMatch = null; break } + } + } if (headerMatch) { + // Help text often underlines section titles with a run of = or - + // characters. Consume the underline: left in the content, a line of + // 4+ = or - is an AsciiDoc block delimiter and breaks the page + // (unterminated example block). + if (i + 1 < lines.length && /^\s*(={3,}|-{3,})\s*$/.test(lines[i + 1])) { + skipNext = true + } // Save previous section if (currentSection) { // Preserve indentation: only remove leading/trailing blank lines, not spaces @@ -1832,6 +1970,13 @@ function parseDescriptionSections(desc) { sections[currentSection] = trimBlankLines(currentContent.join('\n')) } + const delimiterRun = /^\s*(={4,}|-{4,})\s*$/ + for (const [name, body] of Object.entries(sections)) { + if (body.split('\n').some(l => delimiterRun.test(l))) { + console.warn(`Warning: section "${name}" contains a bare =/- delimiter run; it may break AsciiDoc rendering`) + } + } + return { mainDescription: mainLines.join('\n').trim(), sections @@ -1941,6 +2086,45 @@ function dashify(commandPath) { * @param {string} desc - Full description * @returns {string} Short description */ +/** + * Collapse runs of three or more newlines to a single blank line, except + * inside AsciiDoc delimited blocks (----, ====, ...., |===, and -- open + * blocks), where blank lines are content. + * @param {string} text - Rendered page content + * @returns {string} + */ +function collapseBlankLines(text) { + const lines = text.split('\n') + const out = [] + let inBlock = false + let blockDelim = null + let blankRun = 0 + + for (const line of lines) { + const trimmed = line.trimEnd() + const isDelim = /^(-{4,}|={4,}|\.{4,}|\|===|--)$/.test(trimmed) + if (isDelim) { + if (!inBlock) { + inBlock = true + blockDelim = trimmed + } else if (trimmed === blockDelim || (blockDelim === '|===' && trimmed === '|===')) { + inBlock = false + blockDelim = null + } + } + + if (!inBlock && trimmed === '') { + blankRun++ + if (blankRun > 1) continue + } else { + blankRun = 0 + } + out.push(line) + } + + return out.join('\n') +} + function capToTwoSentences(desc) { if (!desc) return '' if (typeof desc !== 'string') return String(desc) @@ -1949,6 +2133,19 @@ function capToTwoSentences(desc) { // Pattern: matches == Section Header and everything after it let cleaned = desc.replace(/\s*==\s+.+$/s, '') + // Delimited code/output blocks never belong in a summary. Indented help + // examples are converted to [,text]/[,bash]/[.no-copy] blocks before this + // runs. A block introduced by a colon ends the summary there (the prose + // after it would read as a non sequitur without the sample); otherwise + // drop the block and keep the surrounding prose. + const blockRe = /\n?\[[^\]\n]*\]\n----\n[\s\S]*?\n----|\n----\n[\s\S]*?\n----/ + const firstBlock = cleaned.match(blockRe) + if (firstBlock && /:\s*$/.test(cleaned.slice(0, firstBlock.index))) { + cleaned = cleaned.slice(0, firstBlock.index) + } else { + cleaned = cleaned.replace(new RegExp(blockRe.source, 'g'), '') + } + // Remove indented content (tables, lists, etc.) after colons before sentence detection // Pattern: text ending with colon, optionally followed by blank line, then indented content // Examples: @@ -1982,11 +2179,26 @@ function capToTwoSentences(desc) { }) } + // Protect decimal points in version-like numbers (OAuth 2.0, HTTP 1.1) so + // they neither split a sentence nor cause the leading fragment to be + // dropped by the sentence matcher + normalized = normalized.replace(/(\d)\.(\d)/g, '$1__DECIMAL__$2') + // Match sentences - const sentences = normalized.match(/[^.!?]+[.!?]+(?:\s|$)/g) + let sentences = normalized.match(/[^.!?]+[.!?]+(?:\s|$)/g) + + // Never drop a leading fragment: if the first matched sentence does not + // start at the beginning of the text (an unterminated prefix was skipped), + // glue the prefix back onto it + if (sentences && sentences.length > 0) { + const firstIdx = normalized.indexOf(sentences[0]) + if (firstIdx > 0) { + sentences = [normalized.slice(0, firstIdx) + sentences[0], ...sentences.slice(1)] + } + } if (!sentences || sentences.length === 0) { // Restore and return - let result = normalized + let result = normalized.replace(/__DECIMAL__/g, '.') placeholders.forEach(({ ph, original }) => { result = result.replace(ph, original) }) @@ -1998,6 +2210,9 @@ function capToTwoSentences(desc) { let result = sentences.slice(0, 2).join('') + // Restore decimal points + result = result.replace(/__DECIMAL__/g, '.') + // Restore abbreviations placeholders.forEach(({ ph, original }) => { result = result.replace(ph, original) @@ -2087,6 +2302,61 @@ function findTopLevelWithSubcommands(tree) { return result } +/** + * Known rpk plugins that are managed separately from rpk core (they have + * install/uninstall/upgrade shim commands, and their real command tree only + * appears when the plugin binary is installed in the generation environment). + * Shared with rpk-docs-handler.js, which iterates this list to install each + * plugin before running --print-tree. + */ +const KNOWN_PLUGINS = ['ai', 'check', 'connect', 'k8s', 'oxla'] + +/** + * Subcommands that rpk core ships as a built-in shim for managed plugins, + * present even when the plugin binary itself is not installed. + */ +const PLUGIN_SHIM_COMMANDS = new Set(['install', 'uninstall', 'upgrade']) + +/** + * Check whether a command path belongs to a protected plugin's subtree + * (including the plugin's own top-level command). + * @param {string} commandPath - Full command path, e.g. 'rpk k8s multicluster' + * @param {Array} protectedPlugins - Plugin names, e.g. ['k8s'] + * @returns {boolean} + */ +function isProtectedCommandPath(commandPath, protectedPlugins) { + return protectedPlugins.some(pl => commandPath === `rpk ${pl}` || commandPath.startsWith(`rpk ${pl} `)) +} + +/** + * Determine which managed plugins must be protected for this run. + * + * Auto-detects plugins whose commands are missing from this run's tree: + * rpk core ships only a shim (install/uninstall/upgrade) for managed + * plugins; the real commands appear only when the plugin binary is + * installed in the generation environment. If a known plugin's subtree is + * absent or shim-only, its existing pages reflect the plugin, not stale + * commands, so they must be preserved. Pre-GA windows hit this every time: + * the plugin publisher only promotes stable X.Y.Z releases, so `rpk + * install` finds nothing until GA (see redpanda-data/docs#1831). + * + * @param {Array} commands - Flat command list from flattenCommands() + * @param {Array} explicitProtected - Plugins the caller already knows + * failed to install this run (merged with auto-detection) + * @returns {Array} Deduplicated protected plugin names + */ +function detectProtectedPlugins(commands, explicitProtected = []) { + const autoProtected = KNOWN_PLUGINS.filter(pl => { + const subNames = commands + .filter(c => c.path.startsWith(`rpk ${pl} `)) + .map(c => c.path.split(' ')[2]) + const hasTopLevel = commands.some(c => c.path === `rpk ${pl}`) + if (!hasTopLevel) return true + return subNames.length === 0 || subNames.every(n => PLUGIN_SHIM_COMMANDS.has(n)) + }) + return [...new Set([...explicitProtected, ...autoProtected])] +} + /** * Static entries that always appear at the top of the rpk nav section, * before the auto-generated command entries. These are hand-written pages @@ -2095,6 +2365,10 @@ function findTopLevelWithSubcommands(tree) { * landing page (the section header), so it is skipped during generation. */ const STATIC_RPK_NAV_ENTRIES = [ + // The root rpk command page is generated but skipped by the entry loop + // (its path has no group segment), so it is listed here to keep it in + // the nav ahead of the hand-written pages. + '*** xref:reference:rpk/rpk.adoc[]', '*** xref:reference:rpk/rpk-commands.adoc[]', '*** xref:reference:rpk/rpk-x-options.adoc[rpk -X]', ] @@ -2109,7 +2383,7 @@ const STATIC_RPK_NAV_ENTRIES = [ * @param {Set} topLevelWithSubcommands - Set of top-level command names with subcommands * @returns {Object} { navUpdated, navEntriesGenerated } */ -function updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands) { +function updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands, protectedPlugins = []) { if (!fs.existsSync(navFile)) { console.warn(`Warning: nav file not found, skipping nav update: ${navFile}`) return { navUpdated: false } @@ -2140,6 +2414,10 @@ function updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcomm const newEntries = [] for (const { path: commandPath } of commands) { if (commandPath === 'rpk') continue + // Protected plugins get no generated entries: this run's tree has at + // most the shim (install/uninstall/upgrade) for them, so their previous + // nav block is preserved in place instead (below). + if (isProtectedCommandPath(commandPath, protectedPlugins)) continue if (shouldExcludeCommand(resolvedOverrides, commandPath)) continue if (shouldUsePartialDir(resolvedOverrides, commandPath)) continue if (commandPath.startsWith('rpk cloud')) continue @@ -2155,12 +2433,65 @@ function updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcomm newEntries.push(`${stars} xref:${xrefPath}[]`) } - // Rebuild: header + static entries + generated entries - const newSection = [ - lines[sectionStart], - ...STATIC_RPK_NAV_ENTRIES, - ...newEntries, - ] + // Preserve existing entries for protected plugins. Their commands are + // absent (or shim-only) in this run's tree, so nothing was generated for + // them above. Each plugin's previous nav block (parent entry plus nested + // children) is kept together and spliced back in at its original position, + // anchored to the nearest preceding entry that survives regeneration — + // never appended at the end, where the children would render under the + // wrong parent. + const stripStars = (line) => line.replace(/^\*+ /, '') + const emittedTargets = new Set([...STATIC_RPK_NAV_ENTRIES, ...newEntries].map(stripStars)) + // Map of anchor target (or null for "before all surviving entries") to the + // preserved lines that follow that anchor, in original order. + const preservedBlocks = new Map() + let preservedCount = 0 + const preservedPlugins = new Set() + if (protectedPlugins.length > 0) { + const patternsFor = (pl) => [`reference:rpk/rpk-${pl}/`, `reference:rpk/rpk-${pl}.adoc`] + const pluginFor = (line) => protectedPlugins.find(pl => patternsFor(pl).some(pat => line.includes(pat))) + let lastAnchor = null + for (const line of lines.slice(sectionStart + 1, sectionEnd)) { + const plugin = pluginFor(line) + if (plugin) { + // Never duplicate an entry that regeneration already emits. + if (emittedTargets.has(stripStars(line))) continue + if (!preservedBlocks.has(lastAnchor)) preservedBlocks.set(lastAnchor, []) + preservedBlocks.get(lastAnchor).push(line) + preservedCount++ + preservedPlugins.add(plugin) + } else if (emittedTargets.has(stripStars(line))) { + lastAnchor = stripStars(line) + } + } + if (preservedCount > 0) { + console.log(` Preserving ${preservedCount} nav entries for plugins: ${[...preservedPlugins].join(', ')}`) + } + } + + // Rebuild: header + static entries + generated entries, splicing each + // preserved block back in directly after its anchor entry. + const newSection = [lines[sectionStart]] + const pushEntry = (line) => { + newSection.push(line) + const target = stripStars(line) + if (preservedBlocks.has(target)) { + newSection.push(...preservedBlocks.get(target)) + preservedBlocks.delete(target) + } + } + // Blocks that preceded every surviving entry in the old nav go right after + // the section header, before the static entries. + if (preservedBlocks.has(null)) { + newSection.push(...preservedBlocks.get(null)) + preservedBlocks.delete(null) + } + for (const line of STATIC_RPK_NAV_ENTRIES) pushEntry(line) + for (const line of newEntries) pushEntry(line) + // Safety net: if an anchor vanished between collection and rebuild (it + // cannot with the logic above, but never silently drop preserved entries), + // append any remaining blocks at the end. + for (const block of preservedBlocks.values()) newSection.push(...block) const newLines = [ ...lines.slice(0, sectionStart), @@ -2187,7 +2518,12 @@ async function generateRpkDocs(options = {}) { pluginVersions = {}, draftMissing = false, flatOutput = false, // If true, output all files flat (legacy behavior) - navFile // Optional: path to nav.adoc for automatic nav updates + navFile, // Optional: path to nav.adoc for automatic nav updates + // Plugins that exist but could not be installed for this run (for example, + // rpk k8s before its GA plugin publishes). Their pages and nav entries are + // preserved instead of treated as stale, because their absence from the + // command tree reflects the generation environment, not the product. + protectedPlugins = [] } = options // Register partials @@ -2225,6 +2561,16 @@ async function generateRpkDocs(options = {}) { // Flatten command tree const commands = flattenCommands(tree) + // Determine which managed plugins are protected this run: explicitly + // passed by the caller (failed installs) merged with auto-detection of + // absent or shim-only plugin subtrees. Protection covers the whole + // pipeline — page writes, the stale-file sweep, and nav updates — so a + // run without the plugin binary leaves the plugin's docs fully untouched. + const effectiveProtectedPlugins = detectProtectedPlugins(commands, protectedPlugins) + if (effectiveProtectedPlugins.length > 0) { + console.log(` Protected plugins this run (pages and nav preserved, not regenerated): ${effectiveProtectedPlugins.join(', ')}`) + } + // Find top-level commands with subcommands (for directory structure) const topLevelWithSubcommands = findTopLevelWithSubcommands(tree) @@ -2246,6 +2592,18 @@ async function generateRpkDocs(options = {}) { continue } + // Protected plugins: skip generating the whole subtree, parent page + // included. This run's tree has at most the shim + // (install/uninstall/upgrade) for these plugins, so regenerating would + // overwrite full-plugin pages from an earlier successful run — for + // example, rewriting rpk-k8s.adoc with a Subcommands table reduced to + // the shim and orphaning the preserved child pages. Existing pages stay + // untouched, exactly as in the fully-absent-plugin case. + if (isProtectedCommandPath(commandPath, effectiveProtectedPlugins)) { + filesSkipped++ + continue + } + // Check if command should be excluded if (shouldExcludeCommand(resolvedOverrides, commandPath)) { filesSkipped++ @@ -2273,8 +2631,9 @@ async function generateRpkDocs(options = {}) { if (mergedCommand.excludeExamples && mergedCommand.excludeExamples.length > 0) { examplesContent = filterExamples(sectionContent, mergedCommand.excludeExamples) } - // Apply text transformations (e.g. rpai → rpk ai) then format - examplesContent = applyTextTransformations(examplesContent, textTransformations) + // Apply text transformations (e.g. rpai → rpk ai) then format. + // Command lines only get code-safe rules. + examplesContent = applyTextTransformationsToExamples(examplesContent, textTransformations) sections[sectionName] = formatExamples(examplesContent) } else { sections[sectionName] = formatDescription(sectionContent, textTransformations) @@ -2291,8 +2650,9 @@ async function generateRpkDocs(options = {}) { if (mergedCommand.excludeExamples && mergedCommand.excludeExamples.length > 0) { examplesContent = filterExamples(examplesContent, mergedCommand.excludeExamples) } - // Apply text transformations (e.g. rpai → rpk ai) then format - examplesContent = applyTextTransformations(examplesContent, textTransformations) + // Apply text transformations (e.g. rpai → rpk ai) then format. + // Command lines only get code-safe rules. + examplesContent = applyTextTransformationsToExamples(examplesContent, textTransformations) sections.EXAMPLES = formatExamples(examplesContent) } @@ -2365,9 +2725,16 @@ async function generateRpkDocs(options = {}) { // Build subcommands with correct xref paths // Filter out excluded and asPartial subcommands — excluded have no file, // asPartial ones live in the partials directory with no linkable xref. + // rpk cloud and rpk security secret are hardcoded-routed to partials + // (single-sourced into cloud docs), so they have no linkable pages + // either: rpk-security.adoc linking rpk-security-secret.adoc was a + // broken xref in the published site. const subcommands = (command.commands || []) .filter(sub => { const subPath = `${commandPath} ${sub.name}` + if (subPath.startsWith('rpk cloud') || subPath.startsWith('rpk security secret')) { + return false + } return !shouldExcludeCommand(resolvedOverrides, subPath) && !shouldUsePartialDir(resolvedOverrides, subPath) }) .map(sub => { @@ -2431,7 +2798,24 @@ async function generateRpkDocs(options = {}) { return `${parentPath} ${alias}` }), aliasNotes: commandOverride.aliasNotes, - flags: (mergedCommand.flags || []).map(flag => ({ + // A curated override section titled "Flags" wins over the extracted + // flag table: rendering both produced duplicate == Flags headings and + // conflicting content (rpk connect run). Curation is authoritative; + // the warning tells maintainers the override can be dropped to adopt + // the extracted table. + flags: (() => { + const hasCuratedFlagsSection = Object.values(processedContent.sections || {}) + .some(items => (items || []).some(item => /^flags$/i.test(item.title || ''))) + if (hasCuratedFlagsSection && (mergedCommand.flags || []).length > 0) { + console.warn( + `Warning: ${commandPath} has a curated "Flags" override section; ` + + `skipping the extracted flag table (${mergedCommand.flags.length} flags). ` + + 'Remove the override section to adopt the extracted table.' + ) + return [] + } + return mergedCommand.flags || [] + })().map(flag => ({ ...flag, name: flag.shorthand ? `-${flag.shorthand}, --${flag.name}` : `--${flag.name}`, type: flag.type || '-', @@ -2542,8 +2926,26 @@ async function generateRpkDocs(options = {}) { hasUnknownSections: unknownSections.length > 0 } - // Render template - const content = template(context) + // Render template, then collapse runs of blank lines left behind by + // absent optional sections (plugin commands have no aliases or flags, so + // the separators between skipped blocks stack up). Blank lines inside + // delimited blocks are preserved: they are content there. + const content = collapseBlankLines(template(context)) + + // Duplicate top-level headings almost always mean an override adds a + // section the page already renders (or embeds its own heading in raw + // content). The page still builds, but anchors collide and readers see + // the same section twice. + const h2Counts = new Map() + for (const line of content.split('\n')) { + const h2 = line.match(/^== (\S.*)$/) + if (h2) h2Counts.set(h2[1], (h2Counts.get(h2[1]) || 0) + 1) + } + for (const [heading, count] of h2Counts) { + if (count > 1) { + console.warn(`⚠️ ${commandPath}: heading "== ${heading}" appears ${count} times. Check the override content for this command.`) + } + } // Determine output path // Check if this command should go to cloudSecretDir @@ -2614,9 +3016,19 @@ async function generateRpkDocs(options = {}) { .map(entry => path.join(baseDir, entry.name)) } + // Protected plugins (computed before the write loop, see + // detectProtectedPlugins): their existing pages reflect the plugin, not + // stale commands, so their directories are excluded from the sweep. + const protectedDirNames = new Set(effectiveProtectedPlugins.map(pl => `rpk-${pl}`)) const scanDirs = new Set() for (const dir of rpkSubdirs(outputDir)) scanDirs.add(dir) for (const dir of rpkSubdirs(cloudSecretDir)) scanDirs.add(dir) + for (const dir of [...scanDirs]) { + if (protectedDirNames.has(path.basename(dir))) { + console.log(` Preserving ${path.basename(dir)}/ (plugin commands absent from this run's tree)`) + scanDirs.delete(dir) + } + } for (const dir of scanDirs) { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { @@ -2644,7 +3056,7 @@ async function generateRpkDocs(options = {}) { // Update nav.adoc if a path was provided let navResult = { navUpdated: false } if (navFile) { - navResult = updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands) + navResult = updateNavFile(navFile, commands, resolvedOverrides, topLevelWithSubcommands, effectiveProtectedPlugins) } return { @@ -2681,9 +3093,12 @@ module.exports = { shouldExcludeCommand, shouldUsePartialDir, updateNavFile, + KNOWN_PLUGINS, + detectProtectedPlugins, getCommandMetadata, processContentArray, // Exported for testing filterExamples, - formatExamples + formatExamples, + applyTextTransformationsToExamples } diff --git a/tools/rpk-docs/helpers/index.js b/tools/rpk-docs/helpers/index.js index f33243ef..e224d6ee 100644 --- a/tools/rpk-docs/helpers/index.js +++ b/tools/rpk-docs/helpers/index.js @@ -52,16 +52,26 @@ function shortDescription(str) { }) } - const sentences = normalized.match(/[^.!?]+[.!?]+(?:\s|$)/g) + // Protect decimal points (OAuth 2.0) so version numbers neither split a + // sentence nor cause the leading fragment to be dropped + normalized = normalized.replace(/(\d)\.(\d)/g, '$1__DECIMAL__$2') + + let sentences = normalized.match(/[^.!?]+[.!?]+(?:\s|$)/g) if (!sentences || sentences.length === 0) { - let result = normalized + let result = normalized.replace(/__DECIMAL__/g, '.') placeholders.forEach(({ ph, original }) => { result = result.replace(ph, original) }) return result.trim() } - let result = sentences.slice(0, 2).join('') + // Never drop an unterminated leading fragment + const firstIdx = normalized.indexOf(sentences[0]) + if (firstIdx > 0) { + sentences = [normalized.slice(0, firstIdx) + sentences[0], ...sentences.slice(1)] + } + + let result = sentences.slice(0, 2).join('').replace(/__DECIMAL__/g, '.') placeholders.forEach(({ ph, original }) => { result = result.replace(ph, original) }) diff --git a/tools/rpk-docs/report-delta.js b/tools/rpk-docs/report-delta.js index f95b4289..a09b8173 100644 --- a/tools/rpk-docs/report-delta.js +++ b/tools/rpk-docs/report-delta.js @@ -132,7 +132,14 @@ function compareFlags(oldFlag, newFlag) { * @returns {Object} Diff report */ function generateRpkDiff(oldTree, newTree, options = {}) { - const { oldVersion = 'old', newVersion = 'new' } = options + const { + oldVersion = 'old', + newVersion = 'new', + // deprecated_commands maps from the snapshots (path -> metadata), as + // produced by scan-deprecated-commands.js + oldDeprecatedCommands = {}, + newDeprecatedCommands = {} + } = options const oldCommands = flattenToMap(oldTree) const newCommands = flattenToMap(newTree) @@ -152,23 +159,95 @@ function generateRpkDiff(oldTree, newTree, options = {}) { } }) - // Find removed commands + // Newly deprecated commands: in the new snapshot's deprecation map but not + // the old one. Detected by source scanning, because deprecated commands + // that are also hidden never appear in --print-tree output. + const newlyDeprecatedRaw = Object.entries(newDeprecatedCommands) + .filter(([path]) => !oldDeprecatedCommands[path]) + .map(([path, info]) => ({ + path, + message: info.deprecatedMessage || '', + replacement: info.replacement || '', + hidden: info.hidden === true || /Hidden:\s*true/.test(info._note || ''), + deprecatedInVersion: newVersion + })) + + // Roll deprecated subcommands up into their nearest deprecated ancestor: + // one entry for `rpk redpanda admin` with its subcommands listed reads + // better than a dozen sibling entries + const newlyDeprecatedDetails = newlyDeprecatedRaw.filter(d => + !newlyDeprecatedRaw.some(other => other !== d && d.path.startsWith(other.path + ' '))) + for (const child of newlyDeprecatedRaw) { + if (newlyDeprecatedDetails.includes(child)) continue + const root = newlyDeprecatedDetails.find(r => child.path.startsWith(r.path + ' ')) + if (root) { + root.affectedSubcommands = root.affectedSubcommands || [] + if (!root.affectedSubcommands.includes(child.path)) { + root.affectedSubcommands.push(child.path) + } + } + } + + // Find removed commands, then separate genuine removals from commands that + // disappeared from the tree because they (or an ancestor) were deprecated + // and hidden — those still work as aliases and should be reported as + // deprecations, not removals. + const deprecatedRoots = newlyDeprecatedDetails.map(d => d.path) + const isUnderNewlyDeprecated = (p) => + deprecatedRoots.some(root => p === root || p.startsWith(root + ' ')) + const removedCommandPaths = [...oldPaths].filter(p => !newPaths.has(p)) - const removedCommandsDetails = removedCommandPaths.map(path => { + const removedCommandsDetails = [] + for (const path of removedCommandPaths) { + if (isUnderNewlyDeprecated(path)) { + const root = deprecatedRoots.find(r => path === r || path.startsWith(r + ' ')) + const entry = newlyDeprecatedDetails.find(d => d.path === root) + if (path !== root) { + entry.affectedSubcommands = entry.affectedSubcommands || [] + if (!entry.affectedSubcommands.includes(path)) { + entry.affectedSubcommands.push(path) + } + } + continue + } const cmd = oldCommands.get(path) - return { + removedCommandsDetails.push({ path, name: cmd.name, description: cmd.description || '', removedInVersion: newVersion - } - }) + }) + } + + for (const dep of newlyDeprecatedDetails) { + if (dep.affectedSubcommands) dep.affectedSubcommands.sort() + } // Find flag changes in existing commands const newFlags = [] const removedFlags = [] const changedDefaults = [] + const changedFlagTypes = [] + const changedFlagRequirements = [] + const changedFlagDescriptions = [] const descriptionChanges = [] + const flagDataBackfilled = [] + + // Baseline gap detection: plugin subtrees captured before flag extraction + // existed record no flags on any of their own commands (the install/ + // uninstall/upgrade shims are rpk-native and do carry flags). Flag + // "additions" inside such a group are newly captured documentation, not + // newly introduced flags, and stamping them "New in " would + // mislabel long-standing flags. Core groups have baseline flag data, so a + // zero-flag command there that gains a flag is a genuine addition. + const groupsWithBaselineFlagData = new Set() + for (const [path, cmd] of oldCommands) { + const parts = path.split(' ') + if (parts.length < 2) continue + // "rpk install|uninstall|upgrade" shims are rpk-native + if (parts.length === 3 && /^(install|uninstall|upgrade)$/.test(parts[2])) continue + if ((cmd.flags || []).length > 0) groupsWithBaselineFlagData.add(parts[1]) + } for (const path of newPaths) { if (!oldPaths.has(path)) continue // Skip new commands @@ -179,8 +258,16 @@ function generateRpkDiff(oldTree, newTree, options = {}) { const oldFlags = getFlagsMap(oldCmd) const newFlagsMap = getFlagsMap(newCmd) - // Find new flags - for (const [flagName, flag] of newFlagsMap) { + const topLevel = path.split(' ')[1] + const isFlagBackfill = topLevel !== undefined && + !groupsWithBaselineFlagData.has(topLevel) && + oldFlags.size === 0 && newFlagsMap.size > 0 + if (isFlagBackfill) { + flagDataBackfilled.push({ commandPath: path, flagCount: newFlagsMap.size }) + } + + // Find new flags (skipped entirely for backfilled commands) + for (const [flagName, flag] of isFlagBackfill ? [] : newFlagsMap) { if (!oldFlags.has(flagName)) { newFlags.push({ commandPath: path, @@ -222,6 +309,30 @@ function generateRpkDiff(oldTree, newTree, options = {}) { newDefault: changes.default.new }) } + if (changes.type) { + changedFlagTypes.push({ + commandPath: path, + flagName, + oldType: changes.type.old, + newType: changes.type.new + }) + } + if (changes.required) { + changedFlagRequirements.push({ + commandPath: path, + flagName, + oldRequired: changes.required.old, + newRequired: changes.required.new + }) + } + if (changes.description) { + changedFlagDescriptions.push({ + commandPath: path, + flagName, + oldDescription: changes.description.old, + newDescription: changes.description.new + }) + } } } @@ -244,19 +355,29 @@ function generateRpkDiff(oldTree, newTree, options = {}) { }, summary: { newCommands: newCommandsDetails.length, + newlyDeprecatedCommands: newlyDeprecatedDetails.length, removedCommands: removedCommandsDetails.length, newFlags: newFlags.length, removedFlags: removedFlags.length, changedDefaults: changedDefaults.length, - descriptionChanges: descriptionChanges.length + changedFlagTypes: changedFlagTypes.length, + changedFlagRequirements: changedFlagRequirements.length, + changedFlagDescriptions: changedFlagDescriptions.length, + descriptionChanges: descriptionChanges.length, + flagDataBackfilled: flagDataBackfilled.length }, details: { newCommands: newCommandsDetails, + newlyDeprecatedCommands: newlyDeprecatedDetails, removedCommands: removedCommandsDetails, newFlags, removedFlags, changedDefaults, - descriptionChanges + changedFlagTypes, + changedFlagRequirements, + changedFlagDescriptions, + descriptionChanges, + flagDataBackfilled } } } @@ -272,11 +393,18 @@ function printDiffReport(diff) { console.log('Summary:') console.log(` New commands: ${diff.summary.newCommands}`) - console.log(` Deprecated commands: ${diff.summary.removedCommands}`) + console.log(` Deprecated commands: ${diff.summary.newlyDeprecatedCommands || 0}`) + console.log(` Removed commands: ${diff.summary.removedCommands}`) console.log(` New flags: ${diff.summary.newFlags}`) - console.log(` Deprecated flags: ${diff.summary.removedFlags}`) + if (diff.summary.flagDataBackfilled) { + console.log(` Flag documentation backfilled: ${diff.summary.flagDataBackfilled} command(s) (baseline had no flag data; not reported as new flags)`) + } + console.log(` Removed flags: ${diff.summary.removedFlags}`) console.log(` Changed defaults: ${diff.summary.changedDefaults}`) - console.log(` Description changes: ${diff.summary.descriptionChanges}`) + console.log(` Changed flag types: ${diff.summary.changedFlagTypes || 0}`) + console.log(` Changed flag requirements: ${diff.summary.changedFlagRequirements || 0}`) + console.log(` Changed flag descriptions: ${diff.summary.changedFlagDescriptions || 0}`) + console.log(` Command description changes: ${diff.summary.descriptionChanges}`) if (diff.details.newCommands.length > 0) { console.log('\nNew Commands:') @@ -289,8 +417,19 @@ function printDiffReport(diff) { } } + if ((diff.details.newlyDeprecatedCommands || []).length > 0) { + console.log('\nDeprecated Commands (still work, hidden or discouraged):') + for (const cmd of diff.details.newlyDeprecatedCommands) { + console.log(` ⚠ ${cmd.path}${cmd.hidden ? ' (hidden)' : ''}`) + if (cmd.message) console.log(` ${cmd.message}`) + if ((cmd.affectedSubcommands || []).length > 0) { + console.log(` affects ${cmd.affectedSubcommands.length} subcommand(s)`) + } + } + } + if (diff.details.removedCommands.length > 0) { - console.log('\nDeprecated Commands (no longer in command tree):') + console.log('\nRemoved Commands (no longer in command tree):') for (const cmd of diff.details.removedCommands) { console.log(` ⚠ ${cmd.path}`) } @@ -304,7 +443,7 @@ function printDiffReport(diff) { } if (diff.details.removedFlags.length > 0) { - console.log('\nDeprecated Flags (no longer in command tree):') + console.log('\nRemoved Flags (no longer in command tree):') for (const flag of diff.details.removedFlags) { console.log(` ⚠ ${flag.commandPath} --${flag.flagName}`) } @@ -318,6 +457,20 @@ function printDiffReport(diff) { } } + if ((diff.details.changedFlagTypes || []).length > 0) { + console.log('\nChanged Flag Types:') + for (const change of diff.details.changedFlagTypes) { + console.log(` ~ ${change.commandPath} --${change.flagName}: ${change.oldType} → ${change.newType}`) + } + } + + if ((diff.details.changedFlagRequirements || []).length > 0) { + console.log('\nChanged Flag Requirements:') + for (const change of diff.details.changedFlagRequirements) { + console.log(` ~ ${change.commandPath} --${change.flagName}: required ${change.oldRequired} → ${change.newRequired}`) + } + } + console.log('') } @@ -339,10 +492,13 @@ function generateMarkdownSummary(diff) { lines.push(`| Category | Count |`) lines.push(`|----------|-------|`) lines.push(`| New commands | ${diff.summary.newCommands} |`) - lines.push(`| Deprecated commands | ${diff.summary.removedCommands} |`) + lines.push(`| Removed commands | ${diff.summary.removedCommands} |`) lines.push(`| New flags | ${diff.summary.newFlags} |`) - lines.push(`| Deprecated flags | ${diff.summary.removedFlags} |`) + lines.push(`| Removed flags | ${diff.summary.removedFlags} |`) lines.push(`| Changed defaults | ${diff.summary.changedDefaults} |`) + if (diff.summary.flagDataBackfilled) { + lines.push(`| Flag docs backfilled (baseline had no flag data) | ${diff.summary.flagDataBackfilled} commands |`) + } lines.push(``) if (diff.details.newCommands.length > 0) { @@ -355,9 +511,9 @@ function generateMarkdownSummary(diff) { } if (diff.details.removedCommands.length > 0) { - lines.push(`### Deprecated Commands`) + lines.push(`### Removed Commands`) lines.push(``) - lines.push(`> Commands no longer in the active command tree. These may still work but are deprecated.`) + lines.push(`> Commands no longer in the active command tree.`) lines.push(``) for (const cmd of diff.details.removedCommands) { lines.push(`- ~~\`${cmd.path}\`~~`) @@ -411,32 +567,67 @@ function commandPathToXref(commandPath) { function generateWhatsNewSection(diff, options = {}) { const lines = [] const version = options.version || diff.comparison.newVersion + // Plugin subtrees may render as partials with no linkable pages, so plugin + // runs disable xrefs and render plain command names instead. + const useXrefs = options.xrefs !== false + // Commands that render as partials or are excluded have no linkable page + const linkable = typeof options.linkable === 'function' ? options.linkable : () => true + // Section heading for the page ("== Redpanda CLI" for core rpk changes, + // "== rpk plugins" for plugin releases). When blockLabel is set, the block + // opens with a "===