diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 50fb9ede08de..a0b9136ca7bc 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -98,11 +98,14 @@ in rec { > $out ''; - optionsCommonMark = pkgs.runCommand "options.md" {} '' - ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ - --format commonmark \ + optionsCommonMark = pkgs.runCommand "options.md" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options commonmark \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ ${optionsJSON}/share/doc/nixos/options.json \ - > $out + $out ''; optionsJSON = pkgs.runCommand "options.json" diff --git a/nixos/lib/make-options-doc/generateDoc.py b/nixos/lib/make-options-doc/generateDoc.py index 07884ed657e4..a41255067bfb 100644 --- a/nixos/lib/make-options-doc/generateDoc.py +++ b/nixos/lib/make-options-doc/generateDoc.py @@ -2,7 +2,7 @@ import argparse import json import sys -formats = ['commonmark', 'asciidoc'] +formats = ['asciidoc'] parser = argparse.ArgumentParser( description = 'Generate documentation for a set of JSON-formatted NixOS options' @@ -38,33 +38,6 @@ class OptionsEncoder(json.JSONEncoder): return super().encode(obj) -def generate_commonmark(options): - for (name, value) in options.items(): - print('##', name.replace('<', '<').replace('>', '>')) - print(value['description']) - print() - if 'type' in value: - print('*_Type_*') - print ('```') - print(value['type']) - print ('```') - print() - print() - if 'default' in value: - print('*_Default_*') - print('```') - print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('```') - print() - print() - if 'example' in value: - print('*_Example_*') - print('```') - print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('```') - print() - print() - # TODO: declarations: link to github def generate_asciidoc(options): for (name, value) in options.items(): @@ -103,9 +76,7 @@ def generate_asciidoc(options): with open(args.nix_options_path) as nix_options_json: options = json.load(nix_options_json) - if args.format == 'commonmark': - generate_commonmark(options) - elif args.format == 'asciidoc': + if args.format == 'asciidoc': generate_asciidoc(options) else: raise Exception(f'Unsupported documentation format `--format {args.format}`') diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py new file mode 100644 index 000000000000..4a708b1f92c6 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py @@ -0,0 +1,231 @@ +from collections.abc import Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from typing import Any, cast, Optional + +from .md import md_escape, md_make_code, Renderer + +import markdown_it +from markdown_it.token import Token +from markdown_it.utils import OptionsDict + +@dataclass(kw_only=True) +class List: + next_idx: Optional[int] = None + compact: bool + first_item_seen: bool = False + +@dataclass +class Par: + indent: str + continuing: bool = False + +class CommonMarkRenderer(Renderer): + __output__ = "commonmark" + + _parstack: list[Par] + _link_stack: list[str] + _list_stack: list[List] + + def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): + super().__init__(manpage_urls, parser) + self._parstack = [ Par("") ] + self._link_stack = [] + self._list_stack = [] + + def _enter_block(self, extra_indent: str) -> None: + self._parstack.append(Par(self._parstack[-1].indent + extra_indent)) + def _leave_block(self) -> None: + self._parstack.pop() + self._parstack[-1].continuing = True + def _break(self) -> str: + self._parstack[-1].continuing = True + return f"\n{self._parstack[-1].indent}" + def _maybe_parbreak(self) -> str: + result = f"\n{self._parstack[-1].indent}" * 2 if self._parstack[-1].continuing else "" + self._parstack[-1].continuing = True + return result + + def _admonition_open(self, kind: str) -> str: + pbreak = self._maybe_parbreak() + self._enter_block("") + return f"{pbreak}**{kind}:** " + def _admonition_close(self) -> str: + self._leave_block() + return "" + + def _indent_raw(self, s: str) -> str: + if '\n' not in s: + return s + return f"\n{self._parstack[-1].indent}".join(s.splitlines()) + + def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return self._indent_raw(md_escape(token.content)) + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._maybe_parbreak() + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f" {self._break()}" + def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return md_make_code(token.content) + def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self.fence(token, tokens, i, options, env) + def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + self._link_stack.append(cast(str, token.attrs['href'])) + return "[" + def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"]({md_escape(self._link_stack.pop())})" + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + lst = self._list_stack[-1] + lbreak = "" if not lst.first_item_seen else self._break() * (1 if lst.compact else 2) + lst.first_item_seen = True + head = " -" + if lst.next_idx is not None: + head = f" {lst.next_idx}." + lst.next_idx += 1 + self._enter_block(" " * (len(head) + 1)) + return f'{lbreak}{head} ' + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append(List(compact=bool(token.meta['compact']))) + return self._maybe_parbreak() + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" + def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "*" + def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "*" + def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + code = token.content + if code.endswith('\n'): + code = code[:-1] + pbreak = self._maybe_parbreak() + return pbreak + self._indent_raw(md_make_code(code, info=token.info, multiline=True)) + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._maybe_parbreak() + self._enter_block("> ") + return pbreak + "> " + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Note") + def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Caution") + def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Important") + def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Tip") + def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("Warning") + def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append(List(compact=False)) + return "" + def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" + def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._maybe_parbreak() + self._enter_block(" ") + # add an opening zero-width non-joiner to separate *our* emphasis from possible + # emphasis in the provided term + return f'{pbreak} - *{chr(0x200C)}' + def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"{chr(0x200C)}*" + def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return "" + def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "" + def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + content = md_make_code(token.content) + if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): + return f"[{content}]({url})" + return content # no roles in regular commonmark + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + # there's no way we can emit attrspans correctly in all cases. we could use inline + # html for ids, but that would not round-trip. same holds for classes. since this + # renderer is only used for approximate options export and all of these things are + # not allowed in options we can ignore them for now. + return "" + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return token.markup + " " + def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "\n" + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.append( + List(next_idx = cast(int, token.attrs.get('start', 1)), + compact = bool(token.meta['compact']))) + return self._maybe_parbreak() + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._list_stack.pop() + return "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 39ea8f8c627f..d8a24b885f87 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -13,6 +13,7 @@ from xml.sax.saxutils import escape, quoteattr import markdown_it from . import parallel +from .commonmark import CommonMarkRenderer from .docbook import DocBookRenderer, make_xml_id from .manpage import ManpageRenderer, man_escape from .md import Converter, md_escape, md_make_code @@ -422,6 +423,59 @@ class ManpageConverter(BaseConverter): return "\n".join(result) +class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer): + pass + +class CommonMarkConverter(BaseConverter): + __renderer__ = OptionsCommonMarkRenderer + __option_block_separator__ = "" + + def _parallel_render_prepare(self) -> Any: + return (self._manpage_urls, self._revision, self._markdown_by_default) + @classmethod + def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter: + return cls(*a) + + def _render_code(self, option: dict[str, Any], key: str) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if lit := option_is(option, key, 'literalDocBook'): + return [ f"*{key.capitalize()}:* {lit['text']}" ] + else: + return super()._render_code(option, key) + + def _render_description(self, desc: str | dict[str, Any]) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if isinstance(desc, str) and not self._markdown_by_default: + return [ desc ] + else: + return super()._render_description(desc) + + def _related_packages_header(self) -> list[str]: + return [ "*Related packages:*" ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ f"*{header}:*" ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + return [ f" - [{md_escape(name)}]({href})" ] + return [ f" - {md_escape(name)}" ] + + def _decl_def_footer(self) -> list[str]: + return [] + + def finalize(self) -> str: + result = [] + + for (name, opt) in self._sorted_options(): + result.append(f"## {md_escape(name)}\n") + result += opt.lines + result.append("\n\n") + + return "\n".join(result) + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) @@ -437,6 +491,13 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None: p.add_argument("infile") p.add_argument("outfile") +def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: + p.add_argument('--manpage-urls', required=True) + p.add_argument('--revision', required=True) + p.add_argument('--markdown-by-default', default=False, action='store_true') + p.add_argument("infile") + p.add_argument("outfile") + def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter( @@ -464,15 +525,30 @@ def _run_cli_manpage(args: argparse.Namespace) -> None: with open(args.outfile, 'w') as f: f.write(md.finalize()) +def _run_cli_commonmark(args: argparse.Namespace) -> None: + with open(args.manpage_urls, 'r') as manpage_urls: + md = CommonMarkConverter( + json.load(manpage_urls), + revision = args.revision, + markdown_by_default = args.markdown_by_default) + + with open(args.infile, 'r') as f: + md.add_options(json.load(f)) + with open(args.outfile, 'w') as f: + f.write(md.finalize()) + def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) _build_cli_db(formats.add_parser('docbook')) _build_cli_manpage(formats.add_parser('manpage')) + _build_cli_commonmark(formats.add_parser('commonmark')) def run_cli(args: argparse.Namespace) -> None: if args.format == 'docbook': _run_cli_db(args) elif args.format == 'manpage': _run_cli_manpage(args) + elif args.format == 'commonmark': + _run_cli_commonmark(args) else: raise RuntimeError('format not hooked up', args) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py new file mode 100644 index 000000000000..5e0d63eb6723 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py @@ -0,0 +1,92 @@ +import nixos_render_docs + +from sample_md import sample1 + +from typing import Mapping, Optional + +import markdown_it + +class Converter(nixos_render_docs.md.Converter): + __renderer__ = nixos_render_docs.commonmark.CommonMarkRenderer + +# NOTE: in these tests we represent trailing spaces by ` ` and replace them with real space later, +# since a number of editors will strip trailing whitespace on save and that would break the tests. + +def test_indented_fence() -> None: + c = Converter({}) + s = """\ +> - ```foo +> thing +>       +> rest +> ```\ +""".replace(' ', ' ') + assert c._render(s) == s + +def test_full() -> None: + c = Converter({ 'man(1)': 'http://example.org' }) + assert c._render(sample1) == f"""\ +**Warning:** foo + +**Note:** nested + +[ +multiline +](link) + +[` man(1) `](http://example.org) reference + +some nested anchors + +*emph* **strong** *nesting emph **and strong** and ` code `* + + - wide bullet + + - list + + 1. wide ordered + + 2. list + + - narrow bullet + - list + + 1. narrow ordered + 2. list + +> quotes +>  +> > with *nesting* +> >  +> > ``` +> > nested code block +> > ``` +>  +> - and lists +> - ``` +> containing code +> ``` +>  +> and more quote + + 100. list starting at 100 + 101. goes on + + - *‌deflist‌* +    + > with a quote + > and stuff +    + ``` + code block + ``` +    + ``` + fenced block + ``` +    + text + + - *‌more stuff in same deflist‌* +    + foo""".replace(' ', ' ')