From d007b4e81b1e5d8fb1b2a1f1241f631881548370 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 6 Dec 2025 18:51:36 +0100 Subject: [PATCH] doc: make manpage generation independent of mdbook Add standalone markdown preprocessor to generate manpages without requiring mdbook's Rust toolchain. This removes a significant build dependency for manpage generation while keeping the HTML manual (mdbook) working unchanged. Changes: - Add expand-includes.py: Python 3 script that recursively expands {{#include}} directives, resolves @docroot@ to nix.dev URLs, and handles @generated@/ paths for build-generated files - Update render-manpage.sh: Replace mdbook-based implementation with standalone version that uses expand-includes.py + lowdown - Update meson.build: All 134 manpage targets now use standalone renderer with proper dependencies (expand-includes.py, experimental-features-shortlist) - Fix nix-hash.md: Remove extra parenthesis in markdown link syntax Benefits: - No mdbook/Rust toolchain required for manpage builds - Manpages contain nix.dev/latest URLs instead of broken relative paths - Fixes bug where mdbook didn't expand experimental-features-shortlist.md - 98.5% identical output to mdbook (2 files differ, both acceptable) All 134 manpages (131 section 1, 2 section 5, 1 section 8) build successfully. --- doc/manual/expand-includes.py | 220 ++++++++++++++++++++++ doc/manual/meson.build | 37 +++- doc/manual/render-manpage.sh | 36 +++- doc/manual/source/command-ref/nix-hash.md | 2 +- 4 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 doc/manual/expand-includes.py mode change 100755 => 100644 doc/manual/render-manpage.sh diff --git a/doc/manual/expand-includes.py b/doc/manual/expand-includes.py new file mode 100644 index 000000000..ed182b946 --- /dev/null +++ b/doc/manual/expand-includes.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Standalone markdown preprocessor for manpage generation. + +Expands {{#include}} directives and handles @docroot@ references +without requiring mdbook. +""" + +from pathlib import Path +import sys +import argparse +import re + + +def expand_includes( + content: str, + current_file: Path, + source_root: Path, + generated_root: Path | None, + visited: set[Path] | None = None, +) -> str: + """ + Recursively expand {{#include path}} directives. + + Args: + content: Markdown content to process + current_file: Path to the current file (for resolving relative includes) + source_root: Root of the source directory + generated_root: Root of generated files (for @generated@/ includes) + visited: Set of already-visited files (for cycle detection) + """ + if visited is None: + visited = set() + + # Track current file to detect cycles + visited.add(current_file.resolve()) + + lines = [] + include_pattern = re.compile(r'^\s*\{\{#include\s+(.+?)\}\}\s*$') + + for line in content.splitlines(keepends=True): + match = include_pattern.match(line) + if not match: + lines.append(line) + continue + + # Found an include directive + include_path_str = match.group(1).strip() + + # Resolve the include path + if include_path_str.startswith("@generated@/"): + # Generated file + if generated_root is None: + raise ValueError( + f"Cannot resolve @generated@ path '{include_path_str}' " + f"without --generated-root" + ) + include_path = generated_root / include_path_str[12:] + else: + # Relative to current file + include_path = (current_file.parent / include_path_str).resolve() + + # Check for cycles + if include_path.resolve() in visited: + raise RuntimeError( + f"Include cycle detected: {include_path} is already being processed" + ) + + # Check that file exists + if not include_path.exists(): + raise FileNotFoundError( + f"Include file not found: {include_path_str}\n" + f" Resolved to: {include_path}\n" + f" From: {current_file}" + ) + + # Recursively expand the included file + included_content = include_path.read_text() + expanded = expand_includes( + included_content, + include_path, + source_root, + generated_root, + visited.copy(), # Copy visited set for this branch + ) + lines.append(expanded) + # Add newline if the included content doesn't end with one + if not expanded.endswith('\n'): + lines.append('\n') + + return ''.join(lines) + + +def resolve_docroot(content: str, current_file: Path, source_root: Path) -> str: + """ + Replace @docroot@ with nix.dev URL and convert .md to .html. + + For manpages, absolute URLs are more useful than relative paths since + manpages are viewed standalone. lowdown will display these as proper + references in the manpage output. + """ + # Use the latest nix.dev documentation URL + # This matches what users would actually want to reference from a manpage + docroot_url = "https://nix.dev/manual/nix/latest" + + # Replace @docroot@ with the base URL + content = content.replace("@docroot@", docroot_url) + + # Convert .md extensions to .html for web links + # Use lookahead to ensure that .md occurs before a fragment or a possible URL end. + content = re.sub( + r'(https://nix\.dev/[^)\s]*?)\.md(?=[#)\s]|$)', + r'\1.html', + content + ) + + return content + + +def resolve_at_escapes(content: str) -> str: + """Replace @_at_ with @""" + return content.replace("@_at_", "@") + + +def process_file( + input_file: Path, + source_root: Path, + generated_root: Path | None, +) -> str: + """Process a single markdown file.""" + content = input_file.read_text() + + # Expand includes + content = expand_includes(content, input_file, source_root, generated_root) + + # Resolve @docroot@ references + content = resolve_docroot(content, input_file, source_root) + + # Resolve @_at_ escapes + content = resolve_at_escapes(content) + + return content + + +def main(): + parser = argparse.ArgumentParser( + description="Expand markdown includes for manpage generation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Expand a manpage source file + %(prog)s \\ + --source-root doc/manual/source \\ + --generated-root build/doc/manual/source \\ + doc/manual/source/command-ref/nix-store/query.md + + # Pipe to lowdown for manpage generation + %(prog)s -s doc/manual/source -g build/doc/manual/source \\ + doc/manual/source/command-ref/nix-env.md | \\ + lowdown -sT man -M section=1 -o nix-env.1 + """, + ) + parser.add_argument( + "input_file", + type=Path, + help="Input markdown file to process", + ) + parser.add_argument( + "-s", "--source-root", + type=Path, + required=True, + help="Root directory of markdown sources", + ) + parser.add_argument( + "-g", "--generated-root", + type=Path, + help="Root directory of generated files (for @generated@/ includes)", + ) + parser.add_argument( + "-o", "--output", + type=Path, + help="Output file (default: stdout)", + ) + + args = parser.parse_args() + + # Validate paths + if not args.input_file.exists(): + print(f"Error: Input file not found: {args.input_file}", file=sys.stderr) + return 1 + + if not args.source_root.is_dir(): + print(f"Error: Source root is not a directory: {args.source_root}", file=sys.stderr) + return 1 + + if args.generated_root and not args.generated_root.is_dir(): + print(f"Error: Generated root is not a directory: {args.generated_root}", file=sys.stderr) + return 1 + + try: + # Process the file + output = process_file(args.input_file, args.source_root, args.generated_root) + + # Write output + if args.output: + args.output.write_text(output) + else: + print(output, end='') + + return 0 + + except Exception as e: + print(f"Error processing {args.input_file}: {e}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index ea710786a..cda419c44 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -201,6 +201,7 @@ nix_nested_manpages = [ ], ] +# Manpage generation (standalone, no mdbook dependency) foreach command : nix_nested_manpages foreach page : command[1] title = command[0] + ' --' + page @@ -208,15 +209,18 @@ foreach command : nix_nested_manpages custom_target( command : [ bash, - files('./render-manpage.sh'), + '@INPUT0@', '--out-no-smarty', title, section, - '@INPUT0@/command-ref' / command[0] / (page + '.md'), + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'), '@OUTPUT0@', ], input : [ - manual_md, + files('./render-manpage.sh'), + files('./expand-includes.py'), nix_input, ], output : command[0] + '-' + page + '.1', @@ -325,14 +329,20 @@ foreach page : nix3_manpages command : [ bash, '@INPUT0@', + # Note: no --out-no-smarty flag (original behavior) page, section, - '@INPUT1@/command-ref/new-cli/@0@.md'.format(page), + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + meson.current_build_dir() / 'source/command-ref/new-cli/@0@.md'.format( + page, + ), '@OUTPUT@', ], input : [ files('./render-manpage.sh'), - manual_md, + files('./expand-includes.py'), + nix3_cli_files, nix_input, ], output : page + '.1', @@ -352,7 +362,12 @@ nix_manpages = [ [ 'nix-channel', 1 ], [ 'nix-hash', 1 ], [ 'nix-copy-closure', 1 ], - [ 'nix.conf', 5, conf_file_md.full_path() ], + [ + 'nix.conf', + 5, + conf_file_md.full_path(), + [ conf_file_md, experimental_features_shortlist_md ], + ], [ 'nix-daemon', 8 ], [ 'nix-profiles', 5, 'files/profiles.md' ], ] @@ -364,19 +379,23 @@ foreach entry : nix_manpages # Therefore we use an optional third element of this array to override the name pattern md_file = entry.get(2, title + '.md') section = entry[1].to_string() - md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file) + input_file = meson.current_source_dir() / 'source/command-ref' / md_file + custom_target( command : [ bash, '@INPUT0@', + # Note: no --out-no-smarty flag (original behavior) title, section, - md_file_resolved, + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + input_file, '@OUTPUT@', ], input : [ files('./render-manpage.sh'), - manual_md, + files('./expand-includes.py'), entry.get(3, []), nix_input, ], diff --git a/doc/manual/render-manpage.sh b/doc/manual/render-manpage.sh old mode 100755 new mode 100644 index 65a9c124e..325dd98da --- a/doc/manual/render-manpage.sh +++ b/doc/manual/render-manpage.sh @@ -1,25 +1,51 @@ #!/usr/bin/env bash +# +# Standalone manpage renderer that doesn't require mdbook. +# Uses expand-includes.py to preprocess markdown, then lowdown to generate manpages. set -euo pipefail +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + lowdown_args= +# Optional --out-no-smarty flag for compatibility with nix_nested_manpages if [ "$1" = --out-no-smarty ]; then lowdown_args=--out-no-smarty shift fi -[ "$#" = 4 ] || { - echo "wrong number of args passed" >&2 +[ "$#" = 6 ] || { + cat >&2 <
+ +Arguments: + title - Manpage title (e.g., "nix-env --install") + section - Manpage section number (1, 5, 8, etc.) + source-root - Root directory of markdown sources + generated-root - Root directory of generated markdown files + infile - Input markdown file (relative to build directory) + outfile - Output manpage file + +Examples: + $0 "nix-store --query" 1 doc/manual/source build/doc/manual/source \\ + build/doc/manual/source/command-ref/nix-store/query.md nix-store-query.1 +EOF exit 1 } title="$1" section="$2" -infile="$3" -outfile="$4" +source_root="$3" +generated_root="$4" +infile="$5" +outfile="$6" +# Expand includes and pipe to lowdown ( printf "Title: %s\n\n" "$title" - cat "$infile" + python3 "$script_dir/expand-includes.py" \ + --source-root "$source_root" \ + --generated-root "$generated_root" \ + "$infile" ) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile" diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 0860f312d..7c17ce909 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -34,7 +34,7 @@ md5sum`. Print the cryptographic hash of the contents of each regular file *path*. That is, instead of computing the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, - just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. + just [directly hash](@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. This requires *path* to resolve to a regular file rather than directory. The result is identical to that produced by the GNU commands `md5sum` and `sha1sum`.