diff --git a/doc/manual/expand-includes.py b/doc/manual/expand-includes.py new file mode 100644 index 000000000..59c687f23 --- /dev/null +++ b/doc/manual/expand-includes.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +""" +Standalone markdown preprocessor for manpage generation. + +Expands {{#include}} directives and handles @docroot@ references +without requiring mdbook. +""" + +from pathlib import Path +import sys +import argparse +import re + + +def expand_includes( + content: str, + current_file: Path, + source_root: Path, + generated_root: Path | None, + visited: set[Path] | None = None, +) -> str: + """ + Recursively expand {{#include path}} directives. + + Args: + content: Markdown content to process + current_file: Path to the current file (for resolving relative includes) + source_root: Root of the source directory + generated_root: Root of generated files (for @generated@/ includes) + visited: Set of already-visited files (for cycle detection) + """ + if visited is None: + visited = set() + + # Track current file to detect cycles + visited.add(current_file.resolve()) + + lines = [] + include_pattern = re.compile(r'^\s*\{\{#include\s+(.+?)\}\}\s*$') + + for line in content.splitlines(keepends=True): + match = include_pattern.match(line) + if not match: + lines.append(line) + continue + + # Found an include directive + include_path_str = match.group(1).strip() + + # Resolve the include path + if include_path_str.startswith("@generated@/"): + # Generated file + if generated_root is None: + raise ValueError( + f"Cannot resolve @generated@ path '{include_path_str}' " + f"without --generated-root" + ) + include_path = generated_root / include_path_str[12:] + else: + # Relative to current file + include_path = (current_file.parent / include_path_str).resolve() + + # Check for cycles + if include_path.resolve() in visited: + raise RuntimeError( + f"Include cycle detected: {include_path} is already being processed" + ) + + # Check that file exists + if not include_path.exists(): + raise FileNotFoundError( + f"Include file not found: {include_path_str}\n" + f" Resolved to: {include_path}\n" + f" From: {current_file}" + ) + + # Recursively expand the included file + included_content = include_path.read_text() + expanded = expand_includes( + included_content, + include_path, + source_root, + generated_root, + visited.copy(), # Copy visited set for this branch + ) + lines.append(expanded) + # Add newline if the included content doesn't end with one + if not expanded.endswith('\n'): + lines.append('\n') + + return ''.join(lines) + + +def resolve_docroot(content: str, current_file: Path, source_root: Path, docroot_url: str) -> str: + """ + Replace @docroot@ with nix.dev URL and convert .md to .html. + + For manpages, absolute URLs are more useful than relative paths since + manpages are viewed standalone. lowdown will display these as proper + references in the manpage output. + """ + # Replace @docroot@ with the base URL + content = content.replace("@docroot@", docroot_url) + + # Convert .md extensions to .html for web links + # Use lookahead to ensure that .md occurs before a fragment or a possible URL end. + content = re.sub( + r'(https://nix\.dev/[^)\s]*?)\.md(?=[#)\s]|$)', + r'\1.html', + content + ) + + return content + + +def resolve_at_escapes(content: str) -> str: + """Replace @_at_ with @""" + return content.replace("@_at_", "@") + + +def process_file( + input_file: Path, + source_root: Path, + generated_root: Path | None, + docroot_url: str, +) -> str: + """Process a single markdown file.""" + content = input_file.read_text() + + # Expand includes + content = expand_includes(content, input_file, source_root, generated_root) + + # Resolve @docroot@ references + content = resolve_docroot(content, input_file, source_root, docroot_url) + + # Resolve @_at_ escapes + content = resolve_at_escapes(content) + + return content + + +def main(): + parser = argparse.ArgumentParser( + description="Expand markdown includes for manpage generation", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Expand a manpage source file + %(prog)s \\ + --source-root doc/manual/source \\ + --generated-root build/doc/manual/source \\ + doc/manual/source/command-ref/nix-store/query.md + + # Pipe to lowdown for manpage generation + %(prog)s -s doc/manual/source -g build/doc/manual/source \\ + doc/manual/source/command-ref/nix-env.md | \\ + lowdown -sT man -M section=1 -o nix-env.1 + """, + ) + parser.add_argument( + "input_file", + type=Path, + help="Input markdown file to process", + ) + parser.add_argument( + "-s", "--source-root", + type=Path, + required=True, + help="Root directory of markdown sources", + ) + parser.add_argument( + "-g", "--generated-root", + type=Path, + help="Root directory of generated files (for @generated@/ includes)", + ) + parser.add_argument( + "-o", "--output", + type=Path, + help="Output file (default: stdout)", + ) + parser.add_argument( + "-u", "--doc-url", + type=str, + default="https://nix.dev/manual/nix/latest", + help="Base URL for documentation links (default: https://nix.dev/manual/nix/latest)", + ) + + args = parser.parse_args() + + # Validate paths + if not args.input_file.exists(): + print(f"Error: Input file not found: {args.input_file}", file=sys.stderr) + return 1 + + if not args.source_root.is_dir(): + print(f"Error: Source root is not a directory: {args.source_root}", file=sys.stderr) + return 1 + + if args.generated_root and not args.generated_root.is_dir(): + print(f"Error: Generated root is not a directory: {args.generated_root}", file=sys.stderr) + return 1 + + try: + # Process the file + output = process_file(args.input_file, args.source_root, args.generated_root, args.doc_url) + + # Write output + if args.output: + args.output.write_text(output) + else: + print(output, end='') + + return 0 + + except Exception as e: + print(f"Error processing {args.input_file}: {e}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index ea710786a..3c3e79541 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -5,11 +5,29 @@ project( license : 'LGPL-2.1-or-later', ) +# Compute documentation URL based on version and release type +version = meson.project_version() +official_release = get_option('official-release') + +if official_release + # For official releases, use versioned URL (dropping patch version) + version_parts = version.split('.') + major_minor = '@0@.@1@'.format(version_parts[0], version_parts[1]) + doc_url = 'https://nix.dev/manual/nix/@0@'.format(major_minor) +else + # For development builds, use /latest + doc_url = 'https://nix.dev/manual/nix/latest' +endif + nix = find_program('nix', native : true) -mdbook = find_program('mdbook', native : true) bash = find_program('bash', native : true) -rsync = find_program('rsync', required : true, native : true) + +# HTML manual dependencies (conditional) +if get_option('html-manual') + mdbook = find_program('mdbook', native : true) + rsync = find_program('rsync', required : true, native : true) +endif pymod = import('python') python = pymod.find_installation('python3') @@ -95,67 +113,71 @@ else nix_input = [] endif -manual = custom_target( - 'manual', - command : [ - bash, - '-euo', - 'pipefail', - '-c', - ''' - @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ - @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md - sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml - @4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ - (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 - rm -rf @2@/manual - mv @2@/html @2@/manual - # Remove Mathjax 2.7, because we will actually use MathJax 3.x - find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d' - find @2@/manual -iname meson.build -delete - '''.format( - python.full_path(), - mdbook.full_path(), - meson.current_build_dir(), - meson.project_version(), - rsync.full_path(), - ), - ], - input : [ - generate_manual_deps, - 'substitute.py', - 'book.toml.in', - 'anchors.jq', - 'custom.css', - redirects_js, - nix3_cli_files, - experimental_features_shortlist_md, - experimental_feature_descriptions_md, - types_dir, - conf_file_md, - builtins_md, - rl_next_generated, - summary_rl_next, - json_schema_generated_files, - nix_input, - ], - output : [ +# HTML manual build (conditional) +if get_option('html-manual') + manual = custom_target( 'manual', - 'markdown', - ], - depfile : 'manual.d', - env : { - 'RUST_LOG' : 'info', - 'MDBOOK_SUBSTITUTE_SEARCH' : meson.current_build_dir() / 'source', - }, -) -manual_html = manual[0] -manual_md = manual[1] + command : [ + bash, + '-euo', + 'pipefail', + '-c', + ''' + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ + @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md + sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml + @4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 + rm -rf @2@/manual + mv @2@/html @2@/manual + # Remove Mathjax 2.7, because we will actually use MathJax 3.x + find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d' + find @2@/manual -iname meson.build -delete + '''.format( + python.full_path(), + mdbook.full_path(), + meson.current_build_dir(), + meson.project_version(), + rsync.full_path(), + ), + ], + input : [ + generate_manual_deps, + 'substitute.py', + 'book.toml.in', + 'anchors.jq', + 'custom.css', + redirects_js, + nix3_cli_files, + experimental_features_shortlist_md, + experimental_feature_descriptions_md, + types_dir, + conf_file_md, + builtins_md, + rl_next_generated, + summary_rl_next, + json_schema_generated_files, + nix_input, + ], + output : [ + 'manual', + 'markdown', + ], + depfile : 'manual.d', + build_by_default : true, + env : { + 'RUST_LOG' : 'info', + 'MDBOOK_SUBSTITUTE_SEARCH' : meson.current_build_dir() / 'source', + }, + ) + manual_html = manual[0] + manual_md = manual[1] -install_subdir( - manual_html.full_path(), - install_dir : get_option('datadir') / 'doc/nix', -) + install_subdir( + manual_html.full_path(), + install_dir : get_option('datadir') / 'doc/nix', + ) +endif nix_nested_manpages = [ [ @@ -201,6 +223,7 @@ nix_nested_manpages = [ ], ] +# Manpage generation (standalone, no mdbook dependency) foreach command : nix_nested_manpages foreach page : command[1] title = command[0] + ' --' + page @@ -208,15 +231,19 @@ foreach command : nix_nested_manpages custom_target( command : [ bash, - files('./render-manpage.sh'), + '@INPUT0@', '--out-no-smarty', title, section, - '@INPUT0@/command-ref' / command[0] / (page + '.md'), + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + doc_url, + meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'), '@OUTPUT0@', ], input : [ - manual_md, + files('./render-manpage.sh'), + files('./expand-includes.py'), nix_input, ], output : command[0] + '-' + page + '.1', @@ -325,14 +352,21 @@ foreach page : nix3_manpages command : [ bash, '@INPUT0@', + # Note: no --out-no-smarty flag (original behavior) page, section, - '@INPUT1@/command-ref/new-cli/@0@.md'.format(page), + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + doc_url, + meson.current_build_dir() / 'source/command-ref/new-cli/@0@.md'.format( + page, + ), '@OUTPUT@', ], input : [ files('./render-manpage.sh'), - manual_md, + files('./expand-includes.py'), + nix3_cli_files, nix_input, ], output : page + '.1', @@ -352,7 +386,12 @@ nix_manpages = [ [ 'nix-channel', 1 ], [ 'nix-hash', 1 ], [ 'nix-copy-closure', 1 ], - [ 'nix.conf', 5, conf_file_md.full_path() ], + [ + 'nix.conf', + 5, + conf_file_md.full_path(), + [ conf_file_md, experimental_features_shortlist_md ], + ], [ 'nix-daemon', 8 ], [ 'nix-profiles', 5, 'files/profiles.md' ], ] @@ -364,19 +403,24 @@ foreach entry : nix_manpages # Therefore we use an optional third element of this array to override the name pattern md_file = entry.get(2, title + '.md') section = entry[1].to_string() - md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file) + input_file = meson.current_source_dir() / 'source/command-ref' / md_file + custom_target( command : [ bash, '@INPUT0@', + # Note: no --out-no-smarty flag (original behavior) title, section, - md_file_resolved, + meson.current_source_dir() / 'source', + meson.current_build_dir() / 'source', + doc_url, + input_file, '@OUTPUT@', ], input : [ files('./render-manpage.sh'), - manual_md, + files('./expand-includes.py'), entry.get(3, []), nix_input, ], diff --git a/doc/manual/meson.options b/doc/manual/meson.options new file mode 100644 index 000000000..1d6b7c73a --- /dev/null +++ b/doc/manual/meson.options @@ -0,0 +1,13 @@ +option( + 'official-release', + type : 'boolean', + value : true, + description : 'Whether this is an official release build (affects documentation URLs)', +) + +option( + 'html-manual', + type : 'boolean', + value : true, + description : 'Whether to build the HTML manual (requires mdbook)', +) diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 78c4aa825..3a90a0faf 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -19,6 +19,11 @@ # Configuration Options version, + /** + Whether to build the HTML manual. + When false, only manpages are built, avoiding the mdbook dependency. + */ + buildHtmlManual ? true, # `tests` attribute testers, @@ -57,9 +62,22 @@ mkMesonDerivation (finalAttrs: { ../../doc/manual/package.nix; # TODO the man pages should probably be separate - outputs = [ - "out" - "man" + outputs = + if buildHtmlManual then + [ + "out" + "man" + ] + else + [ "out" ]; # Only one output when HTML manual is disabled; use "out" for manpages + + # When HTML manual is disabled, install manpages to "out" instead of "man" + mesonFlags = [ + (lib.mesonBool "official-release" officialRelease) + (lib.mesonBool "html-manual" buildHtmlManual) + ] + ++ lib.optionals (!buildHtmlManual) [ + "--mandir=${placeholder "out"}/share/man" ]; nativeBuildInputs = [ @@ -67,14 +85,15 @@ mkMesonDerivation (finalAttrs: { meson ninja (lib.getBin lowdown-unsandboxed) - mdbook jq python3 + ] + ++ lib.optionals buildHtmlManual [ + mdbook rsync json-schema-for-humans - changelog-d ] - ++ lib.optionals (!officialRelease) [ + ++ lib.optionals (!officialRelease && buildHtmlManual) [ # When not an official release, we likely have changelog entries that have # yet to be rendered. # When released, these are rendered into a committed file to save a dependency. @@ -86,45 +105,48 @@ mkMesonDerivation (finalAttrs: { echo ${finalAttrs.version} > ./.version ''; - postInstall = '' + postInstall = lib.optionalString buildHtmlManual '' mkdir -p ''$out/nix-support echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products ''; - /** - The root of the HTML manual. - E.g. "${nix-manual.site}/index.html" exists. - */ - passthru.site = finalAttrs.finalPackage + "/share/doc/nix/manual"; + passthru = lib.optionalAttrs buildHtmlManual { + /** + The root of the HTML manual. + E.g. "${nix-manual.site}/index.html" exists. + */ - passthru.tests = - let - redirect-targets = callPackage ./redirect-targets-html.nix { }; - in - { - # https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck - linkcheck = testers.lycheeLinkCheck { - site = - let - plain = finalAttrs.finalPackage.site; - in - runCommand "nix-manual-with-redirect-targets" { } '' - cp -r ${plain} $out - chmod -R u+w $out - cp ${redirect-targets}/redirect-targets.html $out/redirect-targets.html - ''; - extraConfig = { - exclude = [ - # Exclude auto-generated JSON schema documentation which has - # auto-generated fragment IDs that don't match the link references - ".*/protocols/json/.*\\.html" - # Exclude undocumented builtins - ".*/language/builtins\\.html#builtins-addErrorContext" - ".*/language/builtins\\.html#builtins-appendContext" - ]; + site = finalAttrs.finalPackage + "/share/doc/nix/manual"; + + tests = + let + redirect-targets = callPackage ./redirect-targets-html.nix { }; + in + { + # https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck + linkcheck = testers.lycheeLinkCheck { + site = + let + plain = finalAttrs.finalPackage.site; + in + runCommand "nix-manual-with-redirect-targets" { } '' + cp -r ${plain} $out + chmod -R u+w $out + cp ${redirect-targets}/redirect-targets.html $out/redirect-targets.html + ''; + extraConfig = { + exclude = [ + # Exclude auto-generated JSON schema documentation which has + # auto-generated fragment IDs that don't match the link references + ".*/protocols/json/.*\\.html" + # Exclude undocumented builtins + ".*/language/builtins\\.html#builtins-addErrorContext" + ".*/language/builtins\\.html#builtins-appendContext" + ]; + }; }; }; - }; + }; meta = { platforms = lib.platforms.all; diff --git a/doc/manual/render-manpage.sh b/doc/manual/render-manpage.sh old mode 100755 new mode 100644 index 65a9c124e..6577809b0 --- a/doc/manual/render-manpage.sh +++ b/doc/manual/render-manpage.sh @@ -1,25 +1,55 @@ #!/usr/bin/env bash +# +# Standalone manpage renderer that doesn't require mdbook. +# Uses expand-includes.py to preprocess markdown, then lowdown to generate manpages. set -euo pipefail +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + lowdown_args= +# Optional --out-no-smarty flag for compatibility with nix_nested_manpages if [ "$1" = --out-no-smarty ]; then lowdown_args=--out-no-smarty shift fi -[ "$#" = 4 ] || { - echo "wrong number of args passed" >&2 +[ "$#" = 7 ] || { + cat >&2 <
+ +Arguments: + title - Manpage title (e.g., "nix-env --install") + section - Manpage section number (1, 5, 8, etc.) + source-root - Root directory of markdown sources + generated-root - Root directory of generated markdown files + doc-url - Base URL for documentation links + infile - Input markdown file (relative to build directory) + outfile - Output manpage file + +Examples: + $0 "nix-store --query" 1 doc/manual/source build/doc/manual/source \\ + https://nix.dev/manual/nix/latest \\ + build/doc/manual/source/command-ref/nix-store/query.md nix-store-query.1 +EOF exit 1 } title="$1" section="$2" -infile="$3" -outfile="$4" +source_root="$3" +generated_root="$4" +doc_url="$5" +infile="$6" +outfile="$7" +# Expand includes and pipe to lowdown ( printf "Title: %s\n\n" "$title" - cat "$infile" + python3 "$script_dir/expand-includes.py" \ + --source-root "$source_root" \ + --generated-root "$generated_root" \ + --doc-url "$doc_url" \ + "$infile" ) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile" diff --git a/doc/manual/source/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md index 0860f312d..7c17ce909 100644 --- a/doc/manual/source/command-ref/nix-hash.md +++ b/doc/manual/source/command-ref/nix-hash.md @@ -34,7 +34,7 @@ md5sum`. Print the cryptographic hash of the contents of each regular file *path*. That is, instead of computing the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, - just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. + just [directly hash](@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. This requires *path* to resolve to a regular file rather than directory. The result is identical to that produced by the GNU commands `md5sum` and `sha1sum`. diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build index ab9d76d3e..e32cf0640 100644 --- a/doc/manual/source/protocols/json/meson.build +++ b/doc/manual/source/protocols/json/meson.build @@ -35,27 +35,27 @@ endforeach json_schema_generated_files = [] -# Generate markdown documentation from JSON schema -# Note: output must be just a filename, not a path -gen_file = custom_target( - schema_name + '-schema-docs.tmp', - command : [ - json_schema_for_humans, - '--config-file', - json_schema_config, - meson.current_source_dir() / 'schema', - meson.current_build_dir(), - ], - input : schema_files + [ - json_schema_config, - ], - output : schema_outputs, - capture : false, - build_by_default : true, -) - -idx = 0 if json_schema_for_humans.found() + # Generate markdown documentation from JSON schema + # Note: output must be just a filename, not a path + gen_file = custom_target( + schema_name + '-schema-docs.tmp', + command : [ + json_schema_for_humans, + '--config-file', + json_schema_config, + meson.current_source_dir() / 'schema', + meson.current_build_dir(), + ], + input : schema_files + [ + json_schema_config, + ], + output : schema_outputs, + capture : false, + build_by_default : true, + ) + + idx = 0 foreach schema_name : schemas #schema_file = 'schema' / schema_name + '.yaml' diff --git a/flake.nix b/flake.nix index 56d291d1c..c0155391d 100644 --- a/flake.nix +++ b/flake.nix @@ -369,6 +369,7 @@ # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual; + nix-manual-manpages-only = nixpkgsFor.${system}.native.nixComponents2.nix-manual-manpages-only; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs; } diff --git a/packaging/components.nix b/packaging/components.nix index bbd6208b9..dbf2180e8 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -429,6 +429,15 @@ in The manual as would be published on https://nix.dev/reference/nix-manual */ nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; }; + + /** + Manpages only (no HTML manual, no mdbook dependency) + */ + nix-manual-manpages-only = callPackage ../doc/manual/package.nix { + version = fineVersion; + buildHtmlManual = false; + }; + /** Doxygen pages for C++ code */ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 67e2c0dfd..20605418b 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -70,6 +70,7 @@ let ] ++ lib.optionals enableDocs [ "nix-manual" + "nix-manual-manpages-only" "nix-internal-api-docs" "nix-external-api-docs" ]