1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-12-08 18:11:02 +01:00

Merge pull request #14727 from roberth/issue-14548

Make mdBook dependency optional (#14548), fix manpage links
This commit is contained in:
John Ericson 2025-12-06 22:49:31 +00:00 committed by GitHub
commit 42d7d9676d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 477 additions and 134 deletions

View file

@ -0,0 +1,223 @@
#!/usr/bin/env python3
"""
Standalone markdown preprocessor for manpage generation.
Expands {{#include}} directives and handles @docroot@ references
without requiring mdbook.
"""
from pathlib import Path
import sys
import argparse
import re
def expand_includes(
content: str,
current_file: Path,
source_root: Path,
generated_root: Path | None,
visited: set[Path] | None = None,
) -> str:
"""
Recursively expand {{#include path}} directives.
Args:
content: Markdown content to process
current_file: Path to the current file (for resolving relative includes)
source_root: Root of the source directory
generated_root: Root of generated files (for @generated@/ includes)
visited: Set of already-visited files (for cycle detection)
"""
if visited is None:
visited = set()
# Track current file to detect cycles
visited.add(current_file.resolve())
lines = []
include_pattern = re.compile(r'^\s*\{\{#include\s+(.+?)\}\}\s*$')
for line in content.splitlines(keepends=True):
match = include_pattern.match(line)
if not match:
lines.append(line)
continue
# Found an include directive
include_path_str = match.group(1).strip()
# Resolve the include path
if include_path_str.startswith("@generated@/"):
# Generated file
if generated_root is None:
raise ValueError(
f"Cannot resolve @generated@ path '{include_path_str}' "
f"without --generated-root"
)
include_path = generated_root / include_path_str[12:]
else:
# Relative to current file
include_path = (current_file.parent / include_path_str).resolve()
# Check for cycles
if include_path.resolve() in visited:
raise RuntimeError(
f"Include cycle detected: {include_path} is already being processed"
)
# Check that file exists
if not include_path.exists():
raise FileNotFoundError(
f"Include file not found: {include_path_str}\n"
f" Resolved to: {include_path}\n"
f" From: {current_file}"
)
# Recursively expand the included file
included_content = include_path.read_text()
expanded = expand_includes(
included_content,
include_path,
source_root,
generated_root,
visited.copy(), # Copy visited set for this branch
)
lines.append(expanded)
# Add newline if the included content doesn't end with one
if not expanded.endswith('\n'):
lines.append('\n')
return ''.join(lines)
def resolve_docroot(content: str, current_file: Path, source_root: Path, docroot_url: str) -> str:
"""
Replace @docroot@ with nix.dev URL and convert .md to .html.
For manpages, absolute URLs are more useful than relative paths since
manpages are viewed standalone. lowdown will display these as proper
references in the manpage output.
"""
# Replace @docroot@ with the base URL
content = content.replace("@docroot@", docroot_url)
# Convert .md extensions to .html for web links
# Use lookahead to ensure that .md occurs before a fragment or a possible URL end.
content = re.sub(
r'(https://nix\.dev/[^)\s]*?)\.md(?=[#)\s]|$)',
r'\1.html',
content
)
return content
def resolve_at_escapes(content: str) -> str:
"""Replace @_at_ with @"""
return content.replace("@_at_", "@")
def process_file(
input_file: Path,
source_root: Path,
generated_root: Path | None,
docroot_url: str,
) -> str:
"""Process a single markdown file."""
content = input_file.read_text()
# Expand includes
content = expand_includes(content, input_file, source_root, generated_root)
# Resolve @docroot@ references
content = resolve_docroot(content, input_file, source_root, docroot_url)
# Resolve @_at_ escapes
content = resolve_at_escapes(content)
return content
def main():
parser = argparse.ArgumentParser(
description="Expand markdown includes for manpage generation",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Expand a manpage source file
%(prog)s \\
--source-root doc/manual/source \\
--generated-root build/doc/manual/source \\
doc/manual/source/command-ref/nix-store/query.md
# Pipe to lowdown for manpage generation
%(prog)s -s doc/manual/source -g build/doc/manual/source \\
doc/manual/source/command-ref/nix-env.md | \\
lowdown -sT man -M section=1 -o nix-env.1
""",
)
parser.add_argument(
"input_file",
type=Path,
help="Input markdown file to process",
)
parser.add_argument(
"-s", "--source-root",
type=Path,
required=True,
help="Root directory of markdown sources",
)
parser.add_argument(
"-g", "--generated-root",
type=Path,
help="Root directory of generated files (for @generated@/ includes)",
)
parser.add_argument(
"-o", "--output",
type=Path,
help="Output file (default: stdout)",
)
parser.add_argument(
"-u", "--doc-url",
type=str,
default="https://nix.dev/manual/nix/latest",
help="Base URL for documentation links (default: https://nix.dev/manual/nix/latest)",
)
args = parser.parse_args()
# Validate paths
if not args.input_file.exists():
print(f"Error: Input file not found: {args.input_file}", file=sys.stderr)
return 1
if not args.source_root.is_dir():
print(f"Error: Source root is not a directory: {args.source_root}", file=sys.stderr)
return 1
if args.generated_root and not args.generated_root.is_dir():
print(f"Error: Generated root is not a directory: {args.generated_root}", file=sys.stderr)
return 1
try:
# Process the file
output = process_file(args.input_file, args.source_root, args.generated_root, args.doc_url)
# Write output
if args.output:
args.output.write_text(output)
else:
print(output, end='')
return 0
except Exception as e:
print(f"Error processing {args.input_file}: {e}", file=sys.stderr)
import traceback
traceback.print_exc(file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())

View file

@ -5,11 +5,29 @@ project(
license : 'LGPL-2.1-or-later',
)
# Compute documentation URL based on version and release type
version = meson.project_version()
official_release = get_option('official-release')
if official_release
# For official releases, use versioned URL (dropping patch version)
version_parts = version.split('.')
major_minor = '@0@.@1@'.format(version_parts[0], version_parts[1])
doc_url = 'https://nix.dev/manual/nix/@0@'.format(major_minor)
else
# For development builds, use /latest
doc_url = 'https://nix.dev/manual/nix/latest'
endif
nix = find_program('nix', native : true)
mdbook = find_program('mdbook', native : true)
bash = find_program('bash', native : true)
rsync = find_program('rsync', required : true, native : true)
# HTML manual dependencies (conditional)
if get_option('html-manual')
mdbook = find_program('mdbook', native : true)
rsync = find_program('rsync', required : true, native : true)
endif
pymod = import('python')
python = pymod.find_installation('python3')
@ -95,67 +113,71 @@ else
nix_input = []
endif
manual = custom_target(
'manual',
command : [
bash,
'-euo',
'pipefail',
'-c',
'''
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
rm -rf @2@/manual
mv @2@/html @2@/manual
# Remove Mathjax 2.7, because we will actually use MathJax 3.x
find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d'
find @2@/manual -iname meson.build -delete
'''.format(
python.full_path(),
mdbook.full_path(),
meson.current_build_dir(),
meson.project_version(),
rsync.full_path(),
),
],
input : [
generate_manual_deps,
'substitute.py',
'book.toml.in',
'anchors.jq',
'custom.css',
redirects_js,
nix3_cli_files,
experimental_features_shortlist_md,
experimental_feature_descriptions_md,
types_dir,
conf_file_md,
builtins_md,
rl_next_generated,
summary_rl_next,
json_schema_generated_files,
nix_input,
],
output : [
# HTML manual build (conditional)
if get_option('html-manual')
manual = custom_target(
'manual',
'markdown',
],
depfile : 'manual.d',
env : {
'RUST_LOG' : 'info',
'MDBOOK_SUBSTITUTE_SEARCH' : meson.current_build_dir() / 'source',
},
)
manual_html = manual[0]
manual_md = manual[1]
command : [
bash,
'-euo',
'pipefail',
'-c',
'''
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
rm -rf @2@/manual
mv @2@/html @2@/manual
# Remove Mathjax 2.7, because we will actually use MathJax 3.x
find @2@/manual | grep .html | xargs sed -i -e '/2.7.1.MathJax.js/d'
find @2@/manual -iname meson.build -delete
'''.format(
python.full_path(),
mdbook.full_path(),
meson.current_build_dir(),
meson.project_version(),
rsync.full_path(),
),
],
input : [
generate_manual_deps,
'substitute.py',
'book.toml.in',
'anchors.jq',
'custom.css',
redirects_js,
nix3_cli_files,
experimental_features_shortlist_md,
experimental_feature_descriptions_md,
types_dir,
conf_file_md,
builtins_md,
rl_next_generated,
summary_rl_next,
json_schema_generated_files,
nix_input,
],
output : [
'manual',
'markdown',
],
depfile : 'manual.d',
build_by_default : true,
env : {
'RUST_LOG' : 'info',
'MDBOOK_SUBSTITUTE_SEARCH' : meson.current_build_dir() / 'source',
},
)
manual_html = manual[0]
manual_md = manual[1]
install_subdir(
manual_html.full_path(),
install_dir : get_option('datadir') / 'doc/nix',
)
install_subdir(
manual_html.full_path(),
install_dir : get_option('datadir') / 'doc/nix',
)
endif
nix_nested_manpages = [
[
@ -201,6 +223,7 @@ nix_nested_manpages = [
],
]
# Manpage generation (standalone, no mdbook dependency)
foreach command : nix_nested_manpages
foreach page : command[1]
title = command[0] + ' --' + page
@ -208,15 +231,19 @@ foreach command : nix_nested_manpages
custom_target(
command : [
bash,
files('./render-manpage.sh'),
'@INPUT0@',
'--out-no-smarty',
title,
section,
'@INPUT0@/command-ref' / command[0] / (page + '.md'),
meson.current_source_dir() / 'source',
meson.current_build_dir() / 'source',
doc_url,
meson.current_source_dir() / 'source/command-ref' / command[0] / (page + '.md'),
'@OUTPUT0@',
],
input : [
manual_md,
files('./render-manpage.sh'),
files('./expand-includes.py'),
nix_input,
],
output : command[0] + '-' + page + '.1',
@ -325,14 +352,21 @@ foreach page : nix3_manpages
command : [
bash,
'@INPUT0@',
# Note: no --out-no-smarty flag (original behavior)
page,
section,
'@INPUT1@/command-ref/new-cli/@0@.md'.format(page),
meson.current_source_dir() / 'source',
meson.current_build_dir() / 'source',
doc_url,
meson.current_build_dir() / 'source/command-ref/new-cli/@0@.md'.format(
page,
),
'@OUTPUT@',
],
input : [
files('./render-manpage.sh'),
manual_md,
files('./expand-includes.py'),
nix3_cli_files,
nix_input,
],
output : page + '.1',
@ -352,7 +386,12 @@ nix_manpages = [
[ 'nix-channel', 1 ],
[ 'nix-hash', 1 ],
[ 'nix-copy-closure', 1 ],
[ 'nix.conf', 5, conf_file_md.full_path() ],
[
'nix.conf',
5,
conf_file_md.full_path(),
[ conf_file_md, experimental_features_shortlist_md ],
],
[ 'nix-daemon', 8 ],
[ 'nix-profiles', 5, 'files/profiles.md' ],
]
@ -364,19 +403,24 @@ foreach entry : nix_manpages
# Therefore we use an optional third element of this array to override the name pattern
md_file = entry.get(2, title + '.md')
section = entry[1].to_string()
md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file)
input_file = meson.current_source_dir() / 'source/command-ref' / md_file
custom_target(
command : [
bash,
'@INPUT0@',
# Note: no --out-no-smarty flag (original behavior)
title,
section,
md_file_resolved,
meson.current_source_dir() / 'source',
meson.current_build_dir() / 'source',
doc_url,
input_file,
'@OUTPUT@',
],
input : [
files('./render-manpage.sh'),
manual_md,
files('./expand-includes.py'),
entry.get(3, []),
nix_input,
],

13
doc/manual/meson.options Normal file
View file

@ -0,0 +1,13 @@
option(
'official-release',
type : 'boolean',
value : true,
description : 'Whether this is an official release build (affects documentation URLs)',
)
option(
'html-manual',
type : 'boolean',
value : true,
description : 'Whether to build the HTML manual (requires mdbook)',
)

View file

@ -19,6 +19,11 @@
# Configuration Options
version,
/**
Whether to build the HTML manual.
When false, only manpages are built, avoiding the mdbook dependency.
*/
buildHtmlManual ? true,
# `tests` attribute
testers,
@ -57,9 +62,22 @@ mkMesonDerivation (finalAttrs: {
../../doc/manual/package.nix;
# TODO the man pages should probably be separate
outputs = [
"out"
"man"
outputs =
if buildHtmlManual then
[
"out"
"man"
]
else
[ "out" ]; # Only one output when HTML manual is disabled; use "out" for manpages
# When HTML manual is disabled, install manpages to "out" instead of "man"
mesonFlags = [
(lib.mesonBool "official-release" officialRelease)
(lib.mesonBool "html-manual" buildHtmlManual)
]
++ lib.optionals (!buildHtmlManual) [
"--mandir=${placeholder "out"}/share/man"
];
nativeBuildInputs = [
@ -67,14 +85,15 @@ mkMesonDerivation (finalAttrs: {
meson
ninja
(lib.getBin lowdown-unsandboxed)
mdbook
jq
python3
]
++ lib.optionals buildHtmlManual [
mdbook
rsync
json-schema-for-humans
changelog-d
]
++ lib.optionals (!officialRelease) [
++ lib.optionals (!officialRelease && buildHtmlManual) [
# When not an official release, we likely have changelog entries that have
# yet to be rendered.
# When released, these are rendered into a committed file to save a dependency.
@ -86,45 +105,48 @@ mkMesonDerivation (finalAttrs: {
echo ${finalAttrs.version} > ./.version
'';
postInstall = ''
postInstall = lib.optionalString buildHtmlManual ''
mkdir -p ''$out/nix-support
echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products
'';
/**
The root of the HTML manual.
E.g. "${nix-manual.site}/index.html" exists.
*/
passthru.site = finalAttrs.finalPackage + "/share/doc/nix/manual";
passthru = lib.optionalAttrs buildHtmlManual {
/**
The root of the HTML manual.
E.g. "${nix-manual.site}/index.html" exists.
*/
passthru.tests =
let
redirect-targets = callPackage ./redirect-targets-html.nix { };
in
{
# https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck
linkcheck = testers.lycheeLinkCheck {
site =
let
plain = finalAttrs.finalPackage.site;
in
runCommand "nix-manual-with-redirect-targets" { } ''
cp -r ${plain} $out
chmod -R u+w $out
cp ${redirect-targets}/redirect-targets.html $out/redirect-targets.html
'';
extraConfig = {
exclude = [
# Exclude auto-generated JSON schema documentation which has
# auto-generated fragment IDs that don't match the link references
".*/protocols/json/.*\\.html"
# Exclude undocumented builtins
".*/language/builtins\\.html#builtins-addErrorContext"
".*/language/builtins\\.html#builtins-appendContext"
];
site = finalAttrs.finalPackage + "/share/doc/nix/manual";
tests =
let
redirect-targets = callPackage ./redirect-targets-html.nix { };
in
{
# https://nixos.org/manual/nixpkgs/stable/index.html#tester-lycheeLinkCheck
linkcheck = testers.lycheeLinkCheck {
site =
let
plain = finalAttrs.finalPackage.site;
in
runCommand "nix-manual-with-redirect-targets" { } ''
cp -r ${plain} $out
chmod -R u+w $out
cp ${redirect-targets}/redirect-targets.html $out/redirect-targets.html
'';
extraConfig = {
exclude = [
# Exclude auto-generated JSON schema documentation which has
# auto-generated fragment IDs that don't match the link references
".*/protocols/json/.*\\.html"
# Exclude undocumented builtins
".*/language/builtins\\.html#builtins-addErrorContext"
".*/language/builtins\\.html#builtins-appendContext"
];
};
};
};
};
};
meta = {
platforms = lib.platforms.all;

40
doc/manual/render-manpage.sh Executable file → Normal file
View file

@ -1,25 +1,55 @@
#!/usr/bin/env bash
#
# Standalone manpage renderer that doesn't require mdbook.
# Uses expand-includes.py to preprocess markdown, then lowdown to generate manpages.
set -euo pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
lowdown_args=
# Optional --out-no-smarty flag for compatibility with nix_nested_manpages
if [ "$1" = --out-no-smarty ]; then
lowdown_args=--out-no-smarty
shift
fi
[ "$#" = 4 ] || {
echo "wrong number of args passed" >&2
[ "$#" = 7 ] || {
cat >&2 <<EOF
Usage: $0 [--out-no-smarty] <title> <section> <source-root> <generated-root> <doc-url> <infile> <outfile>
Arguments:
title - Manpage title (e.g., "nix-env --install")
section - Manpage section number (1, 5, 8, etc.)
source-root - Root directory of markdown sources
generated-root - Root directory of generated markdown files
doc-url - Base URL for documentation links
infile - Input markdown file (relative to build directory)
outfile - Output manpage file
Examples:
$0 "nix-store --query" 1 doc/manual/source build/doc/manual/source \\
https://nix.dev/manual/nix/latest \\
build/doc/manual/source/command-ref/nix-store/query.md nix-store-query.1
EOF
exit 1
}
title="$1"
section="$2"
infile="$3"
outfile="$4"
source_root="$3"
generated_root="$4"
doc_url="$5"
infile="$6"
outfile="$7"
# Expand includes and pipe to lowdown
(
printf "Title: %s\n\n" "$title"
cat "$infile"
python3 "$script_dir/expand-includes.py" \
--source-root "$source_root" \
--generated-root "$generated_root" \
--doc-url "$doc_url" \
"$infile"
) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile"

View file

@ -34,7 +34,7 @@ md5sum`.
Print the cryptographic hash of the contents of each regular file *path*.
That is, instead of computing
the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*,
just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is.
just [directly hash](@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is.
This requires *path* to resolve to a regular file rather than directory.
The result is identical to that produced by the GNU commands
`md5sum` and `sha1sum`.

View file

@ -35,27 +35,27 @@ endforeach
json_schema_generated_files = []
# Generate markdown documentation from JSON schema
# Note: output must be just a filename, not a path
gen_file = custom_target(
schema_name + '-schema-docs.tmp',
command : [
json_schema_for_humans,
'--config-file',
json_schema_config,
meson.current_source_dir() / 'schema',
meson.current_build_dir(),
],
input : schema_files + [
json_schema_config,
],
output : schema_outputs,
capture : false,
build_by_default : true,
)
idx = 0
if json_schema_for_humans.found()
# Generate markdown documentation from JSON schema
# Note: output must be just a filename, not a path
gen_file = custom_target(
schema_name + '-schema-docs.tmp',
command : [
json_schema_for_humans,
'--config-file',
json_schema_config,
meson.current_source_dir() / 'schema',
meson.current_build_dir(),
],
input : schema_files + [
json_schema_config,
],
output : schema_outputs,
capture : false,
build_by_default : true,
)
idx = 0
foreach schema_name : schemas
#schema_file = 'schema' / schema_name + '.yaml'

View file

@ -369,6 +369,7 @@
# TODO probably should be `nix-cli`
nix = self.packages.${system}.nix-everything;
nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual;
nix-manual-manpages-only = nixpkgsFor.${system}.native.nixComponents2.nix-manual-manpages-only;
nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs;
nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs;
}

View file

@ -429,6 +429,15 @@ in
The manual as would be published on https://nix.dev/reference/nix-manual
*/
nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; };
/**
Manpages only (no HTML manual, no mdbook dependency)
*/
nix-manual-manpages-only = callPackage ../doc/manual/package.nix {
version = fineVersion;
buildHtmlManual = false;
};
/**
Doxygen pages for C++ code
*/

View file

@ -70,6 +70,7 @@ let
]
++ lib.optionals enableDocs [
"nix-manual"
"nix-manual-manpages-only"
"nix-internal-api-docs"
"nix-external-api-docs"
]