diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 2bfdae17b..0c5c103bf 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -116,6 +116,7 @@ rec { ) nixComponentsInstrumented) // lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) { "${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests; + "${componentTestsPrefix}nix-json-schema-checks" = nixComponentsInstrumented.nix-json-schema-checks; }; codeCoverage = diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 2e372dedd..7090c949c 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -115,6 +115,7 @@ manual = custom_target( builtins_md, rl_next_generated, summary_rl_next, + json_schema_generated_files, nix_input, ], output : [ diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 69b7c0e49..7b94721ae 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -12,6 +12,7 @@ rsync, nix-cli, changelog-d, + json-schema-for-humans, officialRelease, # Configuration Options @@ -55,6 +56,7 @@ mkMesonDerivation (finalAttrs: { jq python3 rsync + json-schema-for-humans changelog-d ] ++ lib.optionals (!officialRelease) [ diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 25e68811d..f74ed7043 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -117,6 +117,7 @@ - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) + - [Hash](protocols/json/hash.md) - [Store Object Info](protocols/json/store-object-info.md) - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) diff --git a/doc/manual/source/meson.build b/doc/manual/source/meson.build index 949d26526..294d57ad9 100644 --- a/doc/manual/source/meson.build +++ b/doc/manual/source/meson.build @@ -1,3 +1,6 @@ +# Process JSON schema documentation +subdir('protocols') + summary_rl_next = custom_target( command : [ bash, diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index cc9389f7c..602ab67e4 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,120 +1,7 @@ -# Derivation JSON Format +{{#include derivation-v3-fixed.md}} -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. + diff --git a/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed new file mode 100644 index 000000000..126e666e9 --- /dev/null +++ b/doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed @@ -0,0 +1,14 @@ +# For some reason, backticks in the JSON schema are being escaped rather +# than being kept as intentional code spans. This removes all backtick +# escaping, which is an ugly solution, but one that is fine, because we +# are not using backticks for any other purpose. +s/\\`/`/g + +# The way that semi-external references are rendered (i.e. ones to +# sibling schema files, as opposed to separate website ones, is not nice +# for humans. Replace it with a nice relative link within the manual +# instead. +# +# As we have more such relative links, more replacements of this nature +# should appear below. +s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g diff --git a/doc/manual/source/protocols/json/hash.md b/doc/manual/source/protocols/json/hash.md new file mode 100644 index 000000000..d2bdf1062 --- /dev/null +++ b/doc/manual/source/protocols/json/hash.md @@ -0,0 +1,7 @@ +{{#include hash-v1-fixed.md}} + + diff --git a/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml b/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml new file mode 100644 index 000000000..cad098053 --- /dev/null +++ b/doc/manual/source/protocols/json/json-schema-for-humans-config.yaml @@ -0,0 +1,17 @@ +# Configuration file for json-schema-for-humans +# +# https://github.com/coveooss/json-schema-for-humans/blob/main/docs/examples/examples_md_default/Configuration.md + +template_name: md +show_toc: true +# impure timestamp and distracting +with_footer: false +recursive_detection_depth: 3 +show_breadcrumbs: false +description_is_markdown: true +template_md_options: + properties_table_columns: + - Property + - Type + - Pattern + - Title/Description diff --git a/doc/manual/source/protocols/json/meson.build b/doc/manual/source/protocols/json/meson.build new file mode 100644 index 000000000..44795599c --- /dev/null +++ b/doc/manual/source/protocols/json/meson.build @@ -0,0 +1,74 @@ +# Tests in: ../../../../src/json-schema-checks + +fs = import('fs') + +# Find json-schema-for-humans if available +json_schema_for_humans = find_program('generate-schema-doc', required : false) + +# Configuration for json-schema-for-humans +json_schema_config = files('json-schema-for-humans-config.yaml') + +schemas = [ + 'hash-v1', + 'derivation-v3', +] + +schema_files = files() +foreach schema_name : schemas + schema_files += files('schema' / schema_name + '.yaml') +endforeach + + +schema_outputs = [] +foreach schema_name : schemas + schema_outputs += schema_name + '.md' +endforeach + +json_schema_generated_files = [] + +# Generate markdown documentation from JSON schema +# Note: output must be just a filename, not a path +gen_file = custom_target( + schema_name + '-schema-docs.tmp', + command : [ + json_schema_for_humans, + '--config-file', + json_schema_config, + meson.current_source_dir() / 'schema', + meson.current_build_dir(), + ], + input : schema_files + [ + json_schema_config, + ], + output : schema_outputs, + capture : false, + build_by_default : true, +) + +idx = 0 +if json_schema_for_humans.found() + foreach schema_name : schemas + #schema_file = 'schema' / schema_name + '.yaml' + + # There is one so-so hack, and one horrible hack being done here. + sedded_file = custom_target( + schema_name + '-schema-docs', + command : [ + 'sed', + '-f', + # Out of line to avoid https://github.com/mesonbuild/meson/issues/1564 + files('fixup-json-schema-generated-doc.sed'), + '@INPUT@', + ], + capture : true, + input : gen_file[idx], + output : schema_name + '-fixed.md', + ) + idx += 1 + json_schema_generated_files += [ sedded_file ] + endforeach +else + warning( + 'json-schema-for-humans not found, skipping JSON schema documentation generation', + ) +endif diff --git a/doc/manual/source/protocols/json/schema/derivation-v3.yaml b/doc/manual/source/protocols/json/schema/derivation-v3.yaml new file mode 100644 index 000000000..7c92d475d --- /dev/null +++ b/doc/manual/source/protocols/json/schema/derivation-v3.yaml @@ -0,0 +1,178 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json +title: Derivation +description: | + Experimental JSON representation of a Nix derivation (version 3). + + This schema describes the JSON representation of Nix's `Derivation` type. + + > **Warning** + > + > This JSON format is currently + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and subject to change. + +type: object +required: + - name + - version + - outputs + - inputSrcs + - inputDrvs + - system + - builder + - args + - env +properties: + name: + type: string + title: Derivation name + description: | + The name of the derivation. + Used when calculating store paths for the derivation’s outputs. + + version: + const: 3 + title: Format version (must be 3) + description: | + Must be `3`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: + + - Version 0: A-Term format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Version 3: Drop store dir from store paths, just include base name. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + + outputs: + type: object + title: Output specifications + description: | + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + additionalProperties: + "$ref": "#/$defs/output" + + inputSrcs: + type: array + title: Input source paths + description: | + List of store paths on which this derivation depends. + + > **Example** + > + > ```json + > "inputSrcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + items: + type: string + + inputDrvs: + type: object + title: Input derivations + description: | + Mapping of derivation paths to lists of output names they provide. + + > **Example** + > + > ```json + > "inputDrvs": { + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + > + > specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + + system: + type: string + title: Build system type + description: | + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + + builder: + type: string + title: Build program path + description: | + Absolute path of the program used to perform the build. + Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + + args: + type: array + title: Builder arguments + description: | + Command-line arguments passed to the `builder`. + items: + type: string + + env: + type: object + title: Environment variables + description: | + Environment variables passed to the `builder`. + additionalProperties: + type: string + + structuredAttrs: + title: Structured attributes + description: | + [Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. + type: object + additionalProperties: true + +"$defs": + output: + type: object + properties: + path: + type: string + title: Output path + description: | + The output path, if known in advance. + + method: + type: string + title: Content addressing method + enum: [flat, nar, text, git] + description: | + For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + hashAlgo: + title: Hash algorithm + "$ref": "./hash-v1.yaml#/$defs/algorithm" + + hash: + type: string + title: Expected hash value + description: | + For fixed-output derivations, the expected content hash in base-16. diff --git a/doc/manual/source/protocols/json/schema/hash-v1.yaml b/doc/manual/source/protocols/json/schema/hash-v1.yaml new file mode 100644 index 000000000..44a59541b --- /dev/null +++ b/doc/manual/source/protocols/json/schema/hash-v1.yaml @@ -0,0 +1,30 @@ +"$schema": http://json-schema.org/draft-04/schema# +"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json +title: Hash +description: | + A cryptographic hash value used throughout Nix for content addressing and integrity verification. + + This schema describes the JSON representation of Nix's `Hash` type. + + TODO Work in progress +type: object +properties: + algorithm: + title: Hash algorithm + "$ref": "#/$defs/algorithm" +required: +- algorithm +additionalProperties: false +"$defs": + algorithm: + type: string + enum: + - blake3 + - md5 + - sha1 + - sha256 + - sha512 + description: | + The hash algorithm used to compute the hash value. + + `blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature. diff --git a/doc/manual/source/protocols/meson.build b/doc/manual/source/protocols/meson.build new file mode 100644 index 000000000..5b5eb900d --- /dev/null +++ b/doc/manual/source/protocols/meson.build @@ -0,0 +1,2 @@ +# Process JSON schema documentation +subdir('json') diff --git a/flake.nix b/flake.nix index fd623c807..8d3d963be 100644 --- a/flake.nix +++ b/flake.nix @@ -413,6 +413,10 @@ supportsCross = false; }; + "nix-json-schema-checks" = { + supportsCross = false; + }; + "nix-perl-bindings" = { supportsCross = false; }; diff --git a/meson.build b/meson.build index 736756157..f3158ea6d 100644 --- a/meson.build +++ b/meson.build @@ -60,3 +60,4 @@ if get_option('unit-tests') subproject('libflake-tests') endif subproject('nix-functional-tests') +subproject('json-schema-checks') diff --git a/packaging/components.nix b/packaging/components.nix index c621b7073..f9d7b109a 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -438,6 +438,11 @@ in */ nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; + /** + JSON schema validation checks + */ + nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { }; + nix-perl-bindings = callPackage ../src/perl/package.nix { }; /** diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 5fb4f14d2..153e7a3eb 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -108,6 +108,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs + ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs ++ lib.optional ( !buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 diff --git a/packaging/hydra.nix b/packaging/hydra.nix index bc75b5dfb..3bbb6c15b 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -62,6 +62,7 @@ let "nix-cmd" "nix-cli" "nix-functional-tests" + "nix-json-schema-checks" ] ++ lib.optionals enableBindings [ "nix-perl-bindings" diff --git a/src/json-schema-checks/.version b/src/json-schema-checks/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/json-schema-checks/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/json-schema-checks/derivation b/src/json-schema-checks/derivation new file mode 120000 index 000000000..3dc1cbe06 --- /dev/null +++ b/src/json-schema-checks/derivation @@ -0,0 +1 @@ +../../src/libstore-tests/data/derivation \ No newline at end of file diff --git a/src/json-schema-checks/meson.build b/src/json-schema-checks/meson.build new file mode 100644 index 000000000..5326ad4c6 --- /dev/null +++ b/src/json-schema-checks/meson.build @@ -0,0 +1,76 @@ +# Run with: +# meson test --suite json-schema +# Run with: (without shell / configure) +# nix build .#nix-json-schema-checks + +project( + 'nix-json-schema-checks', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Note: The 'jsonschema' package provides the 'jv' command +jv = find_program('jv', required : true) + +# The schema directory is a committed symlink to the actual schema location +schema_dir = meson.current_source_dir() / 'schema' + +# Get all example files +schemas = [ + { + 'stem' : 'derivation', + 'schema' : schema_dir / 'derivation-v3.yaml', + 'files' : [ + 'dyn-dep-derivation.json', + 'simple-derivation.json', + ], + }, + # # Not sure how to make subschema work + # { + # 'stem': 'derivation', + # 'schema': schema_dir / 'derivation-v3.yaml#output', + # 'files' : [ + # 'output-caFixedFlat.json', + # 'output-caFixedNAR.json', + # 'output-caFixedText.json', + # 'output-caFloating.json', + # 'output-deferred.json', + # 'output-impure.json', + # 'output-inputAddressed.json', + # ], + # }, +] + +# Validate each example against the schema +foreach schema : schemas + stem = schema['stem'] + schema_file = schema['schema'] + if '#' not in schema_file + # Validate the schema itself against JSON Schema Draft 04 + test( + stem + '-schema-valid', + jv, + args : [ + '--map', + './hash-v1.yaml=' + schema_dir / 'hash-v1.yaml', + 'http://json-schema.org/draft-04/schema', + schema_file, + ], + suite : 'json-schema', + ) + endif + foreach example : schema['files'] + test( + stem + '-example-' + fs.stem(example), + jv, + args : [ + schema_file, + files(stem / example), + ], + suite : 'json-schema', + ) + endforeach +endforeach diff --git a/src/json-schema-checks/package.nix b/src/json-schema-checks/package.nix new file mode 100644 index 000000000..2061672cd --- /dev/null +++ b/src/json-schema-checks/package.nix @@ -0,0 +1,50 @@ +# Run with: nix build .#nix-json-schema-checks +{ + lib, + mkMesonDerivation, + + meson, + ninja, + jsonschema, + + # Configuration Options + + version, +}: + +mkMesonDerivation (finalAttrs: { + pname = "nix-json-schema-checks"; + inherit version; + + workDir = ./.; + fileset = lib.fileset.unions [ + ../../.version + ../../doc/manual/source/protocols/json/schema + ../../src/libstore-tests/data/derivation + ./. + ]; + + outputs = [ "out" ]; + + passthru.externalNativeBuildInputs = [ + jsonschema + ]; + + nativeBuildInputs = [ + meson + ninja + ] + ++ finalAttrs.passthru.externalNativeBuildInputs; + + doCheck = true; + + mesonCheckFlags = [ "--print-errorlogs" ]; + + postInstall = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/json-schema-checks/schema b/src/json-schema-checks/schema new file mode 120000 index 000000000..473e47b1b --- /dev/null +++ b/src/json-schema-checks/schema @@ -0,0 +1 @@ +../../doc/manual/source/protocols/json/schema \ No newline at end of file diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index 35507d9ad..4e37c4e6f 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -12,8 +12,7 @@ a Nix expression evaluates. [store derivation]: @docroot@/glossary.md#gloss-store-derivation -`nix derivation add` takes a single derivation in the following format: - -{{#include ../../protocols/json/derivation.md}} +`nix derivation add` takes a single derivation in the JSON format. +See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format. )"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 9fff58ef9..1784be44c 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -48,10 +48,9 @@ By default, this command only shows top-level derivations, but with [store derivation]: @docroot@/glossary.md#gloss-store-derivation -`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format: +`nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format. +See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format. [store path]: @docroot@/store/store-path.md -{{#include ../../protocols/json/derivation.md}} - )""