nixos/nix
1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-19 16:59:35 +01:00
Code Activity

Merge branch 'master' into flake_show_attr

Browse source
This commit is contained in:
Matthieu Coudron 2025-11-15 23:30:42 +01:00 • committed by GitHub
commit 653d701300
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
466 changed files with 11446 additions and 3038 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/external-api-docs/README.md
View file

@ -15,7 +15,7 @@ programmatically:
1. Embedding the evaluator
2. Writing language plug-ins
Embedding means you link the Nix C libraries in your program and use them from
Embedding means you link the Nix C API libraries in your program and use them from
there. Adding a plug-in means you make a library that gets loaded by the Nix
language evaluator, specified through a configuration option.

1
src/json-schema-checks/build-result Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/build-result

1
src/json-schema-checks/build-trace-entry Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/realisation

1
src/json-schema-checks/content-address Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/content-address

1
src/json-schema-checks/deriving-path Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/derived-path

170
src/json-schema-checks/meson.build
View file

@ -30,28 +30,166 @@ schemas = [
'blake3-base64.json',
],
},
{
'stem' : 'content-address',
'schema' : schema_dir / 'content-address-v1.yaml',
'files' : [
'text.json',
'nar.json',
],
},
{
'stem' : 'store-path',
'schema' : schema_dir / 'store-path-v1.yaml',
'files' : [
'simple.json',
],
},
{
'stem' : 'deriving-path',
'schema' : schema_dir / 'deriving-path-v1.yaml',
'files' : [
'single_opaque.json',
'single_built.json',
'single_built_built.json',
],
},
{
'stem' : 'build-trace-entry',
'schema' : schema_dir / 'build-trace-entry-v1.yaml',
'files' : [
'simple.json',
'with-dependent-realisations.json',
'with-signature.json',
],
},
]
# Derivation and Derivation output
schemas += [
# Match overall
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml',
'schema' : schema_dir / 'derivation-v4.yaml',
'files' : [
'dyn-dep-derivation.json',
'simple-derivation.json',
],
},
# # Not sure how to make subschema work
# {
# 'stem': 'derivation',
# 'schema': schema_dir / 'derivation-v3.yaml#output',
# 'files' : [
# 'output-caFixedFlat.json',
# 'output-caFixedNAR.json',
# 'output-caFixedText.json',
# 'output-caFloating.json',
# 'output-deferred.json',
# 'output-impure.json',
# 'output-inputAddressed.json',
# ],
# },
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/overall',
'files' : [
'output-caFixedFlat.json',
'output-caFixedNAR.json',
'output-caFixedText.json',
'output-caFloating.json',
'output-deferred.json',
'output-impure.json',
'output-inputAddressed.json',
],
},
# Match exact variant
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/inputAddressed',
'files' : [
'output-inputAddressed.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFixed',
'files' : [
'output-caFixedFlat.json',
'output-caFixedNAR.json',
'output-caFixedText.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/caFloating',
'files' : [
'output-caFloating.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/deferred',
'files' : [
'output-deferred.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v4.yaml#/$defs/output/impure',
'files' : [
'output-impure.json',
],
},
]
# Store object info
schemas += [
# Match overall
{
'stem' : 'store-object-info',
'schema' : schema_dir / 'store-object-info-v2.yaml',
'files' : [
'pure.json',
'impure.json',
'empty_pure.json',
'empty_impure.json',
],
},
{
'stem' : 'nar-info',
'schema' : schema_dir / 'store-object-info-v2.yaml',
'files' : [
'pure.json',
'impure.json',
],
},
{
'stem' : 'build-result',
'schema' : schema_dir / 'build-result-v1.yaml',
'files' : [
'success.json',
'output-rejected.json',
'not-deterministic.json',
],
},
# Match exact variant
{
'stem' : 'store-object-info',
'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base',
'files' : [
'pure.json',
'empty_pure.json',
],
},
{
'stem' : 'store-object-info',
'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/impure',
'files' : [
'impure.json',
'empty_impure.json',
],
},
{
'stem' : 'nar-info',
'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/base',
'files' : [
'pure.json',
],
},
{
'stem' : 'nar-info',
'schema' : schema_dir / 'store-object-info-v2.yaml#/$defs/narInfo',
'files' : [
'impure.json',
],
},
]
# Validate each example against the schema
@ -64,8 +202,6 @@ foreach schema : schemas
stem + '-schema-valid',
jv,
args : [
'--map',
'./hash-v1.yaml=' + schema_dir / 'hash-v1.yaml',
'http://json-schema.org/draft-04/schema',
schema_file,
],

1
src/json-schema-checks/nar-info Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/nar-info

15
src/json-schema-checks/package.nix
View file

@ -21,21 +21,24 @@ mkMesonDerivation (finalAttrs: {
../../.version
../../doc/manual/source/protocols/json/schema
../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path
../../src/libstore-tests/data/realisation
../../src/libstore-tests/data/derivation
../../src/libstore-tests/data/derived-path
../../src/libstore-tests/data/path-info
../../src/libstore-tests/data/nar-info
../../src/libstore-tests/data/build-result
./.
];
outputs = [ "out" ];
passthru.externalNativeBuildInputs = [
jsonschema
];
nativeBuildInputs = [
meson
ninja
]
++ finalAttrs.passthru.externalNativeBuildInputs;
jsonschema
];
doCheck = true;

1
src/json-schema-checks/store-object-info Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/path-info

1
src/json-schema-checks/store-path Symbolic link
View file

@ -0,0 +1 @@
../../src/libstore-tests/data/store-path

1
src/kaitai-struct-checks/.version Symbolic link
View file

@ -0,0 +1 @@
../../.version

77
src/kaitai-struct-checks/meson.build Normal file
View file

@ -0,0 +1,77 @@
# Run with:
# meson test --suite kaitai-struct
# Run with: (without shell / configure)
# nix build .#nix-kaitai-struct-checks
project(
'nix-kaitai-struct-checks',
'cpp',
version : files('.version'),
default_options : [
'cpp_std=c++23',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
license : 'LGPL-2.1-or-later',
)
kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true)
gtest_dep = dependency('gtest')
gtest_main_dep = dependency('gtest_main', required : true)
# Find the Kaitai Struct compiler
ksc = find_program('ksc', required : true)
kaitai_generated_srcs = custom_target(
'kaitai-generated-sources',
input : [ 'nar.ksy' ],
output : [ 'nix_nar.cpp', 'nix_nar.h' ],
command : [
ksc,
'@INPUT@',
'--target', 'cpp_stl',
'--outdir',
meson.current_build_dir(),
],
)
nar_kaitai_lib = library(
'nix-nar-kaitai-lib',
kaitai_generated_srcs,
dependencies : [ kaitai_runtime_dep ],
install : true,
)
nar_kaitai_dep = declare_dependency(
link_with : nar_kaitai_lib,
sources : kaitai_generated_srcs[1],
)
# The nar directory is a committed symlink to the actual nars location
nars_dir = meson.current_source_dir() / 'nars'
# Get all example files
nars = [
'dot.nar',
]
test_deps = [
nar_kaitai_dep,
kaitai_runtime_dep,
gtest_main_dep,
]
this_exe = executable(
meson.project_name(),
'test-parse-nar.cc',
dependencies : test_deps,
)
test(
meson.project_name(),
this_exe,
env : [ 'NIX_NARS_DIR=' + nars_dir ],
protocol : 'gtest',
)

1
src/kaitai-struct-checks/nar.ksy Symbolic link
View file

@ -0,0 +1 @@
../../doc/manual/source/protocols/nix-archive/nar.ksy

1
src/kaitai-struct-checks/nars Symbolic link
View file

@ -0,0 +1 @@
../libutil-tests/data/nars

1
src/kaitai-struct-checks/nix-meson-build-support Symbolic link
View file

@ -0,0 +1 @@
../../nix-meson-build-support

70
src/kaitai-struct-checks/package.nix Normal file
View file

@ -0,0 +1,70 @@
# Run with: nix build .#nix-kaitai-struct-checks
# or: `nix develop .#nix-kaitai-struct-checks` to enter a dev shell
{
lib,
mkMesonDerivation,
gtest,
meson,
ninja,
pkg-config,
kaitai-struct-compiler,
fetchzip,
kaitai-struct-cpp-stl-runtime,
# Configuration Options
version,
}:
let
inherit (lib) fileset;
in
mkMesonDerivation (finalAttrs: {
pname = "nix-kaitai-struct-checks";
inherit version;
workDir = ./.;
fileset = lib.fileset.unions [
../../nix-meson-build-support
./nix-meson-build-support
./.version
../../.version
../../doc/manual/source/protocols/nix-archive/nar.ksy
./nars
../../src/libutil-tests/data
./meson.build
./nar.ksy
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
];
outputs = [ "out" ];
buildInputs = [
gtest
kaitai-struct-cpp-stl-runtime
];
nativeBuildInputs = [
meson
ninja
pkg-config
# This can go away when we bump up to 25.11
(kaitai-struct-compiler.overrideAttrs (finalAttrs: {
version = "0.11";
src = fetchzip {
url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip";
sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A=";
};
}))
];
doCheck = true;
mesonCheckFlags = [ "--print-errorlogs" ];
postInstall = ''
touch $out
'';
meta = {
platforms = lib.platforms.all;
};
})

48
src/kaitai-struct-checks/test-parse-nar.cc Normal file
View file

@ -0,0 +1,48 @@
#include <gtest/gtest.h>
#include <filesystem>
#include <fstream>
#include <vector>
#include <string>
#include <kaitai/kaitaistream.h>
#include <fstream>
#include <string>
#include <vector>
#include "nix_nar.h"
static const std::vector<std::string> NarFiles = {
"empty.nar",
"dot.nar",
"dotdot.nar",
"executable-after-contents.nar",
"invalid-tag-instead-of-contents.nar",
"name-after-node.nar",
"nul-character.nar",
"slash.nar",
};
class NarParseTest : public ::testing::TestWithParam<std::string>
{};
TEST_P(NarParseTest, ParseSucceeds)
{
const auto nar_file = GetParam();
const char * nars_dir_env = std::getenv("NIX_NARS_DIR");
if (nars_dir_env == nullptr) {
FAIL() << "NIX_NARS_DIR environment variable not set.";
}
const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar";
ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path;
std::ifstream ifs(nar_file_path, std::ifstream::binary);
ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file;
kaitai::kstream ks(&ifs);
nix_nar_t nar(&ks);
ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file;
}
INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles));

13
src/libcmd/command.cc
View file

@ -2,6 +2,7 @@
#include <nlohmann/json.hpp>
#include "nix/cmd/command.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/markdown.hh"
#include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh"
@ -14,6 +15,18 @@
namespace nix {
RegisterCommand::Commands & RegisterCommand::commands()
{
static RegisterCommand::Commands commands;
return commands;
}
RegisterLegacyCommand::Commands & RegisterLegacyCommand::commands()
{
static RegisterLegacyCommand::Commands commands;
return commands;
}
nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
{
nix::Commands res;

7
src/libcmd/common-eval-args.cc
View file

@ -33,7 +33,8 @@ EvalSettings evalSettings{
// FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto [accessor, lockedRef] =
flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
auto storePath = nix::fetchToStore(
state.fetchSettings,
*state.store,
@ -131,7 +132,7 @@ MixEvalArgs::MixEvalArgs()
fetchers::Attrs extraAttrs;
if (to.subdir != "")
extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs);
fetchers::overrideRegistry(fetchSettings, from.input, to.input, extraAttrs);
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, openStore(), prefix);
@ -187,7 +188,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto [accessor, lockedRef] = flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
auto storePath = nix::fetchToStore(
state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath);

6
src/libcmd/include/nix/cmd/command.hh
View file

@ -286,11 +286,7 @@ struct RegisterCommand
{
typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
static Commands & commands()
{
static Commands commands;
return commands;
}
static Commands & commands();
RegisterCommand(std::vector<std::string> && name, std::function<ref<Command>()> command)
{

6
src/libcmd/include/nix/cmd/legacy.hh
View file

@ -13,11 +13,7 @@ struct RegisterLegacyCommand
{
typedef std::map<std::string, MainFunction> Commands;
static Commands & commands()
{
static Commands commands;
return commands;
}
static Commands & commands();
RegisterLegacyCommand(const std::string & name, MainFunction fun)
{

1
src/libcmd/installables.cc
View file

@ -185,6 +185,7 @@ MixFlakeOptions::MixFlakeOptions()
}
overrideRegistry(
fetchSettings,
fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
input3->lockedRef.input,
extraAttrs);

2
src/libcmd/repl.cc
View file

@ -738,7 +738,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
}
auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
if (evalSettings.pureEval && !flakeRef.input.isLocked())
if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings))
throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
Value v;

43
src/libexpr-c/nix_api_expr.h
View file

@ -4,11 +4,14 @@
* @brief Bindings to the Nix language evaluator
*
* See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
* @{
*/
/** @file
* @brief Main entry for the libexpr C bindings
*/
/** @defgroup libexpr_init Initialization
* @ingroup libexpr
* @{
*/
#include "nix_api_store.h"
#include "nix_api_util.h"
@ -45,7 +48,10 @@ typedef struct nix_eval_state_builder nix_eval_state_builder;
*/
typedef struct EvalState EvalState; // nix::EvalState
/** @} */
/** @brief A Nix language value, or thunk that may evaluate to a value.
* @ingroup value
*
* Values are the primary objects manipulated in the Nix language.
* They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its
@ -56,7 +62,8 @@ typedef struct EvalState EvalState; // nix::EvalState
*
* The evaluator manages its own memory, but your use of the C API must follow the reference counting rules.
*
* @see value_manip
* @struct nix_value
* @see value_create, value_extract
* @see nix_value_incref, nix_value_decref
*/
typedef struct nix_value nix_value;
@ -65,6 +72,7 @@ NIX_DEPRECATED("use nix_value instead") typedef nix_value Value;
// Function prototypes
/**
* @brief Initialize the Nix language evaluator.
* @ingroup libexpr_init
*
* This function must be called at least once,
* at some point before constructing a EvalState for the first time.
@ -77,6 +85,7 @@ nix_err nix_libexpr_init(nix_c_context * context);
/**
* @brief Parses and evaluates a Nix expression from a string.
* @ingroup value_create
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
@ -93,6 +102,7 @@ nix_err nix_expr_eval_from_string(
/**
* @brief Calls a Nix function with an argument.
* @ingroup value_create
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
@ -107,6 +117,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * f
/**
* @brief Calls a Nix function with multiple arguments.
* @ingroup value_create
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
@ -126,10 +137,12 @@ nix_err nix_value_call_multi(
/**
* @brief Calls a Nix function with multiple arguments.
* @ingroup value_create
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
*
* @def NIX_VALUE_CALL
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
* @param[out] value The result of the function call.
@ -147,6 +160,7 @@ nix_err nix_value_call_multi(
/**
* @brief Forces the evaluation of a Nix value.
* @ingroup value_create
*
* The Nix interpreter is lazy, and not-yet-evaluated values can be
* of type NIX_TYPE_THUNK instead of their actual value.
@ -180,18 +194,20 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
/**
* @brief Create a new nix_eval_state_builder
* @ingroup libexpr_init
*
* The settings are initialized to their default value.
* Values can be sourced elsewhere with nix_eval_state_builder_load.
*
* @param[out] context Optional, stores error information
* @param[in] store The Nix store to use.
* @return A new nix_eval_state_builder or NULL on failure.
* @return A new nix_eval_state_builder or NULL on failure. Call nix_eval_state_builder_free() when you're done.
*/
nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store);
/**
* @brief Read settings from the ambient environment
* @ingroup libexpr_init
*
* Settings are sourced from environment variables and configuration files,
* as documented in the Nix manual.
@ -204,6 +220,7 @@ nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_buil
/**
* @brief Set the lookup path for `<...>` expressions
* @ingroup libexpr_init
*
* @param[in] context Optional, stores error information
* @param[in] builder The builder to modify.
@ -214,18 +231,21 @@ nix_err nix_eval_state_builder_set_lookup_path(
/**
* @brief Create a new Nix language evaluator state
* @ingroup libexpr_init
*
* Remember to nix_eval_state_builder_free after building the state.
* The builder becomes unusable after this call. Remember to call nix_eval_state_builder_free()
* after building the state.
*
* @param[out] context Optional, stores error information
* @param[in] builder The builder to use and free
* @return A new Nix state or NULL on failure.
* @return A new Nix state or NULL on failure. Call nix_state_free() when you're done.
* @see nix_eval_state_builder_new, nix_eval_state_builder_free
*/
EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder);
/**
* @brief Free a nix_eval_state_builder
* @ingroup libexpr_init
*
* Does not fail.
*
@ -235,19 +255,21 @@ void nix_eval_state_builder_free(nix_eval_state_builder * builder);
/**
* @brief Create a new Nix language evaluator state
* @ingroup libexpr_init
*
* For more control, use nix_eval_state_builder
*
* @param[out] context Optional, stores error information
* @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
* @param[in] store The Nix store to use.
* @return A new Nix state or NULL on failure.
* @return A new Nix state or NULL on failure. Call nix_state_free() when you're done.
* @see nix_state_builder_new
*/
EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store);
/**
* @brief Frees a Nix state.
* @ingroup libexpr_init
*
* Does not fail.
*
@ -256,6 +278,7 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath,
void nix_state_free(EvalState * state);
/** @addtogroup GC
* @ingroup libexpr
* @brief Reference counting and garbage collector operations
*
* The Nix language evaluator uses a garbage collector. To ease C interop, we implement
@ -286,6 +309,9 @@ nix_err nix_gc_incref(nix_c_context * context, const void * object);
/**
* @brief Decrement the garbage collector reference counter for the given object
*
* @deprecated We are phasing out the general nix_gc_decref() in favor of type-specified free functions, such as
* nix_value_decref().
*
* We also provide typed `nix_*_decref` functions, which are
* - safer to use
* - easier to integrate when deriving bindings
@ -314,12 +340,11 @@ void nix_gc_now();
*/
void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
/** @} */
/** @} */ // doxygen group GC
// cffi end
#ifdef __cplusplus
}
#endif
/** @} */
#endif // NIX_API_EXPR_H

12
src/libexpr-c/nix_api_external.h
View file

@ -2,11 +2,12 @@
#define NIX_API_EXTERNAL_H
/** @ingroup libexpr
* @addtogroup Externals
* @brief Deal with external values
* @brief Externals let Nix expressions work with foreign values that aren't part of the normal Nix value data model
* @{
*/
/** @file
* @brief libexpr C bindings dealing with external values
* @see Externals
*/
#include "nix_api_expr.h"
@ -115,7 +116,7 @@ typedef struct NixCExternalValueDesc
* @brief Try to compare two external values
*
* Optional, the default is always false.
* If the other object was not a Nix C external value, this comparison will
* If the other object was not a Nix C API external value, this comparison will
* also return false
* @param[in] self the void* passed to nix_create_external_value
* @param[in] other the void* passed to the other object's
@ -168,7 +169,7 @@ typedef struct NixCExternalValueDesc
/**
* @brief Create an external value, that can be given to nix_init_external
*
* Owned by the GC. Use nix_gc_decref when you're done with the pointer.
* Call nix_gc_decref() when you're done with the pointer.
*
* @param[out] context Optional, stores error information
* @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
@ -180,10 +181,11 @@ typedef struct NixCExternalValueDesc
ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
/**
* @brief Extract the pointer from a nix c external value.
* @brief Extract the pointer from a Nix C API external value.
* @param[out] context Optional, stores error information
* @param[in] b The external value
* @returns The pointer, or null if the external value was not from nix c.
* @returns The pointer, valid while the external value is valid, or null if the external value was not from the Nix C
* API.
* @see nix_get_external
*/
void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);

2
src/libexpr-c/nix_api_value.cc
View file

@ -235,7 +235,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_
try {
auto & v = check_value_in(value);
assert(v.type() == nix::nString);
call_nix_get_string_callback(v.c_str(), callback, user_data);
call_nix_get_string_callback(v.string_view(), callback, user_data);
}
NIXC_CATCH_ERRS
}

250
src/libexpr-c/nix_api_value.h
View file

@ -1,9 +1,6 @@
#ifndef NIX_API_VALUE_H
#define NIX_API_VALUE_H
/** @addtogroup libexpr
* @{
*/
/** @file
* @brief libexpr C bindings dealing with values
*/
@ -20,18 +17,89 @@ extern "C" {
#endif
// cffi start
/** @defgroup value Value
* @ingroup libexpr
* @brief nix_value type and core operations for working with Nix values
* @see value_create
* @see value_extract
*/
/** @defgroup value_create Value Creation
* @ingroup libexpr
* @brief Functions for allocating and initializing Nix values
*
* Values are usually created with `nix_alloc_value` followed by `nix_init_*` functions.
* In primop callbacks, allocation is already done and only initialization is needed.
*/
/** @defgroup value_extract Value Extraction
* @ingroup libexpr
* @brief Functions for extracting data from Nix values
*/
/** @defgroup primops PrimOps and Builtins
* @ingroup libexpr
*/
// Type definitions
/** @brief Represents the state of a Nix value
*
* Thunk values (NIX_TYPE_THUNK) change to their final, unchanging type when forced.
*
* @see https://nix.dev/manual/nix/latest/language/evaluation.html
* @enum ValueType
* @ingroup value
*/
typedef enum {
/** Unevaluated expression
*
* Thunks often contain an expression and closure, but may contain other
* representations too.
*
* Their state is mutable, unlike that of the other types.
*/
NIX_TYPE_THUNK,
/**
* A 64 bit signed integer.
*/
NIX_TYPE_INT,
/** @brief IEEE 754 double precision floating point number
* @see https://nix.dev/manual/nix/latest/language/types.html#type-float
*/
NIX_TYPE_FLOAT,
/** @brief Boolean true or false value
* @see https://nix.dev/manual/nix/latest/language/types.html#type-bool
*/
NIX_TYPE_BOOL,
/** @brief String value with context
*
* String content may contain arbitrary bytes, not necessarily UTF-8.
* @see https://nix.dev/manual/nix/latest/language/types.html#type-string
*/
NIX_TYPE_STRING,
/** @brief Filesystem path
* @see https://nix.dev/manual/nix/latest/language/types.html#type-path
*/
NIX_TYPE_PATH,
/** @brief Null value
* @see https://nix.dev/manual/nix/latest/language/types.html#type-null
*/
NIX_TYPE_NULL,
/** @brief Attribute set (key-value mapping)
* @see https://nix.dev/manual/nix/latest/language/types.html#type-attrs
*/
NIX_TYPE_ATTRS,
/** @brief Ordered list of values
* @see https://nix.dev/manual/nix/latest/language/types.html#type-list
*/
NIX_TYPE_LIST,
/** @brief Function (lambda or builtin)
* @see https://nix.dev/manual/nix/latest/language/types.html#type-function
*/
NIX_TYPE_FUNCTION,
/** @brief External value from C++ plugins or C API
* @see Externals
*/
NIX_TYPE_EXTERNAL
} ValueType;
@ -39,22 +107,41 @@ typedef enum {
typedef struct nix_value nix_value;
typedef struct EvalState EvalState;
/** @deprecated Use nix_value instead */
[[deprecated("use nix_value instead")]] typedef nix_value Value;
// type defs
/** @brief Stores an under-construction set of bindings
* @ingroup value_manip
* @ingroup value_create
*
* Do not reuse.
* Each builder can only be used once. After calling nix_make_attrs(), the builder
* becomes invalid and must not be used again. Call nix_bindings_builder_free() to release it.
*
* Typical usage pattern:
* 1. Create with nix_make_bindings_builder()
* 2. Insert attributes with nix_bindings_builder_insert()
* 3. Create final attribute set with nix_make_attrs()
* 4. Free builder with nix_bindings_builder_free()
*
* @struct BindingsBuilder
* @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
* @see nix_bindings_builder_insert
*/
typedef struct BindingsBuilder BindingsBuilder;
/** @brief Stores an under-construction list
* @ingroup value_manip
* @ingroup value_create
*
* Do not reuse.
* Each builder can only be used once. After calling nix_make_list(), the builder
* becomes invalid and must not be used again. Call nix_list_builder_free() to release it.
*
* Typical usage pattern:
* 1. Create with nix_make_list_builder()
* 2. Insert elements with nix_list_builder_insert()
* 3. Create final list with nix_make_list()
* 4. Free builder with nix_list_builder_free()
*
* @struct ListBuilder
* @see nix_make_list_builder, nix_list_builder_free, nix_make_list
* @see nix_list_builder_insert
*/
@ -63,25 +150,28 @@ typedef struct ListBuilder ListBuilder;
/** @brief PrimOp function
* @ingroup primops
*
* Owned by the GC
* @see nix_alloc_primop, nix_init_primop
* Can be released with nix_gc_decref() when necessary.
* @struct PrimOp
* @see nix_alloc_primop, nix_init_primop, nix_register_primop
*/
typedef struct PrimOp PrimOp;
/** @brief External Value
* @ingroup Externals
*
* Owned by the GC
* Can be released with nix_gc_decref() when necessary.
* @struct ExternalValue
* @see nix_create_external_value, nix_init_external, nix_get_external
*/
typedef struct ExternalValue ExternalValue;
/** @brief String without placeholders, and realised store paths
* @struct nix_realised_string
* @see nix_string_realise, nix_realised_string_free
*/
typedef struct nix_realised_string nix_realised_string;
/** @defgroup primops Adding primops
* @{
*/
/** @brief Function pointer for primops
* @ingroup primops
*
* When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
*
@ -97,9 +187,9 @@ typedef void (*PrimOpFun)(
void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret);
/** @brief Allocate a PrimOp
* @ingroup primops
*
* Owned by the garbage collector.
* Use nix_gc_decref() when you're done with the returned PrimOp.
* Call nix_gc_decref() when you're done with the returned PrimOp.
*
* @param[out] context Optional, stores error information
* @param[in] fun callback
@ -121,35 +211,38 @@ PrimOp * nix_alloc_primop(
void * user_data);
/** @brief add a primop to the `builtins` attribute set
* @ingroup primops
*
* Only applies to States created after this call.
*
* Moves your PrimOp content into the global evaluator
* registry, meaning your input PrimOp pointer is no longer usable.
* You are free to remove your references to it,
* after which it will be garbage collected.
* Moves your PrimOp content into the global evaluator registry, meaning
* your input PrimOp pointer becomes invalid. The PrimOp must not be used
* with nix_init_primop() before or after this call, as this would cause
* undefined behavior.
* You must call nix_gc_decref() on the original PrimOp pointer
* after this call to release your reference.
*
* @param[out] context Optional, stores error information
* @return primop, or null in case of errors
*
* @param[in] primOp PrimOp to register
* @return error code, NIX_OK on success
*/
nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
/** @} */
// Function prototypes
/** @brief Allocate a Nix value
* @ingroup value_create
*
* Owned by the GC. Use nix_gc_decref() when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] state nix evaluator state
* @return value, or null in case of errors
*
*/
nix_value * nix_alloc_value(nix_c_context * context, EvalState * state);
/**
* @brief Increment the garbage collector reference counter for the given `nix_value`.
* @ingroup value
*
* The Nix language evaluator C API keeps track of alive objects by reference counting.
* When you're done with a refcounted pointer, call nix_value_decref().
@ -161,21 +254,19 @@ nix_err nix_value_incref(nix_c_context * context, nix_value * value);
/**
* @brief Decrement the garbage collector reference counter for the given object
* @ingroup value
*
* When the counter reaches zero, the `nix_value` object becomes invalid.
* The data referenced by `nix_value` may not be deallocated until the memory
* garbage collector has run, but deallocation is not guaranteed.
*
* @param[out] context Optional, stores error information
* @param[in] value The object to stop referencing
*/
nix_err nix_value_decref(nix_c_context * context, nix_value * value);
/** @addtogroup value_manip Manipulating values
* @brief Functions to inspect and change Nix language values, represented by nix_value.
* @{
*/
/** @anchor getters
* @name Getters
*/
/**@{*/
/** @brief Get value type
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return type of nix value
@ -183,14 +274,15 @@ nix_err nix_value_decref(nix_c_context * context, nix_value * value);
ValueType nix_get_type(nix_c_context * context, const nix_value * value);
/** @brief Get type name of value as defined in the evaluator
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return type name, owned string
* @todo way to free the result
* @return type name string, free with free()
*/
const char * nix_get_typename(nix_c_context * context, const nix_value * value);
/** @brief Get boolean value
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return true or false, error info via context
@ -198,6 +290,7 @@ const char * nix_get_typename(nix_c_context * context, const nix_value * value);
bool nix_get_bool(nix_c_context * context, const nix_value * value);
/** @brief Get the raw string
* @ingroup value_extract
*
* This may contain placeholders.
*
@ -205,21 +298,21 @@ bool nix_get_bool(nix_c_context * context, const nix_value * value);
* @param[in] value Nix value to inspect
* @param[in] callback Called with the string value.
* @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
* @return string
* @return error code, NIX_OK on success.
*/
nix_err
nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data);
/** @brief Get path as string
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return string, if the type is NIX_TYPE_PATH
* @return NULL in case of error.
* @return string valid while value is valid, NULL in case of error
*/
const char * nix_get_path_string(nix_c_context * context, const nix_value * value);
/** @brief Get the length of a list
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return length of list, error info via context
@ -227,6 +320,7 @@ const char * nix_get_path_string(nix_c_context * context, const nix_value * valu
unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value);
/** @brief Get the element count of an attrset
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return attrset element count, error info via context
@ -234,6 +328,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value)
unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value);
/** @brief Get float value in 64 bits
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return float contents, error info via context
@ -241,6 +336,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value
double nix_get_float(nix_c_context * context, const nix_value * value);
/** @brief Get int value
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return int contents, error info via context
@ -248,15 +344,18 @@ double nix_get_float(nix_c_context * context, const nix_value * value);
int64_t nix_get_int(nix_c_context * context, const nix_value * value);
/** @brief Get external reference
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @return reference to external, NULL in case of error
* @return reference valid while value is valid. Call nix_gc_incref() if you need it to live longer, then only in that
* case call nix_gc_decref() when done. NULL in case of error
*/
ExternalValue * nix_get_external(nix_c_context * context, nix_value * value);
/** @brief Get the ix'th element of a list
* @ingroup value_extract
*
* Owned by the GC. Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
@ -266,11 +365,12 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value);
nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
/** @brief Get the ix'th element of a list without forcing evaluation of the element
* @ingroup value_extract
*
* Returns the list element without forcing its evaluation, allowing access to lazy values.
* The list value itself must already be evaluated.
*
* Owned by the GC. Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated list)
* @param[in] state nix evaluator state
@ -281,8 +381,9 @@ nix_value *
nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix);
/** @brief Get an attr by name
* @ingroup value_extract
*
* Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
@ -292,11 +393,12 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value
* @ingroup value_extract
*
* Returns the attribute value without forcing its evaluation, allowing access to lazy values.
* The attribute set value itself must already be evaluated.
*
* Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated attribute set)
* @param[in] state nix evaluator state
@ -307,6 +409,7 @@ nix_value *
nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Check if an attribute name exists on a value
* @ingroup value_extract
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
@ -316,6 +419,7 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name);
/** @brief Get an attribute by index
* @ingroup value_extract
*
* Also gives you the name.
*
@ -329,18 +433,19 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
* @param[in] i attribute index
* @param[out] name will store a pointer to the attribute name
* @param[out] name will store a pointer to the attribute name, valid until state is freed
* @return value, NULL in case of errors
*/
nix_value *
nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
/** @brief Get an attribute by index, without forcing evaluation of the attribute's value
* @ingroup value_extract
*
* Also gives you the name.
*
@ -357,18 +462,19 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Use nix_gc_decref when you're done with the pointer
* Call nix_value_decref() when you're done with the pointer
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect (must be an evaluated attribute set)
* @param[in] state nix evaluator state
* @param[in] i attribute index
* @param[out] name will store a pointer to the attribute name
* @param[out] name will store a pointer to the attribute name, valid until state is freed
* @return value, NULL in case of errors
*/
nix_value * nix_get_attr_byidx_lazy(
nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name);
/** @brief Get an attribute name by index
* @ingroup value_extract
*
* Returns the attribute name without forcing evaluation of the attribute's value.
*
@ -382,16 +488,14 @@ nix_value * nix_get_attr_byidx_lazy(
* lexicographic order by Unicode scalar value for valid UTF-8). We recommend
* applying this same ordering for consistency.
*
* Owned by the nix EvalState
* @param[out] context Optional, stores error information
* @param[in] value Nix value to inspect
* @param[in] state nix evaluator state
* @param[in] i attribute index
* @return name, NULL in case of errors
* @return name string valid until state is freed, NULL in case of errors
*/
const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i);
/**@}*/
/** @name Initializers
*
* Values are typically "returned" by initializing already allocated memory that serves as the return value.
@ -401,6 +505,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value,
*/
/**@{*/
/** @brief Set boolean value
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] b the boolean value
@ -409,6 +514,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value,
nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b);
/** @brief Set a string
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] str the string, copied
@ -417,6 +523,7 @@ nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b);
nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str);
/** @brief Set a path
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] str the path string, copied
@ -425,6 +532,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str);
/** @brief Set a float
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] d the float, 64-bits
@ -433,6 +541,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
nix_err nix_init_float(nix_c_context * context, nix_value * value, double d);
/** @brief Set an int
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] i the int
@ -441,6 +550,7 @@ nix_err nix_init_float(nix_c_context * context, nix_value * value, double d);
nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i);
/** @brief Set null
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @return error code, NIX_OK on success.
@ -448,6 +558,7 @@ nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i);
nix_err nix_init_null(nix_c_context * context, nix_value * value);
/** @brief Set the value to a thunk that will perform a function application when needed.
* @ingroup value_create
*
* Thunks may be put into attribute sets and lists to perform some computation lazily; on demand.
* However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp.
@ -464,6 +575,7 @@ nix_err nix_init_null(nix_c_context * context, nix_value * value);
nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg);
/** @brief Set an external value
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] val the external value to set. Will be GC-referenced by the value.
@ -472,18 +584,25 @@ nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * f
nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val);
/** @brief Create a list from a list builder
* @ingroup value_create
*
* After this call, the list builder becomes invalid and cannot be used again.
* The only necessary next step is to free it with nix_list_builder_free().
*
* @param[out] context Optional, stores error information
* @param[in] list_builder list builder to use. Make sure to unref this afterwards.
* @param[in] list_builder list builder to use
* @param[out] value Nix value to modify
* @return error code, NIX_OK on success.
* @see nix_list_builder_free
*/
nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value);
/** @brief Create a list builder
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[in] state nix evaluator state
* @param[in] capacity how many bindings you'll add. Don't exceed.
* @return owned reference to a list builder. Make sure to unref when you're done.
* @return list builder. Call nix_list_builder_free() when you're done.
*/
ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
@ -505,14 +624,21 @@ nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, uns
void nix_list_builder_free(ListBuilder * list_builder);
/** @brief Create an attribute set from a bindings builder
* @ingroup value_create
*
* After this call, the bindings builder becomes invalid and cannot be used again.
* The only necessary next step is to free it with nix_bindings_builder_free().
*
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] b bindings builder to use. Make sure to unref this afterwards.
* @param[in] b bindings builder to use
* @return error code, NIX_OK on success.
* @see nix_bindings_builder_free
*/
nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b);
/** @brief Set primop
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] op primop, will be gc-referenced by the value
@ -521,6 +647,7 @@ nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuild
*/
nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op);
/** @brief Copy from another value
* @ingroup value_create
* @param[out] context Optional, stores error information
* @param[out] value Nix value to modify
* @param[in] source value to copy from
@ -530,12 +657,11 @@ nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_val
/**@}*/
/** @brief Create a bindings builder
* @param[out] context Optional, stores error information
* @param[in] state nix evaluator state
* @param[in] capacity how many bindings you'll add. Don't exceed.
* @return owned reference to a bindings builder. Make sure to unref when you're
done.
*/
* @param[out] context Optional, stores error information
* @param[in] state nix evaluator state
* @param[in] capacity how many bindings you'll add. Don't exceed.
* @return bindings builder. Call nix_bindings_builder_free() when you're done.
*/
BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
/** @brief Insert bindings into a builder
@ -554,7 +680,6 @@ nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder,
* @param[in] builder the builder to free
*/
void nix_bindings_builder_free(BindingsBuilder * builder);
/**@}*/
/** @brief Realise a string context.
*
@ -571,13 +696,13 @@ void nix_bindings_builder_free(BindingsBuilder * builder);
* @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
You should set this to true when this call is part of a primop.
You should set this to false when building for your application's purpose.
* @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
* @return NULL if failed, or a new nix_realised_string, which must be freed with nix_realised_string_free
*/
nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD);
/** @brief Start of the string
* @param[in] realised_string
* @return pointer to the start of the string. It may not be null-terminated.
* @return pointer to the start of the string, valid until realised_string is freed. It may not be null-terminated.
*/
const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
@ -596,7 +721,7 @@ size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_s
/** @brief Get a store path. The store paths are stored in an arbitrary order.
* @param[in] realised_string
* @param[in] index index of the store path, must be less than the count
* @return store path
* @return store path valid until realised_string is freed
*/
const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
@ -610,5 +735,4 @@ void nix_realised_string_free(nix_realised_string * realised_string);
}
#endif
/** @} */
#endif // NIX_API_VALUE_H

3
src/libexpr-test-support/include/nix/expr/tests/libexpr.hh
View file

@ -104,9 +104,10 @@ MATCHER(IsAttrs, "")
MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s))
{
if (arg.type() != nString) {
*result_listener << "Expected a string got " << arg.type();
return false;
}
return std::string_view(arg.c_str()) == s;
return arg.string_view() == s;
}
MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v))

81
src/libexpr-tests/error_traces.cc
View file

@ -139,63 +139,6 @@ TEST_F(ErrorTraceTest, NestedThrows)
#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \
ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo"))
TEST_F(ErrorTraceTest, genericClosure)
{
ASSERT_TRACE2(
"genericClosure 1",
TypeError,
HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
HintFmt("while evaluating the first argument passed to builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure {}",
TypeError,
HintFmt("attribute '%s' missing", "startSet"),
HintFmt("in the attrset passed as argument to builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = 1; }",
TypeError,
HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)),
HintFmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = [{ key = 1;}]; operator = true; }",
TypeError,
HintFmt("expected a function but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)),
HintFmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = [{ key = 1;}]; operator = item: true; }",
TypeError,
HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)),
HintFmt("while evaluating the return value of the `operator` passed to builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }",
TypeError,
HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)),
HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }",
TypeError,
HintFmt("attribute '%s' missing", "key"),
HintFmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure"));
ASSERT_TRACE2(
"genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }",
EvalError,
HintFmt("cannot compare %s with %s", "a string", "an integer"),
HintFmt("while comparing the `key` attributes of two genericClosure elements"));
ASSERT_TRACE2(
"genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }",
TypeError,
HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)),
HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"));
}
TEST_F(ErrorTraceTest, replaceStrings)
{
ASSERT_TRACE2(
@ -1050,17 +993,35 @@ TEST_F(ErrorTraceTest, bitXor)
TEST_F(ErrorTraceTest, lessThan)
{
ASSERT_TRACE1("lessThan 1 \"foo\"", EvalError, HintFmt("cannot compare %s with %s", "an integer", "a string"));
ASSERT_TRACE1(
"lessThan 1 \"foo\"",
EvalError,
HintFmt(
"cannot compare %s with %s; values are %s and %s",
"an integer",
"a string",
Uncolored(ANSI_CYAN "1" ANSI_NORMAL),
Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)));
ASSERT_TRACE1(
"lessThan {} {}",
EvalError,
HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set"));
HintFmt(
"cannot compare %s with %s; values of that type are incomparable (values are %s and %s)",
"a set",
"a set",
Uncolored("{ }"),
Uncolored("{ }")));
ASSERT_TRACE2(
"lessThan [ 1 2 ] [ \"foo\" ]",
EvalError,
HintFmt("cannot compare %s with %s", "an integer", "a string"),
HintFmt(
"cannot compare %s with %s; values are %s and %s",
"an integer",
"a string",
Uncolored(ANSI_CYAN "1" ANSI_NORMAL),
Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)),
HintFmt("while comparing two list elements"));
}

5
src/libexpr-tests/json.cc
View file

@ -1,5 +1,6 @@
#include "nix/expr/tests/libexpr.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/expr/static-string-data.hh"
namespace nix {
// Testing the conversion to JSON
@ -54,7 +55,7 @@ TEST_F(JSONValueTest, IntNegative)
TEST_F(JSONValueTest, String)
{
Value v;
v.mkStringNoCopy("test");
v.mkStringNoCopy("test"_sds);
ASSERT_EQ(getJSONValue(v), "\"test\"");
}
@ -62,7 +63,7 @@ TEST_F(JSONValueTest, StringQuotes)
{
Value v;
v.mkStringNoCopy("test\"");
v.mkStringNoCopy("test\""_sds);
ASSERT_EQ(getJSONValue(v), "\"test\\\"\"");
}

2
src/libexpr-tests/primops.cc
View file

@ -771,7 +771,7 @@ TEST_F(PrimOpTest, derivation)
ASSERT_EQ(v.type(), nFunction);
ASSERT_TRUE(v.isLambda());
ASSERT_NE(v.lambda().fun, nullptr);
ASSERT_TRUE(v.lambda().fun->hasFormals());
ASSERT_TRUE(v.lambda().fun->getFormals());
}
TEST_F(PrimOpTest, currentTime)

20
src/libexpr-tests/trivial.cc
View file

@ -1,4 +1,5 @@
#include "nix/expr/tests/libexpr.hh"
#include "nix/util/tests/gmock-matchers.hh"
namespace nix {
// Testing of trivial expressions
@ -160,7 +161,8 @@ TEST_F(TrivialExpressionTest, assertPassed)
ASSERT_THAT(v, IsIntEq(123));
}
class AttrSetMergeTrvialExpressionTest : public TrivialExpressionTest, public testing::WithParamInterface<const char *>
class AttrSetMergeTrvialExpressionTest : public TrivialExpressionTest,
public ::testing::WithParamInterface<const char *>
{};
TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy)
@ -196,7 +198,7 @@ TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy)
INSTANTIATE_TEST_SUITE_P(
attrsetMergeLazy,
AttrSetMergeTrvialExpressionTest,
testing::Values("{ a.b = 1; a.c = 2; }", "{ a = { b = 1; }; a = { c = 2; }; }"));
::testing::Values("{ a.b = 1; a.c = 2; }", "{ a = { b = 1; }; a = { c = 2; }; }"));
// The following macros ultimately define 48 tests (16 variations on three
// templates). Each template tests an expression that can be written in 2^4
@ -339,4 +341,18 @@ TEST_F(TrivialExpressionTest, orCantBeUsed)
{
ASSERT_THROW(eval("let or = 1; in or"), Error);
}
TEST_F(TrivialExpressionTest, tooManyFormals)
{
std::string expr = "let f = { ";
for (uint32_t i = 0; i <= std::numeric_limits<uint16_t>::max(); ++i) {
expr += fmt("arg%d, ", i);
}
expr += " }: 0 in; f {}";
ASSERT_THAT(
[&]() { eval(expr); },
::testing::ThrowsMessage<Error>(::nix::testing::HasSubstrIgnoreANSIMatcher(
"too many formal arguments, implementation supports at most 65535")));
}
} /* namespace nix */

34
src/libexpr-tests/value/print.cc
View file

@ -1,4 +1,5 @@
#include "nix/expr/tests/libexpr.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/expr/value.hh"
#include "nix/expr/print.hh"
@ -10,7 +11,7 @@ using namespace testing;
struct ValuePrintingTests : LibExprTest
{
template<class... A>
void test(Value v, std::string_view expected, A... args)
void test(Value & v, std::string_view expected, A... args)
{
std::stringstream out;
v.print(state, out, args...);
@ -35,14 +36,14 @@ TEST_F(ValuePrintingTests, tBool)
TEST_F(ValuePrintingTests, tString)
{
Value vString;
vString.mkStringNoCopy("some-string");
vString.mkStringNoCopy("some-string"_sds);
test(vString, "\"some-string\"");
}
TEST_F(ValuePrintingTests, tPath)
{
Value vPath;
vPath.mkStringNoCopy("/foo");
vPath.mkStringNoCopy("/foo"_sds);
test(vPath, "\"/foo\"");
}
@ -110,9 +111,8 @@ TEST_F(ValuePrintingTests, vLambda)
PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1);
auto posIdx = state.positions.add(origin, 0);
auto body = ExprInt(0);
auto formals = Formals{};
ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body);
ExprLambda eLambda(posIdx, createSymbol("a"), &body);
Value vLambda;
vLambda.mkLambda(&env, &eLambda);
@ -290,10 +290,10 @@ TEST_F(StringPrintingTests, maxLengthTruncation)
TEST_F(ValuePrintingTests, attrsTypeFirst)
{
Value vType;
vType.mkStringNoCopy("puppy");
vType.mkStringNoCopy("puppy"_sds);
Value vApple;
vApple.mkStringNoCopy("apple");
vApple.mkStringNoCopy("apple"_sds);
BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.symbols.create("type"), &vType);
@ -334,7 +334,7 @@ TEST_F(ValuePrintingTests, ansiColorsBool)
TEST_F(ValuePrintingTests, ansiColorsString)
{
Value v;
v.mkStringNoCopy("puppy");
v.mkStringNoCopy("puppy"_sds);
test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true});
}
@ -342,7 +342,7 @@ TEST_F(ValuePrintingTests, ansiColorsString)
TEST_F(ValuePrintingTests, ansiColorsStringElided)
{
Value v;
v.mkStringNoCopy("puppy");
v.mkStringNoCopy("puppy"_sds);
test(
v,
@ -390,7 +390,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs)
TEST_F(ValuePrintingTests, ansiColorsDerivation)
{
Value vDerivation;
vDerivation.mkStringNoCopy("derivation");
vDerivation.mkStringNoCopy("derivation"_sds);
BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.s.type, &vDerivation);
@ -413,7 +413,7 @@ TEST_F(ValuePrintingTests, ansiColorsError)
{
Value throw_ = state.getBuiltin("throw");
Value message;
message.mkStringNoCopy("uh oh!");
message.mkStringNoCopy("uh oh!"_sds);
Value vError;
vError.mkApp(&throw_, &message);
@ -430,12 +430,12 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError)
{
Value throw_ = state.getBuiltin("throw");
Value message;
message.mkStringNoCopy("uh oh!");
message.mkStringNoCopy("uh oh!"_sds);
Value vError;
vError.mkApp(&throw_, &message);
Value vDerivation;
vDerivation.mkStringNoCopy("derivation");
vDerivation.mkStringNoCopy("derivation"_sds);
BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.s.type, &vDerivation);
@ -500,9 +500,8 @@ TEST_F(ValuePrintingTests, ansiColorsLambda)
PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1);
auto posIdx = state.positions.add(origin, 0);
auto body = ExprInt(0);
auto formals = Formals{};
ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body);
ExprLambda eLambda(posIdx, createSymbol("a"), &body);
Value vLambda;
vLambda.mkLambda(&env, &eLambda);
@ -625,10 +624,11 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
vThree.mkInt(3);
builder.insert(state.symbols.create("three"), &vThree);
vAttrs.mkAttrs(builder.finish());
Value vAttrs2;
vAttrs2.mkAttrs(builder.finish());
test(
vAttrs,
vAttrs2,
"{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }",
PrintOptions{.ansiColors = true, .maxAttrs = 1});
}

19
src/libexpr-tests/value/value.cc
View file

@ -1,6 +1,8 @@
#include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/store/tests/libstore.hh"
#include <gtest/gtest.h>
namespace nix {
@ -22,4 +24,21 @@ TEST_F(ValueTest, vInt)
ASSERT_EQ(true, vInt.isValid());
}
TEST_F(ValueTest, staticString)
{
Value vStr1;
Value vStr2;
vStr1.mkStringNoCopy("foo"_sds);
vStr2.mkStringNoCopy("foo"_sds);
auto & sd1 = vStr1.string_data();
auto & sd2 = vStr2.string_data();
// The strings should be the same
ASSERT_EQ(sd1.view(), sd2.view());
// The strings should also be backed by the same (static) allocation
ASSERT_EQ(&sd1, &sd2);
}
} // namespace nix

16
src/libexpr/eval-cache.cc
View file

@ -136,17 +136,19 @@ struct AttrDb
});
}
AttrId setString(AttrKey key, std::string_view s, const char ** context = nullptr)
AttrId setString(AttrKey key, std::string_view s, const Value::StringWithContext::Context * context = nullptr)
{
return doSQLite([&]() {
auto state(_state->lock());
if (context) {
std::string ctx;
for (const char ** p = context; *p; ++p) {
if (p != context)
bool first = true;
for (auto * elem : *context) {
if (!first)
ctx.push_back(' ');
ctx.append(*p);
ctx.append(elem->view());
first = false;
}
state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx)
.exec();
@ -406,7 +408,7 @@ Value & AttrCursor::forceValue()
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
if (v.type() == nString)
cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), string_t{v.c_str(), {}}};
cachedValue = {root->db->setString(getKey(), v.string_view(), v.context()), string_t{v.string_view(), {}}};
else if (v.type() == nPath) {
auto path = v.path().path;
cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}};
@ -541,7 +543,7 @@ std::string AttrCursor::getString()
if (v.type() != nString && v.type() != nPath)
root->state.error<TypeError>("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow();
return v.type() == nString ? v.c_str() : v.path().to_string();
return v.type() == nString ? std::string(v.string_view()) : v.path().to_string();
}
string_t AttrCursor::getStringWithContext()
@ -580,7 +582,7 @@ string_t AttrCursor::getStringWithContext()
if (v.type() == nString) {
NixStringContext context;
copyContext(v, context);
return {v.c_str(), std::move(context)};
return {std::string{v.string_view()}, std::move(context)};
} else if (v.type() == nPath)
return {v.path().to_string(), {}};
else

2
src/libexpr/eval-settings.cc
View file

@ -92,7 +92,7 @@ bool EvalSettings::isPseudoUrl(std::string_view s)
std::string EvalSettings::resolvePseudoUrl(std::string_view url)
{
if (hasPrefix(url, "channel:"))
return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
return "https://channels.nixos.org/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
else
return std::string(url);
}

236
src/libexpr/eval.cc
View file

@ -3,6 +3,7 @@
#include "nix/expr/primops.hh"
#include "nix/expr/print-options.hh"
#include "nix/expr/symbol-table.hh"
#include "nix/expr/value.hh"
#include "nix/util/exit.hh"
#include "nix/util/types.hh"
#include "nix/util/util.hh"
@ -28,6 +29,8 @@
#include "parser-tab.hh"
#include <algorithm>
#include <cstddef>
#include <cstdlib>
#include <iostream>
#include <sstream>
#include <cstring>
@ -36,6 +39,7 @@
#include <sys/time.h>
#include <fstream>
#include <functional>
#include <ranges>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>
@ -47,6 +51,9 @@ using json = nlohmann::json;
namespace nix {
/**
* Just for doc strings. Not for regular string values.
*/
static char * allocString(size_t size)
{
char * t;
@ -60,6 +67,9 @@ static char * allocString(size_t size)
// string allocations.
// This function handles makeImmutableString(std::string_view()) by returning
// the empty string.
/**
* Just for doc strings. Not for regular string values.
*/
static const char * makeImmutableString(std::string_view s)
{
const size_t size = s.size();
@ -71,6 +81,25 @@ static const char * makeImmutableString(std::string_view s)
return t;
}
StringData & StringData::alloc(size_t size)
{
void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
if (!t)
throw std::bad_alloc();
auto res = new (t) StringData(size);
return *res;
}
const StringData & StringData::make(std::string_view s)
{
if (s.empty())
return ""_sds;
auto & res = alloc(s.size());
std::memcpy(&res.data_, s.data(), s.size());
res.data_[s.size()] = '\0';
return res;
}
RootValue allocRootValue(Value * v)
{
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
@ -584,7 +613,9 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
.name = name,
.arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
.args = {},
.doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
/* N.B. Can't use StringData here, because that would lead to an interior pointer.
NOTE: memory leak when compiled without GC. */
.doc = makeImmutableString(s.view()),
};
}
if (isFunctor(v)) {
@ -818,36 +849,34 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
void Value::mkString(std::string_view s)
{
mkStringNoCopy(makeImmutableString(s));
mkStringNoCopy(StringData::make(s));
}
static const char ** encodeContext(const NixStringContext & context)
Value::StringWithContext::Context *
Value::StringWithContext::Context::fromBuilder(const NixStringContext & context, EvalMemory & mem)
{
if (!context.empty()) {
size_t n = 0;
auto ctx = (const char **) allocBytes((context.size() + 1) * sizeof(char *));
for (auto & i : context) {
ctx[n++] = makeImmutableString({i.to_string()});
}
ctx[n] = nullptr;
return ctx;
} else
if (context.empty())
return nullptr;
auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
std::ranges::transform(
context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
return ctx;
}
void Value::mkString(std::string_view s, const NixStringContext & context)
void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
{
mkStringNoCopy(makeImmutableString(s), encodeContext(context));
mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
}
void Value::mkStringMove(const char * s, const NixStringContext & context)
void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
{
mkStringNoCopy(s, encodeContext(context));
mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
}
void Value::mkPath(const SourcePath & path)
{
mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
mkPath(&*path.accessor, StringData::make(path.path.abs()));
}
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@ -883,9 +912,9 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
}
}
ListBuilder::ListBuilder(size_t size)
ListBuilder::ListBuilder(EvalMemory & mem, size_t size)
: size(size)
, elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
, elems(size <= 2 ? inlineElems : (Value **) mem.allocBytes(size * sizeof(Value *)))
{
}
@ -925,7 +954,8 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
store->printStorePath(p),
NixStringContext{
NixStringContextElem::Opaque{.path = p},
});
},
mem);
}
std::string EvalState::mkOutputStringRaw(
@ -947,7 +977,7 @@ void EvalState::mkOutputString(
std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings)
{
value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b});
value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b}, mem);
}
std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p)
@ -982,7 +1012,8 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
mkSingleDerivedPathStringRaw(p),
NixStringContext{
std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
});
},
mem);
}
Value * Expr::maybeThunk(EvalState & state, Env & env)
@ -1113,6 +1144,7 @@ void EvalState::resetFileCache()
importResolutionCache->clear();
fileEvalCache->clear();
inputCache->clear();
positions.clear();
}
void EvalState::eval(Expr * e, Value & v)
@ -1496,15 +1528,13 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
ExprLambda & lambda(*vCur.lambda().fun);
auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
auto size = (!lambda.arg ? 0 : 1) + (lambda.getFormals() ? lambda.getFormals()->formals.size() : 0);
Env & env2(mem.allocEnv(size));
env2.up = vCur.lambda().env;
Displacement displ = 0;
if (!lambda.hasFormals())
env2.values[displ++] = args[0];
else {
if (auto formals = lambda.getFormals()) {
try {
forceAttrs(*args[0], lambda.pos, "while evaluating the value passed for the lambda argument");
} catch (Error & e) {
@ -1520,7 +1550,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
there is no matching actual argument but the formal
argument has a default, use the default. */
size_t attrsUsed = 0;
for (auto & i : lambda.formals->formals) {
for (auto & i : formals->formals) {
auto j = args[0]->attrs()->get(i.name);
if (!j) {
if (!i.def) {
@ -1542,13 +1572,13 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
/* Check that each actual argument is listed as a formal
argument (unless the attribute match specifies a `...'). */
if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs()->size()) {
if (!formals->ellipsis && attrsUsed != args[0]->attrs()->size()) {
/* Nope, so show the first unexpected argument to the
user. */
for (auto & i : *args[0]->attrs())
if (!lambda.formals->has(i.name)) {
if (!formals->has(i.name)) {
StringSet formalNames;
for (auto & formal : lambda.formals->formals)
for (auto & formal : formals->formals)
formalNames.insert(std::string(symbols[formal.name]));
auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
error<TypeError>(
@ -1563,6 +1593,8 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
}
unreachable();
}
} else {
env2.values[displ++] = args[0];
}
nrFunctionCalls++;
@ -1747,14 +1779,15 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
}
}
if (!fun.isLambda() || !fun.lambda().fun->hasFormals()) {
if (!fun.isLambda() || !fun.lambda().fun->getFormals()) {
res = fun;
return;
}
auto formals = fun.lambda().fun->getFormals();
auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda().fun->formals->formals.size()), args.size()));
auto attrs = buildBindings(std::max(static_cast<uint32_t>(formals->formals.size()), args.size()));
if (fun.lambda().fun->formals->ellipsis) {
if (formals->ellipsis) {
// If the formals have an ellipsis (eg the function accepts extra args) pass
// all available automatic arguments (which includes arguments specified on
// the command line via --arg/--argstr)
@ -1762,7 +1795,7 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
attrs.insert(v);
} else {
// Otherwise, only pass the arguments that the function accepts
for (auto & i : fun.lambda().fun->formals->formals) {
for (auto & i : formals->formals) {
auto j = args.get(i.name);
if (j) {
attrs.insert(*j);
@ -2020,7 +2053,7 @@ void EvalState::concatLists(
void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
{
NixStringContext context;
std::vector<BackedStringView> s;
std::vector<BackedStringView> strings;
size_t sSize = 0;
NixInt n{0};
NixFloat nf = 0;
@ -2028,32 +2061,11 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
bool first = !forceString;
ValueType firstType = nString;
const auto str = [&] {
std::string result;
result.reserve(sSize);
for (const auto & part : s)
result += *part;
return result;
};
/* c_str() is not str().c_str() because we want to create a string
Value. allocating a GC'd string directly and moving it into a
Value lets us avoid an allocation and copy. */
const auto c_str = [&] {
char * result = allocString(sSize + 1);
char * tmp = result;
for (const auto & part : s) {
memcpy(tmp, part->data(), part->size());
tmp += part->size();
}
*tmp = 0;
return result;
};
// List of returned strings. References to these Values must NOT be persisted.
SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
Value * vTmpP = values.data();
for (auto & [i_pos, i] : *es) {
for (auto & [i_pos, i] : es) {
Value & vTmp = *vTmpP++;
i->eval(state, env, vTmp);
@ -2096,33 +2108,46 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
.withFrame(env, *this)
.debugThrow();
} else {
if (s.empty())
s.reserve(es->size());
if (strings.empty())
strings.reserve(es.size());
/* skip canonization of first path, which would only be not
canonized in the first place if it's coming from a ./${foo} type
path */
auto part = state.coerceToString(
i_pos, vTmp, context, "while evaluating a path segment", false, firstType == nString, !first);
sSize += part->size();
s.emplace_back(std::move(part));
strings.emplace_back(std::move(part));
}
first = false;
}
if (firstType == nInt)
if (firstType == nInt) {
v.mkInt(n);
else if (firstType == nFloat)
} else if (firstType == nFloat) {
v.mkFloat(nf);
else if (firstType == nPath) {
} else if (firstType == nPath) {
if (!context.empty())
state.error<EvalError>("a string that refers to a store path cannot be appended to a path")
.atPos(pos)
.withFrame(env, *this)
.debugThrow();
v.mkPath(state.rootPath(CanonPath(str())));
} else
v.mkStringMove(c_str(), context);
std::string resultStr;
resultStr.reserve(sSize);
for (const auto & part : strings) {
resultStr += *part;
}
v.mkPath(state.rootPath(CanonPath(resultStr)));
} else {
auto & resultStr = StringData::alloc(sSize);
auto * tmp = resultStr.data();
for (const auto & part : strings) {
std::memcpy(tmp, part->data(), part->size());
tmp += part->size();
}
*tmp = '\0';
v.mkStringMove(resultStr, context, state.mem);
}
}
void ExprPos::eval(EvalState & state, Env & env, Value & v)
@ -2160,30 +2185,28 @@ void EvalState::forceValueDeep(Value & v)
{
std::set<const Value *> seen;
std::function<void(Value & v)> recurse;
recurse = [&](Value & v) {
[&, &state(*this)](this const auto & recurse, Value & v) {
if (!seen.insert(&v).second)
return;
forceValue(v, v.determinePos(noPos));
state.forceValue(v, v.determinePos(noPos));
if (v.type() == nAttrs) {
for (auto & i : *v.attrs())
try {
// If the value is a thunk, we're evaling. Otherwise no trace necessary.
auto dts = debugRepl && i.value->isThunk() ? makeDebugTraceStacker(
*this,
*i.value->thunk().expr,
*i.value->thunk().env,
i.pos,
"while evaluating the attribute '%1%'",
symbols[i.name])
: nullptr;
auto dts = state.debugRepl && i.value->isThunk() ? makeDebugTraceStacker(
state,
*i.value->thunk().expr,
*i.value->thunk().env,
i.pos,
"while evaluating the attribute '%1%'",
state.symbols[i.name])
: nullptr;
recurse(*i.value);
} catch (Error & e) {
addErrorTrace(e, i.pos, "while evaluating the attribute '%1%'", symbols[i.name]);
state.addErrorTrace(e, i.pos, "while evaluating the attribute '%1%'", state.symbols[i.name]);
throw;
}
}
@ -2192,9 +2215,7 @@ void EvalState::forceValueDeep(Value & v)
for (auto v2 : v.listView())
recurse(*v2);
}
};
recurse(v);
}(v);
}
NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCtx)
@ -2298,9 +2319,9 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
void copyContext(const Value & v, NixStringContext & context, const ExperimentalFeatureSettings & xpSettings)
{
if (v.context())
for (const char ** p = v.context(); *p; ++p)
context.insert(NixStringContextElem::parse(*p, xpSettings));
if (auto * ctx = v.context())
for (auto * elem : *ctx)
context.insert(NixStringContextElem::parse(elem->view(), xpSettings));
}
std::string_view EvalState::forceString(
@ -2320,7 +2341,9 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
auto s = forceString(v, pos, errorCtx);
if (v.context()) {
error<EvalError>(
"the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0])
"the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string_view(),
(*v.context()->begin())->view())
.withTrace(pos, errorCtx)
.debugThrow();
}
@ -2377,12 +2400,15 @@ BackedStringView EvalState::coerceToString(
}
if (v.type() == nPath) {
return !canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}.
v.pathStr()
: copyToStore ? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs());
if (!canonicalizePath && !copyToStore) {
// FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}.
return v.pathStrView();
} else if (copyToStore) {
return store->printStorePath(copyPathToStore(context, v.path()));
} else {
return std::string{v.path().path.abs()};
}
}
if (v.type() == nAttrs) {
@ -2635,7 +2661,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
return;
case nString:
if (strcmp(v1.c_str(), v2.c_str()) != 0) {
if (v1.string_view() != v2.string_view()) {
error<AssertionError>(
"string '%s' is not equal to string '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
@ -2652,7 +2678,7 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
if (strcmp(v1.pathStr(), v2.pathStr()) != 0) {
if (v1.pathStrView() != v2.pathStrView()) {
error<AssertionError>(
"path '%s' is not equal to path '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
@ -2818,12 +2844,12 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.boolean() == v2.boolean();
case nString:
return strcmp(v1.c_str(), v2.c_str()) == 0;
return v1.string_view() == v2.string_view();
case nPath:
return
// FIXME: compare accessors by their fingerprint.
v1.pathAccessor() == v2.pathAccessor() && strcmp(v1.pathStr(), v2.pathStr()) == 0;
v1.pathAccessor() == v2.pathAccessor() && v1.pathStrView() == v2.pathStrView();
case nNull:
return true;
@ -3067,7 +3093,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
return parseExprFromFile(path, staticBaseEnv);
}
Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
Expr * EvalState::parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv)
{
auto buffer = path.resolveSymlinks().readFile();
// readFile hopefully have left some extra space for terminators
@ -3075,8 +3101,8 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<Sta
return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
}
Expr *
EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
Expr * EvalState::parseExprFromString(
std::string s_, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv)
{
// NOTE this method (and parseStdin) must take care to *fully copy* their input
// into their respective Pos::Origin until the parser stops overwriting its input
@ -3210,7 +3236,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
}
Expr * EvalState::parse(
char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
char * text,
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
const std::shared_ptr<StaticEnv> & staticEnv)
{
DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath
DocCommentMap * docComments = &tmpDocComments;
@ -3220,8 +3250,8 @@ Expr * EvalState::parse(
docComments = &it->second;
}
auto result = parseExprFromBuf(
text, length, origin, basePath, mem.exprs.alloc, symbols, settings, positions, *docComments, rootFS);
auto result =
parseExprFromBuf(text, length, origin, basePath, mem.exprs, symbols, settings, positions, *docComments, rootFS);
result->bindVars(*this, staticEnv);

8
src/libexpr/get-drvs.cc
View file

@ -168,7 +168,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT
for (auto elem : outTI->listView()) {
if (elem->type() != nString)
throw errMsg;
auto out = outputs.find(elem->c_str());
auto out = outputs.find(elem->string_view());
if (out == outputs.end())
throw errMsg;
result.insert(*out);
@ -245,7 +245,7 @@ std::string PackageInfo::queryMetaString(const std::string & name)
Value * v = queryMeta(name);
if (!v || v->type() != nString)
return "";
return v->c_str();
return std::string{v->string_view()};
}
NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
@ -258,7 +258,7 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def)
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
integer meta fields. */
if (auto n = string2Int<NixInt::Inner>(v->c_str()))
if (auto n = string2Int<NixInt::Inner>(v->string_view()))
return NixInt{*n};
}
return def;
@ -274,7 +274,7 @@ NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def)
if (v->type() == nString) {
/* Backwards compatibility with before we had support for
float meta fields. */
if (auto n = string2Float<NixFloat>(v->c_str()))
if (auto n = string2Float<NixFloat>(v->string_view()))
return *n;
}
return def;

45
src/libexpr/include/nix/expr/attr-set.hh
View file

@ -5,6 +5,7 @@
#include "nix/expr/symbol-table.hh"
#include <boost/container/static_vector.hpp>
#include <boost/iterator/function_output_iterator.hpp>
#include <algorithm>
#include <functional>
@ -463,12 +464,48 @@ private:
return bindings->baseLayer;
}
/**
* If the bindings gets "layered" on top of another we need to recalculate
* the number of unique attributes in the chain.
*
* This is done by either iterating over the base "layer" and the newly added
* attributes and counting duplicates. If the base "layer" is big this approach
* is inefficient and we fall back to doing per-element binary search in the base
* "layer".
*/
void finishSizeIfNecessary()
{
if (hasBaseLayer())
/* NOTE: Do not use std::ranges::distance, since Bindings is a sized
range, but we are calculating this size here. */
bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end());
if (!hasBaseLayer())
return;
auto & base = *bindings->baseLayer;
auto attrs = std::span(bindings->attrs, bindings->numAttrs);
Bindings::size_type duplicates = 0;
/* If the base bindings is smaller than the newly added attributes
iterate using std::set_intersection to run in O(|base| + |attrs|) =
O(|attrs|). Otherwise use an O(|attrs| * log(|base|)) per-attr binary
search to check for duplicates. Note that if we are in this code path then
|attrs| <= bindingsUpdateLayerRhsSizeThreshold, which 16 by default. We are
optimizing for the case when a small attribute set gets "layered" on top of
a much larger one. When attrsets are already small it's fine to do a linear
scan, but we should avoid expensive iterations over large "base" attrsets. */
if (attrs.size() > base.size()) {
std::set_intersection(
base.begin(),
base.end(),
attrs.begin(),
attrs.end(),
boost::make_function_output_iterator([&]([[maybe_unused]] auto && _) { ++duplicates; }));
} else {
for (const auto & attr : attrs) {
if (base.get(attr.name))
++duplicates;
}
}
bindings->numAttrsInChain = base.numAttrsInChain + attrs.size() - duplicates;
}
public:

2
src/libexpr/include/nix/expr/eval-inline.hh
View file

@ -12,7 +12,7 @@ namespace nix {
* Note: Various places expect the allocated memory to be zeroed.
*/
[[gnu::always_inline]]
inline void * allocBytes(size_t n)
inline void * EvalMemory::allocBytes(size_t n)
{
void * p;
#if NIX_USE_BOEHMGC

27
src/libexpr/include/nix/expr/eval.hh
View file

@ -191,7 +191,7 @@ std::ostream & operator<<(std::ostream & os, const ValueType t);
struct RegexCache;
std::shared_ptr<RegexCache> makeRegexCache();
ref<RegexCache> makeRegexCache();
struct DebugTrace
{
@ -335,6 +335,7 @@ public:
EvalMemory & operator=(const EvalMemory &) = delete;
EvalMemory & operator=(EvalMemory &&) = delete;
inline void * allocBytes(size_t n);
inline Value * allocValue();
inline Env & allocEnv(size_t size);
@ -348,7 +349,7 @@ public:
ListBuilder buildList(size_t size)
{
stats.nrListElems += size;
return ListBuilder(size);
return ListBuilder(*this, size);
}
const Statistics & getStats() const &
@ -372,6 +373,7 @@ public:
const fetchers::Settings & fetchSettings;
const EvalSettings & settings;
SymbolTable symbols;
PosTable positions;
@ -418,7 +420,7 @@ public:
RootValue vImportedDrvToDerivation = nullptr;
ref<fetchers::InputCache> inputCache;
const ref<fetchers::InputCache> inputCache;
/**
* Debugger
@ -471,18 +473,18 @@ private:
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
const ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
/**
* A cache that maps paths to "resolved" paths for importing Nix
* expressions, i.e. `/foo` to `/foo/default.nix`.
*/
ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
const ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
/**
* A cache from resolved paths to values.
*/
ref<boost::concurrent_flat_map<
const ref<boost::concurrent_flat_map<
SourcePath,
Value *,
std::hash<SourcePath>,
@ -504,7 +506,7 @@ private:
/**
* Cache used by prim_match().
*/
std::shared_ptr<RegexCache> regexCache;
const ref<RegexCache> regexCache;
public:
@ -592,12 +594,13 @@ public:
* Parse a Nix expression from the specified file.
*/
Expr * parseExprFromFile(const SourcePath & path);
Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv);
/**
* Parse a Nix expression from the specified string.
*/
Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
Expr *
parseExprFromString(std::string s, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromString(std::string s, const SourcePath & basePath);
Expr * parseStdin();
@ -766,7 +769,7 @@ public:
#if NIX_USE_BOEHMGC
/** A GC root for the baseEnv reference. */
std::shared_ptr<Env *> baseEnvP;
const std::shared_ptr<Env *> baseEnvP;
#endif
public:
@ -780,7 +783,7 @@ public:
/**
* The same, but used during parsing to resolve variables.
*/
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
const std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
/**
* Internal primops not exposed to the user.
@ -862,7 +865,7 @@ private:
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::shared_ptr<StaticEnv> & staticEnv);
const std::shared_ptr<StaticEnv> & staticEnv);
/**
* Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run

2
src/libexpr/include/nix/expr/get-drvs.hh
View file

@ -15,7 +15,7 @@ namespace nix {
struct PackageInfo
{
public:
typedef std::map<std::string, std::optional<StorePath>> Outputs;
typedef std::map<std::string, std::optional<StorePath>, std::less<>> Outputs;
private:
EvalState * state;

1
src/libexpr/include/nix/expr/meson.build
View file

@ -31,6 +31,7 @@ headers = [ config_pub_h ] + files(
'print.hh',
'repl-exit-status.hh',
'search-path.hh',
'static-string-data.hh',
'symbol-table.hh',
'value-to-json.hh',
'value-to-xml.hh',

211
src/libexpr/include/nix/expr/nixexpr.hh
View file

@ -3,6 +3,7 @@
#include <map>
#include <span>
#include <memory>
#include <vector>
#include <memory_resource>
#include <algorithm>
@ -11,8 +12,11 @@
#include "nix/expr/value.hh"
#include "nix/expr/symbol-table.hh"
#include "nix/expr/eval-error.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/util/pos-idx.hh"
#include "nix/expr/counter.hh"
#include "nix/util/pos-table.hh"
#include "nix/util/error.hh"
namespace nix {
@ -89,13 +93,6 @@ std::string showAttrPath(const SymbolTable & symbols, std::span<const AttrName>
using UpdateQueue = SmallTemporaryValueVector<conservativeStackReservation>;
class Exprs
{
std::pmr::monotonic_buffer_resource buffer;
public:
std::pmr::polymorphic_allocator<char> alloc{&buffer};
};
/* Abstract syntax of Nix expressions. */
struct Expr
@ -191,22 +188,18 @@ struct ExprString : Expr
* This is only for strings already allocated in our polymorphic allocator,
* or that live at least that long (e.g. c++ string literals)
*/
ExprString(const char * s)
ExprString(const StringData & s)
{
v.mkStringNoCopy(s);
};
ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
{
auto len = sv.length();
if (len == 0) {
v.mkStringNoCopy("");
if (sv.size() == 0) {
v.mkStringNoCopy(""_sds);
return;
}
char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkStringNoCopy(s);
v.mkStringNoCopy(StringData::make(*alloc.resource(), sv));
};
Value * maybeThunk(EvalState & state, Env & env) override;
@ -221,11 +214,7 @@ struct ExprPath : Expr
ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv)
: accessor(accessor)
{
auto len = sv.length();
char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkPath(&*accessor, s);
v.mkPath(&*accessor, StringData::make(*alloc.resource(), sv));
}
Value * maybeThunk(EvalState & state, Env & env) override;
@ -350,7 +339,7 @@ struct ExprOpHasAttr : Expr
Expr * e;
std::span<AttrName> attrPath;
ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::vector<AttrName> attrPath)
ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::span<AttrName> attrPath)
: e(e)
, attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()})
{
@ -442,8 +431,14 @@ struct ExprAttrs : Expr
struct ExprList : Expr
{
std::vector<Expr *> elems;
ExprList() {};
std::span<Expr *> elems;
ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::span<Expr *> exprs)
: elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
{
std::ranges::copy(exprs, elems.begin());
};
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env) override;
@ -460,7 +455,7 @@ struct Formal
Expr * def;
};
struct Formals
struct FormalsBuilder
{
typedef std::vector<Formal> Formals_;
/**
@ -475,6 +470,23 @@ struct Formals
formals.begin(), formals.end(), arg, [](const Formal & f, const Symbol & sym) { return f.name < sym; });
return it != formals.end() && it->name == arg;
}
};
struct Formals
{
std::span<Formal> formals;
bool ellipsis;
Formals(std::span<Formal> formals, bool ellipsis)
: formals(formals)
, ellipsis(ellipsis) {};
bool has(Symbol arg) const
{
auto it = std::lower_bound(
formals.begin(), formals.end(), arg, [](const Formal & f, const Symbol & sym) { return f.name < sym; });
return it != formals.end() && it->name == arg;
}
std::vector<Formal> lexicographicOrder(const SymbolTable & symbols) const
{
@ -492,31 +504,71 @@ struct ExprLambda : Expr
PosIdx pos;
Symbol name;
Symbol arg;
Formals * formals;
private:
bool hasFormals;
bool ellipsis;
uint16_t nFormals;
Formal * formalsStart;
public:
std::optional<Formals> getFormals() const
{
if (hasFormals)
return Formals{{formalsStart, nFormals}, ellipsis};
else
return std::nullopt;
}
Expr * body;
DocComment docComment;
ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body)
ExprLambda(
const PosTable & positions,
std::pmr::polymorphic_allocator<char> & alloc,
PosIdx pos,
Symbol arg,
const FormalsBuilder & formals,
Expr * body)
: pos(pos)
, arg(arg)
, formals(formals)
, body(body) {};
ExprLambda(PosIdx pos, Formals * formals, Expr * body)
: pos(pos)
, formals(formals)
, hasFormals(true)
, ellipsis(formals.ellipsis)
, nFormals(formals.formals.size())
, formalsStart(alloc.allocate_object<Formal>(nFormals))
, body(body)
{
}
if (formals.formals.size() > nFormals) [[unlikely]] {
auto err = Error(
"too many formal arguments, implementation supports at most %1%",
std::numeric_limits<decltype(nFormals)>::max());
if (pos)
err.atPos(positions[pos]);
throw err;
}
std::uninitialized_copy_n(formals.formals.begin(), nFormals, formalsStart);
};
ExprLambda(PosIdx pos, Symbol arg, Expr * body)
: pos(pos)
, arg(arg)
, hasFormals(false)
, ellipsis(false)
, nFormals(0)
, formalsStart(nullptr)
, body(body) {};
ExprLambda(
const PosTable & positions,
std::pmr::polymorphic_allocator<char> & alloc,
PosIdx pos,
const FormalsBuilder & formals,
Expr * body)
: ExprLambda(positions, alloc, pos, Symbol(), formals, body) {};
void setName(Symbol name) override;
std::string showNamePos(const EvalState & state) const;
inline bool hasFormals() const
{
return formals != nullptr;
}
PosIdx getPos() const override
{
return pos;
@ -572,8 +624,8 @@ struct ExprLet : Expr
struct ExprWith : Expr
{
PosIdx pos;
uint32_t prevWith;
Expr *attrs, *body;
size_t prevWith;
ExprWith * parentWith;
ExprWith(const PosIdx & pos, Expr * attrs, Expr * body)
: pos(pos)
@ -695,11 +747,31 @@ struct ExprConcatStrings : Expr
{
PosIdx pos;
bool forceString;
std::vector<std::pair<PosIdx, Expr *>> * es;
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
std::span<std::pair<PosIdx, Expr *>> es;
ExprConcatStrings(
std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::span<std::pair<PosIdx, Expr *>> es)
: pos(pos)
, forceString(forceString)
, es(es) {};
, es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
{
std::ranges::copy(es, this->es.begin());
};
ExprConcatStrings(
std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::initializer_list<std::pair<PosIdx, Expr *>> es)
: pos(pos)
, forceString(forceString)
, es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
{
std::ranges::copy(es, this->es.begin());
};
PosIdx getPos() const override
{
@ -737,6 +809,61 @@ struct ExprBlackHole : Expr
extern ExprBlackHole eBlackHole;
class Exprs
{
std::pmr::monotonic_buffer_resource buffer;
public:
std::pmr::polymorphic_allocator<char> alloc{&buffer};
template<class C>
[[gnu::always_inline]]
C * add(auto &&... args)
{
return alloc.new_object<C>(std::forward<decltype(args)>(args)...);
}
// we define some calls to add explicitly so that the argument can be passed in as initializer lists
template<class C>
[[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
requires(std::same_as<C, ExprCall>)
{
return alloc.new_object<C>(pos, fun, std::move(args));
}
template<class C>
[[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
requires(std::same_as<C, ExprCall>)
{
return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));
}
template<class C>
[[gnu::always_inline]]
C *
add(std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::span<std::pair<PosIdx, Expr *>> es)
requires(std::same_as<C, ExprConcatStrings>)
{
return alloc.new_object<C>(alloc, pos, forceString, es);
}
template<class C>
[[gnu::always_inline]]
C *
add(std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::initializer_list<std::pair<PosIdx, Expr *>> es)
requires(std::same_as<C, ExprConcatStrings>)
{
return alloc.new_object<C>(alloc, pos, forceString, es);
}
};
/* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at
runtime. */

48
src/libexpr/include/nix/expr/parser-state.hh
View file

@ -4,6 +4,8 @@
#include <limits>
#include "nix/expr/eval.hh"
#include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
namespace nix {
@ -78,7 +80,7 @@ struct LexerState
struct ParserState
{
const LexerState & lexerState;
std::pmr::polymorphic_allocator<char> & alloc;
Exprs & exprs;
SymbolTable & symbols;
PosTable & positions;
Expr * result;
@ -93,8 +95,8 @@ struct ParserState
void addAttr(
ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
Expr * stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
void validateFormals(FormalsBuilder & formals, PosIdx pos = noPos, Symbol arg = {});
Expr * stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es);
PosIdx at(const ParserLocation & loc);
};
@ -132,11 +134,11 @@ inline void ParserState::addAttr(
dupAttr(attrPath, pos, j->second.pos);
}
} else {
nested = new ExprAttrs;
nested = exprs.add<ExprAttrs>();
attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
}
} else {
nested = new ExprAttrs;
nested = exprs.add<ExprAttrs>();
attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
}
attrs = nested;
@ -213,17 +215,17 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
}
}
inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
inline void ParserState::validateFormals(FormalsBuilder & formals, PosIdx pos, Symbol arg)
{
std::sort(formals->formals.begin(), formals->formals.end(), [](const auto & a, const auto & b) {
std::sort(formals.formals.begin(), formals.formals.end(), [](const auto & a, const auto & b) {
return std::tie(a.name, a.pos) < std::tie(b.name, b.pos);
});
std::optional<std::pair<Symbol, PosIdx>> duplicate;
for (size_t i = 0; i + 1 < formals->formals.size(); i++) {
if (formals->formals[i].name != formals->formals[i + 1].name)
for (size_t i = 0; i + 1 < formals.formals.size(); i++) {
if (formals.formals[i].name != formals.formals[i + 1].name)
continue;
std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos};
std::pair thisDup{formals.formals[i].name, formals.formals[i + 1].pos};
duplicate = std::min(thisDup, duplicate.value_or(thisDup));
}
if (duplicate)
@ -231,18 +233,16 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym
{.msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]),
.pos = positions[duplicate->second]});
if (arg && formals->has(arg))
if (arg && formals.has(arg))
throw ParseError(
{.msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]), .pos = positions[pos]});
return formals;
}
inline Expr *
ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
ParserState::stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es)
{
if (es.empty())
return new ExprString("");
return exprs.add<ExprString>(""_sds);
/* Figure out the minimum indentation. Note that by design
whitespace-only final lines are not taken into account. (So
@ -282,7 +282,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
}
/* Strip spaces from each line. */
auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
std::vector<std::pair<PosIdx, Expr *>> es2{};
atStartOfLine = true;
size_t curDropped = 0;
size_t n = es.size();
@ -290,7 +290,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
const auto trimExpr = [&](Expr * e) {
atStartOfLine = false;
curDropped = 0;
es2->emplace_back(i->first, e);
es2.emplace_back(i->first, e);
};
const auto trimString = [&](const StringToken & t) {
std::string s2;
@ -324,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
// Ignore empty strings for a minor optimisation and AST simplification
if (s2 != "") {
es2->emplace_back(i->first, new ExprString(alloc, s2));
es2.emplace_back(i->first, exprs.add<ExprString>(exprs.alloc, s2));
}
};
for (; i != es.end(); ++i, --n) {
@ -333,19 +333,17 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
// If there is nothing at all, return the empty string directly.
// This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
if (es2->size() == 0) {
auto * const result = new ExprString("");
delete es2;
if (es2.size() == 0) {
auto * const result = exprs.add<ExprString>(""_sds);
return result;
}
/* If this is a single string, then don't do a concatenation. */
if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
auto * const result = (*es2)[0].second;
delete es2;
if (es2.size() == 1 && dynamic_cast<ExprString *>((es2)[0].second)) {
auto * const result = (es2)[0].second;
return result;
}
return new ExprConcatStrings(pos, true, es2);
return exprs.add<ExprConcatStrings>(exprs.alloc, pos, true, es2);
}
inline PosIdx LexerState::at(const ParserLocation & loc)

6
src/libexpr/include/nix/expr/primops.hh
View file

@ -12,11 +12,7 @@ struct RegisterPrimOp
{
typedef std::vector<PrimOp> PrimOps;
static PrimOps & primOps()
{
static PrimOps primOps;
return primOps;
}
static PrimOps & primOps();
/**
* You can register a constant by passing an arity of 0. fun

2
src/libexpr/include/nix/expr/print-options.hh
View file

@ -110,7 +110,7 @@ struct PrintOptions
* `PrintOptions` for unknown and therefore potentially large values in error messages,
* to avoid printing "too much" output.
*/
static PrintOptions errorPrintOptions = PrintOptions{
static constexpr PrintOptions errorPrintOptions = PrintOptions{
.ansiColors = true,
.maxDepth = 10,
.maxAttrs = 10,

44
src/libexpr/include/nix/expr/static-string-data.hh Normal file
View file

@ -0,0 +1,44 @@
#pragma once
///@file
#include "nix/expr/value.hh"
namespace nix {
template<size_t N>
struct StringData::Static
{
/**
* @note Must be first to make layout compatible with StringData.
*/
const size_t size = N - 1;
char data[N];
consteval Static(const char (&str)[N])
{
static_assert(N > 0);
if (str[size] != '\0')
throw;
std::copy_n(str, N, data);
}
operator const StringData &() const &
{
static_assert(sizeof(decltype(*this)) >= sizeof(StringData));
static_assert(alignof(decltype(*this)) == alignof(StringData));
/* NOTE: This cast is somewhat on the fence of what's legal in C++.
The question boils down to whether flexible array members are
layout compatible with fixed-size arrays. This is a gray area, since
FAMs are not standard anyway.
*/
return *reinterpret_cast<const StringData *>(this);
}
};
template<StringData::Static S>
const StringData & operator""_sds()
{
return S;
}
} // namespace nix

35
src/libexpr/include/nix/expr/symbol-table.hh
View file

@ -3,6 +3,7 @@
#include <memory_resource>
#include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/util/chunked-vector.hh"
#include "nix/util/error.hh"
@ -16,7 +17,6 @@ class SymbolValue : protected Value
friend class SymbolStr;
friend class SymbolTable;
uint32_t size_;
uint32_t idx;
SymbolValue() = default;
@ -24,7 +24,7 @@ class SymbolValue : protected Value
public:
operator std::string_view() const noexcept
{
return {c_str(), size_};
return string_view();
}
};
@ -96,13 +96,13 @@ class SymbolStr
SymbolValueStore & store;
std::string_view s;
std::size_t hash;
std::pmr::polymorphic_allocator<char> & alloc;
std::pmr::memory_resource & resource;
Key(SymbolValueStore & store, std::string_view s, std::pmr::polymorphic_allocator<char> & stringAlloc)
Key(SymbolValueStore & store, std::string_view s, std::pmr::memory_resource & stringMemory)
: store(store)
, s(s)
, hash(HashType{}(s))
, alloc(stringAlloc)
, resource(stringMemory)
{
}
};
@ -122,14 +122,10 @@ public:
// for multi-threaded implementations: lock store and allocator here
const auto & [v, idx] = key.store.add(SymbolValue{});
if (size == 0) {
v.mkStringNoCopy("", nullptr);
v.mkStringNoCopy(""_sds, nullptr);
} else {
auto s = key.alloc.allocate(size + 1);
memcpy(s, key.s.data(), size);
s[size] = '\0';
v.mkStringNoCopy(s, nullptr);
v.mkStringNoCopy(StringData::make(key.resource, key.s));
}
v.size_ = size;
v.idx = idx;
this->s = &v;
}
@ -139,6 +135,12 @@ public:
return *s == s2;
}
[[gnu::always_inline]]
const StringData & string_data() const noexcept
{
return s->string_data();
}
[[gnu::always_inline]]
const char * c_str() const noexcept
{
@ -155,13 +157,17 @@ public:
[[gnu::always_inline]]
bool empty() const noexcept
{
return s->size_ == 0;
auto * p = &s->string_data();
// Save a dereference in the sentinel value case
if (p == &""_sds)
return true;
return p->size() == 0;
}
[[gnu::always_inline]]
size_t size() const noexcept
{
return s->size_;
return s->string_data().size();
}
[[gnu::always_inline]]
@ -259,7 +265,6 @@ private:
* During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based.
*/
std::pmr::monotonic_buffer_resource buffer;
std::pmr::polymorphic_allocator<char> stringAlloc{&buffer};
SymbolStr::SymbolValueStore store{16};
/**
@ -282,7 +287,7 @@ public:
// Most symbols are looked up more than once, so we trade off insertion performance
// for lookup performance.
// FIXME: make this thread-safe.
return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first);
return Symbol(*symbols.insert(SymbolStr::Key{store, s, buffer}).first);
}
std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const

200
src/libexpr/include/nix/expr/value.hh
View file

@ -1,8 +1,14 @@
#pragma once
///@file
#include <bit>
#include <cassert>
#include <cstddef>
#include <cstring>
#include <memory>
#include <memory_resource>
#include <span>
#include <string_view>
#include <type_traits>
#include <concepts>
@ -82,6 +88,7 @@ class PosIdx;
struct Pos;
class StorePath;
class EvalState;
class EvalMemory;
class XMLWriter;
class Printer;
@ -155,7 +162,7 @@ class ListBuilder
Value * inlineElems[2] = {nullptr, nullptr};
public:
Value ** elems;
ListBuilder(size_t size);
ListBuilder(EvalMemory & mem, size_t size);
// NOTE: Can be noexcept because we are just copying integral values and
// raw pointers.
@ -186,6 +193,91 @@ public:
friend struct Value;
};
class StringData
{
public:
using size_type = std::size_t;
size_type size_;
char data_[];
/*
* This in particular ensures that we cannot have a `StringData`
* that we use by value, which is just what we want!
*
* Dynamically sized types aren't a thing in C++ and even flexible array
* members are a language extension and beyond the realm of standard C++.
* Technically, sizeof data_ member is 0 and the intended way to use flexible
* array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
* and the compiler will consider alignment restrictions for the FAM.
*
*/
StringData(StringData &&) = delete;
StringData & operator=(StringData &&) = delete;
StringData(const StringData &) = delete;
StringData & operator=(const StringData &) = delete;
~StringData() = default;
private:
StringData() = delete;
explicit StringData(size_type size)
: size_(size)
{
}
public:
/**
* Allocate StringData on the (possibly) GC-managed heap and copy
* the contents of s to it.
*/
static const StringData & make(std::string_view s);
/**
* Allocate StringData on the (possibly) GC-managed heap.
* @param size Length of the string (without the NUL terminator).
*/
static StringData & alloc(size_t size);
size_t size() const
{
return size_;
}
char * data() noexcept
{
return data_;
}
const char * data() const noexcept
{
return data_;
}
const char * c_str() const noexcept
{
return data_;
}
constexpr std::string_view view() const noexcept
{
return std::string_view(data_, size_);
}
template<size_t N>
struct Static;
static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
{
auto & res =
*new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
std::memcpy(res.data_, s.data(), s.size());
res.data_[s.size()] = '\0';
return res;
}
};
namespace detail {
/**
@ -219,14 +311,73 @@ struct ValueBase
*/
struct StringWithContext
{
const char * c_str;
const char ** context; // must be in sorted order
const StringData * str;
/**
* The type of the context itself.
*
* Currently, it is length-prefixed array of pointers to
* null-terminated strings. The strings are specially formatted
* to represent a flattening of the recursive sum type that is a
* context element.
*
* @See NixStringContext for an more easily understood type,
* that of the "builder" for this data structure.
*/
struct Context
{
using value_type = const StringData *;
using size_type = std::size_t;
using iterator = const value_type *;
Context(size_type size)
: size_(size)
{
}
private:
/**
* Number of items in the array
*/
size_type size_;
/**
* @pre must be in sorted order
*/
value_type elems[];
public:
iterator begin() const
{
return elems;
}
iterator end() const
{
return elems + size();
}
size_type size() const
{
return size_;
}
/**
* @return null pointer when context.empty()
*/
static Context * fromBuilder(const NixStringContext & context, EvalMemory & mem);
};
/**
* May be null for a string without context.
*/
const Context * context;
};
struct Path
{
SourceAccessor * accessor;
const char * path;
const StringData * path;
};
struct Null
@ -587,13 +738,13 @@ protected:
void getStorage(StringWithContext & string) const noexcept
{
string.context = untagPointer<decltype(string.context)>(payload[0]);
string.c_str = std::bit_cast<const char *>(payload[1]);
string.str = std::bit_cast<const StringData *>(payload[1]);
}
void getStorage(Path & path) const noexcept
{
path.accessor = untagPointer<decltype(path.accessor)>(payload[0]);
path.path = std::bit_cast<const char *>(payload[1]);
path.path = std::bit_cast<const StringData *>(payload[1]);
}
void setStorage(NixInt integer) noexcept
@ -638,7 +789,7 @@ protected:
void setStorage(StringWithContext string) noexcept
{
setUntaggablePayload<pdString>(string.context, string.c_str);
setUntaggablePayload<pdString>(string.context, string.str);
}
void setStorage(Path path) noexcept
@ -991,22 +1142,22 @@ public:
setStorage(b);
}
void mkStringNoCopy(const char * s, const char ** context = 0) noexcept
void mkStringNoCopy(const StringData & s, const Value::StringWithContext::Context * context = nullptr) noexcept
{
setStorage(StringWithContext{.c_str = s, .context = context});
setStorage(StringWithContext{.str = &s, .context = context});
}
void mkString(std::string_view s);
void mkString(std::string_view s, const NixStringContext & context);
void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
void mkStringMove(const char * s, const NixStringContext & context);
void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
void mkPath(const SourcePath & path);
inline void mkPath(SourceAccessor * accessor, const char * path) noexcept
inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
{
setStorage(Path{.accessor = accessor, .path = path});
setStorage(Path{.accessor = accessor, .path = &path});
}
inline void mkNull() noexcept
@ -1104,20 +1255,26 @@ public:
SourcePath path() const
{
return SourcePath(ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), pathStr()));
return SourcePath(
ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), std::string(pathStrView())));
}
std::string_view string_view() const noexcept
const StringData & string_data() const noexcept
{
return std::string_view(getStorage<StringWithContext>().c_str);
return *getStorage<StringWithContext>().str;
}
const char * c_str() const noexcept
{
return getStorage<StringWithContext>().c_str;
return getStorage<StringWithContext>().str->data();
}
const char ** context() const noexcept
std::string_view string_view() const noexcept
{
return string_data().view();
}
const Value::StringWithContext::Context * context() const noexcept
{
return getStorage<StringWithContext>().context;
}
@ -1174,7 +1331,12 @@ public:
const char * pathStr() const noexcept
{
return getStorage<Path>().path;
return getStorage<Path>().path->c_str();
}
std::string_view pathStrView() const noexcept
{
return getStorage<Path>().path->view();
}
SourceAccessor * pathAccessor() const noexcept

13
src/libexpr/include/nix/expr/value/context.hh
View file

@ -24,6 +24,14 @@ public:
}
};
/**
* @todo This should be renamed to `StringContextBuilderElem`, since:
*
* 1. We use `*Builder` for off-heap temporary data structures
*
* 2. The `Nix*` is totally redundant. (And my mistake from a long time
* ago.)
*/
struct NixStringContextElem
{
/**
@ -77,6 +85,11 @@ struct NixStringContextElem
std::string to_string() const;
};
/**
* @todo This should be renamed to `StringContextBuilder`.
*
* @see NixStringContextElem for explanation why.
*/
typedef std::set<NixStringContextElem> NixStringContext;
} // namespace nix

42
src/libexpr/lexer.l
View file

@ -142,11 +142,11 @@ or { return OR_KW; }
return PIPE_INTO;
}
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
{ID} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return ID; }
{INT} { errno = 0;
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
if (numMay.has_value()) {
yylval->n = NixInt{*numMay};
yylval->emplace<NixInt>(*numMay);
} else {
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
@ -156,7 +156,7 @@ or { return OR_KW; }
return INT_LIT;
}
{FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0);
yylval->emplace<NixFloat>(strtod(yytext, 0));
if (errno != 0)
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid float '%1%'", yytext),
@ -183,7 +183,7 @@ or { return OR_KW; }
/* It is impossible to match strings ending with '$' with one
regex because trailing contexts are only valid at the end
of a rule. (A sane but undocumented limitation.) */
yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; });
yylval->emplace<StringToken>(unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }));
return STR;
}
<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@ -198,27 +198,27 @@ or { return OR_KW; }
\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
yylval->str = {yytext, (size_t) yyleng, true};
forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; });
yylval->emplace<StringToken>(yytext, (size_t) yyleng, true);
forceNoNullByte(yylval->as<StringToken>(), [&]() { return state->positions[CUR_POS]; });
return IND_STR;
}
<IND_STRING>\'\'\$ |
<IND_STRING>\$ {
yylval->str = {"$", 1};
yylval->emplace<StringToken>("$", 1);
return IND_STR;
}
<IND_STRING>\'\'\' {
yylval->str = {"''", 2};
yylval->emplace<StringToken>("''", 2);
return IND_STR;
}
<IND_STRING>\'\'\\{ANY} {
yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; });
yylval->emplace<StringToken>(unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }));
return IND_STR;
}
<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
<IND_STRING>\' {
yylval->str = {"'", 1};
yylval->emplace<StringToken>("'", 1);
return IND_STR;
}
@ -232,23 +232,31 @@ or { return OR_KW; }
<PATH_START>{PATH_SEG} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->path = {yytext, (size_t) yyleng};
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
return PATH;
}
<PATH_START>{HPATH_START} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->path = {yytext, (size_t) yyleng};
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
return HPATH;
}
<PATH_START>{ANY} |
<PATH_START><<EOF>> {
/* This should be unreachable: PATH_START is only entered after matching
PATH_SEG or HPATH_START, and we rewind to re-parse those same patterns.
This rule exists to satisfy flex's %option nodefault requirement. */
unreachable();
}
{PATH} {
if (yytext[yyleng-1] == '/')
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->path = {yytext, (size_t) yyleng};
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
return PATH;
}
{HPATH} {
@ -256,7 +264,7 @@ or { return OR_KW; }
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->path = {yytext, (size_t) yyleng};
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
return HPATH;
}
@ -272,7 +280,7 @@ or { return OR_KW; }
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->str = {yytext, (size_t) yyleng};
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
return STR;
}
<INPATH>{ANY} |
@ -294,8 +302,8 @@ or { return OR_KW; }
});
}
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
{SPATH} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return SPATH; }
{URI} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return URI; }
%{
// Doc comment rule

49
src/libexpr/meson.build
View file

@ -183,17 +183,62 @@ subdir('primops')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
# Turns out that Bison/Flex are particularly sensitive to compilers
# failing to inline functions. For that reason we crank up the inlining
# threshold manually for optimized builds. Yes, this can be considered 'ricing'
# the compiler, but it does pay off.
#
# NOTE: missed inlining can be spotted (for Clang) using -Rpass-missed=inline
# and -fdump-ipa-inline-missed (for GCC).
parser_library_cpp_args = []
if not get_option('debug')
if cxx.get_id() == 'clang'
# The default as of LLVM 21 is 225:
# llc --help-hidden | grep inline-threshold
parser_library_cpp_args += [
'-mllvm',
'-inline-threshold=5000',
]
elif cxx.get_id() == 'gcc'
parser_library_cpp_args += [
'--param=max-inline-insns-single=1000',
'--param=max-inline-insns-auto=1000',
'--param=inline-unit-growth=400',
]
endif
endif
# Working around https://github.com/mesonbuild/meson/issues/1367.
parser_library = static_library(
'nixexpr-parser',
parser_tab,
lexer_tab,
cpp_args : parser_library_cpp_args,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
# 1. Stdlib and regular assertions regress parser performance significantly, so build without
# them for this one library when building in a release configuration.
# 2. Disable LTO for GCC because then inlining flags won't apply, since LTO in GCC is done
# by plonking down GIMPLE in the archive.
override_options : [
'b_ndebug=@0@'.format(not get_option('debug')),
'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'),
],
)
this_library = library(
'nixexpr',
sources,
config_priv_h,
parser_tab,
lexer_tab,
parser_tab[1],
lexer_tab[1],
generated_headers,
soversion : nix_soversion,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
link_args : linker_export_flags,
link_whole : [ parser_library ],
prelink : true, # For C++ static initializers
install : true,
cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],

15
src/libexpr/nixexpr.cc
View file

@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
{
str << v.pathStr();
str << v.pathStrView();
}
void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
@ -154,7 +154,7 @@ void ExprList::show(const SymbolTable & symbols, std::ostream & str) const
void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
{
str << "(";
if (hasFormals()) {
if (auto formals = getFormals()) {
str << "{ ";
bool first = true;
// the natural Symbol ordering is by creation time, which can lead to the
@ -171,7 +171,7 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
i.def->show(symbols, str);
}
}
if (formals->ellipsis) {
if (ellipsis) {
if (!first)
str << ", ";
str << "...";
@ -246,7 +246,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
{
bool first = true;
str << "(";
for (auto & i : *es) {
for (auto & i : es) {
if (first)
first = false;
else
@ -452,14 +452,14 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
es.exprEnvs.insert(std::make_pair(this, env));
auto newEnv =
std::make_shared<StaticEnv>(nullptr, env, (hasFormals() ? formals->formals.size() : 0) + (!arg ? 0 : 1));
std::make_shared<StaticEnv>(nullptr, env, (getFormals() ? getFormals()->formals.size() : 0) + (!arg ? 0 : 1));
Displacement displ = 0;
if (arg)
newEnv->vars.emplace_back(arg, displ++);
if (hasFormals()) {
if (auto formals = getFormals()) {
for (auto & i : formals->formals)
newEnv->vars.emplace_back(i.name, displ++);
@ -523,6 +523,7 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
prevWith = 0;
for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up.get(), level++)
if (curEnv->isWith) {
assert(level <= std::numeric_limits<uint32_t>::max());
prevWith = level;
break;
}
@ -564,7 +565,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
for (auto & i : *this->es)
for (auto & i : this->es)
i.second->bindVars(es, env);
}

5
src/libexpr/package.nix
View file

@ -70,11 +70,6 @@ mkMesonLibrary (finalAttrs: {
nix-util
nix-store
nix-fetchers
]
++ finalAttrs.passthru.externalPropagatedBuildInputs;
# Hack for sake of the dev shell
passthru.externalPropagatedBuildInputs = [
boost
nlohmann_json
]

296
src/libexpr/parser.y
View file

@ -14,6 +14,10 @@
%code requires {
// bison adds a bunch of switch statements with default:
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
#ifndef BISON_HEADER
#define BISON_HEADER
@ -59,7 +63,7 @@ Expr * parseExprFromBuf(
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::pmr::polymorphic_allocator<char> & alloc,
Exprs & exprs,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
@ -109,57 +113,39 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
}
}
static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
e2->args.push_back(arg);
return fn;
}
return new ExprCall(pos, fn, {arg});
return exprs.add<ExprCall>(pos, fn, {arg});
}
%}
%union {
// !!! We're probably leaking stuff here.
nix::Expr * e;
nix::ExprList * list;
nix::ExprAttrs * attrs;
nix::Formals * formals;
nix::Formal * formal;
nix::NixInt n;
nix::NixFloat nf;
nix::StringToken id; // !!! -> Symbol
nix::StringToken path;
nix::StringToken uri;
nix::StringToken str;
std::vector<nix::AttrName> * attrNames;
std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
std::variant<nix::Expr *, std::string_view> * to_be_string;
std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
}
%define api.value.type variant
%type <e> start expr expr_function expr_if expr_op
%type <e> expr_select expr_simple expr_app
%type <e> expr_pipe_from expr_pipe_into
%type <list> expr_list
%type <attrs> binds binds1
%type <formals> formals formal_set
%type <formal> formal
%type <attrNames> attrpath
%type <inheritAttrs> attrs
%type <string_parts> string_parts_interpolated
%type <ind_string_parts> ind_string_parts
%type <e> path_start
%type <to_be_string> string_parts string_attr
%type <id> attr
%token <id> ID
%token <str> STR IND_STR
%token <n> INT_LIT
%token <nf> FLOAT_LIT
%token <path> PATH HPATH SPATH PATH_END
%token <uri> URI
%type <Expr *> start expr expr_function expr_if expr_op
%type <Expr *> expr_select expr_simple expr_app
%type <Expr *> expr_pipe_from expr_pipe_into
%type <std::vector<Expr *>> list
%type <ExprAttrs *> binds binds1
%type <FormalsBuilder> formals formal_set
%type <Formal> formal
%type <std::vector<AttrName>> attrpath
%type <std::vector<std::pair<AttrName, PosIdx>>> attrs
%type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated
%type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts
%type <Expr *> path_start
%type <std::variant<Expr *, std::string_view>> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID
%token <StringToken> STR IND_STR
%token <NixInt> INT_LIT
%token <NixFloat> FLOAT_LIT
%token <StringToken> PATH HPATH SPATH PATH_END
%token <StringToken> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
@ -193,86 +179,90 @@ expr: expr_function;
expr_function
: ID ':' expr_function
{ auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3);
{ auto me = state->exprs.add<ExprLambda>(CUR_POS, state->symbols.create($1), $3);
$$ = me;
SET_DOC_POS(me, @1);
}
| formal_set ':' expr_function[body]
{ auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body);
{
state->validateFormals($formal_set);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, $formal_set, $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| formal_set '@' ID ':' expr_function[body]
{
auto arg = state->symbols.create($ID);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
state->validateFormals($formal_set, CUR_POS, arg);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| ID '@' formal_set ':' expr_function[body]
{
auto arg = state->symbols.create($ID);
auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body);
state->validateFormals($formal_set, CUR_POS, arg);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body);
$$ = me;
SET_DOC_POS(me, @1);
}
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
{ $$ = state->exprs.add<ExprAssert>(CUR_POS, $2, $4); }
| WITH expr ';' expr_function
{ $$ = new ExprWith(CUR_POS, $2, $4); }
{ $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
| LET binds IN_KW expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in let"),
.pos = state->positions[CUR_POS]
});
$$ = new ExprLet($2, $4);
$$ = state->exprs.add<ExprLet>($2, $4);
}
| expr_if
;
expr_if
: IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
: IF expr THEN expr ELSE expr { $$ = state->exprs.add<ExprIf>(CUR_POS, $2, $4, $6); }
| expr_pipe_from
| expr_pipe_into
| expr_op
;
expr_pipe_from
: expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); }
| expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); }
: expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->exprs, state->at(@2), $1, $3); }
| expr_op PIPE_FROM expr_op { $$ = makeCall(state->exprs, state->at(@2), $1, $3); }
;
expr_pipe_into
: expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
| expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); }
: expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->exprs, state->at(@2), $3, $1); }
| expr_op PIPE_INTO expr_op { $$ = makeCall(state->exprs, state->at(@2), $3, $1); }
;
expr_op
: '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
| '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); }
| expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
| expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
| expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); }
| expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); }
| expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); }
| expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); }
| expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); }
| expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
| expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
| expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; }
: '!' expr_op %prec NOT { $$ = state->exprs.add<ExprOpNot>($2); }
| '-' expr_op %prec NEGATE { $$ = state->exprs.add<ExprCall>(CUR_POS, state->exprs.add<ExprVar>(state->s.sub), {state->exprs.add<ExprInt>(0), $2}); }
| expr_op EQ expr_op { $$ = state->exprs.add<ExprOpEq>($1, $3); }
| expr_op NEQ expr_op { $$ = state->exprs.add<ExprOpNEq>($1, $3); }
| expr_op '<' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.lessThan), {$1, $3}); }
| expr_op LEQ expr_op { $$ = state->exprs.add<ExprOpNot>(state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.lessThan), {$3, $1})); }
| expr_op '>' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.lessThan), {$3, $1}); }
| expr_op GEQ expr_op { $$ = state->exprs.add<ExprOpNot>(state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.lessThan), {$1, $3})); }
| expr_op AND expr_op { $$ = state->exprs.add<ExprOpAnd>(state->at(@2), $1, $3); }
| expr_op OR expr_op { $$ = state->exprs.add<ExprOpOr>(state->at(@2), $1, $3); }
| expr_op IMPL expr_op { $$ = state->exprs.add<ExprOpImpl>(state->at(@2), $1, $3); }
| expr_op UPDATE expr_op { $$ = state->exprs.add<ExprOpUpdate>(state->at(@2), $1, $3); }
| expr_op '?' attrpath { $$ = state->exprs.add<ExprOpHasAttr>(state->exprs.alloc, $1, $3); }
| expr_op '+' expr_op
{ $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
| expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
| expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
| expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
| expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); }
{ $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); }
| expr_op '-' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.sub), {$1, $3}); }
| expr_op '*' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.mul), {$1, $3}); }
| expr_op '/' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.div), {$1, $3}); }
| expr_op CONCAT expr_op { $$ = state->exprs.add<ExprOpConcatLists>(state->at(@2), $1, $3); }
| expr_app
;
expr_app
: expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); $2->warnIfCursedOr(state->symbols, state->positions); }
: expr_app expr_select { $$ = makeCall(state->exprs, CUR_POS, $1, $2); $2->warnIfCursedOr(state->symbols, state->positions); }
| /* Once a ‘cursed or’ reaches this nonterminal, it is no longer cursed,
because the uncursed parse would also produce an expr_app. But we need
to remove the cursed status in order to prevent valid things like
@ -282,9 +272,9 @@ expr_app
expr_select
: expr_simple '.' attrpath
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
{ $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, nullptr); }
| expr_simple '.' attrpath OR_KW expr_select
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
{ $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, $5); $5->warnIfCursedOr(state->symbols, state->positions); }
| /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
allow stuff like ‘map or [...]’. This production is problematic (see
https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@ -293,7 +283,7 @@ expr_select
the ExprCall with data (establishing that it is a ‘cursed or’) that can
be used to emit a warning when an affected expression is parsed. */
expr_simple OR_KW
{ $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}, state->positions.add(state->origin, @$.endOffset)); }
{ $$ = state->exprs.add<ExprCall>(CUR_POS, $1, {state->exprs.add<ExprVar>(CUR_POS, state->s.or_)}, state->positions.add(state->origin, @$.endOffset)); }
| expr_simple
;
@ -301,34 +291,32 @@ expr_simple
: ID {
std::string_view s = "__curPos";
if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
$$ = new ExprPos(CUR_POS);
$$ = state->exprs.add<ExprPos>(CUR_POS);
else
$$ = new ExprVar(CUR_POS, state->symbols.create($1));
$$ = state->exprs.add<ExprVar>(CUR_POS, state->symbols.create($1));
}
| INT_LIT { $$ = new ExprInt($1); }
| FLOAT_LIT { $$ = new ExprFloat($1); }
| INT_LIT { $$ = state->exprs.add<ExprInt>($1); }
| FLOAT_LIT { $$ = state->exprs.add<ExprFloat>($1); }
| '"' string_parts '"' {
std::visit(overloaded{
[&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
[&](std::string_view str) { $$ = state->exprs.add<ExprString>(state->exprs.alloc, str); },
[&](Expr * expr) { $$ = expr; }},
*$2);
delete $2;
$2);
}
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, std::move(*$2));
delete $2;
$$ = state->stripIndentation(CUR_POS, $2);
}
| path_start PATH_END
| path_start string_parts_interpolated PATH_END {
$2->insert($2->begin(), {state->at(@1), $1});
$$ = new ExprConcatStrings(CUR_POS, false, $2);
$2.insert($2.begin(), {state->at(@1), $1});
$$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, false, $2);
}
| SPATH {
std::string_view path($1.p + 1, $1.l - 2);
$$ = new ExprCall(CUR_POS,
new ExprVar(state->s.findFile),
{new ExprVar(state->s.nixPath),
new ExprString(state->alloc, path)});
$$ = state->exprs.add<ExprCall>(CUR_POS,
state->exprs.add<ExprVar>(state->s.findFile),
{state->exprs.add<ExprVar>(state->s.nixPath),
state->exprs.add<ExprString>(state->exprs.alloc, path)});
}
| URI {
static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals);
@ -337,37 +325,36 @@ expr_simple
.msg = HintFmt("URL literals are disabled"),
.pos = state->positions[CUR_POS]
});
$$ = new ExprString(state->alloc, $1);
$$ = state->exprs.add<ExprString>(state->exprs.alloc, $1);
}
| '(' expr ')' { $$ = $2; }
/* Let expressions `let {..., body = ...}' are just desugared
into `(rec {..., body = ...}).body'. */
| LET '{' binds '}'
{ $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(state->alloc, noPos, $3, state->s.body); }
{ $3->recursive = true; $3->pos = CUR_POS; $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, noPos, $3, state->s.body); }
| REC '{' binds '}'
{ $3->recursive = true; $3->pos = CUR_POS; $$ = $3; }
| '{' binds1 '}'
{ $2->pos = CUR_POS; $$ = $2; }
| '{' '}'
{ $$ = new ExprAttrs(CUR_POS); }
| '[' expr_list ']' { $$ = $2; }
{ $$ = state->exprs.add<ExprAttrs>(CUR_POS); }
| '[' list ']' { $$ = state->exprs.add<ExprList>(state->exprs.alloc, $2); }
;
string_parts
: STR { $$ = new std::variant<Expr *, std::string_view>($1); }
| string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
| { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
: STR { $$ = $1; }
| string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); }
| { $$ = std::string_view(); }
;
string_parts_interpolated
: string_parts_interpolated STR
{ $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
| DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
{ $$ = std::move($1); $$.emplace_back(state->at(@2), state->exprs.add<ExprString>(state->exprs.alloc, $2)); }
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
| DOLLAR_CURLY expr '}' { $$.emplace_back(state->at(@1), $2); }
| STR DOLLAR_CURLY expr '}' {
$$ = new std::vector<std::pair<PosIdx, Expr *>>;
$$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
$$->emplace_back(state->at(@2), $3);
$$.emplace_back(state->at(@1), state->exprs.add<ExprString>(state->exprs.alloc, $1));
$$.emplace_back(state->at(@2), $3);
}
;
@ -392,8 +379,8 @@ path_start
root filesystem accessor, rather than the accessor of the
current Nix expression. */
literal.front() == '/'
? new ExprPath(state->alloc, state->rootFS, path)
: new ExprPath(state->alloc, state->basePath.accessor, path);
? state->exprs.add<ExprPath>(state->exprs.alloc, state->rootFS, path)
: state->exprs.add<ExprPath>(state->exprs.alloc, state->basePath.accessor, path);
}
| HPATH {
if (state->settings.pureEval) {
@ -403,99 +390,91 @@ path_start
);
}
Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(state->alloc, ref<SourceAccessor>(state->rootFS), path);
$$ = state->exprs.add<ExprPath>(state->exprs.alloc, ref<SourceAccessor>(state->rootFS), path);
}
;
ind_string_parts
: ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
| { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
: ind_string_parts IND_STR { $$ = std::move($1); $$.emplace_back(state->at(@2), $2); }
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
| { }
;
binds
: binds1
| { $$ = new ExprAttrs; }
| { $$ = state->exprs.add<ExprAttrs>(); }
;
binds1
: binds1[accum] attrpath '=' expr ';'
{ $$ = $accum;
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
}
| binds[accum] INHERIT attrs ';'
{ $$ = $accum;
for (auto & [i, iPos] : *$attrs) {
for (auto & [i, iPos] : $attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
}
delete $attrs;
}
| binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum;
if (!$accum->inheritFromExprs)
$accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr);
auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : *$attrs) {
auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : $attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(
new ExprSelect(state->alloc, iPos, from, i.symbol),
state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),
iPos,
ExprAttrs::AttrDef::Kind::InheritedFrom));
}
delete $attrs;
}
| attrpath '=' expr ';'
{ $$ = new ExprAttrs;
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
{ $$ = state->exprs.add<ExprAttrs>();
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
}
;
attrs
: attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
: attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
| attrs string_attr
{ $$ = $1;
{ $$ = std::move($1);
std::visit(overloaded {
[&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
[&](Expr * expr) {
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
}, *$2);
delete $2;
}, $2);
}
| { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
| { }
;
attrpath
: attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
: attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
| attrpath '.' string_attr
{ $$ = $1;
{ $$ = std::move($1);
std::visit(overloaded {
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$3);
delete $3;
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($3));
}
| attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
| attr { $$.emplace_back(state->symbols.create($1)); }
| string_attr
{ $$ = new std::vector<AttrName>;
std::visit(overloaded {
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$1);
delete $1;
{ std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($1));
}
;
@ -505,33 +484,33 @@ attr
;
string_attr
: '"' string_parts '"' { $$ = $2; }
| DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
: '"' string_parts '"' { $$ = std::move($2); }
| DOLLAR_CURLY expr '}' { $$ = $2; }
;
expr_list
: expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
| { $$ = new ExprList; }
list
: list expr_select { $$ = std::move($1); $$.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
| { }
;
formal_set
: '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; }
| '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; }
| '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; }
| '{' formals '}' { $$ = $formals; $$->ellipsis = false; }
| '{' '}' { $$ = new Formals; $$->ellipsis = false; }
: '{' formals ',' ELLIPSIS '}' { $$ = std::move($formals); $$.ellipsis = true; }
| '{' ELLIPSIS '}' { $$.ellipsis = true; }
| '{' formals ',' '}' { $$ = std::move($formals); $$.ellipsis = false; }
| '{' formals '}' { $$ = std::move($formals); $$.ellipsis = false; }
| '{' '}' { $$.ellipsis = false; }
;
formals
: formals[accum] ',' formal
{ $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
{ $$ = std::move($accum); $$.formals.emplace_back(std::move($formal)); }
| formal
{ $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; }
{ $$.formals.emplace_back(std::move($formal)); }
;
formal
: ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
| ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
: ID { $$ = Formal{CUR_POS, state->symbols.create($1), 0}; }
| ID '?' expr { $$ = Formal{CUR_POS, state->symbols.create($1), $3}; }
;
%%
@ -546,7 +525,7 @@ Expr * parseExprFromBuf(
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::pmr::polymorphic_allocator<char> & alloc,
Exprs & exprs,
SymbolTable & symbols,
const EvalSettings & settings,
PosTable & positions,
@ -561,7 +540,7 @@ Expr * parseExprFromBuf(
};
ParserState state {
.lexerState = lexerState,
.alloc = alloc,
.exprs = exprs,
.symbols = symbols,
.positions = positions,
.basePath = basePath,
@ -582,3 +561,4 @@ Expr * parseExprFromBuf(
}
#pragma GCC diagnostic pop // end ignored "-Wswitch-enum"

228
src/libexpr/primops.cc
View file

@ -5,6 +5,7 @@
#include "nix/expr/eval-settings.hh"
#include "nix/expr/gc-small-vector.hh"
#include "nix/expr/json-to-value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/store/globals.hh"
#include "nix/store/names.hh"
#include "nix/store/path-references.hh"
@ -39,6 +40,12 @@
namespace nix {
RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
{
static RegisterPrimOp::PrimOps primOps;
return primOps;
}
/*************************************************************
* Miscellaneous
*************************************************************/
@ -221,7 +228,8 @@ void derivationToValue(
path2,
{
NixStringContextElem::DrvDeep{.drvPath = storePath},
});
},
state.mem);
attrs.alloc(state.s.name).mkString(drv.env["name"]);
auto list = state.buildList(drv.outputs.size());
@ -487,34 +495,34 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
state.forceValue(*args[0], pos);
switch (args[0]->type()) {
case nInt:
v.mkStringNoCopy("int");
v.mkStringNoCopy("int"_sds);
break;
case nBool:
v.mkStringNoCopy("bool");
v.mkStringNoCopy("bool"_sds);
break;
case nString:
v.mkStringNoCopy("string");
v.mkStringNoCopy("string"_sds);
break;
case nPath:
v.mkStringNoCopy("path");
v.mkStringNoCopy("path"_sds);
break;
case nNull:
v.mkStringNoCopy("null");
v.mkStringNoCopy("null"_sds);
break;
case nAttrs:
v.mkStringNoCopy("set");
v.mkStringNoCopy("set"_sds);
break;
case nList:
v.mkStringNoCopy("list");
v.mkStringNoCopy("list"_sds);
break;
case nFunction:
v.mkStringNoCopy("lambda");
v.mkStringNoCopy("lambda"_sds);
break;
case nExternal:
v.mkString(args[0]->external()->typeOf());
break;
case nFloat:
v.mkStringNoCopy("float");
v.mkStringNoCopy("float"_sds);
break;
case nThunk:
unreachable();
@ -681,7 +689,14 @@ struct CompareValues
if (v1->type() == nInt && v2->type() == nFloat)
return v1->integer().value < v2->fpoint();
if (v1->type() != v2->type())
state.error<EvalError>("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow();
state
.error<EvalError>(
"cannot compare %s with %s; values are %s and %s",
showType(*v1),
showType(*v2),
ValuePrinter(state, *v1, errorPrintOptions),
ValuePrinter(state, *v2, errorPrintOptions))
.debugThrow();
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
@ -691,12 +706,12 @@ struct CompareValues
case nFloat:
return v1->fpoint() < v2->fpoint();
case nString:
return strcmp(v1->c_str(), v2->c_str()) < 0;
return v1->string_view() < v2->string_view();
case nPath:
// Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
// reproducible way.
return strcmp(v1->pathStr(), v2->pathStr()) < 0;
return v1->pathStrView() < v2->pathStrView();
case nList:
// Lexicographic comparison
for (size_t i = 0;; i++) {
@ -711,7 +726,11 @@ struct CompareValues
default:
state
.error<EvalError>(
"cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2))
"cannot compare %s with %s; values of that type are incomparable (values are %s and %s)",
showType(*v1),
showType(*v2),
ValuePrinter(state, *v1, errorPrintOptions),
ValuePrinter(state, *v2, errorPrintOptions))
.debugThrow();
#pragma GCC diagnostic pop
}
@ -757,42 +776,79 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar
`workSet', adding the result to `workSet', continuing until
no new elements are found. */
ValueList res;
// `doneKeys' doesn't need to be a GC root, because its values are
// reachable from res.
auto cmp = CompareValues(state, noPos, "while comparing the `key` attributes of two genericClosure elements");
std::set<Value *, decltype(cmp)> doneKeys(cmp);
// Track which element each key came from
auto cmp = CompareValues(state, noPos, "");
std::map<Value *, Value *, decltype(cmp)> keyToElem(cmp);
while (!workSet.empty()) {
Value * e = *(workSet.begin());
workSet.pop_front();
state.forceAttrs(
*e,
noPos,
"while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure");
try {
state.forceAttrs(*e, noPos, "");
} catch (Error & err) {
err.addTrace(nullptr, "in genericClosure element %s", ValuePrinter(state, *e, errorPrintOptions));
throw;
}
auto key = state.getAttr(
state.s.key,
e->attrs(),
"in one of the attrsets generated by (or initially passed to) builtins.genericClosure");
const Attr * key;
try {
key = state.getAttr(state.s.key, e->attrs(), "");
} catch (Error & err) {
err.addTrace(nullptr, "in genericClosure element %s", ValuePrinter(state, *e, errorPrintOptions));
throw;
}
state.forceValue(*key->value, noPos);
if (!doneKeys.insert(key->value).second)
continue;
try {
auto [it, inserted] = keyToElem.insert({key->value, e});
if (!inserted)
continue;
} catch (Error & err) {
// Try to find which element we're comparing against
Value * otherElem = nullptr;
for (auto & [otherKey, elem] : keyToElem) {
try {
cmp(key->value, otherKey);
} catch (Error &) {
// Found the element we're comparing against
otherElem = elem;
break;
}
}
if (otherElem) {
// Traces are printed in reverse order; pre-swap them.
err.addTrace(nullptr, "with element %s", ValuePrinter(state, *otherElem, errorPrintOptions));
err.addTrace(nullptr, "while comparing element %s", ValuePrinter(state, *e, errorPrintOptions));
} else {
// Couldn't find the specific element, just show current
err.addTrace(nullptr, "while checking key of element %s", ValuePrinter(state, *e, errorPrintOptions));
}
throw;
}
res.push_back(e);
/* Call the `operator' function with `e' as argument. */
Value newElements;
state.callFunction(*op->value, {&e, 1}, newElements, noPos);
state.forceList(
newElements,
noPos,
"while evaluating the return value of the `operator` passed to builtins.genericClosure");
try {
state.callFunction(*op->value, {&e, 1}, newElements, noPos);
state.forceList(
newElements,
noPos,
"while evaluating the return value of the `operator` passed to builtins.genericClosure");
/* Add the values returned by the operator to the work set. */
for (auto elem : newElements.listView()) {
state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator`
// passed to builtins.genericClosure");
workSet.push_back(elem);
/* Add the values returned by the operator to the work set. */
for (auto elem : newElements.listView()) {
state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator`
// passed to builtins.genericClosure");
workSet.push_back(elem);
}
} catch (Error & err) {
err.addTrace(
nullptr,
"while calling %s on genericClosure element %s",
state.symbols[state.s.operator_],
ValuePrinter(state, *e, errorPrintOptions));
throw;
}
}
@ -825,10 +881,10 @@ static RegisterPrimOp primop_genericClosure(
- [Int](@docroot@/language/types.md#type-int)
- [Float](@docroot@/language/types.md#type-float)
- [Boolean](@docroot@/language/types.md#type-boolean)
- [Boolean](@docroot@/language/types.md#type-bool)
- [String](@docroot@/language/types.md#type-string)
- [Path](@docroot@/language/types.md#type-path)
- [List](@docroot@/language/types.md#list)
- [List](@docroot@/language/types.md#type-list)
The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
Items are compared by their `key` attribute.
@ -1374,7 +1430,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
pos,
"while evaluating the `__structuredAttrs` "
"attribute passed to builtins.derivationStrict"))
jsonObject = StructuredAttrs{.structuredAttrs = json::object()};
jsonObject = StructuredAttrs{};
/* Check whether null attributes should be ignored. */
bool ignoreNulls = false;
@ -1762,7 +1818,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drvPathS,
{
NixStringContextElem::DrvDeep{.drvPath = drvPath},
});
},
state.mem);
for (auto & i : drv.outputs)
mkOutputString(state, result, drvPath, i);
@ -1815,7 +1872,7 @@ static void prim_toPath(EvalState & state, const PosIdx pos, Value ** args, Valu
NixStringContext context;
auto path =
state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath");
v.mkString(path.path.abs(), context);
v.mkString(path.path.abs(), context, state.mem);
}
static RegisterPrimOp primop_toPath({
@ -1858,7 +1915,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V
if (!settings.readOnlyMode)
state.store->ensurePath(path2);
context.insert(NixStringContextElem::Opaque{.path = path2});
v.mkString(path.abs(), context);
v.mkString(path.abs(), context, state.mem);
}
static RegisterPrimOp primop_storePath({
@ -1940,7 +1997,8 @@ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value ** args,
v.mkString(
legacyBaseNameOf(*state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)),
context);
context,
state.mem);
}
static RegisterPrimOp primop_baseNameOf({
@ -1974,8 +2032,13 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
NixStringContext context;
auto path = state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false);
auto dir = dirOf(*path);
v.mkString(dir, context);
auto pos = path->rfind('/');
if (pos == path->npos)
v.mkStringMove("."_sds, context, state.mem);
else if (pos == 0)
v.mkStringMove("/"_sds, context, state.mem);
else
v.mkString(path->substr(0, pos), context, state.mem);
}
}
@ -2017,7 +2080,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Va
.path = std::move((StorePath &&) p),
});
}
v.mkString(s, context);
v.mkString(s, context, state.mem);
}
static RegisterPrimOp primop_readFile({
@ -2103,7 +2166,7 @@ static RegisterPrimOp primop_findFile(
builtins.findFile builtins.nixPath "nixpkgs"
```
A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes:
A search path is represented as a list of [attribute sets](./types.md#type-attrs) with two attributes:
- `prefix` is a relative path.
- `path` denotes a file system location
@ -2186,7 +2249,7 @@ static RegisterPrimOp primop_findFile(
> - ```
> {
> prefix = "nixpkgs";
> path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz";
> path = "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz";
> }
> ```
@ -2256,10 +2319,10 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
static const Constants stringValues = []() {
Constants res;
res.regular.mkStringNoCopy("regular");
res.directory.mkStringNoCopy("directory");
res.symlink.mkStringNoCopy("symlink");
res.unknown.mkStringNoCopy("unknown");
res.regular.mkStringNoCopy("regular"_sds);
res.directory.mkStringNoCopy("directory"_sds);
res.symlink.mkStringNoCopy("symlink"_sds);
res.unknown.mkStringNoCopy("unknown"_sds);
return res;
}();
@ -2395,7 +2458,7 @@ static RegisterPrimOp primop_outputOf({
returns an input placeholder for the output of the output of `myDrv`.
This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-paths), e.g. as part of installable syntax on the command line.
This primop corresponds to the `^` sigil for [deriving paths](@docroot@/glossary.md#gloss-deriving-path), e.g. as part of installable syntax on the command line.
)",
.fun = prim_outputOf,
.experimentalFeature = Xp::DynamicDerivations,
@ -2413,7 +2476,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
std::ostringstream out;
NixStringContext context;
printValueAsXML(state, true, false, *args[0], out, context, pos);
v.mkString(out.view(), context);
v.mkString(out.view(), context, state.mem);
}
static RegisterPrimOp primop_toXML({
@ -2521,7 +2584,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
std::ostringstream out;
NixStringContext context;
printValueAsJSON(state, true, *args[0], pos, out, context);
v.mkString(out.view(), context);
v.mkString(out.view(), context, state.mem);
}
static RegisterPrimOp primop_toJSON({
@ -2930,7 +2993,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value ** args, V
for (const auto & [n, i] : enumerate(*args[0]->attrs()))
list[n] = Value::toPtr(state.symbols[i.name]);
std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; });
std::sort(list.begin(), list.end(), [](Value * v1, Value * v2) { return v1->string_view() < v2->string_view(); });
v.mkList(list);
}
@ -3363,21 +3426,20 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value ** args
if (!args[0]->isLambda())
state.error<TypeError>("'functionArgs' requires a function").atPos(pos).debugThrow();
if (!args[0]->lambda().fun->hasFormals()) {
if (const auto & formals = args[0]->lambda().fun->getFormals()) {
auto attrs = state.buildBindings(formals->formals.size());
for (auto & i : formals->formals)
attrs.insert(i.name, state.getBool(i.def), i.pos);
/* Optimization: avoid sorting bindings. `formals` must already be sorted according to
(std::tie(a.name, a.pos) < std::tie(b.name, b.pos)) predicate, so the following assertion
always holds:
assert(std::is_sorted(attrs.alreadySorted()->begin(), attrs.alreadySorted()->end()));
.*/
v.mkAttrs(attrs.alreadySorted());
} else {
v.mkAttrs(&Bindings::emptyBindings);
return;
}
const auto & formals = args[0]->lambda().fun->formals->formals;
auto attrs = state.buildBindings(formals.size());
for (auto & i : formals)
attrs.insert(i.name, state.getBool(i.def), i.pos);
/* Optimization: avoid sorting bindings. `formals` must already be sorted according to
(std::tie(a.name, a.pos) < std::tie(b.name, b.pos)) predicate, so the following assertion
always holds:
assert(std::is_sorted(attrs.alreadySorted()->begin(), attrs.alreadySorted()->end()));
.*/
v.mkAttrs(attrs.alreadySorted());
}
static RegisterPrimOp primop_functionArgs({
@ -4351,7 +4413,7 @@ static void prim_toString(EvalState & state, const PosIdx pos, Value ** args, Va
NixStringContext context;
auto s = state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false);
v.mkString(*s, context);
v.mkString(*s, context, state.mem);
}
static RegisterPrimOp primop_toString({
@ -4411,7 +4473,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
if (len == 0) {
state.forceValue(*args[2], pos);
if (args[2]->type() == nString) {
v.mkStringNoCopy("", args[2]->context());
v.mkStringNoCopy(""_sds, args[2]->context());
return;
}
}
@ -4424,7 +4486,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
auto s = state.coerceToString(
pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context);
v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context, state.mem);
}
static RegisterPrimOp primop_substring({
@ -4611,9 +4673,9 @@ struct RegexCache
}
};
std::shared_ptr<RegexCache> makeRegexCache()
ref<RegexCache> makeRegexCache()
{
return std::make_shared<RegexCache>();
return make_ref<RegexCache>();
}
void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@ -4822,7 +4884,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value **
"while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep");
}
v.mkString(res, context);
v.mkString(res, context, state.mem);
}
static RegisterPrimOp primop_concatStringsSep({
@ -4897,7 +4959,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar
}
}
v.mkString(res, context);
v.mkString(res, context, state.mem);
}
static RegisterPrimOp primop_replaceStrings({
@ -4967,7 +5029,7 @@ static RegisterPrimOp primop_compareVersions({
version *s1* is older than version *s2*, `0` if they are the same,
and `1` if *s1* is newer than *s2*. The version comparison
algorithm is the same as the one used by [`nix-env
-u`](../command-ref/nix-env.md#operation---upgrade).
-u`](../command-ref/nix-env/upgrade.md).
)",
.fun = prim_compareVersions,
});
@ -4996,7 +5058,7 @@ static RegisterPrimOp primop_splitVersion({
.doc = R"(
Split a string representing a version into its components, by the
same version splitting logic underlying the version comparison in
[`nix-env -u`](../command-ref/nix-env.md#operation---upgrade).
[`nix-env -u`](../command-ref/nix-env/upgrade.md).
)",
.fun = prim_splitVersion,
});
@ -5046,9 +5108,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
Primitive value.
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
[comparison operators](@docroot@/language/operators.md#comparison)
and used in
[conditional expressions](@docroot@/language/syntax.md#Conditionals).
[conditional expressions](@docroot@/language/syntax.md#conditionals).
The name `true` is not special, and can be shadowed:
@ -5069,9 +5131,9 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
Primitive value.
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
[comparison operators](@docroot@/language/operators.md#comparison)
and used in
[conditional expressions](@docroot@/language/syntax.md#Conditionals).
[conditional expressions](@docroot@/language/syntax.md#conditionals).
The name `false` is not special, and can be shadowed:

10
src/libexpr/primops/context.cc
View file

@ -69,7 +69,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
}
}
v.mkString(*s, context2);
v.mkString(*s, context2, state.mem);
}
static RegisterPrimOp primop_unsafeDiscardOutputDependency(
@ -79,7 +79,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency(
Create a copy of the given string where every
[derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
string context element is turned into a
[constant](@docroot@/language/string-context.md#string-context-element-constant)
[constant](@docroot@/language/string-context.md#string-context-constant)
string context element.
This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
@ -137,7 +137,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
context.begin()->raw)}),
};
v.mkString(*s, context2);
v.mkString(*s, context2, state.mem);
}
static RegisterPrimOp primop_addDrvOutputDependencies(
@ -145,7 +145,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies(
.args = {"s"},
.doc = R"(
Create a copy of the given string where a single
[constant](@docroot@/language/string-context.md#string-context-element-constant)
[constant](@docroot@/language/string-context.md#string-context-constant)
string context element is turned into a
[derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
string context element.
@ -321,7 +321,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg
}
}
v.mkString(orig, context);
v.mkString(orig, context, state.mem);
}
static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .fun = prim_appendContext});

6
src/libexpr/primops/fetchClosure.cc
View file

@ -64,6 +64,8 @@ static void runFetchClosureWithRewrite(
.pos = state.positions[pos]});
}
state.allowClosure(toPath);
state.mkStorePathString(toPath, v);
}
@ -91,6 +93,8 @@ static void runFetchClosureWithContentAddressedPath(
.pos = state.positions[pos]});
}
state.allowClosure(fromPath);
state.mkStorePathString(fromPath, v);
}
@ -115,6 +119,8 @@ static void runFetchClosureWithInputAddressedPath(
.pos = state.positions[pos]});
}
state.allowClosure(fromPath);
state.mkStorePathString(fromPath, v);
}

2
src/libexpr/primops/fetchMercurial.cc
View file

@ -81,7 +81,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.store);
auto [storePath, input2] = input.fetchToStore(state.fetchSettings, state.store);
auto attrs2 = state.buildBindings(8);
state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));

141
src/libexpr/primops/fetchTree.cc
View file

@ -82,7 +82,7 @@ struct FetchTreeParams
static void fetchTree(
EvalState & state, const PosIdx pos, Value ** args, Value & v, const FetchTreeParams & params = FetchTreeParams{})
{
fetchers::Input input{state.fetchSettings};
fetchers::Input input{};
NixStringContext context;
std::optional<std::string> type;
auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree";
@ -194,9 +194,9 @@ static void fetchTree(
}
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first;
input = lookupInRegistries(state.fetchSettings, state.store, input, fetchers::UseRegistries::Limited).first;
if (state.settings.pureEval && !input.isLocked()) {
if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) {
if (input.getNarHash())
warn(
"Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@ -219,7 +219,8 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
}
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
auto cachedInput =
state.inputCache->getAccessor(state.fetchSettings, state.store, input, fetchers::UseRegistries::No);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
@ -317,77 +318,11 @@ static RegisterPrimOp primop_fetchTree({
> }
> ```
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
@ -405,6 +340,26 @@ static RegisterPrimOp primop_fetchTree({
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
@ -412,6 +367,52 @@ static RegisterPrimOp primop_fetchTree({
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
The following input types are still subject to change:
- `"path"`

10
src/libexpr/primops/fromTOML.cc
View file

@ -1,5 +1,6 @@
#include "nix/expr/primops.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/expr/static-string-data.hh"
#include "expr-config-private.hh"
@ -92,7 +93,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
std::istringstream tomlStream(std::string{toml});
auto visit = [&](auto & self, Value & v, toml::value t) -> void {
auto visit = [&](this auto & self, Value & v, toml::value t) -> void {
switch (t.type()) {
case toml::value_t::table: {
auto table = toml::get<toml::table>(t);
@ -100,7 +101,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
for (auto & elem : table) {
forceNoNullByte(elem.first);
self(self, attrs.alloc(elem.first), elem.second);
self(attrs.alloc(elem.first), elem.second);
}
v.mkAttrs(attrs);
@ -110,7 +111,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
auto list = state.buildList(array.size());
for (const auto & [n, v] : enumerate(list))
self(self, *(v = state.allocValue()), array[n]);
self(*(v = state.allocValue()), array[n]);
v.mkList(list);
} break;
case toml::value_t::boolean:
@ -136,7 +137,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
normalizeDatetimeFormat(t);
#endif
auto attrs = state.buildBindings(2);
attrs.alloc("_type").mkStringNoCopy("timestamp");
attrs.alloc("_type").mkStringNoCopy("timestamp"_sds);
std::ostringstream s;
s << t;
auto str = s.view();
@ -155,7 +156,6 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
try {
visit(
visit,
val,
toml::parse(
tomlStream,

2
src/libexpr/value-to-json.cc
View file

@ -33,7 +33,7 @@ json printValueAsJSON(
case nString:
copyContext(v, context);
out = v.c_str();
out = v.string_view();
break;
case nPath:

12
src/libexpr/value-to-xml.cc
View file

@ -82,7 +82,7 @@ static void printValueAsXML(
case nString:
/* !!! show the context? */
copyContext(v, context);
doc.writeEmptyElement("string", singletonAttrs("value", v.c_str()));
doc.writeEmptyElement("string", singletonAttrs("value", v.string_view()));
break;
case nPath:
@ -102,14 +102,14 @@ static void printValueAsXML(
if (strict)
state.forceValue(*a->value, a->pos);
if (a->value->type() == nString)
xmlAttrs["drvPath"] = drvPath = a->value->c_str();
xmlAttrs["drvPath"] = drvPath = a->value->string_view();
}
if (auto a = v.attrs()->get(state.s.outPath)) {
if (strict)
state.forceValue(*a->value, a->pos);
if (a->value->type() == nString)
xmlAttrs["outPath"] = a->value->c_str();
xmlAttrs["outPath"] = a->value->string_view();
}
XMLOpenElement _(doc, "derivation", xmlAttrs);
@ -145,14 +145,14 @@ static void printValueAsXML(
posToXML(state, xmlAttrs, state.positions[v.lambda().fun->pos]);
XMLOpenElement _(doc, "function", xmlAttrs);
if (v.lambda().fun->hasFormals()) {
if (auto formals = v.lambda().fun->getFormals()) {
XMLAttrs attrs;
if (v.lambda().fun->arg)
attrs["name"] = state.symbols[v.lambda().fun->arg];
if (v.lambda().fun->formals->ellipsis)
if (formals->ellipsis)
attrs["ellipsis"] = "1";
XMLOpenElement _(doc, "attrspat", attrs);
for (auto & i : v.lambda().fun->formals->lexicographicOrder(state.symbols))
for (auto & i : formals->lexicographicOrder(state.symbols))
doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name]));
} else
doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda().fun->arg]));

3
src/libexpr/value/context.cc
View file

@ -9,8 +9,7 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0, const Expe
{
std::string_view s = s0;
std::function<SingleDerivedPath()> parseRest;
parseRest = [&]() -> SingleDerivedPath {
auto parseRest = [&](this auto & parseRest) -> SingleDerivedPath {
// Case on whether there is a '!'
size_t index = s.find("!");
if (index == std::string_view::npos) {

2
src/libfetchers-tests/git.cc
View file

@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
{"ref", "main"},
});
auto [accessor, i] = input.getAccessor(store);
auto [accessor, i] = input.getAccessor(settings, store);
ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
}

61
src/libfetchers-tests/input.cc Normal file
View file

@ -0,0 +1,61 @@
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh"
#include <gtest/gtest.h>
#include <string>
namespace nix {
using fetchers::Attr;
struct InputFromAttrsTestCase
{
fetchers::Attrs attrs;
std::string expectedUrl;
std::string description;
fetchers::Attrs expectedAttrs = attrs;
};
class InputFromAttrsTest : public ::testing::WithParamInterface<InputFromAttrsTestCase>, public ::testing::Test
{};
TEST_P(InputFromAttrsTest, attrsAreCorrectAndRoundTrips)
{
fetchers::Settings fetchSettings;
const auto & testCase = GetParam();
auto input = fetchers::Input::fromAttrs(fetchSettings, fetchers::Attrs(testCase.attrs));
EXPECT_EQ(input.toAttrs(), testCase.expectedAttrs);
EXPECT_EQ(input.toURLString(), testCase.expectedUrl);
auto input2 = fetchers::Input::fromAttrs(fetchSettings, input.toAttrs());
EXPECT_EQ(input, input2);
EXPECT_EQ(input.toAttrs(), input2.toAttrs());
}
INSTANTIATE_TEST_SUITE_P(
InputFromAttrs,
InputFromAttrsTest,
::testing::Values(
// Test for issue #14429.
InputFromAttrsTestCase{
.attrs =
{
{"url", Attr("git+ssh://git@github.com/NixOS/nixpkgs")},
{"type", Attr("git")},
},
.expectedUrl = "git+ssh://git@github.com/NixOS/nixpkgs",
.description = "strips_git_plus_prefix",
.expectedAttrs =
{
{"url", Attr("ssh://git@github.com/NixOS/nixpkgs")},
{"type", Attr("git")},
},
}),
[](const ::testing::TestParamInfo<InputFromAttrsTestCase> & info) { return info.param.description; });
} // namespace nix

1
src/libfetchers-tests/meson.build
View file

@ -42,6 +42,7 @@ sources = files(
'access-tokens.cc',
'git-utils.cc',
'git.cc',
'input.cc',
'nix_api_fetchers.cc',
'public-key.cc',
)

40
src/libfetchers/fetchers.cc
View file

@ -5,6 +5,7 @@
#include "nix/util/json-utils.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/url.hh"
#include <nlohmann/json.hpp>
@ -65,6 +66,12 @@ Input Input::fromURL(const Settings & settings, const ParsedURL & url, bool requ
}
}
// Provide a helpful hint when user tries file+git instead of git+file
auto parsedScheme = parseUrlScheme(url.scheme);
if (parsedScheme.application == "file" && parsedScheme.transport == "git") {
throw Error("input '%s' is unsupported; did you mean 'git+file' instead of 'file+git'?", url);
}
throw Error("input '%s' is unsupported", url);
}
@ -82,7 +89,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
// but not all of them. Doing this is to support those other
// operations which are supposed to be robust on
// unknown/uninterpretable inputs.
Input input{settings};
Input input;
input.attrs = attrs;
fixupInput(input);
return input;
@ -152,9 +159,9 @@ bool Input::isDirect() const
return !scheme || scheme->isDirect(*this);
}
bool Input::isLocked() const
bool Input::isLocked(const Settings & settings) const
{
return scheme && scheme->isLocked(*this);
return scheme && scheme->isLocked(settings, *this);
}
bool Input::isFinal() const
@ -191,17 +198,17 @@ bool Input::contains(const Input & other) const
}
// FIXME: remove
std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
try {
auto [accessor, result] = getAccessorUnchecked(store);
auto [accessor, result] = getAccessorUnchecked(settings, store);
auto storePath =
nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
nix::fetchToStore(settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
@ -290,10 +297,10 @@ void Input::checkLocks(Input specified, Input & result)
}
}
std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, ref<Store> store) const
{
try {
auto [accessor, result] = getAccessorUnchecked(store);
auto [accessor, result] = getAccessorUnchecked(settings, store);
result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@ -306,7 +313,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
}
}
std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> store) const
std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, ref<Store> store) const
{
// FIXME: cache the accessor
@ -342,7 +349,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
if (accessor->fingerprint) {
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings->getCache()->upsert(cacheKey, *store, {}, storePath);
settings.getCache()->upsert(cacheKey, *store, {}, storePath);
}
accessor->setPathDisplay("«" + to_string() + "»");
@ -353,7 +360,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
}
}
auto [accessor, result] = scheme->getAccessor(store, *this);
auto [accessor, result] = scheme->getAccessor(settings, store, *this);
if (!accessor->fingerprint)
accessor->fingerprint = result.getFingerprint(store);
@ -370,10 +377,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
return scheme->applyOverrides(*this, ref, rev);
}
void Input::clone(const Path & destDir) const
void Input::clone(const Settings & settings, const Path & destDir) const
{
assert(scheme);
scheme->clone(*this, destDir);
scheme->clone(settings, *this, destDir);
}
std::optional<std::filesystem::path> Input::getSourcePath() const
@ -486,7 +493,7 @@ void InputScheme::putFile(
throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
}
void InputScheme::clone(const Input & input, const Path & destDir) const
void InputScheme::clone(const Settings & settings, const Input & input, const Path & destDir) const
{
throw Error("do not know how to clone input '%s'", input.to_string());
}
@ -512,10 +519,11 @@ using namespace nix;
fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json)
{
fetchers::PublicKey res = {};
if (auto type = optionalValueAt(json, "type"))
auto & obj = getObject(json);
if (auto * type = optionalValueAt(obj, "type"))
res.type = getString(*type);
res.key = getString(valueAt(json, "key"));
res.key = getString(valueAt(obj, "key"));
return res;
}

6
src/libfetchers/git-lfs-fetch.cc
View file

@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
auto url = api.endpoint + "/objects/batch";
const auto & authHeader = api.authHeader;
FileTransferRequest request(parseURL(url));
request.post = true;
request.method = HttpMethod::POST;
Headers headers;
if (authHeader.has_value())
headers.push_back({"Authorization", *authHeader});
@ -219,7 +219,9 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
nlohmann::json oidList = pointerToPayload(pointers);
nlohmann::json data = {{"operation", "download"}};
data["objects"] = oidList;
request.data = data.dump();
auto payload = data.dump();
StringSource source{payload};
request.data = {source};
FileTransferResult result = getFileTransfer()->upload(request);
auto responseString = result.data;

93
src/libfetchers/git-utils.cc
View file

@ -9,6 +9,9 @@
#include "nix/util/users.hh"
#include "nix/util/fs-sink.hh"
#include "nix/util/sync.hh"
#include "nix/util/util.hh"
#include "nix/util/thread-pool.hh"
#include "nix/util/pool.hh"
#include <git2/attr.h>
#include <git2/blob.h>
@ -32,12 +35,14 @@
#include <git2/tag.h>
#include <git2/tree.h>
#include <boost/unordered/concurrent_flat_set.hpp>
#include <boost/unordered/unordered_flat_map.hpp>
#include <boost/unordered/unordered_flat_set.hpp>
#include <iostream>
#include <queue>
#include <regex>
#include <span>
#include <ranges>
namespace std {
@ -226,12 +231,16 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
/** Location of the repository on disk. */
std::filesystem::path path;
bool bare;
/**
* libgit2 repository. Note that new objects are not written to disk,
* because we are using a mempack backend. For writing to disk, see
* `flush()`, which is also called by `GitFileSystemObjectSink::sync()`.
*/
Repository repo;
/**
* In-memory object store for efficient batched writing to packfiles.
* Owned by `repo`.
@ -240,6 +249,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
GitRepoImpl(std::filesystem::path _path, bool create, bool bare)
: path(std::move(_path))
, bare(bare)
{
initLibGit2();
@ -316,32 +326,56 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
checkInterrupt();
}
/**
* Return a connection pool for this repo. Useful for
* multithreaded access.
*/
Pool<GitRepoImpl> getPool()
{
// TODO: as an optimization, it would be nice to include `this` in the pool.
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
return make_ref<GitRepoImpl>(path, false, bare);
});
}
uint64_t getRevCount(const Hash & rev) override
{
boost::unordered_flat_set<git_oid, std::hash<git_oid>> done;
std::queue<Commit> todo;
boost::concurrent_flat_set<git_oid, std::hash<git_oid>> done;
todo.push(peelObject<Commit>(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
auto startCommit = peelObject<Commit>(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT);
auto startOid = *git_commit_id(startCommit.get());
done.insert(startOid);
while (auto commit = pop(todo)) {
if (!done.insert(*git_commit_id(commit->get())).second)
continue;
auto repoPool(getPool());
for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
git_commit * parent;
if (git_commit_parent(&parent, commit->get(), n)) {
ThreadPool pool;
auto process = [&done, &pool, &repoPool](this const auto & process, const git_oid & oid) -> void {
auto repo(repoPool.get());
auto _commit = lookupObject(*repo, oid, GIT_OBJECT_COMMIT);
auto commit = (const git_commit *) &*_commit;
for (auto n : std::views::iota(0U, git_commit_parentcount(commit))) {
auto parentOid = git_commit_parent_id(commit, n);
if (!parentOid) {
throw Error(
"Failed to retrieve the parent of Git commit '%s': %s. "
"This may be due to an incomplete repository history. "
"To resolve this, either enable the shallow parameter in your flake URL (?shallow=1) "
"or add set the shallow parameter to true in builtins.fetchGit, "
"or fetch the complete history for this branch.",
*git_commit_id(commit->get()),
*git_commit_id(commit),
git_error_last()->message);
}
todo.push(Commit(parent));
if (done.insert(*parentOid))
pool.enqueue(std::bind(process, *parentOid));
}
}
};
pool.enqueue(std::bind(process, startOid));
pool.process();
return done.size();
}
@ -530,12 +564,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
auto act = (Activity *) payload;
act->result(
resFetchStatus,
fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
fmt("%d/%d objects received, %d/%d deltas indexed, %s",
stats->received_objects,
stats->total_objects,
stats->indexed_deltas,
stats->total_deltas,
stats->received_bytes / (1024.0 * 1024.0)));
renderSize(stats->received_bytes)));
return getInterrupted() ? -1 : 0;
}
@ -548,25 +582,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// then use code that was removed in this commit (see blame)
auto dir = this->path;
Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"};
Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--progress", "--force"};
if (shallow)
append(gitArgs, {"--depth", "1"});
append(gitArgs, {std::string("--"), url, refspec});
auto [status, output] = runProgram(
RunOptions{
.program = "git",
.lookupPath = true,
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
.args = gitArgs,
.input = {},
.mergeStderrToStdout = true,
.isInteractive = true});
auto status = runProgram(RunOptions{.program = "git", .args = gitArgs, .isInteractive = true}).first;
if (status > 0) {
throw Error("Failed to fetch git repository %s : %s", url, output);
}
if (status > 0)
throw Error("Failed to fetch git repository '%s'", url);
}
void verifyCommit(const Hash & rev, const std::vector<fetchers::PublicKey> & publicKeys) override
@ -1304,13 +1328,18 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
return result;
}
ref<GitRepo> getTarballCache()
{
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
namespace fetchers {
return GitRepo::openRepo(repoDir, true, true);
ref<GitRepo> Settings::getTarballCache() const
{
auto tarballCache(_tarballCache.lock());
if (!*tarballCache)
*tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
return ref<GitRepo>(*tarballCache);
}
} // namespace fetchers
GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
{
static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;

49
src/libfetchers/git.cc
View file

@ -164,13 +164,10 @@ struct GitInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override
{
auto parsedScheme = parseUrlScheme(url.scheme);
if (parsedScheme.application != "git")
if (url.scheme != "git" && parseUrlScheme(url.scheme).application != "git")
return {};
auto url2(url);
if (hasPrefix(url2.scheme, "git+"))
url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
@ -232,7 +229,7 @@ struct GitInputScheme : InputScheme
if (auto ref = maybeGetStrAttr(attrs, "ref"); ref && !isLegalRefName(*ref))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
Input input{settings};
Input input{};
input.attrs = attrs;
input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string();
getShallowAttr(input);
@ -281,7 +278,7 @@ struct GitInputScheme : InputScheme
return res;
}
void clone(const Input & input, const Path & destDir) const override
void clone(const Settings & settings, const Input & input, const Path & destDir) const override
{
auto repoInfo = getRepoInfo(input);
@ -626,7 +623,7 @@ struct GitInputScheme : InputScheme
}
std::pair<ref<SourceAccessor>, Input>
getAccessorFromCommit(ref<Store> store, RepoInfo & repoInfo, Input && input) const
getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
{
assert(!repoInfo.workdirInfo.isDirty);
@ -736,13 +733,10 @@ struct GitInputScheme : InputScheme
auto rev = *input.getRev();
Attrs infoAttrs({
{"rev", rev.gitRev()},
{"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
});
input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
if (!getShallowAttr(input))
infoAttrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev));
input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@ -785,8 +779,8 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("submodules", Explicit<bool>{true});
attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs});
attrs.insert_or_assign("allRefs", Explicit<bool>{true});
auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
mounts.insert_or_assign(submodule.path, submoduleAccessor);
}
@ -798,15 +792,12 @@ struct GitInputScheme : InputScheme
}
assert(!origRev || origRev == rev);
if (!getShallowAttr(input))
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {accessor, std::move(input)};
}
std::pair<ref<SourceAccessor>, Input>
getAccessorFromWorkdir(ref<Store> store, RepoInfo & repoInfo, Input && input) const
getAccessorFromWorkdir(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
{
auto repoPath = repoInfo.getPath().value();
@ -838,8 +829,8 @@ struct GitInputScheme : InputScheme
// TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
// attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
/* If the submodule is dirty, mark this repo dirty as
@ -866,12 +857,12 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev());
if (!getShallowAttr(input)) {
input.attrs.insert_or_assign(
"revCount", rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
"revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev));
}
verifyCommit(input, repo);
} else {
repoInfo.warnDirty(*input.settings);
repoInfo.warnDirty(settings);
if (repoInfo.workdirInfo.headRev) {
input.attrs.insert_or_assign("dirtyRev", repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
@ -883,14 +874,14 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign(
"lastModified",
repoInfo.workdirInfo.headRev
? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
repoInfo.workdirInfo.headRev ? getLastModified(settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
return {accessor, std::move(input)};
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
Input input(_input);
@ -906,8 +897,8 @@ struct GitInputScheme : InputScheme
}
auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(store, repoInfo, std::move(input));
? getAccessorFromCommit(settings, store, repoInfo, std::move(input))
: getAccessorFromWorkdir(settings, store, repoInfo, std::move(input));
return {accessor, std::move(final)};
}
@ -943,7 +934,7 @@ struct GitInputScheme : InputScheme
}
}
bool isLocked(const Input & input) const override
bool isLocked(const Settings & settings, const Input & input) const override
{
auto rev = input.getRev();
return rev && rev != nullRev;

84
src/libfetchers/github.cc
View file

@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
Input input{settings};
Input input{};
input.attrs.insert_or_assign("type", std::string{schemeName()});
input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]);
@ -129,7 +129,7 @@ struct GitArchiveInputScheme : InputScheme
getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo");
Input input{settings};
Input input{};
input.attrs = attrs;
return input;
}
@ -233,9 +233,9 @@ struct GitArchiveInputScheme : InputScheme
std::optional<Hash> treeHash;
};
virtual RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0;
struct TarballInfo
{
@ -243,7 +243,7 @@ struct GitArchiveInputScheme : InputScheme
time_t lastModified;
};
std::pair<Input, TarballInfo> downloadArchive(ref<Store> store, Input input) const
std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, ref<Store> store, Input input) const
{
if (!maybeGetStrAttr(input.attrs, "ref"))
input.attrs.insert_or_assign("ref", "HEAD");
@ -252,7 +252,7 @@ struct GitArchiveInputScheme : InputScheme
auto rev = input.getRev();
if (!rev) {
auto refInfo = getRevFromRef(store, input);
auto refInfo = getRevFromRef(settings, store, input);
rev = refInfo.rev;
upstreamTreeHash = refInfo.treeHash;
debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev());
@ -261,7 +261,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev());
auto cache = input.settings->getCache();
auto cache = settings.getCache();
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@ -270,7 +270,7 @@ struct GitArchiveInputScheme : InputScheme
if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (getTarballCache()->hasObject(treeHash))
if (settings.getTarballCache()->hasObject(treeHash))
return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
else
debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
@ -278,7 +278,7 @@ struct GitArchiveInputScheme : InputScheme
}
/* Stream the tarball into the tarball cache. */
auto url = getDownloadUrl(input);
auto url = getDownloadUrl(settings, input);
auto source = sinkToSource([&](Sink & sink) {
FileTransferRequest req(url.url);
@ -290,7 +290,7 @@ struct GitArchiveInputScheme : InputScheme
*logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive{*source};
auto tarballCache = getTarballCache();
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush();
@ -315,28 +315,29 @@ struct GitArchiveInputScheme : InputScheme
return {std::move(input), tarballInfo};
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
auto [input, tarballInfo] = downloadArchive(store, _input);
auto [input, tarballInfo] = downloadArchive(settings, store, _input);
#if 0
input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
#endif
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
auto accessor =
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
return {accessor, input};
}
bool isLocked(const Input & input) const override
bool isLocked(const Settings & settings, const Input & input) const override
{
/* Since we can't verify the integrity of the tarball from the
Git revision alone, we also require a NAR hash for
locking. FIXME: in the future, we may want to require a Git
tree hash instead of a NAR hash. */
return input.getRev().has_value()
&& (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value());
return input.getRev().has_value() && (settings.trustTarballsFromGitForges || input.getNarHash().has_value());
}
std::optional<ExperimentalFeature> experimentalFeature() const override
@ -386,7 +387,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
return getStrAttr(input.attrs, "repo");
}
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{
auto host = getHost(input);
auto url = fmt(
@ -396,9 +397,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
getRepo(input),
*input.getRef());
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto downloadResult = downloadFile(store, settings, url, "source", headers);
auto json = nlohmann::json::parse(
store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
@ -407,11 +408,11 @@ struct GitHubInputScheme : GitArchiveInputScheme
.treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)};
}
DownloadUrl getDownloadUrl(const Input & input) const override
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override
{
auto host = getHost(input);
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
// If we have no auth headers then we default to the public archive
// urls so we do not run into rate limits.
@ -425,12 +426,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers};
}
void clone(const Input & input, const Path & destDir) const override
void clone(const Settings & settings, const Input & input, const Path & destDir) const override
{
auto host = getHost(input);
Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
.clone(settings, destDir);
}
};
@ -460,7 +461,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
}
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below
@ -471,9 +472,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"),
*input.getRef());
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto downloadResult = downloadFile(store, settings, url, "source", headers);
auto json = nlohmann::json::parse(
store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
@ -487,7 +488,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
}
}
DownloadUrl getDownloadUrl(const Input & input) const override
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override
{
// This endpoint has a rate limit threshold that may be
// server-specific and vary based whether the user is
@ -502,19 +503,19 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
return DownloadUrl{parseURL(url), headers};
}
void clone(const Input & input, const Path & destDir) const override
void clone(const Settings & settings, const Input & input, const Path & destDir) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(
*input.settings,
settings,
fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
.clone(settings, destDir);
}
};
@ -535,7 +536,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
// Once it is implemented, however, should work as expected.
}
RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead.
@ -546,11 +547,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme
auto base_url =
fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
std::string refUri;
if (ref == "HEAD") {
auto downloadFileResult = downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers);
auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
@ -563,8 +564,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
}
std::regex refRegex(refUri);
auto downloadFileResult =
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers);
auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
std::istringstream is(contents);
@ -582,7 +582,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)};
}
DownloadUrl getDownloadUrl(const Input & input) const override
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url =
@ -592,18 +592,18 @@ struct SourceHutInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input);
return DownloadUrl{parseURL(url), headers};
}
void clone(const Input & input, const Path & destDir) const override
void clone(const Settings & settings, const Input & input, const Path & destDir) const override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
Input::fromURL(
*input.settings,
settings,
fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
.clone(settings, destDir);
}
};

10
src/libfetchers/include/nix/fetchers/fetch-settings.hh
View file

@ -11,6 +11,12 @@
#include <sys/types.h>
namespace nix {
struct GitRepo;
}
namespace nix::fetchers {
struct Cache;
@ -125,8 +131,12 @@ struct Settings : public Config
ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;
private:
mutable Sync<std::shared_ptr<Cache>> _cache;
mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
};
} // namespace nix::fetchers

24
src/libfetchers/include/nix/fetchers/fetchers.hh
View file

@ -36,13 +36,6 @@ struct Input
{
friend struct InputScheme;
const Settings * settings;
Input(const Settings & settings)
: settings{&settings}
{
}
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
@ -87,7 +80,7 @@ public:
* attributes like a Git revision or NAR hash that uniquely
* identify its contents.
*/
bool isLocked() const;
bool isLocked(const Settings & settings) const;
/**
* Only for relative path flakes, i.e. 'path:./foo', returns the
@ -120,7 +113,7 @@ public:
* Fetch the entire input into the Nix store, returning the
* location in the Nix store and the locked input.
*/
std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
std::pair<StorePath, Input> fetchToStore(const Settings & settings, ref<Store> store) const;
/**
* Check the locking attributes in `result` against
@ -140,17 +133,17 @@ public:
* input without copying it to the store. Also return a possibly
* unlocked input.
*/
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store) const;
std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, ref<Store> store) const;
private:
std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(ref<Store> store) const;
std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, ref<Store> store) const;
public:
Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
void clone(const Path & destDir) const;
void clone(const Settings & settings, const Path & destDir) const;
std::optional<std::filesystem::path> getSourcePath() const;
@ -223,7 +216,7 @@ struct InputScheme
virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
virtual void clone(const Input & input, const Path & destDir) const;
virtual void clone(const Settings & settings, const Input & input, const Path & destDir) const;
virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
@ -233,7 +226,8 @@ struct InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const;
virtual std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const = 0;
virtual std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & input) const = 0;
/**
* Is this `InputScheme` part of an experimental feature?
@ -250,7 +244,7 @@ struct InputScheme
return std::nullopt;
}
virtual bool isLocked(const Input & input) const
virtual bool isLocked(const Settings & settings, const Input & input) const
{
return false;
}

2
src/libfetchers/include/nix/fetchers/git-utils.hh
View file

@ -120,8 +120,6 @@ struct GitRepo
virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0;
};
ref<GitRepo> getTarballCache();
// A helper to ensure that the `git_*_free` functions get called.
template<auto del>
struct Deleter

4
src/libfetchers/include/nix/fetchers/input-cache.hh
View file

@ -3,6 +3,7 @@
namespace nix::fetchers {
enum class UseRegistries : int;
struct Settings;
struct InputCache
{
@ -14,7 +15,8 @@ struct InputCache
Attrs extraAttrs;
};
CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
CachedResult
getAccessor(const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
struct CachedInput
{

8
src/libfetchers/include/nix/fetchers/registry.hh
View file

@ -2,6 +2,7 @@
///@file
#include "nix/util/types.hh"
#include "nix/util/source-path.hh"
#include "nix/fetchers/fetchers.hh"
namespace nix {
@ -39,7 +40,7 @@ struct Registry
{
}
static std::shared_ptr<Registry> read(const Settings & settings, const Path & path, RegistryType type);
static std::shared_ptr<Registry> read(const Settings & settings, const SourcePath & path, RegistryType type);
void write(const Path & path);
@ -58,7 +59,7 @@ Path getUserRegistryPath();
Registries getRegistries(const Settings & settings, ref<Store> store);
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
enum class UseRegistries : int {
No,
@ -70,6 +71,7 @@ enum class UseRegistries : int {
* Rewrite a flakeref using the registries. If `filter` is set, only
* use the registries for which the filter function returns true.
*/
std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & input, UseRegistries useRegistries);
std::pair<Input, Attrs>
lookupInRegistries(const Settings & settings, ref<Store> store, const Input & input, UseRegistries useRegistries);
} // namespace nix::fetchers

7
src/libfetchers/indirect.cc
View file

@ -44,7 +44,7 @@ struct IndirectInputScheme : InputScheme
// FIXME: forbid query params?
Input input{settings};
Input input{};
input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id);
if (rev)
@ -76,7 +76,7 @@ struct IndirectInputScheme : InputScheme
if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id);
Input input{settings};
Input input{};
input.attrs = attrs;
return input;
}
@ -106,7 +106,8 @@ struct IndirectInputScheme : InputScheme
return input;
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & input) const override
{
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
}

10
src/libfetchers/input-cache.cc
View file

@ -5,23 +5,23 @@
namespace nix::fetchers {
InputCache::CachedResult
InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
InputCache::CachedResult InputCache::getAccessor(
const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
{
auto fetched = lookup(originalInput);
Input resolvedInput = originalInput;
if (!fetched) {
if (originalInput.isDirect()) {
auto [accessor, lockedInput] = originalInput.getAccessor(store);
auto [accessor, lockedInput] = originalInput.getAccessor(settings, store);
fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
} else {
if (useRegistries != UseRegistries::No) {
auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries);
auto [res, extraAttrs] = lookupInRegistries(settings, store, originalInput, useRegistries);
resolvedInput = std::move(res);
fetched = lookup(resolvedInput);
if (!fetched) {
auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
auto [accessor, lockedInput] = resolvedInput.getAccessor(settings, store);
fetched.emplace(
CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs});
}

25
src/libfetchers/mercurial.cc
View file

@ -89,7 +89,7 @@ struct MercurialInputScheme : InputScheme
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
}
Input input{settings};
Input input{};
input.attrs = attrs;
return input;
}
@ -154,7 +154,7 @@ struct MercurialInputScheme : InputScheme
return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
}
StorePath fetchToStore(ref<Store> store, Input & input) const
StorePath fetchToStore(const Settings & settings, ref<Store> store, Input & input) const
{
auto origRev = input.getRev();
@ -176,10 +176,10 @@ struct MercurialInputScheme : InputScheme
/* This is an unclean working tree. So copy all tracked
files. */
if (!input.settings->allowDirty)
if (!settings.allowDirty)
throw Error("Mercurial tree '%s' is unclean", actualUrl);
if (input.settings->warnDirty)
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl})));
@ -240,13 +240,13 @@ struct MercurialInputScheme : InputScheme
Cache::Key refToRevKey{"hgRefToRev", {{"url", actualUrl}, {"ref", *input.getRef()}}};
if (!input.getRev()) {
if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
if (auto res = settings.getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
}
/* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) {
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath);
}
@ -300,7 +300,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a
cached store path. */
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir();
@ -317,18 +317,19 @@ struct MercurialInputScheme : InputScheme
});
if (!origRev)
input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
settings.getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath));
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
Input input(_input);
auto storePath = fetchToStore(store, input);
auto storePath = fetchToStore(settings, store, input);
auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->setPathDisplay("«" + input.to_string() + "»");
@ -336,7 +337,7 @@ struct MercurialInputScheme : InputScheme
return {accessor, input};
}
bool isLocked(const Input & input) const override
bool isLocked(const Settings & settings, const Input & input) const override
{
return (bool) input.getRev();
}

11
src/libfetchers/path.cc
View file

@ -17,7 +17,7 @@ struct PathInputScheme : InputScheme
if (url.authority && url.authority->host.size())
throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority);
Input input{settings};
Input input{};
input.attrs.insert_or_assign("type", "path");
input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path));
@ -60,7 +60,7 @@ struct PathInputScheme : InputScheme
{
getStrAttr(attrs, "path");
Input input{settings};
Input input{};
input.attrs = attrs;
return input;
}
@ -101,7 +101,7 @@ struct PathInputScheme : InputScheme
return path;
}
bool isLocked(const Input & input) const override
bool isLocked(const Settings & settings, const Input & input) const override
{
return (bool) input.getNarHash();
}
@ -116,7 +116,8 @@ struct PathInputScheme : InputScheme
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
Input input(_input);
auto path = getStrAttr(input.attrs, "path");
@ -145,7 +146,7 @@ struct PathInputScheme : InputScheme
auto info = store->queryPathInfo(*storePath);
accessor->fingerprint =
fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
input.settings->getCache()->upsert(
settings.getCache()->upsert(
makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
*store,

49
src/libfetchers/registry.cc
View file

@ -10,18 +10,18 @@
namespace nix::fetchers {
std::shared_ptr<Registry> Registry::read(const Settings & settings, const Path & path, RegistryType type)
std::shared_ptr<Registry> Registry::read(const Settings & settings, const SourcePath & path, RegistryType type)
{
debug("reading registry '%s'", path);
auto registry = std::make_shared<Registry>(settings, type);
if (!pathExists(path))
if (!path.pathExists())
return std::make_shared<Registry>(settings, type);
try {
auto json = nlohmann::json::parse(readFile(path));
auto json = nlohmann::json::parse(path.readFile());
auto version = json.value("version", 0);
@ -97,7 +97,10 @@ static Path getSystemRegistryPath()
static std::shared_ptr<Registry> getSystemRegistry(const Settings & settings)
{
static auto systemRegistry = Registry::read(settings, getSystemRegistryPath(), Registry::System);
static auto systemRegistry = Registry::read(
settings,
SourcePath{getFSSourceAccessor(), CanonPath{getSystemRegistryPath()}}.resolveSymlinks(),
Registry::System);
return systemRegistry;
}
@ -108,13 +111,17 @@ Path getUserRegistryPath()
std::shared_ptr<Registry> getUserRegistry(const Settings & settings)
{
static auto userRegistry = Registry::read(settings, getUserRegistryPath(), Registry::User);
static auto userRegistry = Registry::read(
settings,
SourcePath{getFSSourceAccessor(), CanonPath{getUserRegistryPath()}}.resolveSymlinks(),
Registry::User);
return userRegistry;
}
std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Path & p)
{
static auto customRegistry = Registry::read(settings, p, Registry::Custom);
static auto customRegistry =
Registry::read(settings, SourcePath{getFSSourceAccessor(), CanonPath{p}}.resolveSymlinks(), Registry::Custom);
return customRegistry;
}
@ -124,9 +131,9 @@ std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
return flagRegistry;
}
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs)
{
getFlagRegistry(*from.settings)->add(from, to, extraAttrs);
getFlagRegistry(settings)->add(from, to, extraAttrs);
}
static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
@ -137,14 +144,19 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
return std::make_shared<Registry>(settings, Registry::Global); // empty registry
}
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
path = store->toRealPath(storePath);
}
return Registry::read(settings, path, Registry::Global);
return Registry::read(
settings,
[&] -> SourcePath {
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
return {store->requireStoreObjectAccessor(storePath)};
} else {
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
}
}(),
Registry::Global);
}();
return reg;
@ -160,7 +172,8 @@ Registries getRegistries(const Settings & settings, ref<Store> store)
return registries;
}
std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & _input, UseRegistries useRegistries)
std::pair<Input, Attrs>
lookupInRegistries(const Settings & settings, ref<Store> store, const Input & _input, UseRegistries useRegistries)
{
Attrs extraAttrs;
int n = 0;
@ -175,7 +188,7 @@ restart:
if (n > 100)
throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(*input.settings, store)) {
for (auto & registry : getRegistries(settings, store)) {
if (useRegistries == UseRegistries::Limited
&& !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global))
continue;

29
src/libfetchers/tarball.cc
View file

@ -136,11 +136,11 @@ static DownloadTarballResult downloadTarball_(
.treeHash = treeHash,
.lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
.accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix),
.accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
};
};
if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash")))
if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash")))
cached.reset();
if (cached && !cached->expired)
@ -179,7 +179,7 @@ static DownloadTarballResult downloadTarball_(
TarArchive{path};
})
: TarArchive{*source};
auto tarballCache = getTarballCache();
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush();
@ -224,7 +224,7 @@ ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings,
auto input = Input::fromAttrs(settings, std::move(attrs));
return input.getAccessor(store).first;
return input.getAccessor(settings, store).first;
}
// An input scheme corresponding to a curl-downloadable resource.
@ -252,7 +252,7 @@ struct CurlInputScheme : InputScheme
if (!isValidURL(_url, requireTree))
return std::nullopt;
Input input{settings};
Input input{};
auto url = _url;
@ -302,7 +302,7 @@ struct CurlInputScheme : InputScheme
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
{
Input input{settings};
Input input{};
input.attrs = attrs;
// input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
@ -319,7 +319,7 @@ struct CurlInputScheme : InputScheme
return url;
}
bool isLocked(const Input & input) const override
bool isLocked(const Settings & settings, const Input & input) const override
{
return (bool) input.getNarHash();
}
@ -340,7 +340,8 @@ struct FileInputScheme : CurlInputScheme
: (!requireTree && !hasTarballExtension(url)));
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
auto input(_input);
@ -348,7 +349,7 @@ struct FileInputScheme : CurlInputScheme
the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like
tarballs. */
auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
@ -377,15 +378,15 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url)));
}
std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{
auto input(_input);
auto result =
downloadTarball_(*input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»");
auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»");
if (result.immutableUrl) {
auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl);
auto immutableInput = Input::fromURL(settings, *result.immutableUrl);
// FIXME: would be nice to support arbitrary flakerefs
// here, e.g. git flakes.
if (immutableInput.getType() != "tarball")
@ -398,7 +399,7 @@ struct TarballInputScheme : CurlInputScheme
input.attrs.insert_or_assign(
"narHash",
getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true));
return {result.accessor, input};
}

22
src/libflake-tests/flakeref.cc
View file

@ -206,6 +206,28 @@ INSTANTIATE_TEST_SUITE_P(
.description = "flake_id_ref_branch_ignore_empty_segments_ref_rev",
.expectedUrl = "flake:nixpkgs/branch/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
},
InputFromURLTestCase{
.url = "git://somewhere/repo?ref=branch",
.attrs =
{
{"type", Attr("git")},
{"ref", Attr("branch")},
{"url", Attr("git://somewhere/repo")},
},
.description = "plain_git_with_ref",
.expectedUrl = "git://somewhere/repo?ref=branch",
},
InputFromURLTestCase{
.url = "git+https://somewhere.aaaaaaa/repo?ref=branch",
.attrs =
{
{"type", Attr("git")},
{"ref", Attr("branch")},
{"url", Attr("https://somewhere.aaaaaaa/repo")},
},
.description = "git_https_with_ref",
.expectedUrl = "git+https://somewhere.aaaaaaa/repo?ref=branch",
},
InputFromURLTestCase{
// Note that this is different from above because the "flake id" shorthand
// doesn't allow this.

2
src/libflake/flake-primops.cc
View file

@ -38,7 +38,7 @@ PrimOp getFlake(const Settings & settings)
std::string flakeRefS(
state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
if (state.settings.pureEval && !flakeRef.input.isLocked())
if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings))
throw Error(
"cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)",
flakeRefS,

36
src/libflake/flake.cc
View file

@ -97,7 +97,7 @@ static void parseFlakeInputAttr(EvalState & state, const Attr & attr, fetchers::
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (attr.value->type()) {
case nString:
attrs.emplace(state.symbols[attr.name], attr.value->c_str());
attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view()));
break;
case nBool:
attrs.emplace(state.symbols[attr.name], Explicit<bool>{attr.value->boolean()});
@ -177,7 +177,7 @@ static FlakeInput parseFlakeInput(
parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first;
} else if (attr.name == sFollows) {
expectType(state, nString, *attr.value, attr.pos);
auto follows(parseInputAttrPath(attr.value->c_str()));
auto follows(parseInputAttrPath(attr.value->string_view()));
follows.insert(follows.begin(), lockRootAttrPath.begin(), lockRootAttrPath.end());
input.follows = follows;
} else
@ -264,7 +264,7 @@ static Flake readFlake(
if (auto description = vInfo.attrs()->get(state.s.description)) {
expectType(state, nString, *description->value, description->pos);
flake.description = description->value->c_str();
flake.description = description->value->string_view();
}
auto sInputs = state.symbols.create("inputs");
@ -281,12 +281,15 @@ static Flake readFlake(
if (auto outputs = vInfo.attrs()->get(sOutputs)) {
expectType(state, nFunction, *outputs->value, outputs->pos);
if (outputs->value->isLambda() && outputs->value->lambda().fun->hasFormals()) {
for (auto & formal : outputs->value->lambda().fun->formals->formals) {
if (formal.name != state.s.self)
flake.inputs.emplace(
state.symbols[formal.name],
FlakeInput{.ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name]))});
if (outputs->value->isLambda()) {
if (auto formals = outputs->value->lambda().fun->getFormals()) {
for (auto & formal : formals->formals) {
if (formal.name != state.s.self)
flake.inputs.emplace(
state.symbols[formal.name],
FlakeInput{
.ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name]))});
}
}
}
@ -369,7 +372,8 @@ static Flake getFlake(
const InputAttrPath & lockRootAttrPath)
{
// Fetch a lazy tree first.
auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
auto cachedInput =
state.inputCache->getAccessor(state.fetchSettings, state.store, originalRef.input, useRegistries);
auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@ -385,7 +389,8 @@ static Flake getFlake(
debug("refetching input '%s' due to self attribute", newLockedRef);
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash");
auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No);
auto cachedInput2 = state.inputCache->getAccessor(
state.fetchSettings, state.store, newLockedRef.input, fetchers::UseRegistries::No);
cachedInput.accessor = cachedInput2.accessor;
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
}
@ -502,8 +507,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
std::function<void(const FlakeInput & input, const InputAttrPath & prefix)> addOverrides;
addOverrides = [&](const FlakeInput & input, const InputAttrPath & prefix) {
auto addOverrides =
[&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputAttrPath(prefix);
inputAttrPath.push_back(idOverride);
@ -701,7 +706,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
this input. */
debug("creating new input '%s'", inputAttrPathS);
if (!lockFlags.allowUnlocked && !input.ref->input.isLocked() && !input.ref->input.isRelative())
if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings)
&& !input.ref->input.isRelative())
throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS);
/* Note: in case of an --override-input, we use
@ -750,7 +756,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
return {*resolvedPath, *input.ref};
} else {
auto cachedInput = state.inputCache->getAccessor(
state.store, input.ref->input, useRegistriesInputs);
state.fetchSettings, state.store, input.ref->input, useRegistriesInputs);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);

13
src/libflake/flakeref.cc
View file

@ -64,9 +64,10 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
return str;
}
FlakeRef FlakeRef::resolve(ref<Store> store, fetchers::UseRegistries useRegistries) const
FlakeRef FlakeRef::resolve(
const fetchers::Settings & fetchSettings, ref<Store> store, fetchers::UseRegistries useRegistries) const
{
auto [input2, extraAttrs] = lookupInRegistries(store, input, useRegistries);
auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
}
@ -109,7 +110,8 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
std::smatch match;
auto succeeds = std::regex_match(url, match, pathFlakeRegex);
assert(succeeds);
if (!succeeds)
throw Error("invalid flakeref '%s'", url);
auto path = match[1].str();
auto query = decodeQuery(match[3].str(), /*lenient=*/true);
auto fragment = percentDecode(match[5].str());
@ -286,9 +288,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
std::pair<ref<SourceAccessor>, FlakeRef>
FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const
{
auto [accessor, lockedInput] = input.getAccessor(store);
auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store);
return {accessor, FlakeRef(std::move(lockedInput), subdir)};
}

8
src/libflake/include/nix/flake/flakeref.hh
View file

@ -71,11 +71,15 @@ struct FlakeRef
fetchers::Attrs toAttrs() const;
FlakeRef resolve(ref<Store> store, fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
FlakeRef resolve(
const fetchers::Settings & fetchSettings,
ref<Store> store,
fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
std::pair<ref<SourceAccessor>, FlakeRef>
lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const;
/**
* Canonicalize a flakeref for the purpose of comparing "old" and

37
src/libflake/lockfile.cc
View file

@ -74,7 +74,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann:
, parentInputAttrPath(
json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt)
{
if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) {
if (lockedRef.input.getNarHash())
warn(
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@ -147,11 +147,10 @@ LockFile::LockFile(const fetchers::Settings & fetchSettings, std::string_view co
if (version < 5 || version > 7)
throw Error("lock file '%s' has unsupported version %d", path, version);
std::map<std::string, ref<Node>> nodeMap;
std::string rootKey = json["root"];
std::map<std::string, ref<Node>> nodeMap{{rootKey, root}};
std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
getInputs = [&](Node & node, const nlohmann::json & jsonNode) {
[&](this const auto & getInputs, Node & node, const nlohmann::json & jsonNode) {
if (jsonNode.find("inputs") == jsonNode.end())
return;
for (auto & i : jsonNode["inputs"].items()) {
@ -179,11 +178,7 @@ LockFile::LockFile(const fetchers::Settings & fetchSettings, std::string_view co
throw Error("lock file contains cycle to root node");
}
}
};
std::string rootKey = json["root"];
nodeMap.insert_or_assign(rootKey, root);
getInputs(*root, json["nodes"][rootKey]);
}(*root, json["nodes"][rootKey]);
// FIXME: check that there are no cycles in version >= 7. Cycles
// between inputs are only possible using 'follows' indirections.
@ -197,9 +192,7 @@ std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
KeyMap nodeKeys;
boost::unordered_flat_set<std::string> keys;
std::function<std::string(const std::string & key, ref<const Node> node)> dumpNode;
dumpNode = [&](std::string key, ref<const Node> node) -> std::string {
auto dumpNode = [&](this auto & dumpNode, std::string key, ref<const Node> node) -> std::string {
auto k = nodeKeys.find(node);
if (k != nodeKeys.end())
return k->second;
@ -276,24 +269,20 @@ std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSet
{
std::set<ref<const Node>> nodes;
std::function<void(ref<const Node> node)> visit;
visit = [&](ref<const Node> node) {
[&](this const auto & visit, ref<const Node> node) {
if (!nodes.insert(node).second)
return;
for (auto & i : node->inputs)
if (auto child = std::get_if<0>(&i.second))
visit(*child);
};
visit(root);
}(root);
/* Return whether the input is either locked, or, if
`allow-dirty-locks` is enabled, it has a NAR hash. In the
latter case, we can verify the input but we may not be able to
fetch it from anywhere. */
auto isConsideredLocked = [&](const fetchers::Input & input) {
return input.isLocked() || (fetchSettings.allowDirtyLocks && input.getNarHash());
return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash());
};
for (auto & i : nodes) {
@ -332,9 +321,7 @@ std::map<InputAttrPath, Node::Edge> LockFile::getAllInputs() const
std::set<ref<Node>> done;
std::map<InputAttrPath, Node::Edge> res;
std::function<void(const InputAttrPath & prefix, ref<Node> node)> recurse;
recurse = [&](const InputAttrPath & prefix, ref<Node> node) {
[&](this const auto & recurse, const InputAttrPath & prefix, ref<Node> node) {
if (!done.insert(node).second)
return;
@ -345,9 +332,7 @@ std::map<InputAttrPath, Node::Edge> LockFile::getAllInputs() const
if (auto child = std::get_if<0>(&input))
recurse(inputAttrPath, *child);
}
};
recurse({}, root);
}({}, root);
return res;
}

2
src/libmain/include/nix/main/shared.hh
View file

@ -89,8 +89,6 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
std::string showBytes(uint64_t bytes);
struct GCResults;
struct PrintFreed

67
src/libmain/progress-bar.cc
View file

@ -467,8 +467,6 @@ public:
std::string getStatus(State & state)
{
auto MiB = 1024.0 * 1024.0;
std::string res;
auto renderActivity =
@ -516,6 +514,65 @@ public:
return s;
};
auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
done += j.second->done;
expected += j.second->expected;
running += j.second->running;
failed += j.second->failed;
}
expected = std::max(expected, act.expected);
std::optional<SizeUnit> commonUnit;
std::string s;
if (running || done || expected || failed) {
if (running)
if (expected != 0) {
commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done, (int64_t) expected});
s =
fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL "/%s",
commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
} else {
commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done});
s =
fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL,
commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done));
}
else if (expected != done)
if (expected != 0) {
commonUnit = getCommonSizeUnit({(int64_t) done, (int64_t) expected});
s =
fmt(ANSI_GREEN "%s" ANSI_NORMAL "/%s",
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
} else {
commonUnit = getSizeUnit(done);
s = fmt(ANSI_GREEN "%s" ANSI_NORMAL, renderSizeWithoutUnit(done, *commonUnit));
}
else {
commonUnit = getSizeUnit(done);
s = fmt(done ? ANSI_GREEN "%s" ANSI_NORMAL : "%s", renderSizeWithoutUnit(done, *commonUnit));
}
if (commonUnit)
s = fmt("%s %siB", s, getSizeUnitSuffix(*commonUnit));
s = fmt(itemFmt, s);
if (failed)
s += fmt(" (" ANSI_RED "%s failed" ANSI_NORMAL ")", renderSize(failed));
}
return s;
};
auto showActivity =
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
auto s = renderActivity(type, itemFmt, numberFmt, unit);
@ -529,7 +586,7 @@ public:
showActivity(actBuilds, "%s built");
auto s1 = renderActivity(actCopyPaths, "%s copied");
auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB);
auto s2 = renderSizeActivity(actCopyPath);
if (!s1.empty() || !s2.empty()) {
if (!res.empty())
@ -545,12 +602,12 @@ public:
}
}
showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
renderSizeActivity(actFileTransfer, "%s DL");
{
auto s = renderActivity(actOptimiseStore, "%s paths optimised");
if (s != "") {
s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
s += fmt(", %s / %d inodes freed", renderSize(state.bytesLinked), state.filesLinked);
if (!res.empty())
res += ", ";
res += s;

65
src/libmain/shared.cc
View file

@ -6,6 +6,7 @@
#include "nix/main/loggers.hh"
#include "nix/main/progress-bar.hh"
#include "nix/util/signals.hh"
#include "nix/util/util.hh"
#include <algorithm>
#include <exception>
@ -64,18 +65,19 @@ void printMissing(ref<Store> store, const MissingPaths & missing, Verbosity lvl)
}
if (!missing.willSubstitute.empty()) {
const float downloadSizeMiB = missing.downloadSize / (1024.f * 1024.f);
const float narSizeMiB = missing.narSize / (1024.f * 1024.f);
if (missing.willSubstitute.size() == 1) {
printMsg(
lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, narSizeMiB);
lvl,
"this path will be fetched (%s download, %s unpacked):",
renderSize(missing.downloadSize),
renderSize(missing.narSize));
} else {
printMsg(
lvl,
"these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
"these %d paths will be fetched (%s download, %s unpacked):",
missing.willSubstitute.size(),
downloadSizeMiB,
narSizeMiB);
renderSize(missing.downloadSize),
renderSize(missing.narSize));
}
std::vector<const StorePath *> willSubstituteSorted = {};
std::for_each(missing.willSubstitute.begin(), missing.willSubstitute.end(), [&](const StorePath & p) {
@ -320,34 +322,29 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
try {
try {
fun();
} catch (...) {
/* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
}
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
fun();
} catch (...) {
/* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
}
} catch (...) {
/* In case logger also throws just give up. */
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
@ -411,7 +408,7 @@ RunPager::~RunPager()
PrintFreed::~PrintFreed()
{
if (show)
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), showBytes(results.bytesFreed));
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
}
} // namespace nix

42
src/libstore-c/nix_api_store.cc
View file

@ -7,6 +7,7 @@
#include "nix/store/store-api.hh"
#include "nix/store/store-open.hh"
#include "nix/store/build-result.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/globals.hh"
@ -109,7 +110,8 @@ nix_err nix_store_real_path(
if (context)
context->last_err_code = NIX_OK;
try {
auto res = store->ptr->toRealPath(path->path);
auto store2 = store->ptr.dynamic_pointer_cast<nix::LocalFSStore>();
auto res = store2 ? store2->toRealPath(path->path) : store->ptr->printStorePath(path->path);
return call_nix_get_string_callback(res, callback, user_data);
}
NIXC_CATCH_ERRS
@ -126,6 +128,36 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_get_fs_closure(
nix_c_context * context,
Store * store,
const StorePath * store_path,
bool flip_direction,
bool include_outputs,
bool include_derivers,
void * userdata,
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path))
{
if (context)
context->last_err_code = NIX_OK;
try {
const auto nixStore = store->ptr;
nix::StorePathSet set;
nixStore->computeFSClosure(store_path->path, set, flip_direction, include_outputs, include_derivers);
if (callback) {
for (const auto & path : set) {
const StorePath tmp{path};
callback(context, userdata, &tmp);
if (context && context->last_err_code != NIX_OK)
return context->last_err_code;
}
}
}
NIXC_CATCH_ERRS
}
nix_err nix_store_realise(
nix_c_context * context,
Store * store,
@ -143,6 +175,14 @@ nix_err nix_store_realise(
const auto nixStore = store->ptr;
auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore);
assert(results.size() == 1);
// Check if any builds failed
for (auto & result : results) {
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
if (auto * success = result.tryGetSuccess()) {

31
src/libstore-c/nix_api_store.h
View file

@ -186,6 +186,8 @@ nix_err nix_store_real_path(
* @param[in] path Path to build
* @param[in] userdata data to pass to every callback invocation
* @param[in] callback called for every realised output
* @return NIX_OK if the build succeeded, or an error code if the build/scheduling/outputs/copying/etc failed.
* On error, the callback is never invoked and error information is stored in context.
*/
nix_err nix_store_realise(
nix_c_context * context,
@ -245,6 +247,35 @@ void nix_derivation_free(nix_derivation * drv);
*/
nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path);
/**
* @brief Gets the closure of a specific store path
*
* @note The callback borrows each StorePath only for the duration of the call.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] store_path The path to compute from
* @param[in] flip_direction If false, compute the forward closure (paths referenced by any store path in the closure).
* If true, compute the backward closure (paths that reference any store path in the closure).
* @param[in] include_outputs If flip_direction is false: for any derivation in the closure, include its outputs.
* If flip_direction is true: for any output in the closure, include derivations that produce
* it.
* @param[in] include_derivers If flip_direction is false: for any output in the closure, include the derivation that
* produced it.
* If flip_direction is true: for any derivation in the closure, include its outputs.
* @param[in] callback The function to call for every store path, in no particular order
* @param[in] userdata The userdata to pass to the callback
*/
nix_err nix_store_get_fs_closure(
nix_c_context * context,
Store * store,
const StorePath * store_path,
bool flip_direction,
bool include_outputs,
bool include_derivers,
void * userdata,
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path));
// cffi end
#ifdef __cplusplus
}

52
src/libstore-test-support/include/nix/store/tests/protocol.hh
View file

@ -6,6 +6,7 @@
#include "nix/store/tests/libstore.hh"
#include "nix/util/tests/characterization.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@ -16,12 +17,30 @@ class ProtoTest : public CharacterizationTest
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / (std::string{testStem + ".bin"});
return unitTestData / testStem;
}
public:
Path storeDir = "/nix/store";
StoreDirConfig store{storeDir};
/**
* Golden test for `T` JSON reading
*/
template<typename T>
void readJsonTest(PathView testStem, const T & expected)
{
nix::readJsonTest(*this, testStem, expected);
}
/**
* Golden test for `T` JSON write
*/
template<typename T>
void writeJsonTest(PathView testStem, const T & decoded)
{
nix::writeJsonTest(*this, testStem, decoded);
}
};
template<class Proto, const char * protocolDir>
@ -34,7 +53,7 @@ public:
template<typename T>
void readProtoTest(PathView testStem, typename Proto::Version version, T expected)
{
CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) {
T got = ({
StringSource from{encoded};
Proto::template Serialise<T>::read(
@ -55,7 +74,7 @@ public:
template<typename T>
void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded)
{
CharacterizationTest::writeTest(testStem, [&]() {
CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() {
StringSink to;
Proto::template Serialise<T>::write(
this->store,
@ -69,14 +88,25 @@ public:
}
};
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_read) \
{ \
readProtoTest(STEM, VERSION, VALUE); \
} \
TEST_F(FIXTURE, NAME##_write) \
{ \
writeProtoTest(STEM, VERSION, VALUE); \
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_read) \
{ \
readProtoTest(STEM, VERSION, VALUE); \
} \
TEST_F(FIXTURE, NAME##_write) \
{ \
writeProtoTest(STEM, VERSION, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
} \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
} // namespace nix

108
src/libstore-tests/build-result.cc Normal file
View file

@ -0,0 +1,108 @@
#include <gtest/gtest.h>
#include "nix/store/build-result.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
class BuildResultTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "build-result";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
using nlohmann::json;
struct BuildResultJsonTest : BuildResultTest,
JsonCharacterizationTest<BuildResult>,
::testing::WithParamInterface<std::pair<std::string_view, BuildResult>>
{};
TEST_P(BuildResultJsonTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_P(BuildResultJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
using namespace std::literals::chrono_literals;
INSTANTIATE_TEST_SUITE_P(
BuildResultJSON,
BuildResultJsonTest,
::testing::Values(
std::pair{
"not-deterministic",
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::NotDeterministic,
.errorMsg = "no idea why",
.isNonDeterministic = false, // Note: This field is separate from the status
}},
.timesBuilt = 1,
},
},
std::pair{
"output-rejected",
BuildResult{
.inner{BuildResult::Failure{
.status = BuildResult::Failure::OutputRejected,
.errorMsg = "no idea why",
.isNonDeterministic = false,
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
},
},
std::pair{
"success",
BuildResult{
.inner{BuildResult::Success{
.status = BuildResult::Success::Built,
.builtOutputs{
{
"foo",
{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
},
},
{
"bar",
{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
},
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
},
},
},
}},
.timesBuilt = 3,
.startTime = 30,
.stopTime = 50,
.cpuUser = std::chrono::microseconds(500s),
.cpuSystem = std::chrono::microseconds(604s),
},
}));
} // namespace nix

32
src/libstore-tests/common-protocol.cc
View file

@ -3,6 +3,7 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include "nix/util/json-utils.hh"
#include "nix/store/common-protocol.hh"
#include "nix/store/common-protocol-impl.hh"
#include "nix/store/build-result.hh"
@ -22,7 +23,7 @@ public:
template<typename T>
void readProtoTest(PathView testStem, const T & expected)
{
CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) {
T got = ({
StringSource from{encoded};
CommonProto::Serialise<T>::read(store, CommonProto::ReadConn{.from = from});
@ -38,7 +39,7 @@ public:
template<typename T>
void writeProtoTest(PathView testStem, const T & decoded)
{
CharacterizationTest::writeTest(testStem, [&]() -> std::string {
CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() -> std::string {
StringSink to;
CommonProto::Serialise<T>::write(store, CommonProto::WriteConn{.to = to}, decoded);
return to.s;
@ -54,6 +55,14 @@ public:
TEST_F(CommonProtoTest, NAME##_write) \
{ \
writeProtoTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
CHARACTERIZATION_TEST(
@ -114,13 +123,28 @@ CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
},
DrvOutput{
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
},
}))
CHARACTERIZATION_TEST(
realisation_with_deps,
"realisation-with-deps",
(std::tuple<Realisation>{
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},

97
src/libstore-tests/content-address.cc
View file

@ -1,6 +1,7 @@
#include <gtest/gtest.h>
#include "nix/store/content-address.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@ -8,33 +9,93 @@ namespace nix {
* ContentAddressMethod::parse, ContentAddressMethod::render
* --------------------------------------------------------------------------*/
TEST(ContentAddressMethod, testRoundTripPrintParse_1)
static auto methods = ::testing::Values(
std::pair{ContentAddressMethod::Raw::Text, "text"},
std::pair{ContentAddressMethod::Raw::Flat, "flat"},
std::pair{ContentAddressMethod::Raw::NixArchive, "nar"},
std::pair{ContentAddressMethod::Raw::Git, "git"});
struct ContentAddressMethodTest : ::testing::Test,
::testing::WithParamInterface<std::pair<ContentAddressMethod, std::string_view>>
{};
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1)
{
for (ContentAddressMethod cam : {
ContentAddressMethod::Raw::Text,
ContentAddressMethod::Raw::Flat,
ContentAddressMethod::Raw::NixArchive,
ContentAddressMethod::Raw::Git,
}) {
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
}
auto & [cam, _] = GetParam();
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
}
TEST(ContentAddressMethod, testRoundTripPrintParse_2)
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2)
{
for (const std::string_view camS : {
"text",
"flat",
"nar",
"git",
}) {
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
}
auto & [cam, camS] = GetParam();
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
}
INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods);
TEST(ContentAddressMethod, testParseContentAddressMethodOptException)
{
EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
}
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
class ContentAddressTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "content-address";
public:
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
using nlohmann::json;
struct ContentAddressJsonTest : ContentAddressTest,
JsonCharacterizationTest<ContentAddress>,
::testing::WithParamInterface<std::pair<std::string_view, ContentAddress>>
{};
TEST_P(ContentAddressJsonTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_P(ContentAddressJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
INSTANTIATE_TEST_SUITE_P(
ContentAddressJSON,
ContentAddressJsonTest,
::testing::Values(
std::pair{
"text",
ContentAddress{
.method = ContentAddressMethod::Raw::Text,
.hash = hashString(HashAlgorithm::SHA256, "asdf"),
},
},
std::pair{
"nar",
ContentAddress{
.method = ContentAddressMethod::Raw::NixArchive,
.hash = hashString(HashAlgorithm::SHA256, "qwer"),
},
}));
} // namespace nix

9
src/libstore-tests/data/build-result/not-deterministic.json Normal file
View file

@ -0,0 +1,9 @@
{
"errorMsg": "no idea why",
"isNonDeterministic": false,
"startTime": 0,
"status": "NotDeterministic",
"stopTime": 0,
"success": false,
"timesBuilt": 1
}

Some files were not shown because too many files have changed in this diff Show more