mirror of
https://github.com/NixOS/nix.git
synced 2025-11-09 03:56:01 +01:00
Implement support for Git hashing with SHA-256
SHA-256 is Git's next hash algorithm. The world is still basically stuck on SHA-1 with git, but shouldn't be. We can at least do our part to get ready. On the C++ implementation side, only a little bit of generalization was needed, and that was fairly straight-forward. The tests (unit and system) were actually bigger, and care was taken to make sure they were all cover both algorithms equally.
This commit is contained in:
parent
7f4acb9f10
commit
d21e3f88ec
20 changed files with 350 additions and 181 deletions
|
|
@ -262,8 +262,6 @@
|
||||||
''^tests/functional/gc-concurrent\.sh$''
|
''^tests/functional/gc-concurrent\.sh$''
|
||||||
''^tests/functional/gc-concurrent2\.builder\.sh$''
|
''^tests/functional/gc-concurrent2\.builder\.sh$''
|
||||||
''^tests/functional/gc-non-blocking\.sh$''
|
''^tests/functional/gc-non-blocking\.sh$''
|
||||||
''^tests/functional/git-hashing/common\.sh$''
|
|
||||||
''^tests/functional/git-hashing/simple\.sh$''
|
|
||||||
''^tests/functional/hash-convert\.sh$''
|
''^tests/functional/hash-convert\.sh$''
|
||||||
''^tests/functional/impure-derivations\.sh$''
|
''^tests/functional/impure-derivations\.sh$''
|
||||||
''^tests/functional/impure-eval\.sh$''
|
''^tests/functional/impure-eval\.sh$''
|
||||||
|
|
@ -339,7 +337,6 @@
|
||||||
''^tests/functional/user-envs\.builder\.sh$''
|
''^tests/functional/user-envs\.builder\.sh$''
|
||||||
''^tests/functional/user-envs\.sh$''
|
''^tests/functional/user-envs\.sh$''
|
||||||
''^tests/functional/why-depends\.sh$''
|
''^tests/functional/why-depends\.sh$''
|
||||||
''^src/libutil-tests/data/git/check-data\.sh$''
|
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -107,8 +107,11 @@ static std::string makeType(const MixStoreDirMethods & store, std::string && typ
|
||||||
|
|
||||||
StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
|
StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
|
||||||
{
|
{
|
||||||
if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1)
|
if (info.method == FileIngestionMethod::Git
|
||||||
throw Error("Git file ingestion must use SHA-1 hash");
|
&& !(info.hash.algo == HashAlgorithm::SHA1 || info.hash.algo == HashAlgorithm::SHA256)) {
|
||||||
|
throw Error(
|
||||||
|
"Git file ingestion must use SHA-1 or SHA-256 hash, but instead using: %s", printHashAlgo(info.hash.algo));
|
||||||
|
}
|
||||||
|
|
||||||
if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) {
|
if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) {
|
||||||
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
|
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
|
||||||
|
|
|
||||||
44
src/libutil-tests/data/git/check-data.sh
Normal file → Executable file
44
src/libutil-tests/data/git/check-data.sh
Normal file → Executable file
|
|
@ -2,30 +2,34 @@
|
||||||
|
|
||||||
set -eu -o pipefail
|
set -eu -o pipefail
|
||||||
|
|
||||||
export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data
|
TEST_ROOT=$(realpath "${TMPDIR:-/tmp}/nix-test")/git-hashing/check-data
|
||||||
mkdir -p $TEST_ROOT
|
export TEST_ROOT
|
||||||
|
mkdir -p "$TEST_ROOT"
|
||||||
|
|
||||||
repo="$TEST_ROOT/scratch"
|
for hash in sha1 sha256; do
|
||||||
git init "$repo"
|
repo="$TEST_ROOT/scratch-$hash"
|
||||||
|
git init "$repo" --object-format="$hash"
|
||||||
|
|
||||||
git -C "$repo" config user.email "you@example.com"
|
git -C "$repo" config user.email "you@example.com"
|
||||||
git -C "$repo" config user.name "Your Name"
|
git -C "$repo" config user.name "Your Name"
|
||||||
|
|
||||||
# `-w` to write for tree test
|
# `-w` to write for tree test
|
||||||
freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin")
|
freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin")
|
||||||
encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40)
|
encodingHash=$("${hash}sum" -b < "./hello-world-blob.bin" | sed 's/ .*//')
|
||||||
|
|
||||||
# If the hashes match, then `hello-world-blob.bin` must be the encoding
|
# If the hashes match, then `hello-world-blob.bin` must be the encoding
|
||||||
# of `hello-world.bin`.
|
# of `hello-world.bin`.
|
||||||
[[ "$encodingHash" == "$freshlyAddedHash" ]]
|
[[ "$encodingHash" == "$freshlyAddedHash" ]]
|
||||||
|
|
||||||
# Create empty directory object for tree test
|
# Create empty directory object for tree test
|
||||||
echo -n | git -C "$repo" hash-object -w -t tree --stdin
|
echo -n | git -C "$repo" hash-object -w -t tree --stdin
|
||||||
|
|
||||||
# Relies on both child hashes already existing in the git store
|
# Relies on both child hashes already existing in the git store
|
||||||
freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt")
|
tree=tree-${hash}
|
||||||
encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40)
|
freshlyAddedHash=$(git -C "$repo" mktree < "${tree}.txt")
|
||||||
|
encodingHash=$("${hash}sum" -b < "${tree}.bin" | sed 's/ .*//')
|
||||||
|
|
||||||
# If the hashes match, then `tree.bin` must be the encoding of the
|
# If the hashes match, then `tree.bin` must be the encoding of the
|
||||||
# directory denoted by `tree.txt` interpreted as git directory listing.
|
# directory denoted by `tree.txt` interpreted as git directory listing.
|
||||||
[[ "$encodingHash" == "$freshlyAddedHash" ]]
|
[[ "$encodingHash" == "$freshlyAddedHash" ]]
|
||||||
|
done
|
||||||
|
|
|
||||||
BIN
src/libutil-tests/data/git/tree-sha256.bin
Normal file
BIN
src/libutil-tests/data/git/tree-sha256.bin
Normal file
Binary file not shown.
4
src/libutil-tests/data/git/tree-sha256.txt
Normal file
4
src/libutil-tests/data/git/tree-sha256.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
100644 blob ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1 Foo
|
||||||
|
100755 blob ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1 bAr
|
||||||
|
040000 tree 6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321 baZ
|
||||||
|
120000 blob ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1 quuX
|
||||||
|
|
@ -97,7 +97,7 @@ TEST_F(GitTest, blob_write)
|
||||||
* so that we can check our test data in a small shell script test test
|
* so that we can check our test data in a small shell script test test
|
||||||
* (`src/libutil-tests/data/git/check-data.sh`).
|
* (`src/libutil-tests/data/git/check-data.sh`).
|
||||||
*/
|
*/
|
||||||
const static Tree tree = {
|
const static Tree treeSha1 = {
|
||||||
{
|
{
|
||||||
"Foo",
|
"Foo",
|
||||||
{
|
{
|
||||||
|
|
@ -133,9 +133,48 @@ const static Tree tree = {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
TEST_F(GitTest, tree_read)
|
/**
|
||||||
|
* Same conceptual object as `treeSha1`, just different hash algorithm.
|
||||||
|
* See that one for details.
|
||||||
|
*/
|
||||||
|
const static Tree treeSha256 = {
|
||||||
|
{
|
||||||
|
"Foo",
|
||||||
|
{
|
||||||
|
.mode = Mode::Regular,
|
||||||
|
.hash = Hash::parseAny(
|
||||||
|
"ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"bAr",
|
||||||
|
{
|
||||||
|
.mode = Mode::Executable,
|
||||||
|
.hash = Hash::parseAny(
|
||||||
|
"ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"baZ/",
|
||||||
|
{
|
||||||
|
.mode = Mode::Directory,
|
||||||
|
.hash = Hash::parseAny(
|
||||||
|
"6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321", HashAlgorithm::SHA256),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"quuX",
|
||||||
|
{
|
||||||
|
.mode = Mode::Symlink,
|
||||||
|
.hash = Hash::parseAny(
|
||||||
|
"ce60f5ad78a08ac24872ef74d78b078f077be212e7a246893a1a5d957dfbc8b1", HashAlgorithm::SHA256),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
static auto mkTreeReadTest(HashAlgorithm hashAlgo, Tree tree, const ExperimentalFeatureSettings & mockXpSettings)
|
||||||
{
|
{
|
||||||
readTest("tree.bin", [&](const auto & encoded) {
|
return [hashAlgo, tree, mockXpSettings](const auto & encoded) {
|
||||||
StringSource in{encoded};
|
StringSource in{encoded};
|
||||||
NullFileSystemObjectSink out;
|
NullFileSystemObjectSink out;
|
||||||
Tree got;
|
Tree got;
|
||||||
|
|
@ -144,6 +183,7 @@ TEST_F(GitTest, tree_read)
|
||||||
out,
|
out,
|
||||||
CanonPath::root,
|
CanonPath::root,
|
||||||
in,
|
in,
|
||||||
|
hashAlgo,
|
||||||
[&](auto & name, auto entry) {
|
[&](auto & name, auto entry) {
|
||||||
auto name2 = std::string{name.rel()};
|
auto name2 = std::string{name.rel()};
|
||||||
if (entry.mode == Mode::Directory)
|
if (entry.mode == Mode::Directory)
|
||||||
|
|
@ -153,14 +193,33 @@ TEST_F(GitTest, tree_read)
|
||||||
mockXpSettings);
|
mockXpSettings);
|
||||||
|
|
||||||
ASSERT_EQ(got, tree);
|
ASSERT_EQ(got, tree);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(GitTest, tree_sha1_read)
|
||||||
|
{
|
||||||
|
readTest("tree-sha1.bin", mkTreeReadTest(HashAlgorithm::SHA1, treeSha1, mockXpSettings));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(GitTest, tree_sha256_read)
|
||||||
|
{
|
||||||
|
readTest("tree-sha256.bin", mkTreeReadTest(HashAlgorithm::SHA256, treeSha256, mockXpSettings));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(GitTest, tree_sha1_write)
|
||||||
|
{
|
||||||
|
writeTest("tree-sha1.bin", [&]() {
|
||||||
|
StringSink s;
|
||||||
|
dumpTree(treeSha1, s, mockXpSettings);
|
||||||
|
return s.s;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(GitTest, tree_write)
|
TEST_F(GitTest, tree_sha256_write)
|
||||||
{
|
{
|
||||||
writeTest("tree.bin", [&]() {
|
writeTest("tree-sha256.bin", [&]() {
|
||||||
StringSink s;
|
StringSink s;
|
||||||
dumpTree(tree, s, mockXpSettings);
|
dumpTree(treeSha256, s, mockXpSettings);
|
||||||
return s.s;
|
return s.s;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -202,51 +261,54 @@ TEST_F(GitTest, both_roundrip)
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
std::map<Hash, std::string> cas;
|
for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
|
||||||
|
std::map<Hash, std::string> cas;
|
||||||
|
|
||||||
std::function<DumpHook> dumpHook;
|
std::function<DumpHook> dumpHook;
|
||||||
dumpHook = [&](const SourcePath & path) {
|
dumpHook = [&](const SourcePath & path) {
|
||||||
StringSink s;
|
StringSink s;
|
||||||
HashSink hashSink{HashAlgorithm::SHA1};
|
HashSink hashSink{hashAlgo};
|
||||||
TeeSink s2{s, hashSink};
|
TeeSink s2{s, hashSink};
|
||||||
auto mode = dump(path, s2, dumpHook, defaultPathFilter, mockXpSettings);
|
auto mode = dump(path, s2, dumpHook, defaultPathFilter, mockXpSettings);
|
||||||
auto hash = hashSink.finish().first;
|
auto hash = hashSink.finish().first;
|
||||||
cas.insert_or_assign(hash, std::move(s.s));
|
cas.insert_or_assign(hash, std::move(s.s));
|
||||||
return TreeEntry{
|
return TreeEntry{
|
||||||
.mode = mode,
|
.mode = mode,
|
||||||
.hash = hash,
|
.hash = hash,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
};
|
|
||||||
|
|
||||||
auto root = dumpHook({files});
|
auto root = dumpHook({files});
|
||||||
|
|
||||||
auto files2 = make_ref<MemorySourceAccessor>();
|
auto files2 = make_ref<MemorySourceAccessor>();
|
||||||
|
|
||||||
MemorySink sinkFiles2{*files2};
|
MemorySink sinkFiles2{*files2};
|
||||||
|
|
||||||
std::function<void(const CanonPath, const Hash &, BlobMode)> mkSinkHook;
|
std::function<void(const CanonPath, const Hash &, BlobMode)> mkSinkHook;
|
||||||
mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) {
|
mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) {
|
||||||
StringSource in{cas[hash]};
|
StringSource in{cas[hash]};
|
||||||
parse(
|
parse(
|
||||||
sinkFiles2,
|
sinkFiles2,
|
||||||
prefix,
|
prefix,
|
||||||
in,
|
in,
|
||||||
blobMode,
|
blobMode,
|
||||||
[&](const CanonPath & name, const auto & entry) {
|
hashAlgo,
|
||||||
mkSinkHook(
|
[&](const CanonPath & name, const auto & entry) {
|
||||||
prefix / name,
|
mkSinkHook(
|
||||||
entry.hash,
|
prefix / name,
|
||||||
// N.B. this cast would not be acceptable in real
|
entry.hash,
|
||||||
// code, because it would make an assert reachable,
|
// N.B. this cast would not be acceptable in real
|
||||||
// but it should harmless in this test.
|
// code, because it would make an assert reachable,
|
||||||
static_cast<BlobMode>(entry.mode));
|
// but it should harmless in this test.
|
||||||
},
|
static_cast<BlobMode>(entry.mode));
|
||||||
mockXpSettings);
|
},
|
||||||
};
|
mockXpSettings);
|
||||||
|
};
|
||||||
|
|
||||||
mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular);
|
mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular);
|
||||||
|
|
||||||
ASSERT_EQ(files->root, files2->root);
|
EXPECT_EQ(files->root, files2->root);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(GitLsRemote, parseSymrefLineWithReference)
|
TEST(GitLsRemote, parseSymrefLineWithReference)
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ void parseBlob(
|
||||||
{
|
{
|
||||||
xpSettings.require(Xp::GitHashing);
|
xpSettings.require(Xp::GitHashing);
|
||||||
|
|
||||||
unsigned long long size = std::stoi(getStringUntil(source, 0));
|
const unsigned long long size = std::stoi(getStringUntil(source, 0));
|
||||||
|
|
||||||
auto doRegularFile = [&](bool executable) {
|
auto doRegularFile = [&](bool executable) {
|
||||||
sink.createRegularFile(sinkPath, [&](auto & crf) {
|
sink.createRegularFile(sinkPath, [&](auto & crf) {
|
||||||
|
|
@ -114,10 +114,11 @@ void parseTree(
|
||||||
FileSystemObjectSink & sink,
|
FileSystemObjectSink & sink,
|
||||||
const CanonPath & sinkPath,
|
const CanonPath & sinkPath,
|
||||||
Source & source,
|
Source & source,
|
||||||
|
HashAlgorithm hashAlgo,
|
||||||
std::function<SinkHook> hook,
|
std::function<SinkHook> hook,
|
||||||
const ExperimentalFeatureSettings & xpSettings)
|
const ExperimentalFeatureSettings & xpSettings)
|
||||||
{
|
{
|
||||||
unsigned long long size = std::stoi(getStringUntil(source, 0));
|
const unsigned long long size = std::stoi(getStringUntil(source, 0));
|
||||||
unsigned long long left = size;
|
unsigned long long left = size;
|
||||||
|
|
||||||
sink.createDirectory(sinkPath);
|
sink.createDirectory(sinkPath);
|
||||||
|
|
@ -137,10 +138,15 @@ void parseTree(
|
||||||
left -= name.size();
|
left -= name.size();
|
||||||
left -= 1;
|
left -= 1;
|
||||||
|
|
||||||
std::string hashs = getString(source, 20);
|
const auto hashSize = regularHashSize(hashAlgo);
|
||||||
left -= 20;
|
std::string hashs = getString(source, hashSize);
|
||||||
|
left -= hashSize;
|
||||||
|
|
||||||
Hash hash(HashAlgorithm::SHA1);
|
if (!(hashAlgo == HashAlgorithm::SHA1 || hashAlgo == HashAlgorithm::SHA256)) {
|
||||||
|
throw Error("Unsupported hash algorithm for git trees: %s", printHashAlgo(hashAlgo));
|
||||||
|
}
|
||||||
|
|
||||||
|
Hash hash(hashAlgo);
|
||||||
std::copy(hashs.begin(), hashs.end(), hash.hash);
|
std::copy(hashs.begin(), hashs.end(), hash.hash);
|
||||||
|
|
||||||
hook(
|
hook(
|
||||||
|
|
@ -171,6 +177,7 @@ void parse(
|
||||||
const CanonPath & sinkPath,
|
const CanonPath & sinkPath,
|
||||||
Source & source,
|
Source & source,
|
||||||
BlobMode rootModeIfBlob,
|
BlobMode rootModeIfBlob,
|
||||||
|
HashAlgorithm hashAlgo,
|
||||||
std::function<SinkHook> hook,
|
std::function<SinkHook> hook,
|
||||||
const ExperimentalFeatureSettings & xpSettings)
|
const ExperimentalFeatureSettings & xpSettings)
|
||||||
{
|
{
|
||||||
|
|
@ -183,7 +190,7 @@ void parse(
|
||||||
parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings);
|
parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings);
|
||||||
break;
|
break;
|
||||||
case ObjectType::Tree:
|
case ObjectType::Tree:
|
||||||
parseTree(sink, sinkPath, source, hook, xpSettings);
|
parseTree(sink, sinkPath, source, hashAlgo, hook, xpSettings);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
assert(false);
|
assert(false);
|
||||||
|
|
@ -210,9 +217,9 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook)
|
void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook)
|
||||||
{
|
{
|
||||||
parse(sink, CanonPath::root, source, BlobMode::Regular, [&](CanonPath name, TreeEntry entry) {
|
parse(sink, CanonPath::root, source, BlobMode::Regular, hashAlgo, [&](CanonPath name, TreeEntry entry) {
|
||||||
auto [accessor, from] = hook(entry.hash);
|
auto [accessor, from] = hook(entry.hash);
|
||||||
auto stat = accessor->lstat(from);
|
auto stat = accessor->lstat(from);
|
||||||
auto gotOpt = convertMode(stat.type);
|
auto gotOpt = convertMode(stat.type);
|
||||||
|
|
|
||||||
|
|
@ -20,23 +20,6 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static size_t regularHashSize(HashAlgorithm type)
|
|
||||||
{
|
|
||||||
switch (type) {
|
|
||||||
case HashAlgorithm::BLAKE3:
|
|
||||||
return blake3HashSize;
|
|
||||||
case HashAlgorithm::MD5:
|
|
||||||
return md5HashSize;
|
|
||||||
case HashAlgorithm::SHA1:
|
|
||||||
return sha1HashSize;
|
|
||||||
case HashAlgorithm::SHA256:
|
|
||||||
return sha256HashSize;
|
|
||||||
case HashAlgorithm::SHA512:
|
|
||||||
return sha512HashSize;
|
|
||||||
}
|
|
||||||
unreachable();
|
|
||||||
}
|
|
||||||
|
|
||||||
const StringSet hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512"};
|
const StringSet hashAlgorithms = {"blake3", "md5", "sha1", "sha256", "sha512"};
|
||||||
|
|
||||||
const StringSet hashFormats = {"base64", "nix32", "base16", "sri"};
|
const StringSet hashFormats = {"base64", "nix32", "base16", "sri"};
|
||||||
|
|
|
||||||
|
|
@ -94,10 +94,14 @@ void parseBlob(
|
||||||
BlobMode blobMode,
|
BlobMode blobMode,
|
||||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now.
|
||||||
|
*/
|
||||||
void parseTree(
|
void parseTree(
|
||||||
FileSystemObjectSink & sink,
|
FileSystemObjectSink & sink,
|
||||||
const CanonPath & sinkPath,
|
const CanonPath & sinkPath,
|
||||||
Source & source,
|
Source & source,
|
||||||
|
HashAlgorithm hashAlgo,
|
||||||
std::function<SinkHook> hook,
|
std::function<SinkHook> hook,
|
||||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||||
|
|
||||||
|
|
@ -107,12 +111,15 @@ void parseTree(
|
||||||
* @param rootModeIfBlob How to interpret a root blob, for which there is no
|
* @param rootModeIfBlob How to interpret a root blob, for which there is no
|
||||||
* disambiguating dir entry to answer that questino. If the root it not
|
* disambiguating dir entry to answer that questino. If the root it not
|
||||||
* a blob, this is ignored.
|
* a blob, this is ignored.
|
||||||
|
*
|
||||||
|
* @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now.
|
||||||
*/
|
*/
|
||||||
void parse(
|
void parse(
|
||||||
FileSystemObjectSink & sink,
|
FileSystemObjectSink & sink,
|
||||||
const CanonPath & sinkPath,
|
const CanonPath & sinkPath,
|
||||||
Source & source,
|
Source & source,
|
||||||
BlobMode rootModeIfBlob,
|
BlobMode rootModeIfBlob,
|
||||||
|
HashAlgorithm hashAlgo,
|
||||||
std::function<SinkHook> hook,
|
std::function<SinkHook> hook,
|
||||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||||
|
|
||||||
|
|
@ -131,8 +138,10 @@ using RestoreHook = SourcePath(Hash);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Wrapper around `parse` and `RestoreSink`
|
* Wrapper around `parse` and `RestoreSink`
|
||||||
|
*
|
||||||
|
* @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now.
|
||||||
*/
|
*/
|
||||||
void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook);
|
void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dumps a single file to a sink
|
* Dumps a single file to a sink
|
||||||
|
|
|
||||||
|
|
@ -12,11 +12,26 @@ MakeError(BadHash, Error);
|
||||||
|
|
||||||
enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512, BLAKE3 };
|
enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512, BLAKE3 };
|
||||||
|
|
||||||
const int blake3HashSize = 32;
|
/**
|
||||||
const int md5HashSize = 16;
|
* @return the size of a hash for the given algorithm
|
||||||
const int sha1HashSize = 20;
|
*/
|
||||||
const int sha256HashSize = 32;
|
constexpr inline size_t regularHashSize(HashAlgorithm type)
|
||||||
const int sha512HashSize = 64;
|
{
|
||||||
|
switch (type) {
|
||||||
|
case HashAlgorithm::BLAKE3:
|
||||||
|
return 32;
|
||||||
|
case HashAlgorithm::MD5:
|
||||||
|
return 16;
|
||||||
|
case HashAlgorithm::SHA1:
|
||||||
|
return 20;
|
||||||
|
case HashAlgorithm::SHA256:
|
||||||
|
return 32;
|
||||||
|
case HashAlgorithm::SHA512:
|
||||||
|
return 64;
|
||||||
|
default:
|
||||||
|
assert(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
extern const StringSet hashAlgorithms;
|
extern const StringSet hashAlgorithms;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -74,5 +74,10 @@ rec {
|
||||||
nar-not-recursive = f2 "foo" ./fixed.builder2.sh "nar" "md5" "3670af73070fa14077ad74e0f5ea4e42";
|
nar-not-recursive = f2 "foo" ./fixed.builder2.sh "nar" "md5" "3670af73070fa14077ad74e0f5ea4e42";
|
||||||
|
|
||||||
# Experimental feature
|
# Experimental feature
|
||||||
git = f2 "foo" ./fixed.builder2.sh "git" "sha1" "cd44baf36915d5dec8374232ea7e2057f3b4494e";
|
|
||||||
|
git-sha1 = f2 "foo" ./fixed.builder2.sh "git" "sha1" "cd44baf36915d5dec8374232ea7e2057f3b4494e";
|
||||||
|
|
||||||
|
git-sha256 =
|
||||||
|
f2 "foo" ./fixed.builder2.sh "git" "sha256"
|
||||||
|
"3c957653f90c34c0a8badf343b61393936cddf4a2ca93f64b21f02303ddedcc2";
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
# shellcheck shell=bash
|
||||||
|
|
||||||
source ../common.sh
|
source ../common.sh
|
||||||
|
|
||||||
TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear
|
TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear
|
||||||
|
|
|
||||||
|
|
@ -5,4 +5,8 @@ source common.sh
|
||||||
# Store layer needs bugfix
|
# Store layer needs bugfix
|
||||||
requireDaemonNewerThan "2.27pre20250122"
|
requireDaemonNewerThan "2.27pre20250122"
|
||||||
|
|
||||||
nix-build ../fixed.nix -A git --no-out-link
|
nix-build ../fixed.nix -A git-sha1 --no-out-link
|
||||||
|
|
||||||
|
if isDaemonNewer "2.31pre20250724"; then
|
||||||
|
nix-build ../fixed.nix -A git-sha256 --no-out-link
|
||||||
|
fi
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,8 @@ suites += {
|
||||||
'name': 'git-hashing',
|
'name': 'git-hashing',
|
||||||
'deps': [],
|
'deps': [],
|
||||||
'tests': [
|
'tests': [
|
||||||
'simple.sh',
|
'simple-sha1.sh',
|
||||||
|
'simple-sha256.sh',
|
||||||
'fixed.sh',
|
'fixed.sh',
|
||||||
],
|
],
|
||||||
'workdir': meson.current_source_dir(),
|
'workdir': meson.current_source_dir(),
|
||||||
|
|
|
||||||
96
tests/functional/git-hashing/simple-common.sh
Normal file
96
tests/functional/git-hashing/simple-common.sh
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
# shellcheck shell=bash
|
||||||
|
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
# Assert is set
|
||||||
|
[[ ${hashAlgo+x} ]]
|
||||||
|
|
||||||
|
repo="$TEST_ROOT/scratch"
|
||||||
|
|
||||||
|
initRepo () {
|
||||||
|
git init "$repo" --object-format="$hashAlgo"
|
||||||
|
|
||||||
|
git -C "$repo" config user.email "you@example.com"
|
||||||
|
git -C "$repo" config user.name "Your Name"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Compare Nix's and git's implementation of git hashing
|
||||||
|
try () {
|
||||||
|
local expected="$1"
|
||||||
|
|
||||||
|
local hash
|
||||||
|
hash=$(nix hash path --mode git --format base16 --algo "$hashAlgo" "$TEST_ROOT/hash-path")
|
||||||
|
[[ "$hash" == "$expected" ]]
|
||||||
|
|
||||||
|
git -C "$repo" rm -rf hash-path || true
|
||||||
|
cp -r "$TEST_ROOT/hash-path" "$repo/hash-path"
|
||||||
|
git -C "$repo" add hash-path
|
||||||
|
git -C "$repo" commit -m "x"
|
||||||
|
git -C "$repo" status
|
||||||
|
local hash2
|
||||||
|
hash2=$(git -C "$repo" rev-parse HEAD:hash-path)
|
||||||
|
[[ "$hash2" = "$expected" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check Nix added object has matching git hash
|
||||||
|
try2 () {
|
||||||
|
local hashPath="$1"
|
||||||
|
local expected="$2"
|
||||||
|
|
||||||
|
local path
|
||||||
|
path=$(nix store add --mode git --hash-algo "$hashAlgo" "$repo/$hashPath")
|
||||||
|
|
||||||
|
git -C "$repo" add "$hashPath"
|
||||||
|
git -C "$repo" commit -m "x"
|
||||||
|
git -C "$repo" status
|
||||||
|
local hashFromGit
|
||||||
|
hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath")
|
||||||
|
[[ "$hashFromGit" == "$expected" ]]
|
||||||
|
|
||||||
|
local caFromNix
|
||||||
|
caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca")
|
||||||
|
[[ "fixed:git:$hashAlgo:$(nix hash convert --to nix32 "$hashAlgo:$hashFromGit")" = "$caFromNix" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
test0 () {
|
||||||
|
rm -rf "$TEST_ROOT/hash-path"
|
||||||
|
echo "Hello World" > "$TEST_ROOT/hash-path"
|
||||||
|
}
|
||||||
|
|
||||||
|
test1 () {
|
||||||
|
rm -rf "$TEST_ROOT/hash-path"
|
||||||
|
mkdir "$TEST_ROOT/hash-path"
|
||||||
|
echo "Hello World" > "$TEST_ROOT/hash-path/hello"
|
||||||
|
echo "Run Hello World" > "$TEST_ROOT/hash-path/executable"
|
||||||
|
chmod +x "$TEST_ROOT/hash-path/executable"
|
||||||
|
}
|
||||||
|
|
||||||
|
test2 () {
|
||||||
|
rm -rf "$repo/dummy1"
|
||||||
|
echo Hello World! > "$repo/dummy1"
|
||||||
|
}
|
||||||
|
|
||||||
|
test3 () {
|
||||||
|
rm -rf "$repo/dummy2"
|
||||||
|
mkdir -p "$repo/dummy2"
|
||||||
|
echo Hello World! > "$repo/dummy2/hello"
|
||||||
|
}
|
||||||
|
|
||||||
|
test4 () {
|
||||||
|
rm -rf "$repo/dummy3"
|
||||||
|
mkdir -p "$repo/dummy3"
|
||||||
|
mkdir -p "$repo/dummy3/dir"
|
||||||
|
touch "$repo/dummy3/dir/file"
|
||||||
|
echo Hello World! > "$repo/dummy3/dir/file"
|
||||||
|
touch "$repo/dummy3/dir/executable"
|
||||||
|
chmod +x "$repo/dummy3/dir/executable"
|
||||||
|
echo Run Hello World! > "$repo/dummy3/dir/executable"
|
||||||
|
}
|
||||||
|
|
||||||
|
test5 () {
|
||||||
|
rm -rf "$repo/dummy4"
|
||||||
|
mkdir -p "$repo/dummy4"
|
||||||
|
mkdir -p "$repo/dummy4/dir"
|
||||||
|
touch "$repo/dummy4/dir/file"
|
||||||
|
ln -s './hello/world.txt' "$repo/dummy4/dir/symlink"
|
||||||
|
}
|
||||||
27
tests/functional/git-hashing/simple-sha1.sh
Executable file
27
tests/functional/git-hashing/simple-sha1.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
hashAlgo=sha1
|
||||||
|
|
||||||
|
source simple-common.sh
|
||||||
|
|
||||||
|
initRepo
|
||||||
|
|
||||||
|
# blob
|
||||||
|
test0
|
||||||
|
try "557db03de997c86a4a028e1ebd3a1ceb225be238"
|
||||||
|
|
||||||
|
# tree with children
|
||||||
|
test1
|
||||||
|
try "e5c0a11a556801a5c9dcf330ca9d7e2c572697f4"
|
||||||
|
|
||||||
|
test2
|
||||||
|
try2 dummy1 "980a0d5f19a64b4b30a87d4206aade58726b60e3"
|
||||||
|
|
||||||
|
test3
|
||||||
|
try2 dummy2 "8b8e43b937854f4083ea56777821abda2799e850"
|
||||||
|
|
||||||
|
test4
|
||||||
|
try2 dummy3 "f227adfaf60d2778aabbf93df6dd061272d2dc85"
|
||||||
|
|
||||||
|
test5
|
||||||
|
try2 dummy4 "06f3e789820fc488d602358f03e3a1cbf993bf33"
|
||||||
29
tests/functional/git-hashing/simple-sha256.sh
Executable file
29
tests/functional/git-hashing/simple-sha256.sh
Executable file
|
|
@ -0,0 +1,29 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
hashAlgo=sha256
|
||||||
|
|
||||||
|
source simple-common.sh
|
||||||
|
|
||||||
|
requireDaemonNewerThan 2.31pre20250724
|
||||||
|
|
||||||
|
initRepo
|
||||||
|
|
||||||
|
# blob
|
||||||
|
test0
|
||||||
|
try "7c5c8610459154bdde4984be72c48fb5d9c1c4ac793a6b5976fe38fd1b0b1284"
|
||||||
|
|
||||||
|
# tree with children
|
||||||
|
test1
|
||||||
|
try "cd79952f42462467d0ea574b0283bb6eb77e15b2b86891e29f2b981650365474"
|
||||||
|
|
||||||
|
test2
|
||||||
|
try2 dummy1 "f5b5cec05fb6f9302b507a48c1573e6f36075e954d97caa8667f784e9cdb0d13"
|
||||||
|
|
||||||
|
test3
|
||||||
|
try2 dummy2 "399d851c74ceac2c2b61b53b13dcf5e88df3b6135c7df1f248a323c3c2f9aa78"
|
||||||
|
|
||||||
|
test4
|
||||||
|
try2 dummy3 "d3ae8fc87e76b9b871bd06a58c925c5fb5f83b5393f9f58e4f6dba3f59470289"
|
||||||
|
|
||||||
|
test5
|
||||||
|
try2 dummy4 "8c090dd057e8e01ffe1fec24a3133dfe52ba4eda822e67ee7fefc2af7c6a2906"
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
source common.sh
|
|
||||||
|
|
||||||
repo="$TEST_ROOT/scratch"
|
|
||||||
git init "$repo"
|
|
||||||
|
|
||||||
git -C "$repo" config user.email "you@example.com"
|
|
||||||
git -C "$repo" config user.name "Your Name"
|
|
||||||
|
|
||||||
# Compare Nix's and git's implementation of git hashing
|
|
||||||
try () {
|
|
||||||
local hash=$(nix hash path --mode git --format base16 --algo sha1 $TEST_ROOT/hash-path)
|
|
||||||
[[ "$hash" == "$1" ]]
|
|
||||||
|
|
||||||
git -C "$repo" rm -rf hash-path || true
|
|
||||||
cp -r "$TEST_ROOT/hash-path" "$TEST_ROOT/scratch/hash-path"
|
|
||||||
git -C "$repo" add hash-path
|
|
||||||
git -C "$repo" commit -m "x"
|
|
||||||
git -C "$repo" status
|
|
||||||
local hash2=$(git -C "$TEST_ROOT/scratch" rev-parse HEAD:hash-path)
|
|
||||||
[[ "$hash2" = "$1" ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
# blob
|
|
||||||
rm -rf $TEST_ROOT/hash-path
|
|
||||||
echo "Hello World" > $TEST_ROOT/hash-path
|
|
||||||
try "557db03de997c86a4a028e1ebd3a1ceb225be238"
|
|
||||||
|
|
||||||
# tree with children
|
|
||||||
rm -rf $TEST_ROOT/hash-path
|
|
||||||
mkdir $TEST_ROOT/hash-path
|
|
||||||
echo "Hello World" > $TEST_ROOT/hash-path/hello
|
|
||||||
echo "Run Hello World" > $TEST_ROOT/hash-path/executable
|
|
||||||
chmod +x $TEST_ROOT/hash-path/executable
|
|
||||||
try "e5c0a11a556801a5c9dcf330ca9d7e2c572697f4"
|
|
||||||
|
|
||||||
# Check Nix added object has matching git hash
|
|
||||||
try2 () {
|
|
||||||
local hashPath="$1"
|
|
||||||
local expected="$2"
|
|
||||||
|
|
||||||
local path=$(nix store add --mode git --hash-algo sha1 "$repo/$hashPath")
|
|
||||||
|
|
||||||
git -C "$repo" add "$hashPath"
|
|
||||||
git -C "$repo" commit -m "x"
|
|
||||||
git -C "$repo" status
|
|
||||||
local hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath")
|
|
||||||
[[ "$hashFromGit" == "$2" ]]
|
|
||||||
|
|
||||||
local caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca")
|
|
||||||
[[ "fixed:git:sha1:$(nix hash convert --to nix32 "sha1:$hashFromGit")" = "$caFromNix" ]]
|
|
||||||
}
|
|
||||||
|
|
||||||
rm -rf "$repo/dummy1"
|
|
||||||
echo Hello World! > "$repo/dummy1"
|
|
||||||
try2 dummy1 "980a0d5f19a64b4b30a87d4206aade58726b60e3"
|
|
||||||
|
|
||||||
rm -rf "$repo/dummy2"
|
|
||||||
mkdir -p "$repo/dummy2"
|
|
||||||
echo Hello World! > "$repo/dummy2/hello"
|
|
||||||
try2 dummy2 "8b8e43b937854f4083ea56777821abda2799e850"
|
|
||||||
|
|
||||||
rm -rf "$repo/dummy3"
|
|
||||||
mkdir -p "$repo/dummy3"
|
|
||||||
mkdir -p "$repo/dummy3/dir"
|
|
||||||
touch "$repo/dummy3/dir/file"
|
|
||||||
echo Hello World! > "$repo/dummy3/dir/file"
|
|
||||||
touch "$repo/dummy3/dir/executable"
|
|
||||||
chmod +x "$repo/dummy3/dir/executable"
|
|
||||||
echo Run Hello World! > "$repo/dummy3/dir/executable"
|
|
||||||
try2 dummy3 "f227adfaf60d2778aabbf93df6dd061272d2dc85"
|
|
||||||
|
|
||||||
rm -rf "$repo/dummy4"
|
|
||||||
mkdir -p "$repo/dummy4"
|
|
||||||
mkdir -p "$repo/dummy4/dir"
|
|
||||||
touch "$repo/dummy4/dir/file"
|
|
||||||
ln -s './hello/world.txt' "$repo/dummy4/dir/symlink"
|
|
||||||
try2 dummy4 "06f3e789820fc488d602358f03e3a1cbf993bf33"
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue