1
1
Fork 0
mirror of https://github.com/NixOS/nix.git synced 2025-11-30 22:20:59 +01:00

Merge pull request #13693 from obsidiansystems/more-base-files

Clean up Base* code
This commit is contained in:
John Ericson 2025-08-06 13:02:01 -04:00 committed by GitHub
commit e25ab029ae
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 357 additions and 244 deletions

View file

@ -2,6 +2,7 @@
#include "nix/fetchers/git-lfs-fetch.hh"
#include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/base-n.hh"
#include "nix/util/finally.hh"
#include "nix/util/processes.hh"
#include "nix/util/signals.hh"
@ -608,7 +609,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
std::string keyDecoded;
try {
keyDecoded = base64Decode(k.key);
keyDecoded = base64::decode(k.key);
} catch (Error & e) {
e.addTrace({}, "while decoding public key '%s' used for git signature", k.key);
}

View file

@ -1,3 +1,4 @@
#include "nix/util/base-n.hh"
#include "nix/store/machines.hh"
#include "nix/store/globals.hh"
#include "nix/store/store-open.hh"
@ -158,7 +159,7 @@ static Machine parseBuilderLine(const StringSet & defaultSystems, const std::str
auto ensureBase64 = [&](size_t fieldIndex) {
const auto & str = tokens[fieldIndex];
try {
base64Decode(str);
base64::decode(str);
} catch (FormatError & e) {
e.addTrace({}, "while parsing machine specification at a column #%lu in a row: '%s'", fieldIndex, line);
throw;

View file

@ -4,13 +4,14 @@
#include "nix/util/environment-variables.hh"
#include "nix/util/util.hh"
#include "nix/util/exec.hh"
#include "nix/util/base-n.hh"
namespace nix {
static std::string parsePublicHostKey(std::string_view host, std::string_view sshPublicHostKey)
{
try {
return base64Decode(sshPublicHostKey);
return base64::decode(sshPublicHostKey);
} catch (Error & e) {
e.addTrace({}, "while decoding ssh public host key for host '%s'", host);
throw;

View file

@ -0,0 +1,68 @@
#include <gtest/gtest.h>
#include <numeric>
#include "nix/util/base-n.hh"
#include "nix/util/error.hh"
namespace nix {
static const std::span<const std::byte> stringToByteSpan(const std::string_view s)
{
return {(const std::byte *) s.data(), s.size()};
}
/* ----------------------------------------------------------------------------
* base64::encode
* --------------------------------------------------------------------------*/
TEST(base64Encode, emptyString)
{
ASSERT_EQ(base64::encode(stringToByteSpan("")), "");
}
TEST(base64Encode, encodesAString)
{
ASSERT_EQ(base64::encode(stringToByteSpan("quod erat demonstrandum")), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
}
TEST(base64Encode, encodeAndDecode)
{
auto s = "quod erat demonstrandum";
auto encoded = base64::encode(stringToByteSpan(s));
auto decoded = base64::decode(encoded);
ASSERT_EQ(decoded, s);
}
TEST(base64Encode, encodeAndDecodeNonPrintable)
{
char s[256];
std::iota(std::rbegin(s), std::rend(s), 0);
auto encoded = base64::encode(std::as_bytes(std::span<const char>{std::string_view{s}}));
auto decoded = base64::decode(encoded);
EXPECT_EQ(decoded.length(), 255u);
ASSERT_EQ(decoded, s);
}
/* ----------------------------------------------------------------------------
* base64::decode
* --------------------------------------------------------------------------*/
TEST(base64Decode, emptyString)
{
ASSERT_EQ(base64::decode(""), "");
}
TEST(base64Decode, decodeAString)
{
ASSERT_EQ(base64::decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
}
TEST(base64Decode, decodeThrowsOnInvalidChar)
{
ASSERT_THROW(base64::decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
}
} // namespace nix

View file

@ -44,6 +44,7 @@ subdir('nix-meson-build-support/common')
sources = files(
'args.cc',
'base-n.cc',
'canon-path.cc',
'checked-arithmetic.cc',
'chunked-vector.cc',

View file

@ -3,6 +3,7 @@
#include "nix/util/file-system.hh"
#include "nix/util/terminal.hh"
#include "nix/util/strings.hh"
#include "nix/util/base-n.hh"
#include <limits.h>
#include <gtest/gtest.h>
@ -48,60 +49,6 @@ TEST(hasSuffix, trivialCase)
ASSERT_TRUE(hasSuffix("foobar", "bar"));
}
/* ----------------------------------------------------------------------------
* base64Encode
* --------------------------------------------------------------------------*/
TEST(base64Encode, emptyString)
{
ASSERT_EQ(base64Encode(""), "");
}
TEST(base64Encode, encodesAString)
{
ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
}
TEST(base64Encode, encodeAndDecode)
{
auto s = "quod erat demonstrandum";
auto encoded = base64Encode(s);
auto decoded = base64Decode(encoded);
ASSERT_EQ(decoded, s);
}
TEST(base64Encode, encodeAndDecodeNonPrintable)
{
char s[256];
std::iota(std::rbegin(s), std::rend(s), 0);
auto encoded = base64Encode(s);
auto decoded = base64Decode(encoded);
EXPECT_EQ(decoded.length(), 255u);
ASSERT_EQ(decoded, s);
}
/* ----------------------------------------------------------------------------
* base64Decode
* --------------------------------------------------------------------------*/
TEST(base64Decode, emptyString)
{
ASSERT_EQ(base64Decode(""), "");
}
TEST(base64Decode, decodeAString)
{
ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
}
TEST(base64Decode, decodeThrowsOnInvalidChar)
{
ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
}
/* ----------------------------------------------------------------------------
* getLine
* --------------------------------------------------------------------------*/

114
src/libutil/base-n.cc Normal file
View file

@ -0,0 +1,114 @@
#include <string_view>
#include "nix/util/array-from-string-literal.hh"
#include "nix/util/util.hh"
#include "nix/util/base-n.hh"
using namespace std::literals;
namespace nix {
constexpr static const std::array<char, 16> base16Chars = "0123456789abcdef"_arrayNoNull;
std::string base16::encode(std::span<const std::byte> b)
{
std::string buf;
buf.reserve(b.size() * 2);
for (size_t i = 0; i < b.size(); i++) {
buf.push_back(base16Chars[(uint8_t) b.data()[i] >> 4]);
buf.push_back(base16Chars[(uint8_t) b.data()[i] & 0x0f]);
}
return buf;
}
std::string base16::decode(std::string_view s)
{
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
throw FormatError("invalid character in Base16 string: '%c'", c);
};
assert(s.size() % 2 == 0);
auto decodedSize = s.size() / 2;
std::string res;
res.reserve(decodedSize);
for (unsigned int i = 0; i < decodedSize; i++) {
res.push_back(parseHexDigit(s[i * 2]) << 4 | parseHexDigit(s[i * 2 + 1]));
}
return res;
}
constexpr static const std::array<char, 64> base64Chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"_arrayNoNull;
std::string base64::encode(std::span<const std::byte> s)
{
std::string res;
res.reserve((s.size() + 2) / 3 * 4);
int data = 0, nbits = 0;
for (std::byte c : s) {
data = data << 8 | (uint8_t) c;
nbits += 8;
while (nbits >= 6) {
nbits -= 6;
res.push_back(base64Chars[data >> nbits & 0x3f]);
}
}
if (nbits)
res.push_back(base64Chars[data << (6 - nbits) & 0x3f]);
while (res.size() % 4)
res.push_back('=');
return res;
}
std::string base64::decode(std::string_view s)
{
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&] {
std::array<char, 256> result{};
for (auto & c : result)
c = npos;
for (int i = 0; i < 64; i++)
result[base64Chars[i]] = i;
return result;
}();
std::string res;
// Some sequences are missing the padding consisting of up to two '='.
// vvv
res.reserve((s.size() + 2) / 4 * 3);
unsigned int d = 0, bits = 0;
for (char c : s) {
if (c == '=')
break;
if (c == '\n')
continue;
char digit = base64DecodeChars[(unsigned char) c];
if (digit == npos)
throw FormatError("invalid character in Base64 string: '%c'", c);
bits += 6;
d = d << 6 | digit;
if (bits >= 8) {
res.push_back(d >> (bits - 8) & 0xff);
bits -= 8;
}
}
return res;
}
} // namespace nix

View file

@ -1,6 +1,7 @@
#include <cassert>
#include "nix/util/base-nix-32.hh"
#include "nix/util/util.hh"
namespace nix {
@ -16,12 +17,12 @@ constexpr const std::array<unsigned char, 256> BaseNix32::reverseMap = [] {
return map;
}();
std::string BaseNix32::encode(std::span<const uint8_t> originalData)
std::string BaseNix32::encode(std::span<const std::byte> bs)
{
if (originalData.size() == 0)
if (bs.size() == 0)
return {};
size_t len = encodedLength(originalData.size());
size_t len = encodedLength(bs.size());
assert(len);
std::string s;
@ -31,12 +32,42 @@ std::string BaseNix32::encode(std::span<const uint8_t> originalData)
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
unsigned char c =
(originalData.data()[i] >> j) | (i >= originalData.size() - 1 ? 0 : originalData.data()[i + 1] << (8 - j));
s.push_back(characters[c & 0x1f]);
std::byte c = (bs.data()[i] >> j) | (i >= bs.size() - 1 ? std::byte{0} : bs.data()[i + 1] << (8 - j));
s.push_back(characters[uint8_t(c & std::byte{0x1f})]);
}
return s;
}
std::string BaseNix32::decode(std::string_view s)
{
std::string res;
res.reserve((s.size() * 5 + 7) / 8); // ceiling(size * 5/8)
for (unsigned int n = 0; n < s.size(); ++n) {
char c = s[s.size() - n - 1];
auto digit_opt = BaseNix32::lookupReverse(c);
if (!digit_opt)
throw FormatError("invalid character in Nix32 (Nix's Base32 variation) string: '%c'", c);
uint8_t digit = *digit_opt;
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
// Ensure res has enough space
res.resize(i + 1);
res[i] |= digit << j;
if (digit >> (8 - j)) {
res.resize(i + 2);
res[i + 1] |= digit >> (8 - j);
}
}
return res;
}
} // namespace nix

View file

@ -11,6 +11,7 @@
#include "nix/util/archive.hh"
#include "nix/util/configuration.hh"
#include "nix/util/split.hh"
#include "nix/util/base-n.hh"
#include "nix/util/base-nix-32.hh"
#include <sys/types.h>
@ -59,25 +60,6 @@ std::strong_ordering Hash::operator<=>(const Hash & h) const noexcept
return std::strong_ordering::equivalent;
}
const std::string base16Chars = "0123456789abcdef";
static std::string printHash16(const Hash & hash)
{
std::string buf;
buf.reserve(hash.hashSize * 2);
for (unsigned int i = 0; i < hash.hashSize; i++) {
buf.push_back(base16Chars[hash.hash[i] >> 4]);
buf.push_back(base16Chars[hash.hash[i] & 0x0f]);
}
return buf;
}
static std::string printHash32(const Hash & hash)
{
assert(hash.hashSize);
return BaseNix32::encode({&hash.hash[0], hash.hashSize});
}
std::string printHash16or32(const Hash & hash)
{
assert(static_cast<char>(hash.algo));
@ -91,16 +73,20 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
s += printHashAlgo(algo);
s += hashFormat == HashFormat::SRI ? '-' : ':';
}
const auto bytes = std::as_bytes(std::span<const uint8_t>{&hash[0], hashSize});
switch (hashFormat) {
case HashFormat::Base16:
s += printHash16(*this);
assert(hashSize);
s += base16::encode(bytes);
break;
case HashFormat::Nix32:
s += printHash32(*this);
assert(hashSize);
s += BaseNix32::encode(bytes);
break;
case HashFormat::Base64:
case HashFormat::SRI:
s += base64Encode(std::string_view((const char *) hash, hashSize));
assert(hashSize);
s += base64::encode(bytes);
break;
}
return s;
@ -180,63 +166,38 @@ Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
: Hash(algo)
{
if (!isSRI && rest.size() == base16Len()) {
auto [decode, formatName] = [&]() -> std::pair<decltype(base16::decode) *, std::string_view> {
if (isSRI) {
/* In the SRI case, we always are using Base64. If the
length is wrong, get an error later. */
return {base64::decode, "SRI"};
} else {
/* Otherwise, decide via the length of the hash (for the
given algorithm) what base encoding it is. */
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
throw BadHash("invalid base-16 hash '%s'", rest);
};
if (rest.size() == base16::encodedLength(hashSize))
return {base16::decode, "base16"};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] = parseHexDigit(rest[i * 2]) << 4 | parseHexDigit(rest[i * 2 + 1]);
if (rest.size() == BaseNix32::encodedLength(hashSize))
return {BaseNix32::decode, "nix32"};
if (rest.size() == base64::encodedLength(hashSize))
return {base64::decode, "Base64"};
}
}
else if (!isSRI && rest.size() == base32Len()) {
for (unsigned int n = 0; n < rest.size(); ++n) {
char c = rest[rest.size() - n - 1];
auto digit_opt = BaseNix32::lookupReverse(c);
if (!digit_opt)
throw BadHash("invalid base-32 hash: '%s'", rest);
uint8_t digit = std::move(*digit_opt);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
hash[i] |= digit << j;
if (i < hashSize - 1) {
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j))
throw BadHash("invalid base-32 hash '%s'", rest);
}
}
}
else if (isSRI || rest.size() == base64Len()) {
std::string d;
try {
d = base64Decode(rest);
} catch (Error & e) {
e.addTrace({}, "While decoding hash '%s'", rest);
}
if (d.size() != hashSize)
throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
else
throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
}();
std::string d;
try {
d = decode(rest);
} catch (Error & e) {
e.addTrace({}, "While decoding hash '%s'", rest);
}
if (d.size() != hashSize)
throw BadHash("invalid %s hash '%s' %d %d", formatName, rest);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
Hash Hash::random(HashAlgorithm algo)

View file

@ -0,0 +1,27 @@
#pragma once
///@file
#include <algorithm>
#include <array>
namespace nix {
template<size_t sizeWithNull>
struct ArrayNoNullAdaptor
{
std::array<char, sizeWithNull - 1> data;
constexpr ArrayNoNullAdaptor(const char (&init)[sizeWithNull])
{
static_assert(sizeWithNull > 0);
std::copy_n(init, sizeWithNull - 1, data.data());
}
};
template<ArrayNoNullAdaptor str>
constexpr auto operator""_arrayNoNull()
{
return str.data;
}
} // namespace nix

View file

@ -0,0 +1,53 @@
#pragma once
///@file
#include <string>
#include <span>
namespace nix {
namespace base16 {
/**
* Returns the length of a base-16 representation of this many bytes.
*/
[[nodiscard]] constexpr static inline size_t encodedLength(size_t origSize)
{
return origSize * 2;
}
/**
* Encode arbitrary bytes as Base16.
*/
std::string encode(std::span<const std::byte> b);
/**
* Decode arbitrary Base16 string to bytes.
*/
std::string decode(std::string_view s);
} // namespace base16
namespace base64 {
/**
* Returns the length of a base-64 representation of this many bytes.
*/
[[nodiscard]] constexpr static inline size_t encodedLength(size_t origSize)
{
return ((4 * origSize / 3) + 3) & ~3;
}
/**
* Encode arbitrary bytes as Base64.
*/
std::string encode(std::span<const std::byte> b);
/**
* Decode arbitrary Base64 string to bytes.
*/
std::string decode(std::string_view s);
} // namespace base64
} // namespace nix

View file

@ -7,14 +7,14 @@
#include <string>
#include <span>
#include "nix/util/array-from-string-literal.hh"
namespace nix {
struct BaseNix32
{
/// omitted: E O U T
constexpr static std::array<char, 32> characters = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a',
'b', 'c', 'd', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'p', 'q', 'r', 's', 'v', 'w', 'x', 'y', 'z'};
constexpr static std::array<char, 32> characters = "0123456789abcdfghijklmnpqrsvwxyz"_arrayNoNull;
private:
static const std::array<uint8_t, 256> reverseMap;
@ -34,12 +34,14 @@ public:
/**
* Returns the length of a base-32 representation of this hash.
*/
static size_t encodedLength(size_t originalLength)
[[nodiscard]] constexpr static inline size_t encodedLength(size_t originalLength)
{
return (originalLength * 8 - 1) / 5 + 1;
}
static std::string encode(std::span<const uint8_t> originalData);
static std::string encode(std::span<const std::byte> originalData);
static std::string decode(std::string_view s);
};
} // namespace nix

View file

@ -110,30 +110,6 @@ public:
*/
std::strong_ordering operator<=>(const Hash & h2) const noexcept;
/**
* Returns the length of a base-16 representation of this hash.
*/
[[nodiscard]] size_t base16Len() const
{
return hashSize * 2;
}
/**
* Returns the length of a base-32 representation of this hash.
*/
[[nodiscard]] size_t base32Len() const
{
return (hashSize * 8 - 1) / 5 + 1;
}
/**
* Returns the length of a base-64 representation of this hash.
*/
[[nodiscard]] size_t base64Len() const
{
return ((4 * hashSize / 3) + 3) & ~3;
}
/**
* Return a string representation of the hash, in base-16, base-32
* or base-64. By default, this is prefixed by the hash algo

View file

@ -8,6 +8,8 @@ headers = files(
'archive.hh',
'args.hh',
'args/root.hh',
'array-from-string-literal.hh',
'base-n.hh',
'base-nix-32.hh',
'callback.hh',
'canon-path.hh',

View file

@ -179,16 +179,6 @@ constexpr char treeLast[] = "└───";
constexpr char treeLine[] = "";
constexpr char treeNull[] = " ";
/**
* Encode arbitrary bytes as Base64.
*/
std::string base64Encode(std::string_view s);
/**
* Decode arbitrary bytes to Base64.
*/
std::string base64Decode(std::string_view s);
/**
* Remove common leading whitespace from the lines in the string
* 's'. For example, if every line is indented by at least 3 spaces,

View file

@ -112,6 +112,7 @@ subdir('nix-meson-build-support/common')
sources = [config_priv_h] + files(
'archive.cc',
'args.cc',
'base-n.cc',
'base-nix-32.cc',
'canon-path.cc',
'compression.cc',

View file

@ -1,6 +1,7 @@
#include "nix/util/signature/local-keys.hh"
#include "nix/util/file-system.hh"
#include "nix/util/base-n.hh"
#include "nix/util/util.hh"
#include <sodium.h>
@ -25,7 +26,7 @@ Key::Key(std::string_view s, bool sensitiveValue)
if (name == "" || key == "")
throw FormatError("key is corrupt");
key = base64Decode(key);
key = base64::decode(key);
} catch (Error & e) {
std::string extra;
if (!sensitiveValue)
@ -37,7 +38,7 @@ Key::Key(std::string_view s, bool sensitiveValue)
std::string Key::to_string() const
{
return name + ":" + base64Encode(key);
return name + ":" + base64::encode(std::as_bytes(std::span<const char>{key}));
}
SecretKey::SecretKey(std::string_view s)
@ -52,7 +53,7 @@ std::string SecretKey::signDetached(std::string_view data) const
unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen;
crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data());
return name + ":" + base64Encode(std::string((char *) sig, sigLen));
return name + ":" + base64::encode(std::as_bytes(std::span<const unsigned char>{sig, sigLen}));
}
PublicKey SecretKey::toPublicKey() const
@ -93,7 +94,7 @@ bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig)
{
std::string sig2;
try {
sig2 = base64Decode(sig);
sig2 = base64::decode(sig);
} catch (Error & e) {
e.addTrace({}, "while decoding signature '%s'", sig);
}

View file

@ -204,70 +204,6 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl)
}
}
constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
std::string base64Encode(std::string_view s)
{
std::string res;
res.reserve((s.size() + 2) / 3 * 4);
int data = 0, nbits = 0;
for (char c : s) {
data = data << 8 | (unsigned char) c;
nbits += 8;
while (nbits >= 6) {
nbits -= 6;
res.push_back(base64Chars[data >> nbits & 0x3f]);
}
}
if (nbits)
res.push_back(base64Chars[data << (6 - nbits) & 0x3f]);
while (res.size() % 4)
res.push_back('=');
return res;
}
std::string base64Decode(std::string_view s)
{
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&] {
std::array<char, 256> result{};
for (auto & c : result)
c = npos;
for (int i = 0; i < 64; i++)
result[base64Chars[i]] = i;
return result;
}();
std::string res;
// Some sequences are missing the padding consisting of up to two '='.
// vvv
res.reserve((s.size() + 2) / 4 * 3);
unsigned int d = 0, bits = 0;
for (char c : s) {
if (c == '=')
break;
if (c == '\n')
continue;
char digit = base64DecodeChars[(unsigned char) c];
if (digit == npos)
throw FormatError("invalid character in Base64 string: '%c'", c);
bits += 6;
d = d << 6 | digit;
if (bits >= 8) {
res.push_back(d >> (bits - 8) & 0xff);
bits -= 8;
}
}
return res;
}
std::string stripIndentation(std::string_view s)
{
size_t minIndent = 10000;