Merge remote-tracking branch 'upstream/master' into support-hardlinks-in-tarballs

src/libfetchers/.version

@@ -0,0 +1 @@
+../../.version

src/libfetchers/build-utils-meson

@@ -0,0 +1 @@
+../../build-utils-meson

src/libfetchers/fetch-settings.cc

@@ -1,4 +1,5 @@
 #include "fetch-settings.hh"
+#include "config-global.hh"
 
 namespace nix {
 

src/libfetchers/fetch-settings.hh

@@ -70,30 +70,6 @@ struct FetchSettings : public Config
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
 
-    Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
-        R"(
-          Path or URI of the global flake registry.
-
-          When empty, disables the global flake registry.
-        )",
-        {}, true, Xp::Flakes};
-
-    Setting<bool> useRegistries{this, true, "use-registries",
-        "Whether to use flake registries to resolve flake references.",
-        {}, true, Xp::Flakes};
-
-    Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
-        "Whether to accept nix configuration from a flake without prompting.",
-        {}, true, Xp::Flakes};
-
-    Setting<std::string> commitLockFileSummary{
-        this, "", "commit-lock-file-summary",
-        R"(
-          The commit summary to use when committing changed flake lock files. If
-          empty, the summary is generated based on the action performed.
-        )",
-        {"commit-lockfile-summary"}, true, Xp::Flakes};
-
     Setting<bool> trustTarballsFromGitForges{
         this, true, "trust-tarballs-from-git-forges",
         R"(
@@ -108,7 +84,6 @@ struct FetchSettings : public Config
           `narHash` attribute is specified,
           e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`.
         )"};
-
 };
 
 // FIXME: don't use a global variable.

src/libfetchers/fetch-to-store.hh

@@ -18,7 +18,7 @@ StorePath fetchToStore(
     const SourcePath & path,
     FetchMode mode,
     std::string_view name = "source",
-    ContentAddressMethod method = FileIngestionMethod::Recursive,
+    ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
     PathFilter * filter = nullptr,
     RepairFlag repair = NoRepair);
 

src/libfetchers/fetchers.cc

@@ -260,6 +260,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
 
     auto [accessor, final] = scheme->getAccessor(store, *this);
 
+    assert(!accessor->fingerprint);
     accessor->fingerprint = scheme->getFingerprint(store, final);
 
     return {accessor, std::move(final)};
@@ -305,7 +306,7 @@ StorePath Input::computeStorePath(Store & store) const
     if (!narHash)
         throw Error("cannot compute store path for unlocked input '%s'", to_string());
     return store.makeFixedOutputPath(getName(), FixedOutputInfo {
-        .method = FileIngestionMethod::Recursive,
+        .method = FileIngestionMethod::NixArchive,
         .hash = *narHash,
         .references = {},
     });
@@ -418,7 +419,7 @@ namespace nlohmann {
 using namespace nix;
 
 fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json) {
-    fetchers::PublicKey res = {  };
+    fetchers::PublicKey res = { };
     if (auto type = optionalValueAt(json, "type"))
         res.type = getString(*type);
 

src/libfetchers/git-utils.cc

@@ -851,10 +851,10 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 
     void createRegularFile(
-        const Path & path,
+        const CanonPath & path,
         std::function<void(CreateRegularFileSink &)> func) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
+        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
         if (!prepareDirs(pathComponents, false)) return;
 
         git_writestream * stream = nullptr;
@@ -862,11 +862,11 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
             throw Error("creating a blob stream object: %s", git_error_last()->message);
 
         struct CRF : CreateRegularFileSink {
-            const Path & path;
+            const CanonPath & path;
             GitFileSystemObjectSinkImpl & back;
             git_writestream * stream;
             bool executable = false;
-            CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream)
+            CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream)
                 : path(path), back(back), stream(stream)
             {}
             void operator () (std::string_view data) override
@@ -891,15 +891,15 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
             : GIT_FILEMODE_BLOB);
     }
 
-    void createDirectory(const Path & path) override
+    void createDirectory(const CanonPath & path) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
+        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
         (void) prepareDirs(pathComponents, true);
     }
 
-    void createSymlink(const Path & path, const std::string & target) override
+    void createSymlink(const CanonPath & path, const std::string & target) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path, "/");
+        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
         if (!prepareDirs(pathComponents, false)) return;
 
         git_oid oid;

src/libfetchers/git.cc

@@ -41,21 +41,6 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st)
     return st.st_mtime + settings.tarballTtl > now;
 }
 
-bool touchCacheFile(const Path & path, time_t touch_time)
-{
-#ifndef _WIN32 // TODO implement
-    struct timeval times[2];
-    times[0].tv_sec = touch_time;
-    times[0].tv_usec = 0;
-    times[1].tv_sec = touch_time;
-    times[1].tv_usec = 0;
-
-    return lutimes(path.c_str(), times) == 0;
-#else
-    return false;
-#endif
-}
-
 Path getCachePath(std::string_view key, bool shallow)
 {
     return getCacheDir()
@@ -594,8 +579,11 @@ struct GitInputScheme : InputScheme
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
                 }
 
-                if (!touchCacheFile(localRefFile, now))
-                    warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
+                try {
+                    setWriteTime(localRefFile, now, now);
+                } catch (Error & e) {
+                    warn("could not update mtime for file '%s': %s", localRefFile, e.msg());
+                }
                 if (!originalRef && !storeCachedHead(repoInfo.url, ref))
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
             }

src/libfetchers/github.cc

@@ -433,7 +433,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
                 store->toRealPath(
                     downloadFile(store, url, "source", headers).storePath)));
 
-        if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) {
+        if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
           return RefInfo {
               .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
           };

src/libfetchers/mercurial.cc

@@ -213,7 +213,7 @@ struct MercurialInputScheme : InputScheme
                 auto storePath = store->addToStore(
                     input.getName(),
                     {getFSSourceAccessor(), CanonPath(actualPath)},
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
+                    ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {},
                     filter);
 
                 return storePath;

src/libfetchers/meson.build

@@ -0,0 +1,93 @@
+project('nix-fetchers', 'cpp',
+  version : files('.version'),
+  default_options : [
+    'cpp_std=c++2a',
+    # TODO(Qyriad): increase the warning level
+    'warning_level=1',
+    'debug=true',
+    'optimization=2',
+    'errorlogs=true', # Please print logs for tests that fail
+  ],
+  meson_version : '>= 1.1',
+  license : 'LGPL-2.1-or-later',
+)
+
+cxx = meson.get_compiler('cpp')
+
+subdir('build-utils-meson/deps-lists')
+
+configdata = configuration_data()
+
+deps_private_maybe_subproject = [
+]
+deps_public_maybe_subproject = [
+  dependency('nix-util'),
+  dependency('nix-store'),
+]
+subdir('build-utils-meson/subprojects')
+
+nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
+deps_public += nlohmann_json
+
+libgit2 = dependency('libgit2')
+deps_private += libgit2
+
+add_project_arguments(
+  # TODO(Qyriad): Yes this is how the autoconf+Make system did it.
+  # It would be nice for our headers to be idempotent instead.
+  '-include', 'config-util.hh',
+  '-include', 'config-store.hh',
+  # '-include', 'config-fetchers.h',
+  language : 'cpp',
+)
+
+subdir('build-utils-meson/diagnostics')
+
+sources = files(
+  'attrs.cc',
+  'cache.cc',
+  'fetch-settings.cc',
+  'fetch-to-store.cc',
+  'fetchers.cc',
+  'filtering-source-accessor.cc',
+  'git.cc',
+  'git-utils.cc',
+  'github.cc',
+  'indirect.cc',
+  'mercurial.cc',
+  'mounted-source-accessor.cc',
+  'path.cc',
+  'store-path-accessor.cc',
+  'registry.cc',
+  'tarball.cc',
+)
+
+include_dirs = [include_directories('.')]
+
+headers = files(
+  'attrs.hh',
+  'cache.hh',
+  'fetch-settings.hh',
+  'fetch-to-store.hh',
+  'filtering-source-accessor.hh',
+  'git-utils.hh',
+  'mounted-source-accessor.hh',
+  'fetchers.hh',
+  'registry.hh',
+  'store-path-accessor.hh',
+  'tarball.hh',
+)
+
+this_library = library(
+  'nixfetchers',
+  sources,
+  dependencies : deps_public + deps_private + deps_other,
+  prelink : true, # For C++ static initializers
+  install : true,
+)
+
+install_headers(headers, subdir : 'nix', preserve_path : true)
+
+libraries_private = []
+
+subdir('build-utils-meson/export')

src/libfetchers/package.nix

@@ -0,0 +1,84 @@
+{ lib
+, stdenv
+, mkMesonDerivation
+, releaseTools
+
+, meson
+, ninja
+, pkg-config
+
+, nix-util
+, nix-store
+, nlohmann_json
+, libgit2
+, man
+
+# Configuration Options
+
+, version
+}:
+
+let
+  inherit (lib) fileset;
+in
+
+mkMesonDerivation (finalAttrs: {
+  pname = "nix-fetchers";
+  inherit version;
+
+  workDir = ./.;
+  fileset = fileset.unions [
+    ../../build-utils-meson
+    ./build-utils-meson
+    ../../.version
+    ./.version
+    ./meson.build
+    (fileset.fileFilter (file: file.hasExt "cc") ./.)
+    (fileset.fileFilter (file: file.hasExt "hh") ./.)
+  ];
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    libgit2
+  ];
+
+  propagatedBuildInputs = [
+    nix-store
+    nix-util
+    nlohmann_json
+  ];
+
+  preConfigure =
+    # "Inline" .version so it's not a symlink, and includes the suffix.
+    # Do the meson utils, without modification.
+    ''
+      chmod u+w ./.version
+      echo ${version} > ../../.version
+    '';
+
+  env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) {
+    LDFLAGS = "-fuse-ld=gold";
+  };
+
+  enableParallelBuilding = true;
+
+  separateDebugInfo = !stdenv.hostPlatform.isStatic;
+
+  # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated
+  # to work with `strictDeps`.
+  strictDeps = true;
+
+  hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+
+  meta = {
+    platforms = lib.platforms.unix ++ lib.platforms.windows;
+  };
+
+})

src/libfetchers/registry.cc

@@ -1,12 +1,11 @@
 #include "registry.hh"
 #include "tarball.hh"
 #include "users.hh"
+#include "config-global.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 
-#include "fetch-settings.hh"
-
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -149,10 +148,25 @@ void overrideRegistry(
     flagRegistry->add(from, to, extraAttrs);
 }
 
+struct RegistrySettings : Config
+{
+    Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
+        R"(
+          Path or URI of the global flake registry.
+
+          When empty, disables the global flake registry.
+        )",
+        {}, true, Xp::Flakes};
+};
+
+RegistrySettings registrySettings;
+
+static GlobalConfig::Register rRegistrySettings(&registrySettings);
+
 static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
 {
     static auto reg = [&]() {
-        auto path = fetchSettings.flakeRegistry.get();
+        auto path = registrySettings.flakeRegistry.get();
         if (path == "") {
             return std::make_shared<Registry>(Registry::Global); // empty registry
         }

src/libfetchers/tarball.cc

@@ -365,6 +365,16 @@ struct TarballInputScheme : CurlInputScheme
 
         return {result.accessor, input};
     }
+
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
+    {
+        if (auto narHash = input.getNarHash())
+            return narHash->to_string(HashFormat::SRI, true);
+        else if (auto rev = input.getRev())
+            return rev->gitRev();
+        else
+            return std::nullopt;
+    }
 };
 
 static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });