Grab proot from bootstrap zip rather than including its nix path directly.

This means that the cachix substituter (or already having the package in your nix store somehow) is no longer required to build.

This required reworking the deploy script. As a bonus you can now omit the second argument and it will tell you what it would have copied instead of copying anything.

This is fixes one source of impurity, but for now flake builds will still require the --impure flag

.github/workflows/emulator.yml

@@ -47,7 +47,7 @@ jobs:
           rm -rf n-o-d
           mkdir -p n-o-d
           git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > n-o-d/archive.tar.gz
-          ARCHES=x86_64 nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz n-o-d/
+          ARCHES=x86_64 nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target n-o-d/
           tar cf n-o-d.tar n-o-d
 
       - name: Store zipball and channel tarball to inject (n-o-d)

flake.nix

@@ -59,7 +59,7 @@
 
         deploy = {
           type = "app";
-          program = toString (import ./scripts/deploy.nix { inherit nixpkgs system; });
+          program = import ./scripts/deploy.nix { inherit nixpkgs system; };
         };
       });
 

modules/environment/login/default.nix

@@ -37,7 +37,7 @@ in
 
       prootStatic = mkOption {
         type = types.package;
-        readOnly = true;
+        # not readOnly, this needs to be overridden when building bootstrap zip
         internal = true;
         description = "<literal>proot-static</literal> package.";
       };
@@ -84,14 +84,27 @@ in
     environment.files = {
       inherit login loginInner;
 
+      # Ideally this would build the static proot binary, but doing that on aarch64 is HARD so instead pull it from the bootstrap tarball
       prootStatic =
         let
-          crossCompiledPaths = {
+          attrs = (import ./proot-attrs).${targetSystem};
-            aarch64-linux = "/nix/store/7qd99m1w65x2vgqg453nd70y60sm3kay-proot-termux-static-aarch64-unknown-linux-android-unstable-2024-05-04";
+          prootFile = pkgs.fetchurl {
-            x86_64-linux = "/nix/store/pakj3svvw84rhkzdc6211yhc2cgvc21f-proot-termux-static-x86_64-unknown-linux-android-unstable-2024-05-04";
+            name = "proot-static-file";
+            inherit (attrs) url hash;
+
+            downloadToTemp = true;
+            executable = true;
+            postFetch = ''
+              ${pkgs.unzip}/bin/unzip -u $downloadedFile bin/proot-static
+              echo $PWD >&2
+              mv bin/proot-static $out
+            '';
           };
         in
-        "${crossCompiledPaths.${targetSystem}}";
+        pkgs.runCommand "proot-static" { } ''
+          mkdir -p $out/bin
+          cp ${prootFile} $out/bin/proot-static
+        '';
     };
 
   };

modules/environment/login/proot-attrs/aarch64.nix

@@ -0,0 +1,5 @@
+# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+{
+  url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-aarch64.zip";
+  hash = "sha256-fZyqldmHWbv2e6543mwb5UPcKxcODRg+PDvZNcjVyUU=";
+}

modules/environment/login/proot-attrs/default.nix

@@ -0,0 +1,4 @@
+{
+  x86_64-linux = import ./x86_64.nix;
+  aarch64-linux = import ./aarch64.nix;
+}

modules/environment/login/proot-attrs/x86_64.nix

@@ -0,0 +1,5 @@
+# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+{
+  url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-x86_64.zip";
+  hash = "sha256-1WZBmFNEmZucOwuzDAFO+Sl+b2XO7Lp1aQr/4egl4wU=";
+}

pkgs/default.nix

@@ -42,6 +42,8 @@ let
       # Fix invoking bash after initial build.
       user.shell = "${initialPackageInfo.bash}/bin/bash";
 
+      environment.files.prootStatic = pkgs.lib.mkForce customPkgs.prootTermux;
+
       build = {
         channel = {
           nixpkgs = urlOptionValue nixpkgsChannelURL "NIXPKGS_CHANNEL_URL";

scripts/deploy.nix

@@ -4,31 +4,44 @@
 
 let
   pkgs = nixpkgs.legacyPackages.${system};
-
+  pypkgs = pkgs.python311Packages;
-  runtimePackages = with pkgs; [
+  disablePyLints = [
-    coreutils
+    "line-too-long"
-    git
+    "missing-module-docstring"
-    gnugrep
+    "wrong-import-position" # import should be at top of file: we purposefully don't import click and such so that users that try to run the script directly get a friendly error
-    gnused
+    "missing-function-docstring"
-    gnutar
+    # c'mon, it's a script
-    gzip
+    "too-many-locals"
-    jq
+    "too-many-branches"
-    nix
+    "too-many-statements"
-    openssh
-    rsync
   ];
+  deriv = pypkgs.buildPythonApplication {
+    pname = "deploy";
+    version = "0.0";
+    src = ./.;
+
+    inherit (pkgs) nix git rsync;
+
+    propagatedBuildInputs = [ pypkgs.click ];
+
+    doCheck = true;
+    nativeCheckInputs = with pypkgs; [ mypy pylint black ];
+    checkPhase = ''
+      mypy --strict --no-color deploy.py
+      PYLINTHOME="$PWD/.pylint" pylint \
+        --score=n \
+        --clear-cache-post-run=y \
+        --disable=${pkgs.lib.concatStringsSep "," disablePyLints} \
+        deploy.py
+      black --check --diff deploy.py
+    '';
+
+    patchPhase = ''
+      substituteInPlace deploy.py \
+        --subst-var nix \
+        --subst-var git \
+        --subst-var rsync
+    '';
+  };
 in
-
+"${deriv}/bin/deploy"
-pkgs.runCommand
-  "deploy"
-{
-  preferLocalBuild = true;
-  allowSubstitutes = false;
-}
-  ''
-    install -D -m755  ${./deploy.sh} $out
-
-    substituteInPlace $out \
-      --subst-var-by bash "${pkgs.bash}" \
-      --subst-var-by path "${pkgs.lib.makeBinPath runtimePackages}"
-  ''

scripts/deploy.py

@@ -0,0 +1,181 @@
+import os
+import sys
+
+GIT = "@git@/bin/git"
+NIX = "@nix@/bin/nix"
+NIX_HASH = "@nix@/bin/nix-hash"
+RSYNC = "@rsync@/bin/rsync"
+
+if GIT.startswith("@"):
+    sys.stderr.write(
+        "Do not run this script directly, instead try: nix run .#deploy -- --help"
+    )
+    sys.exit(1)
+
+import subprocess
+import re
+import inspect
+from typing import Never
+from pathlib import Path
+
+import click
+
+
+def err(text: str) -> Never:
+    sys.stderr.write(text)
+    sys.exit(1)
+
+
+def run(*args: str) -> None:
+    subprocess.run(args, check=True)
+
+
+def run_capture(*args: str, env: dict[str, str] | None = None) -> str:
+    proc = subprocess.run(
+        args, check=True, stdout=subprocess.PIPE, stderr=None, env=env
+    )
+    return proc.stdout.decode("utf-8").strip()
+
+
+def log(msg: str) -> None:
+    print(f"> {msg}")
+
+
+@click.command()
+@click.option(
+    "--rsync-target",
+    help="Where bootstrap zipballs and source tarball will be copied to. If given, this is passed directly to rsync so it can be a local folder, ssh path, etc. For production builds this should be a webroot directory that will be served at bootstrap-url",
+)
+@click.option(
+    "--bootstrap-url",
+    help="URL where bootstrap zip files are available. Defaults to folder part of public-url if not given.",
+)
+@click.option(
+    "--arches",
+    default="aarch64,x86_64",
+    help="Which architectures to build for, comma-separated.",
+)
+@click.argument("public-url")
+def go(
+    public_url: str,
+    rsync_target: str | None,
+    bootstrap_url: str | None,
+    arches: str,
+) -> None:
+    """
+    Builds bootstrap zip balls and source code tar ball (for usage as a channel or flake). If rsync_target is specified, uploads it to the directory specified in rsync_target. The contents of this directory should be reachable by the android device with public_url.
+
+    Examples:
+
+    \b
+    $ nix run .#deploy -- \\
+        'https://example.com/bootstrap/source.tar.gz' \\
+        --rsync-target 'user@host:/path/to/bootstrap'
+
+    \b
+    $ nix run .#deploy -- \\
+        'github:USER/nix-on-droid/BRANCH' \\
+        --rsync-target 'user@host:/path/to/bootstrap' \\
+        --bootstrap-url 'https://example.com/bootstrap/'
+
+    \b
+    $ nix run .#deploy -- \\
+        'file:///data/local/tmp/n-o-d/archive.tar.gz'
+
+    ^ useful for testing. Note this is a path on the android device running the APK, not on the build machine
+    """
+    repo_dir = run_capture(GIT, "rev-parse", "--show-toplevel")
+    os.chdir(repo_dir)
+    source_file = "source.tar.gz"
+    if (m := re.search("^github:(.*)/(.*)/(.*)", public_url)) is not None:
+        channel_url = f"https://github.com/{m[1]}/{m[2]}/archive/{m[3]}.tar.gz"
+        if bootstrap_url is None:
+            err("--botstrap-url must be provided for github URLs")
+    elif re.search("^(https?|file)://", public_url):
+        channel_url = public_url
+    else:
+        err(f"unsupported url {public_url}")
+
+    # for CI and local testing
+    if (m := re.search("^file:///(.*)/archive.tar.gz$", public_url)) is not None:
+        flake_url = f"/{m[1]}/unpacked"
+    else:
+        flake_url = public_url
+    base_url = re.sub("/[^/]*$", "", public_url)
+    if bootstrap_url is None:
+        bootstrap_url = base_url
+
+    log(f"channel_url = {channel_url}")
+    log(f"flake_url = {flake_url}")
+    log(f"base_url = {base_url}")
+    log(f"bootstrap_url = {bootstrap_url}")
+
+    uploads: list[str] = []
+
+    for arch in arches.split(","):
+        log(f"building {arch} proot...")
+        proot = run_capture(
+            NIX, "build", "--no-link", "--print-out-paths", f".#prootTermux-{arch}"
+        )
+        proot_hash = run_capture(
+            NIX_HASH, "--type", "sha256", "--sri", f"{proot}/bin/proot-static"
+        )
+        attrs_file = Path(f"modules/environment/login/proot-attrs/{arch}.nix")
+        attrs_text = inspect.cleandoc(
+            f"""
+            # WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+            {{
+              url = "{bootstrap_url}/bootstrap-{arch}.zip";
+              hash = "{proot_hash}";
+            }}
+        """
+        )
+        # nixpkgs-fmt insists files must end with a newline
+        attrs_text = attrs_text + "\n"
+        write_attrs_file = True
+        if not attrs_file.exists():
+            log(f"warn: {attrs_file} not present; creating")
+        elif (old_attrs_text := attrs_file.read_text(encoding="utf-8")) != attrs_text:
+            log(f"updating contents of {attrs_file}")
+            print("<<<<<<")
+            print(old_attrs_text)
+            print("======")
+            print(attrs_text)
+            print(">>>>>>")
+        else:
+            write_attrs_file = False
+            log(f"no changes needed to {attrs_file}")
+
+        if write_attrs_file:
+            attrs_file.write_text(attrs_text, newline="\n", encoding="utf-8")
+            log(f"adding {attrs_file} to git index")
+            run(GIT, "add", str(attrs_file))
+
+        bootstrap_zip_store_path = run_capture(
+            NIX,
+            "build",
+            "--no-link",
+            "--print-out-paths",
+            "--impure",
+            f".#bootstrapZip-{arch}",
+            env={
+                "NIX_ON_DROID_CHANNEL_URL": channel_url,
+                "NIX_ON_DROID_FLAKE_URL": flake_url,
+            },
+        )
+        uploads.append(bootstrap_zip_store_path + f"/bootstrap-{arch}.zip")
+
+    log("creating tarball of current HEAD")
+    run(GIT, "archive", "--prefix", "nix-on-droid/", "--output", source_file, "HEAD")
+    uploads.append(source_file)
+
+    if rsync_target is not None:
+        log("uploading artifacts...")
+        run(RSYNC, "--progress", *uploads, rsync_target)
+    else:
+        log(f"Would have uploaded {uploads}")
+
+
+if __name__ == "__main__":
+    # pylint: disable = no-value-for-parameter
+    go()

scripts/deploy.sh

@@ -1,89 +0,0 @@
-#!@bash@/bin/bash
-set -euo pipefail
-
-PATH=@path@
-
-if [[ $# -ne 2 ]]; then
-    cat >&2 <<EOF
-
-USAGE: nix run .#deploy -- <public_url> <rsync_target>
-
-Builds bootstrap zip ball and source code tar ball (for usage as a channel or
-flake) and uploads it to the directory specified in <rsync_target>. The
-contents of this directory should be reachable by the android device with
-<public_url>.
-
-Examples:
-$ nix run .#deploy -- 'https://example.com/bootstrap/source.tar.gz' 'user@host:/path/to/bootstrap'
-$ nix run .#deploy -- 'github:USER/nix-on-droid/BRANCH' 'user@host:/path/to/bootstrap'
-
-EOF
-    exit 1
-fi
-
-PUBLIC_URL="$1"
-RSYNC_TARGET="$2"
-: ${ARCHES:=aarch64 x86_64}
-
-# this allows to run this script from every place in this git repo
-REPO_DIR="$(git rev-parse --show-toplevel)"
-
-cd "$REPO_DIR"
-
-SOURCE_FILE="source.tar.gz"
-
-function log() {
-    echo "> $*"
-}
-
-
-if [[ "$PUBLIC_URL" =~ ^github:(.*)/(.*)/(.*) ]]; then
-    export NIX_ON_DROID_CHANNEL_URL="https://github.com/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/archive/${BASH_REMATCH[3]}.tar.gz"
-else
-    [[ "$PUBLIC_URL" =~ ^https?:// ]] || \
-    [[ "$PUBLIC_URL" =~ ^file:/// ]] || \
-        { echo "unsupported url $PUBLIC_URL" >&2; exit 1; }
-    export NIX_ON_DROID_CHANNEL_URL="$PUBLIC_URL"
-fi
-# special case for local / CI testing
-if [[ "$PUBLIC_URL" =~ ^file:///(.*)/archive.tar.gz ]]; then
-    export NIX_ON_DROID_FLAKE_URL="/${BASH_REMATCH[1]}/unpacked"
-else
-    export NIX_ON_DROID_FLAKE_URL="$PUBLIC_URL"
-fi
-log "NIX_ON_DROID_CHANNEL_URL=$NIX_ON_DROID_CHANNEL_URL"
-log "NIX_ON_DROID_FLAKE_URL=$NIX_ON_DROID_FLAKE_URL"
-
-
-PROOT_HASH_FILE="modules/environment/login/default.nix"
-UPLOADS=()
-for arch in $ARCHES; do
-    log "building $arch proot..."
-    proot="$(nix build --no-link --print-out-paths ".#prootTermux-${arch}")"
-
-    if grep -q "$arch-linux = \"$proot\";" "$PROOT_HASH_FILE"; then
-        log "keeping $arch proot path in $PROOT_HASH_FILE"
-    elif grep -q "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"; then
-        log "patching $arch proot path in $PROOT_HASH_FILE..."
-        grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"
-        sed -i "s|$arch-linux = \"/nix/store/.*\";|$arch-linux = \"$proot\";|" "$PROOT_HASH_FILE"
-        log "            ->"
-        grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"
-    else
-        log "no $arch proot hash found in $PROOT_HASH_FILE!"
-        exit 1
-    fi
-
-    log "building $arch bootstrapZip..."
-    BOOTSTRAP_ZIP="$(nix build --no-link --print-out-paths --impure ".#bootstrapZip-${arch}")"
-    UPLOADS+=($BOOTSTRAP_ZIP/bootstrap-$arch.zip)
-done
-
-
-log "creating tar ball of current HEAD..."
-git archive --prefix nix-on-droid/ --output "$SOURCE_FILE" HEAD
-UPLOADS+=($SOURCE_FILE)
-
-
-log "uploading artifacts..."
-rsync --progress "${UPLOADS[@]}" "$RSYNC_TARGET"

scripts/local-deploy.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -xevo pipefail
+tmp=`mktemp -d`/n-o-d
+mkdir $tmp
+cd "$(dirname "$0")"/..
+git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > $tmp/archive.tar.gz
+ARCHES=x86_64 nix run .#deploy -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target $tmp/
+adb shell 'rm -rf /data/local/tmp/n-o-d'
+adb push $tmp /data/local/tmp/
+adb shell 'cd /data/local/tmp/n-o-d && tar xzof archive.tar.gz && mv n-o-d unpacked'

scripts/setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='nix-on-droid-deploy-script',
+    version='0.0',
+    packages=[],
+    py_modules=["deploy"],
+    entry_points={
+        'console_scripts': [
+            'deploy=deploy:go',
+        ],
+    },
+)