Merge f0f979e436 into 1c306c07b3

.github/workflows/emulator.yml

@@ -53,7 +53,7 @@ jobs:
           rm -rf n-o-d
           mkdir -p n-o-d
           git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > n-o-d/archive.tar.gz
-          ARCHES=x86_64 nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz n-o-d/
+          nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target n-o-d/ --arches x86_64
           tar cf n-o-d.tar n-o-d
 
       - name: Store zipball and channel tarball to inject (n-o-d)

flake.nix

@@ -59,7 +59,7 @@
 
         deploy = {
           type = "app";
-          program = toString (import ./scripts/deploy.nix { inherit nixpkgs system; });
+          program = import ./scripts/deploy.nix { inherit nixpkgs system; };
         };
       });
 

modules/environment/login/default.nix

@@ -37,7 +37,7 @@ in
 
       prootStatic = mkOption {
         type = types.package;
-        readOnly = true;
+        # not readOnly, this needs to be overridden when building bootstrap zip
         internal = true;
         description = "<literal>proot-static</literal> package.";
       };
@@ -84,14 +84,27 @@ in
     environment.files = {
       inherit login loginInner;
 
+      # Ideally this would build the static proot binary, but doing that on aarch64 is HARD so instead pull it from the bootstrap tarball
       prootStatic =
         let
-          crossCompiledPaths = {
+          attrs = (import ./proot-attrs).${targetSystem};
-            aarch64-linux = "/nix/store/7qd99m1w65x2vgqg453nd70y60sm3kay-proot-termux-static-aarch64-unknown-linux-android-unstable-2024-05-04";
+          prootFile = pkgs.fetchurl {
-            x86_64-linux = "/nix/store/pakj3svvw84rhkzdc6211yhc2cgvc21f-proot-termux-static-x86_64-unknown-linux-android-unstable-2024-05-04";
+            name = "proot-static-file";
+            inherit (attrs) url hash;
+
+            downloadToTemp = true;
+            executable = true;
+            postFetch = ''
+              ${pkgs.unzip}/bin/unzip -u $downloadedFile bin/proot-static
+              echo $PWD >&2
+              mv bin/proot-static $out
+            '';
           };
         in
-        "${crossCompiledPaths.${targetSystem}}";
+        pkgs.runCommand "proot-static" { } ''
+          mkdir -p $out/bin
+          cp ${prootFile} $out/bin/proot-static
+        '';
     };
 
   };

modules/environment/login/proot-attrs/aarch64.nix

@@ -0,0 +1,5 @@
+# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+{
+  url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-aarch64.zip";
+  hash = "sha256-fZyqldmHWbv2e6543mwb5UPcKxcODRg+PDvZNcjVyUU=";
+}

modules/environment/login/proot-attrs/default.nix

@@ -0,0 +1,4 @@
+{
+  x86_64-linux = import ./x86_64.nix;
+  aarch64-linux = import ./aarch64.nix;
+}

modules/environment/login/proot-attrs/x86_64.nix

@@ -0,0 +1,5 @@
+# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+{
+  url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-x86_64.zip";
+  hash = "sha256-1WZBmFNEmZucOwuzDAFO+Sl+b2XO7Lp1aQr/4egl4wU=";
+}

pkgs/default.nix

@@ -42,6 +42,8 @@ let
       # Fix invoking bash after initial build.
       user.shell = "${initialPackageInfo.bash}/bin/bash";
 
+      environment.files.prootStatic = pkgs.lib.mkForce customPkgs.prootTermux;
+
       build = {
         channel = {
           nixpkgs = urlOptionValue nixpkgsChannelURL "NIXPKGS_CHANNEL_URL";

scripts/deploy.nix

@@ -4,31 +4,44 @@
 
 let
   pkgs = nixpkgs.legacyPackages.${system};
-
+  pypkgs = pkgs.python311Packages;
-  runtimePackages = with pkgs; [
+  disablePyLints = [
-    coreutils
+    "line-too-long"
-    git
+    "missing-module-docstring"
-    gnugrep
+    "wrong-import-position" # import should be at top of file: we purposefully don't import click and such so that users that try to run the script directly get a friendly error
-    gnused
+    "missing-function-docstring"
-    gnutar
+    # c'mon, it's a script
-    gzip
+    "too-many-locals"
-    jq
+    "too-many-branches"
-    nix
+    "too-many-statements"
-    openssh
-    rsync
   ];
+  deriv = pypkgs.buildPythonApplication {
+    pname = "deploy";
+    version = "0.0";
+    src = ./.;
+
+    inherit (pkgs) nix git rsync;
+
+    propagatedBuildInputs = [ pypkgs.click ];
+
+    doCheck = true;
+    nativeCheckInputs = with pypkgs; [ mypy pylint black ];
+    checkPhase = ''
+      mypy --strict --no-color deploy.py
+      PYLINTHOME="$PWD/.pylint" pylint \
+        --score=n \
+        --clear-cache-post-run=y \
+        --disable=${pkgs.lib.concatStringsSep "," disablePyLints} \
+        deploy.py
+      black --check --diff deploy.py
+    '';
+
+    patchPhase = ''
+      substituteInPlace deploy.py \
+        --subst-var nix \
+        --subst-var git \
+        --subst-var rsync
+    '';
+  };
 in
-
+"${deriv}/bin/deploy"
-pkgs.runCommand
-  "deploy"
-{
-  preferLocalBuild = true;
-  allowSubstitutes = false;
-}
-  ''
-    install -D -m755  ${./deploy.sh} $out
-
-    substituteInPlace $out \
-      --subst-var-by bash "${pkgs.bash}" \
-      --subst-var-by path "${pkgs.lib.makeBinPath runtimePackages}"
-  ''

scripts/deploy.py

@@ -0,0 +1,181 @@
+import os
+import sys
+
+GIT = "@git@/bin/git"
+NIX = "@nix@/bin/nix"
+NIX_HASH = "@nix@/bin/nix-hash"
+RSYNC = "@rsync@/bin/rsync"
+
+if GIT.startswith("@"):
+    sys.stderr.write(
+        "Do not run this script directly, instead try: nix run .#deploy -- --help"
+    )
+    sys.exit(1)
+
+import subprocess
+import re
+import inspect
+from typing import Never
+from pathlib import Path
+
+import click
+
+
+def err(text: str) -> Never:
+    sys.stderr.write(text)
+    sys.exit(1)
+
+
+def run(*args: str) -> None:
+    subprocess.run(args, check=True)
+
+
+def run_capture(*args: str, env: dict[str, str] | None = None) -> str:
+    proc = subprocess.run(
+        args, check=True, stdout=subprocess.PIPE, stderr=None, env=env
+    )
+    return proc.stdout.decode("utf-8").strip()
+
+
+def log(msg: str) -> None:
+    print(f"> {msg}")
+
+
+@click.command()
+@click.option(
+    "--rsync-target",
+    help="Where bootstrap zipballs and source tarball will be copied to. If given, this is passed directly to rsync so it can be a local folder, ssh path, etc. For production builds this should be a webroot directory that will be served at bootstrap-url",
+)
+@click.option(
+    "--bootstrap-url",
+    help="URL where bootstrap zip files are available. Defaults to folder part of public-url if not given.",
+)
+@click.option(
+    "--arches",
+    default="aarch64,x86_64",
+    help="Which architectures to build for, comma-separated.",
+)
+@click.argument("public-url")
+def go(
+    public_url: str,
+    rsync_target: str | None,
+    bootstrap_url: str | None,
+    arches: str,
+) -> None:
+    """
+    Builds bootstrap zip balls and source code tar ball (for usage as a channel or flake). If rsync_target is specified, uploads it to the directory specified in rsync_target. The contents of this directory should be reachable by the android device with public_url.
+
+    Examples:
+
+    \b
+    $ nix run .#deploy -- \\
+        'https://example.com/bootstrap/source.tar.gz' \\
+        --rsync-target 'user@host:/path/to/bootstrap'
+
+    \b
+    $ nix run .#deploy -- \\
+        'github:USER/nix-on-droid/BRANCH' \\
+        --rsync-target 'user@host:/path/to/bootstrap' \\
+        --bootstrap-url 'https://example.com/bootstrap/'
+
+    \b
+    $ nix run .#deploy -- \\
+        'file:///data/local/tmp/n-o-d/archive.tar.gz'
+
+    ^ useful for testing. Note this is a path on the android device running the APK, not on the build machine
+    """
+    repo_dir = run_capture(GIT, "rev-parse", "--show-toplevel")
+    os.chdir(repo_dir)
+    source_file = "source.tar.gz"
+    if (m := re.search("^github:(.*)/(.*)/(.*)", public_url)) is not None:
+        channel_url = f"https://github.com/{m[1]}/{m[2]}/archive/{m[3]}.tar.gz"
+        if bootstrap_url is None:
+            err("--botstrap-url must be provided for github URLs")
+    elif re.search("^(https?|file)://", public_url):
+        channel_url = public_url
+    else:
+        err(f"unsupported url {public_url}")
+
+    # for CI and local testing
+    if (m := re.search("^file:///(.*)/archive.tar.gz$", public_url)) is not None:
+        flake_url = f"/{m[1]}/unpacked"
+    else:
+        flake_url = public_url
+    base_url = re.sub("/[^/]*$", "", public_url)
+    if bootstrap_url is None:
+        bootstrap_url = base_url
+
+    log(f"channel_url = {channel_url}")
+    log(f"flake_url = {flake_url}")
+    log(f"base_url = {base_url}")
+    log(f"bootstrap_url = {bootstrap_url}")
+
+    uploads: list[str] = []
+
+    for arch in arches.split(","):
+        log(f"building {arch} proot...")
+        proot = run_capture(
+            NIX, "build", "--no-link", "--print-out-paths", f".#prootTermux-{arch}"
+        )
+        proot_hash = run_capture(
+            NIX_HASH, "--type", "sha256", "--sri", f"{proot}/bin/proot-static"
+        )
+        attrs_file = Path(f"modules/environment/login/proot-attrs/{arch}.nix")
+        attrs_text = inspect.cleandoc(
+            f"""
+            # WARNING: This file is autogenerated by the deploy script. Any changes will be overridden
+            {{
+              url = "{bootstrap_url}/bootstrap-{arch}.zip";
+              hash = "{proot_hash}";
+            }}
+        """
+        )
+        # nixpkgs-fmt insists files must end with a newline
+        attrs_text = attrs_text + "\n"
+        write_attrs_file = True
+        if not attrs_file.exists():
+            log(f"warn: {attrs_file} not present; creating")
+        elif (old_attrs_text := attrs_file.read_text(encoding="utf-8")) != attrs_text:
+            log(f"updating contents of {attrs_file}")
+            print("<<<<<<")
+            print(old_attrs_text)
+            print("======")
+            print(attrs_text)
+            print(">>>>>>")
+        else:
+            write_attrs_file = False
+            log(f"no changes needed to {attrs_file}")
+
+        if write_attrs_file:
+            attrs_file.write_text(attrs_text, newline="\n", encoding="utf-8")
+            log(f"adding {attrs_file} to git index")
+            run(GIT, "add", str(attrs_file))
+
+        bootstrap_zip_store_path = run_capture(
+            NIX,
+            "build",
+            "--no-link",
+            "--print-out-paths",
+            "--impure",
+            f".#bootstrapZip-{arch}",
+            env={
+                "NIX_ON_DROID_CHANNEL_URL": channel_url,
+                "NIX_ON_DROID_FLAKE_URL": flake_url,
+            },
+        )
+        uploads.append(bootstrap_zip_store_path + f"/bootstrap-{arch}.zip")
+
+    log("creating tarball of current HEAD")
+    run(GIT, "archive", "--prefix", "nix-on-droid/", "--output", source_file, "HEAD")
+    uploads.append(source_file)
+
+    if rsync_target is not None:
+        log("uploading artifacts...")
+        run(RSYNC, "--progress", *uploads, rsync_target)
+    else:
+        log(f"Would have uploaded {uploads}")
+
+
+if __name__ == "__main__":
+    # pylint: disable = no-value-for-parameter
+    go()

scripts/deploy.sh

@@ -1,89 +0,0 @@
-#!@bash@/bin/bash
-set -euo pipefail
-
-PATH=@path@
-
-if [[ $# -ne 2 ]]; then
-    cat >&2 <<EOF
-
-USAGE: nix run .#deploy -- <public_url> <rsync_target>
-
-Builds bootstrap zip ball and source code tar ball (for usage as a channel or
-flake) and uploads it to the directory specified in <rsync_target>. The
-contents of this directory should be reachable by the android device with
-<public_url>.
-
-Examples:
-$ nix run .#deploy -- 'https://example.com/bootstrap/source.tar.gz' 'user@host:/path/to/bootstrap'
-$ nix run .#deploy -- 'github:USER/nix-on-droid/BRANCH' 'user@host:/path/to/bootstrap'
-
-EOF
-    exit 1
-fi
-
-PUBLIC_URL="$1"
-RSYNC_TARGET="$2"
-: ${ARCHES:=aarch64 x86_64}
-
-# this allows to run this script from every place in this git repo
-REPO_DIR="$(git rev-parse --show-toplevel)"
-
-cd "$REPO_DIR"
-
-SOURCE_FILE="source.tar.gz"
-
-function log() {
-    echo "> $*"
-}
-
-
-if [[ "$PUBLIC_URL" =~ ^github:(.*)/(.*)/(.*) ]]; then
-    export NIX_ON_DROID_CHANNEL_URL="https://github.com/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/archive/${BASH_REMATCH[3]}.tar.gz"
-else
-    [[ "$PUBLIC_URL" =~ ^https?:// ]] || \
-    [[ "$PUBLIC_URL" =~ ^file:/// ]] || \
-        { echo "unsupported url $PUBLIC_URL" >&2; exit 1; }
-    export NIX_ON_DROID_CHANNEL_URL="$PUBLIC_URL"
-fi
-# special case for local / CI testing
-if [[ "$PUBLIC_URL" =~ ^file:///(.*)/archive.tar.gz ]]; then
-    export NIX_ON_DROID_FLAKE_URL="/${BASH_REMATCH[1]}/unpacked"
-else
-    export NIX_ON_DROID_FLAKE_URL="$PUBLIC_URL"
-fi
-log "NIX_ON_DROID_CHANNEL_URL=$NIX_ON_DROID_CHANNEL_URL"
-log "NIX_ON_DROID_FLAKE_URL=$NIX_ON_DROID_FLAKE_URL"
-
-
-PROOT_HASH_FILE="modules/environment/login/default.nix"
-UPLOADS=()
-for arch in $ARCHES; do
-    log "building $arch proot..."
-    proot="$(nix build --no-link --print-out-paths ".#prootTermux-${arch}")"
-
-    if grep -q "$arch-linux = \"$proot\";" "$PROOT_HASH_FILE"; then
-        log "keeping $arch proot path in $PROOT_HASH_FILE"
-    elif grep -q "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"; then
-        log "patching $arch proot path in $PROOT_HASH_FILE..."
-        grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"
-        sed -i "s|$arch-linux = \"/nix/store/.*\";|$arch-linux = \"$proot\";|" "$PROOT_HASH_FILE"
-        log "            ->"
-        grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"
-    else
-        log "no $arch proot hash found in $PROOT_HASH_FILE!"
-        exit 1
-    fi
-
-    log "building $arch bootstrapZip..."
-    BOOTSTRAP_ZIP="$(nix build --no-link --print-out-paths --impure ".#bootstrapZip-${arch}")"
-    UPLOADS+=($BOOTSTRAP_ZIP/bootstrap-$arch.zip)
-done
-
-
-log "creating tar ball of current HEAD..."
-git archive --prefix nix-on-droid/ --output "$SOURCE_FILE" HEAD
-UPLOADS+=($SOURCE_FILE)
-
-
-log "uploading artifacts..."
-rsync --progress "${UPLOADS[@]}" "$RSYNC_TARGET"

scripts/local-deploy.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -xevo pipefail
+tmp=`mktemp -d`/n-o-d
+mkdir $tmp
+cd "$(dirname "$0")"/..
+git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > $tmp/archive.tar.gz
+ARCHES=x86_64 nix run .#deploy -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target $tmp/
+adb shell 'rm -rf /data/local/tmp/n-o-d'
+adb push $tmp /data/local/tmp/
+adb shell 'cd /data/local/tmp/n-o-d && tar xzof archive.tar.gz && mv n-o-d unpacked'

scripts/setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='nix-on-droid-deploy-script',
+    version='0.0',
+    packages=[],
+    py_modules=["deploy"],
+    entry_points={
+        'console_scripts': [
+            'deploy=deploy:go',
+        ],
+    },
+)