diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8baa6127f..971874733 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,13 @@ on: workflow_call: inputs: - os: + system: required: true type: string - system: + runner: + required: true + type: string + runner_small: required: true type: string if: @@ -15,13 +18,21 @@ on: required: false default: true type: boolean + run_vm_tests: + required: false + default: false + type: boolean + run_regression_tests: + required: false + default: false + type: boolean jobs: build: if: ${{ inputs.if }} strategy: fail-fast: false - runs-on: ${{ inputs.os }} + runs-on: ${{ inputs.runner }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -33,15 +44,164 @@ jobs: with: name: ${{ inputs.system }} path: ./tarball/*.xz + test: if: ${{ inputs.if && inputs.run_tests}} needs: build strategy: fail-fast: false - runs-on: ${{ inputs.os }} + runs-on: ${{ inputs.runner }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/determinate-nix-action@main - uses: DeterminateSystems/flakehub-cache-action@main - run: nix flake check -L --system ${{ inputs.system }} + + vm_tests_smoke: + if: inputs.run_vm_tests && github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L \ + .#hydraJobs.tests.functional_user \ + .#hydraJobs.tests.githubFlakes \ + .#hydraJobs.tests.nix-docker \ + .#hydraJobs.tests.tarballFlakes \ + ; + + vm_tests_all: + if: inputs.run_vm_tests && github.event_name == 'merge_group' + needs: build + runs-on: ${{ inputs.runner }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L --keep-going \ + $(nix flake show --json \ + | jq -r ' + .hydraJobs.tests + | with_entries(select(.value.type == "derivation")) + | keys[] + | ".#hydraJobs.tests." + .') + + flake_regressions: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh + + flake_regressions_lazy: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh + + manual: + if: github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_small }} + permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Build manual + run: nix build .#hydraJobs.manual + - uses: nwtgck/actions-netlify@v3.0 + with: + publish-dir: "./result/share/doc/nix/manual" + production-branch: detsys-main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but + # its conversation is locked, so this PR comment can never be posted. + # https://github.com/DeterminateSystems/nix-src/pull/4 + enable-pull-request-comment: ${{ github.event.pull_request.number != 4 }} + enable-commit-comment: true + enable-commit-status: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + + success: + needs: + - build + - test + - vm_tests_smoke + - vm_tests_all + - flake_regressions + - flake_regressions_lazy + - manual + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fcbf9360..82f54ddf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,18 +4,26 @@ on: pull_request: push: branches: + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail - detsys-main - main - master merge_group: + release: + types: + - published permissions: id-token: "write" contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" jobs: eval: - runs-on: blacksmith-32vcpu-ubuntu-2204 + runs-on: UbuntuLatest32Cores128G steps: - uses: actions/checkout@v4 with: @@ -26,150 +34,103 @@ jobs: build_x86_64-linux: uses: ./.github/workflows/build.yml with: - os: blacksmith-32vcpu-ubuntu-2204 system: x86_64-linux + runner: UbuntuLatest32Cores128G + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: true + run_regression_tests: true build_aarch64-linux: uses: ./.github/workflows/build.yml with: if: ${{ github.event_name == 'merge_group' }} - os: blacksmith-32vcpu-ubuntu-2204-arm system: aarch64-linux + runner: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm build_x86_64-darwin: uses: ./.github/workflows/build.yml with: if: ${{ github.event_name == 'merge_group' }} - os: macos-latest-large system: x86_64-darwin + runner: macos-latest-large + runner_small: macos-latest-large build_aarch64-darwin: uses: ./.github/workflows/build.yml with: - os: namespace-profile-mac-m2-12c28g system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge - vm_tests_smoke: - if: github.event_name != 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith-32vcpu-ubuntu-2204 + success: + runs-on: ubuntu-latest + needs: + - eval + - build_x86_64-linux + - build_aarch64-linux + - build_x86_64-darwin + - build_aarch64-darwin + if: ${{ always() }} steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main + - run: "true" - run: | - nix build -L \ - .#hydraJobs.tests.functional_user \ - .#hydraJobs.tests.githubFlakes \ - .#hydraJobs.tests.nix-docker \ - .#hydraJobs.tests.tarballFlakes \ - ; - - vm_tests_all: - if: github.event_name == 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith-32vcpu-ubuntu-2204 - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: | - nix build -L --keep-going \ - $(nix flake show --json \ - | jq -r ' - .hydraJobs.tests - | with_entries(select(.value.type == "derivation")) - | keys[] - | ".#hydraJobs.tests." + .') - - flake_regressions: - if: | - github.event_name == 'merge_group' - || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' - && ( - (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') - || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) - ) - ) - needs: build_x86_64-linux - runs-on: namespace-profile-x86-32cpu-64gb - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh - - flake_regressions_lazy: - if: | - github.event_name == 'merge_group' - || ( - github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' - && ( - (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') - || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) - ) - ) - needs: build_x86_64-linux - runs-on: namespace-profile-x86-32cpu-64gb - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 - with: - repository: DeterminateSystems/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh - - manual: - if: github.event_name != 'merge_group' - needs: build_x86_64-linux - runs-on: blacksmith - permissions: - id-token: "write" - contents: "read" - pull-requests: "write" - statuses: "write" - deployments: "write" - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-cache-action@main - - name: Build manual - run: nix build .#hydraJobs.manual - - uses: nwtgck/actions-netlify@v3.0 - with: - publish-dir: "./result/share/doc/nix/manual" - production-branch: detsys-main - github-token: ${{ secrets.GITHUB_TOKEN }} - deploy-message: "Deploy from GitHub Actions" - # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but - # its conversation is locked, so this PR comment can never be posted. - # https://github.com/DeterminateSystems/nix-src/pull/4 - enable-pull-request-comment: ${{ github.event.pull_request.number != 4 }} - enable-commit-comment: true - enable-commit-status: true - overwrites-pull-request-comment: true + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 env: - NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + + - name: Create artifacts directory + run: mkdir -p ./artifacts + + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + path: downloaded + - name: Move downloaded artifacts to artifacts directory + run: | + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done + + - name: Build fallback-paths.nix + run: | + nix build .#fallbackPathsNix --out-link fallback + cat fallback > ./artifacts/fallback-paths.nix + + - uses: DeterminateSystems/push-artifact-ids@main + with: + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true + allowed_branches: '["detsys-main"]' + + publish: + needs: + - success + if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) + environment: ${{ github.event_name == 'release' && 'production' || '' }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-push@main + with: + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + visibility: "public" + tag: "${{ github.ref_name }}" diff --git a/.github/workflows/upload-release.yml b/.github/workflows/upload-release.yml deleted file mode 100644 index 9e173c34f..000000000 --- a/.github/workflows/upload-release.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Upload release - -concurrency: - group: upload-release - -on: - workflow_call: - push: - branches: - # NOTE: make sure any branches here are also valid directory names, - # otherwise creating the directory and uploading to s3 will fail - - "detsys-main" - pull_request: - types: - - opened - - reopened - - synchronize - - labeled - release: - types: - - published - -permissions: - id-token: "write" - contents: "read" - -jobs: - build-x86_64-linux: - uses: ./.github/workflows/build.yml - with: - os: blacksmith-32vcpu-ubuntu-2204 - system: x86_64-linux - run_tests: false - - build-aarch64-linux: - uses: ./.github/workflows/build.yml - with: - os: blacksmith-32vcpu-ubuntu-2204-arm - system: aarch64-linux - run_tests: false - - build-x86_64-darwin: - uses: ./.github/workflows/build.yml - with: - os: macos-latest-large - system: x86_64-darwin - run_tests: false - - build-aarch64-darwin: - uses: ./.github/workflows/build.yml - with: - os: macos-latest-xlarge - system: aarch64-darwin - run_tests: false - - release: - runs-on: ubuntu-latest - needs: - - build-x86_64-linux - - build-aarch64-linux - - build-x86_64-darwin - - build-aarch64-darwin - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - - name: Create artifacts directory - run: mkdir -p ./artifacts - - - name: Fetch artifacts - uses: actions/download-artifact@v4 - with: - path: downloaded - - name: Move downloaded artifacts to artifacts directory - run: | - for dir in ./downloaded/*; do - arch="$(basename "$dir")" - mv "$dir"/*.xz ./artifacts/"${arch}" - done - - - name: Build fallback-paths.nix - run: | - nix build .#fallbackPathsNix --out-link fallback - cat fallback > ./artifacts/fallback-paths.nix - - - uses: DeterminateSystems/push-artifact-ids@main - with: - s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} - bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} - directory: ./artifacts - ids_project_name: determinate-nix - ids_binary_prefix: determinate-nix - skip_acl: true - allowed_branches: '["detsys-main"]' - - publish: - needs: - - release - if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) - environment: ${{ github.event_name == 'release' && 'production' || '' }} - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/determinate-nix-action@main - - uses: DeterminateSystems/flakehub-push@main - with: - rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - visibility: "public" - tag: "${{ github.ref_name }}"