Merge pull request #10603 from edolstra/flake-regressions

Run the flake regressions test suite

.github/workflows/ci.yml

@@ -208,3 +208,23 @@ jobs:
       # Only meson packages that don't have a tests.run derivation.
       # Those that have it are already built and tested as part of nix flake check.
       - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g')
+
+  flake_regressions:
+    needs: vm_tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout nix
+        uses: actions/checkout@v4
+      - name: Checkout flake-regressions
+        uses: actions/checkout@v4
+        with:
+          repository: NixOS/flake-regressions
+          path: flake-regressions
+      - name: Checkout flake-regressions-data
+        uses: actions/checkout@v4
+        with:
+          repository: NixOS/flake-regressions-data
+          path: flake-regressions/tests
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh

scripts/flake-regressions.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+
+set -e
+
+echo "Nix version:"
+nix --version
+
+cd flake-regressions
+
+status=0
+
+flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25)
+
+echo "Running flake tests..."
+
+for flake in $flakes; do
+
+    if ! REGENERATE=0 ./eval-flake.sh "$flake"; then
+        status=1
+        echo "❌ $flake"
+    else
+        echo "✅ $flake"
+    fi
+
+done
+
+exit "$status"