Merge 3e6372a597 into c69e30122b

Nix 2.24 upgrade wip

flake.lock

@@ -37,33 +37,50 @@
         "type": "github"
       }
     },
-    "flake-utils": {
+    "git-hooks-nix": {
+      "inputs": {
+        "flake-compat": [
+          "nix"
+        ],
+        "gitignore": [
+          "nix"
+        ],
+        "nixpkgs": [
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "nix",
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "lastModified": 1726745158,
+        "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
         "type": "github"
       }
     },
     "libgit2": {
       "flake": false,
       "locked": {
-        "lastModified": 1697646580,
-        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "lastModified": 1715853528,
+        "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=",
         "owner": "libgit2",
         "repo": "libgit2",
-        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96",
         "type": "github"
       },
       "original": {
         "owner": "libgit2",
+        "ref": "v1.8.1",
         "repo": "libgit2",
         "type": "github"
       }
@@ -72,44 +89,61 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
+        "git-hooks-nix": "git-hooks-nix",
         "libgit2": "libgit2",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "nixpkgs-regression": "nixpkgs-regression",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "nixpkgs-23-11": "nixpkgs-23-11",
+        "nixpkgs-regression": "nixpkgs-regression"
       },
       "locked": {
-        "lastModified": 1713874370,
-        "narHash": "sha256-gW1mO/CvsQQ5gvgiwzxsGhPFI/tx30NING+qgF5Do0s=",
+        "lastModified": 1726787955,
+        "narHash": "sha256-XFznzb8L4SdUm9u+w3DPpMWJhffuv+/6+aiVl00slns=",
         "owner": "NixOS",
         "repo": "nix",
-        "rev": "1c8150ac312b5f9ba1b3f6768ff43b09867e5883",
+        "rev": "a7fdef6858dd45b9d7bda7c92324c63faee7f509",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "2.22-maintenance",
+        "ref": "2.24-maintenance",
         "repo": "nix",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712848736,
-        "narHash": "sha256-CzZwhqyLlebljv1zFS2KWVH/3byHND0LfaO1jKsGuVo=",
+        "lastModified": 1726688310,
+        "narHash": "sha256-Xc9lEtentPCEtxc/F1e6jIZsd4MPDYv4Kugl9WtXlz0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1d6a23f11e44d0fb64b3237569b87658a9eb5643",
+        "rev": "dbebdd67a6006bb145d98c8debf9140ac7e651d0",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11-small",
+        "ref": "nixos-24.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "nixpkgs-23-11": {
+      "locked": {
+        "lastModified": 1717159533,
+        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -126,38 +160,6 @@
         "type": "github"
       }
     },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": [
-          "nix"
-        ],
-        "flake-utils": "flake-utils",
-        "gitignore": [
-          "nix"
-        ],
-        "nixpkgs": [
-          "nix",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712897695,
-        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "nix": "nix",

flake.nix

@@ -1,8 +1,8 @@
 {
   description = "A Nix-based continuous build system";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11-small";
-  inputs.nix.url = "github:NixOS/nix/2.22-maintenance";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05-small";
+  inputs.nix.url = "github:NixOS/nix/2.24-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
@@ -25,6 +25,7 @@
         hydra = final.callPackage ./package.nix {
           inherit (nixpkgs.lib) fileset;
           rawSrc = self;
+          nix-perl-bindings = final.nixComponents.nix-perl-bindings;
         };
       };
 

nixos-modules/hydra.nix

@@ -464,10 +464,12 @@ in
           ''
             set -eou pipefail
             compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
-            if [[ $compression == zstd ]]; then
+            if [[ $compression == "" ]]; then
+              compression="bzip2"
+            elif [[ $compression == zstd ]]; then
               compression="zstd --rm"
             fi
-            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r $compression --force --quiet
+            find ${baseDir}/build-logs -type f -name "*.drv" -mtime +3 -size +0c | xargs -r "$compression" --force --quiet
           '';
         startAt = "Sun 01:45";
       };

package.nix

@@ -9,6 +9,7 @@
 , perlPackages
 
 , nix
+, nix-perl-bindings
 , git
 
 , makeWrapper
@@ -59,7 +60,7 @@ let
     name = "hydra-perl-deps";
     paths = lib.closePropagation
       ([
-        nix.perl-bindings
+        nix-perl-bindings
         git
       ] ++ (with perlPackages; [
         AuthenSASL

src/hydra-eval-jobs/Makefile.am

@@ -1,5 +1,5 @@
 bin_PROGRAMS = hydra-eval-jobs
 
 hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
-hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd
+hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixcmd -lnixflake
 hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -6,6 +6,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "eval.hh"
+#include "eval-gc.hh"
 #include "eval-inline.hh"
 #include "eval-settings.hh"
 #include "signals.hh"
@@ -125,11 +126,14 @@ static void worker(
     if (myArgs.flake) {
         using namespace flake;
 
-        auto flakeRef = parseFlakeRef(myArgs.releaseExpr);
+        auto [flakeRef, fragment, outputSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(fetchSettings, myArgs.releaseExpr, absPath("."));
 
         auto vFlake = state.allocValue();
 
-        auto lockedFlake = lockFlake(state, flakeRef,
+        auto lockedFlake = lockFlake(
+                flakeSettings,
+                state,
+                flakeRef,
             LockFlags {
                 .updateLockFile = false,
                 .useRegistries = false,
@@ -368,7 +372,11 @@ int main(int argc, char * * argv)
                             ]()
                             {
                                 try {
-                                    EvalState state(myArgs.lookupPath, openStore());
+                                    auto evalStore = myArgs.evalStoreUrl
+                                             ? openStore(*myArgs.evalStoreUrl)
+                                             : openStore();
+                                    EvalState state(myArgs.lookupPath,
+                                            evalStore, fetchSettings, evalSettings);
                                     Bindings & autoArgs = *myArgs.getAutoArgs(state);
                                     worker(state, autoArgs, *to, *from);
                                 } catch (Error & e) {

src/hydra-queue-runner/dispatcher.cc

@@ -2,6 +2,7 @@
 #include <cmath>
 #include <thread>
 #include <unordered_map>
+#include <unordered_set>
 
 #include "state.hh"
 

src/hydra-queue-runner/hydra-queue-runner.cc

@@ -164,7 +164,7 @@ void State::parseMachines(const std::string & contents)
                 ? string2Int<MaxJobs>(tokens[3]).value()
                 : 1,
             // `speedFactor`
-            atof(tokens[4].c_str()),
+            std::stof(tokens[4].c_str()),
             // `supportedFeatures`
             std::move(supportedFeatures),
             // `mandatoryFeatures`

src/hydra-queue-runner/nar-extractor.cc

@@ -54,32 +54,40 @@ struct Extractor : FileSystemObjectSink
     };
 
     NarMemberDatas & members;
-    Path prefix;
+    std::filesystem::path prefix;
+
+    Path toKey(const CanonPath & path)
+    {
+        std::filesystem::path p = prefix;
+        // Conditional to avoid trailing slash
+        if (!path.isRoot()) p /= path.rel();
+        return p;
+    }
 
     Extractor(NarMemberDatas & members, const Path & prefix)
         : members(members), prefix(prefix)
     { }
 
-    void createDirectory(const Path & path) override
+    void createDirectory(const CanonPath & path) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tDirectory });
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tDirectory });
     }
 
-    void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
+    void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
     {
         NarMemberConstructor nmc {
-            members.insert_or_assign(prefix + path, NarMemberData {
+            members.insert_or_assign(toKey(path), NarMemberData {
                 .type = SourceAccessor::Type::tRegular,
                 .fileSize = 0,
-                .contents = filesToKeep.count(path) ? std::optional("") : std::nullopt,
+                .contents = filesToKeep.count(path.abs()) ? std::optional("") : std::nullopt,
             }).first->second,
         };
         func(nmc);
     }
 
-    void createSymlink(const Path & path, const std::string & target) override
+    void createSymlink(const CanonPath & path, const std::string & target) override
     {
-        members.insert_or_assign(prefix + path, NarMemberData { .type = SourceAccessor::Type::tSymlink });
+        members.insert_or_assign(toKey(path), NarMemberData { .type = SourceAccessor::Type::tSymlink });
     }
 };
 

src/hydra-queue-runner/state.hh

@@ -23,6 +23,7 @@
 #include "nar-extractor.hh"
 #include "serve-protocol.hh"
 #include "serve-protocol-impl.hh"
+#include "serve-protocol-connection.hh"
 #include "machines.hh"
 
 

src/lib/Hydra/Controller/Root.pm

@@ -330,7 +330,7 @@ sub nar :Local :Args(1) {
     else {
         $path = $Nix::Config::storeDir . "/$path";
 
-        gone($c, "Path " . $path . " is no longer available.") unless isValidPath($path);
+        gone($c, "Path " . $path . " is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($path);
 
         $c->stash->{current_view} = 'NixNAR';
         $c->stash->{storePath} = $path;

src/lib/Hydra/Plugin/S3Backup.pm

@@ -92,7 +92,7 @@ sub buildFinished {
         my $hash = substr basename($path), 0, 32;
         my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($path, 0);
         my $system;
-        if (defined $deriver and isValidPath($deriver)) {
+        if (defined $deriver and $MACHINE_LOCAL_STORE->isValidPath($deriver)) {
             $system = derivationFromPath($deriver)->{platform};
         }
         foreach my $reference (@{$refs}) {

src/lib/Hydra/Plugin/SubversionInput.pm

@@ -46,7 +46,7 @@ sub fetchInput {
 
     $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;
 
-    if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
+    if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
         $storePath = $cachedInput->storepath;
         $sha256 = $cachedInput->sha256hash;
     } else {

src/root/overview.tt

@@ -53,11 +53,13 @@
   </thead>
   <tbody>
     [% FOREACH p IN projects %]
-    <tr class="project [% IF !p.enabled %]disabled-project[% END %]">
-      <td><span class="[% IF !p.enabled %]disabled-project[% END %] [%+ IF p.hidden %]hidden-project[% END %]">[% INCLUDE renderProjectName project=p.name inRow=1 %]</span></td>
-      <td>[% HTML.escape(p.displayname) %]</td>
-      <td>[% WRAPPER maybeLink uri=p.homepage %][% HTML.escape(p.description) %][% END %]</td>
-    </tr>
+      [% IF !p.hidden || c.user_exists %]
+        <tr class="project [% IF !p.enabled %]disabled-project[% END %]">
+          <td><span class="[% IF !p.enabled %]disabled-project[% END %] [%+ IF p.hidden %]hidden-project[% END %]">[% INCLUDE renderProjectName project=p.name inRow=1 %]</span></td>
+          <td>[% HTML.escape(p.displayname) %]</td>
+          <td>[% WRAPPER maybeLink uri=p.homepage %][% HTML.escape(p.description) %][% END %]</td>
+        </tr>
+      [% END %]
     [% END %]
   </tbody>
 </table>

t/Hydra/Controller/Job/builds.t

@@ -54,13 +54,14 @@ subtest "/job/PROJECT/JOBSET/JOB/shield" => sub {
 
 subtest "/job/PROJECT/JOBSET/JOB/prometheus" => sub {
     my $response = request(GET '/job/' . $project->name . '/' . $jobset->name . '/' . $build->job . '/prometheus');
-    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
-    my $metrics = $response->content;
 
-    ok($metrics =~ m/hydra_job_failed\{.*\} 0/);
-    ok($metrics =~ m/hydra_job_completion_time\{.*\} [\d]+/);
-    ok($metrics =~ m/hydra_build_closure_size\{.*\} 96/);
-    ok($metrics =~ m/hydra_build_output_size\{.*\} 96/);
+    ok($response->is_success, "The page showing the job's prometheus data returns 200.");
+
+    my $metrics = $response->content;
+    like($metrics, qr/hydra_job_failed\{.*\} 0/);
+    like($metrics, qr/hydra_job_completion_time\{.*\} [\d]+/);
+    like($metrics, qr/hydra_build_closure_size\{.*\} 96/);
+    like($metrics, qr/hydra_build_output_size\{.*\} 96/);
 };
 
 done_testing;

t/Hydra/Controller/Jobset/http.t

@@ -186,7 +186,7 @@ subtest 'Update jobset "job" to have an invalid input type' => sub {
       })
   );
   ok(!$jobsetupdate->is_success);
-  ok($jobsetupdate->content =~ m/Invalid input type.*valid types:/);
+  like($jobsetupdate->content, qr/Invalid input type.*valid types:/);
 };
 
 

t/Hydra/Controller/User/dashboard.t

@@ -24,7 +24,7 @@ my $cookie = $login->header("set-cookie");
 my $my_jobs = request(GET '/dashboard/alice/my-jobs-tab', Accept => 'application/json', Cookie => $cookie);
 ok($my_jobs->is_success);
 my $content = $my_jobs->content();
-ok($content =~ /empty_dir/);
+like($content, qr/empty_dir/);
 ok(!($content =~ /fails/));
 ok(!($content =~ /succeed_with_failed/));
 done_testing;

t/api-test.t

@@ -115,7 +115,7 @@ subtest "evaluation" => sub {
         my $build = decode_json(request_json({ uri => "/build/" . $evals->[0]->{builds}->[0] })->content());
         is($build->{job}, "job", "The build's job name is job");
         is($build->{finished}, 0, "The build isn't finished yet");
-        ok($build->{buildoutputs}->{out}->{path} =~ /\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
+        like($build->{buildoutputs}->{out}->{path}, qr/\/nix\/store\/[a-zA-Z0-9]{32}-job$/, "The build's outpath is in the Nix store and named 'job'");
 
         subtest "search" => sub {
             my $search_project = decode_json(request_json({ uri => "/search/?query=sample" })->content());