From 9516b256f1ad9cbc685354271c91b17b07e93b28 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Fri, 22 Jan 2021 07:14:24 -0500
Subject: [PATCH] Normalize nixexpr{input,path} from builds to jobsetevals.

Duplicating this data on every record of the builds table cost
approximately 4G of duplication.

Note that the database migration included took about 4h45m on an
untuned server which uses very slow rotational disks in a RAID5 setup,
with not a lot of RAM. I imagine in production it might take an hour
or two, but not 4. If this should become a chunked migration, I can do
that.

Note: Because of the question about chunked migrations, I have NOT
YET tested this migration thoroughly enough for merge.
---
 src/lib/Hydra/Schema/Builds.pm      | 18 ++---------------
 src/lib/Hydra/Schema/JobsetEvals.pm | 18 +++++++++++++++--
 src/lib/Hydra/Schema/Jobsets.pm     |  4 ++--
 src/lib/Hydra/Schema/Projects.pm    |  4 ++--
 src/root/build.tt                   |  6 +++---
 src/root/reproduce.tt               |  4 ++--
 src/script/hydra-eval-jobset        |  4 ++--
 src/sql/hydra.sql                   |  9 ++-------
 src/sql/upgrade-71.sql              | 31 +++++++++++++++++++++++++++++
 9 files changed, 62 insertions(+), 36 deletions(-)
 create mode 100644 src/sql/upgrade-71.sql

diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm
index 014d07ce..4e3c9523 100644
--- a/src/lib/Hydra/Schema/Builds.pm
+++ b/src/lib/Hydra/Schema/Builds.pm
@@ -134,16 +134,6 @@ __PACKAGE__->table("builds");
   default_value: 0
   is_nullable: 1
 
-=head2 nixexprinput
-
-  data_type: 'text'
-  is_nullable: 1
-
-=head2 nixexprpath
-
-  data_type: 'text'
-  is_nullable: 1
-
 =head2 priority
 
   data_type: 'integer'
@@ -246,10 +236,6 @@ __PACKAGE__->add_columns(
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "iscurrent",
   { data_type => "integer", default_value => 0, is_nullable => 1 },
-  "nixexprinput",
-  { data_type => "text", is_nullable => 1 },
-  "nixexprpath",
-  { data_type => "text", is_nullable => 1 },
   "priority",
   { data_type => "integer", default_value => 0, is_nullable => 0 },
   "globalpriority",
@@ -542,8 +528,8 @@ __PACKAGE__->many_to_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RIKKFfcKXFWIUeM8ma++iw
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Df5N0EByYJqoSUqA0dld/A
 
 __PACKAGE__->has_many(
   "dependents",
diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm
index b8f6c2a2..36bab6c1 100644
--- a/src/lib/Hydra/Schema/JobsetEvals.pm
+++ b/src/lib/Hydra/Schema/JobsetEvals.pm
@@ -89,6 +89,16 @@ __PACKAGE__->table("jobsetevals");
   data_type: 'text'
   is_nullable: 0
 
+=head2 nixexprinput
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 nixexprpath
+
+  data_type: 'text'
+  is_nullable: 1
+
 =head2 nrbuilds
 
   data_type: 'integer'
@@ -132,6 +142,10 @@ __PACKAGE__->add_columns(
   { data_type => "integer", is_nullable => 0 },
   "hash",
   { data_type => "text", is_nullable => 0 },
+  "nixexprinput",
+  { data_type => "text", is_nullable => 1 },
+  "nixexprpath",
+  { data_type => "text", is_nullable => 1 },
   "nrbuilds",
   { data_type => "integer", is_nullable => 1 },
   "nrsucceeded",
@@ -215,8 +229,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA
 
 __PACKAGE__->has_many(
   "buildIds",
diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm
index 2cf18aab..6ca83dbb 100644
--- a/src/lib/Hydra/Schema/Jobsets.pm
+++ b/src/lib/Hydra/Schema/Jobsets.pm
@@ -375,8 +375,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aDW78MCelU/ma953aTcHvA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw
 
 
 =head2 builds
diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm
index 0a529933..d3bd1911 100644
--- a/src/lib/Hydra/Schema/Projects.pm
+++ b/src/lib/Hydra/Schema/Projects.pm
@@ -258,8 +258,8 @@ Composing rels: L</projectmembers> -> username
 __PACKAGE__->many_to_many("usernames", "projectmembers", "username");
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iBGJjFWiI9Wy9zwT7xGOEA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ
 
 my %hint = (
     columns => [
diff --git a/src/root/build.tt b/src/root/build.tt
index 5a55844d..f6264704 100644
--- a/src/root/build.tt
+++ b/src/root/build.tt
@@ -120,7 +120,7 @@ END;
       <b class="caret"></b>
     </a>
     <ul class="dropdown-menu">
-      [% IF build.nixexprinput || eval.flake %]
+      [% IF eval.nixexprinput || eval.flake %]
         <li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
       [% END %]
       [% IF c.user_exists %]
@@ -346,10 +346,10 @@ END;
           <td>[% build.priority %]</td>
         </tr>
       [% END %]
-      [% IF build.nixexprinput %]
+      [% IF eval.nixexprinput %]
         <tr>
           <th>Nix expression:</th>
-          <td>file <tt>[% HTML.escape(build.nixexprpath) %]</tt> in input <tt>[% HTML.escape(build.nixexprinput) %]</tt></td>
+          <td>file <tt>[% HTML.escape(eval.nixexprpath) %]</tt> in input <tt>[% HTML.escape(eval.nixexprinput) %]</tt></td>
         </tr>
       [% END %]
       <tr>
diff --git a/src/root/reproduce.tt b/src/root/reproduce.tt
index 20ce7596..11eaccc9 100644
--- a/src/root/reproduce.tt
+++ b/src/root/reproduce.tt
@@ -169,7 +169,7 @@ echo "$0: input ‘[% input.name %]’ has unsupported type ‘[% input.type %]
 exit 1
 [% END %]
 
-[% IF input.name == build.nixexprinput +%]
+[% IF input.name == eval.nixexprinput +%]
 nixExprInputDir="$inputDir"
 [%+ END %]
 
@@ -197,7 +197,7 @@ args+=(--option extra-binary-caches '[% c.uri_for('/') %]')
 # when evaluating jobs that rely on builtins.currentSystem.
 args+=(--option system x86_64-linux)
 
-args+=("$nixExprInputDir/[% build.nixexprpath %]" -A '[% build.job.name %]')
+args+=("$nixExprInputDir/[% eval.nixexprpath %]" -A '[% build.job.name %]')
 
 if [ -n "$printFlags" ]; then
     first=1
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index 78938758..dbf03499 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -460,8 +460,6 @@ sub checkBuild {
             , nixname => $buildInfo->{nixName}
             , drvpath => $drvPath
             , system => $buildInfo->{system}
-            , nixexprinput => $jobset->nixexprinput
-            , nixexprpath => $jobset->nixexprpath
             , priority => $buildInfo->{schedulingPriority}
             , finished => 0
             , iscurrent => 1
@@ -724,6 +722,8 @@ sub checkJobsetWrapped {
             , hasnewbuilds => $jobsetChanged ? 1 : 0
             , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
             , flake => $flakeRef
+            , nixexprinput => $jobset->nixexprinput
+            , nixexprpath => $jobset->nixexprpath
             });
 
         $db->storage->dbh->do("notify eval_added, ?", undef,
diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index d8e2805c..07d15ddc 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -162,13 +162,6 @@ create table Builds (
     isChannel     integer not null default 0, -- meta.isHydraChannel
     isCurrent     integer default 0,
 
-    -- Copy of the nixExprInput/nixExprPath fields of the jobset that
-    -- instantiated this build.  Needed if we want to reproduce this
-    -- build.  FIXME: this should be stored in JobsetEvals, storing it
-    -- here is denormal.
-    nixExprInput  text,
-    nixExprPath   text,
-
     -- Priority within a jobset, set via meta.schedulingPriority.
     priority      integer not null default 0,
 
@@ -466,6 +459,8 @@ create table JobsetEvals (
     nrSucceeded   integer, -- set lazily when all builds are finished
 
     flake         text, -- immutable flake reference
+    nixExprInput  text, -- name of the jobsetInput containing the Nix or Guix expression
+    nixExprPath   text, -- relative path of the Nix or Guix expression
 
     foreign key   (project) references Projects(name) on delete cascade on update cascade,
     foreign key   (project, jobset) references Jobsets(project, name) on delete cascade on update cascade
diff --git a/src/sql/upgrade-71.sql b/src/sql/upgrade-71.sql
new file mode 100644
index 00000000..7f4d43fb
--- /dev/null
+++ b/src/sql/upgrade-71.sql
@@ -0,0 +1,31 @@
+ALTER TABLE JobsetEvals
+    ADD COLUMN nixExprInput text,
+    ADD COLUMN nixExprPath text;
+
+
+-- This migration took 4.5 hours on a server
+-- with 5400RPM drives, against a copy of hydra's
+-- production dataset. It might take a significantly
+-- less amount of time there, and not justify a
+-- batched migration.
+UPDATE jobsetevals
+SET (nixexprinput, nixexprpath) = (
+    SELECT builds.nixexprinput, builds.nixexprpath
+    FROM builds
+    LEFT JOIN jobsetevalmembers
+      ON jobsetevalmembers.build = builds.id
+    WHERE jobsetevalmembers.eval = jobsetevals.id
+    LIMIT 1
+)
+WHERE jobsetevals.id in (
+  SELECT jobsetevalsprime.id
+  FROM jobsetevals as jobsetevalsprime
+  WHERE jobsetevalsprime.nixexprinput IS NULL
+  -- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
+  ORDER BY jobsetevalsprime.id ASC  --  /
+  -- LIMIT ?  --  ----------------------
+);
+
+ALTER TABLE builds
+    DROP COLUMN nixexprinput,
+    DROP COLUMN nixexprpath;
\ No newline at end of file