mirror of
https://github.com/NixOS/nix
synced 2024-10-18 00:16:11 -04:00
Merge 34f2477d2e
into 806a91f7bf
This commit is contained in:
commit
2dd53c1de7
4
doc/manual/src/release-notes/rl-next.md
Normal file
4
doc/manual/src/release-notes/rl-next.md
Normal file
|
@ -0,0 +1,4 @@
|
|||
# Release X.Y (202?-??-??)
|
||||
|
||||
- Add a `load-limit` setting to control builder parallelism. This has
|
||||
also been backported to the 2.18 and later release branches.
|
|
@ -199,6 +199,7 @@ struct ClientSettings
|
|||
time_t maxSilentTime;
|
||||
bool verboseBuild;
|
||||
unsigned int buildCores;
|
||||
std::optional<unsigned int> loadLimit;
|
||||
bool useSubstitutes;
|
||||
StringMap overrides;
|
||||
|
||||
|
@ -212,6 +213,7 @@ struct ClientSettings
|
|||
settings.maxSilentTime = maxSilentTime;
|
||||
settings.verboseBuild = verboseBuild;
|
||||
settings.buildCores = buildCores;
|
||||
settings.loadLimit.assign(loadLimit);
|
||||
settings.useSubstitutes = useSubstitutes;
|
||||
|
||||
for (auto & i : overrides) {
|
||||
|
|
|
@ -161,7 +161,6 @@ public:
|
|||
R"(
|
||||
Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation.
|
||||
The `builder` executable can use this variable to control its own maximum amount of parallelism.
|
||||
|
||||
<!--
|
||||
FIXME(@fricklerhandwerk): I don't think this should even be mentioned here.
|
||||
A very generic example using `derivation` and `xargs` may be more appropriate to explain the mechanism.
|
||||
|
@ -171,6 +170,8 @@ public:
|
|||
|
||||
The value `0` means that the `builder` should use all available CPU cores in the system.
|
||||
|
||||
The [`load-limit`](#conf-load-limit) setting can be used to limit the total amount of build parallelism based on system load average.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting.
|
||||
|
@ -179,6 +180,33 @@ public:
|
|||
// Don't document the machine-specific default value
|
||||
false};
|
||||
|
||||
Setting<std::optional<unsigned int>> loadLimit{
|
||||
this,
|
||||
{ getDefaultCores() },
|
||||
"load-limit",
|
||||
R"(
|
||||
Sets the value of the `NIX_LOAD_LIMIT` environment variable in the
|
||||
invocation of builders. Builders can use this value at their discretion
|
||||
to dynamically control the amount of parallelism with respect to the
|
||||
machine's load average.
|
||||
|
||||
For instance, a builder could use the value to set the `-l` flag to GNU
|
||||
Make. In this case, if the load average of the machine exceeds
|
||||
`NIX_LOAD_LIMIT`, the amount of parallelism will be dynamically
|
||||
reduced.
|
||||
|
||||
By default, it is set to the number of cores on the machine.
|
||||
|
||||
On busy machines where Nix co-exists with other workloads, or where
|
||||
build throughput is paramount and memory usage is not a bottleneck, the
|
||||
default value may not work as intended. In this case, `load-limit`
|
||||
should be set to a higher value, or to `none` to prevent the
|
||||
`NIX_LOAD_LIMIT` variable being set at all.
|
||||
)",
|
||||
{},
|
||||
// Don't document the machine-specific default value
|
||||
false};
|
||||
|
||||
/**
|
||||
* Read-only mode. Don't copy stuff to the store, don't change
|
||||
* the database.
|
||||
|
|
|
@ -1202,6 +1202,10 @@ void LocalDerivationGoal::initEnv()
|
|||
/* The maximum number of cores to utilize for parallel building. */
|
||||
env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores);
|
||||
|
||||
/* Provide a load average limit for build tools to throttle jobs. */
|
||||
if (settings.loadLimit.get())
|
||||
env["NIX_LOAD_LIMIT"] = fmt("%d", settings.loadLimit.get().value());
|
||||
|
||||
initTmpDir();
|
||||
|
||||
/* Compatibility hack with Nix <= 0.7: if this is a fixed-output
|
||||
|
|
|
@ -107,6 +107,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
|
|||
|
||||
DECLARE_CONFIG_SERIALISER(std::string)
|
||||
DECLARE_CONFIG_SERIALISER(std::optional<std::string>)
|
||||
DECLARE_CONFIG_SERIALISER(std::optional<unsigned int>)
|
||||
DECLARE_CONFIG_SERIALISER(bool)
|
||||
DECLARE_CONFIG_SERIALISER(Strings)
|
||||
DECLARE_CONFIG_SERIALISER(StringSet)
|
||||
|
|
|
@ -283,6 +283,25 @@ template<> std::string BaseSetting<std::optional<std::string>>::to_string() cons
|
|||
return value ? *value : "";
|
||||
}
|
||||
|
||||
template<> std::optional<unsigned int> BaseSetting<std::optional<unsigned int>>::parse(const std::string & str) const
|
||||
{
|
||||
if (str == "none") return std::nullopt;
|
||||
else {
|
||||
if (auto n = string2Int<unsigned int>(str))
|
||||
return { *n };
|
||||
else
|
||||
throw UsageError("configuration setting '%s' should be 'none' or an integer", name);
|
||||
}
|
||||
}
|
||||
|
||||
template<> std::string BaseSetting<std::optional<unsigned int>>::to_string() const
|
||||
{
|
||||
if (value)
|
||||
return std::to_string(value.value());
|
||||
else
|
||||
return "none";
|
||||
}
|
||||
|
||||
template<> bool BaseSetting<bool>::parse(const std::string & str) const
|
||||
{
|
||||
if (str == "true" || str == "yes" || str == "1")
|
||||
|
|
|
@ -541,6 +541,8 @@ static void main_nix_build(int argc, char * * argv)
|
|||
env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
|
||||
env["NIX_STORE"] = store->storeDir;
|
||||
env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
|
||||
if (settings.loadLimit.get())
|
||||
env["NIX_LOAD_LIMIT"] = std::to_string(settings.loadLimit.get().value());
|
||||
|
||||
auto passAsFile = tokenizeString<StringSet>(getOr(drv.env, "passAsFile", ""));
|
||||
|
||||
|
|
8
tests/functional/load-limit.nix
Normal file
8
tests/functional/load-limit.nix
Normal file
|
@ -0,0 +1,8 @@
|
|||
with import ./config.nix;
|
||||
|
||||
mkDerivation {
|
||||
name = "load-limit";
|
||||
buildCommand = ''
|
||||
printf '%s' "''${NIX_LOAD_LIMIT-unset}" > "$out"
|
||||
'';
|
||||
}
|
23
tests/functional/load-limit.sh
Normal file
23
tests/functional/load-limit.sh
Normal file
|
@ -0,0 +1,23 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
source common.sh
|
||||
|
||||
TODO_NixOS
|
||||
|
||||
clearStore
|
||||
|
||||
outPath=$(nix-build --load-limit 100 load-limit.nix --no-out-link)
|
||||
text=$(cat "$outPath")
|
||||
if test "$text" != "100"; then exit 1; fi
|
||||
|
||||
clearStore
|
||||
|
||||
outPath=$(nix-build --load-limit 0 load-limit.nix --no-out-link)
|
||||
text=$(cat "$outPath")
|
||||
if test "$text" != "0"; then exit 1; fi
|
||||
|
||||
clearStore
|
||||
|
||||
outPath=$(nix-build --load-limit none load-limit.nix --no-out-link)
|
||||
text=$(cat "$outPath")
|
||||
if test "$text" != "unset"; then exit 1; fi
|
|
@ -95,6 +95,7 @@ nix_tests = \
|
|||
ssh-relay.sh \
|
||||
build.sh \
|
||||
build-delete.sh \
|
||||
load-limit.sh \
|
||||
output-normalization.sh \
|
||||
selfref-gc.sh \
|
||||
db-migration.sh \
|
||||
|
|
Loading…
Reference in a new issue