[Nix-dev] [PATCH] t/num_cores

[Marc Weber]

pass NIX_TARGET_LOAD, NIX_MAX_PARALLELIZATION to the builder

both values are taken from config. If not set they're taken from env.
If env is not set they default to 1.

Don't set NIX_MAX_PARALLELIZATION too high. This would create load on cpu
without gaining anything. Moreover it takes longer until load drops so in the
end you would be slower.

I recommend setting both values to nproc * 1.5
---
 src/libmain/shared.cc   |   16 ++++++++++++++++
 src/libstore/build.cc   |    3 +++
 src/libstore/globals.cc |    3 +++
 src/libstore/globals.hh |   11 +++++++++++
 4 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 3fbec4b..c00ed3a 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -137,6 +137,22 @@ static void initAndRun(int argc, char * * argv)
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
+    /* initialise num_cores: if it was not set in config try getting value
+     * from environments. Value should have been set by nproc in the upstart
+     * job - I don't want to think about a portable way to get this value.
+     * In fact num_cores can change eg when you pull cores off or online ..
+     * */
+#define SET_FROM_ENV(var, setting, env, min, max) \
+    var = atoi(querySetting(setting, "-1").c_str()); \
+    if (var == -1 && getenv(env)) \
+      var = atoi(getenv(env)); \
+    if (var >max) var = max; \
+    if (var < min) var = min; \
+
+    SET_FROM_ENV(target_load, "target_load", "NIX_TARGET_LOAD", 1, 500);
+    SET_FROM_ENV(max_parallelization, "max_parallelization", "NIX_MAX_PARALLELIZATION", 1, 500);
+
+
     /* Catch SIGINT. */
     struct sigaction act;
     act.sa_handler = sigintHandler;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4126406..f0c1a1c 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1415,6 +1415,9 @@ void DerivationGoal::startBuilder()
     foreach (StringPairs::iterator, i, drv.env)
         env[i->first] = i->second;
 
+    env["NIX_TARGET_LOAD"] = (format("%d") % target_load).str();
+    env["NIX_MAX_PARALLELIZATION"] = (format("%d") % max_parallelization).str();
+
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index cc0e44e..da35ae9 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -25,6 +25,9 @@ unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 time_t maxSilentTime = 0;
+int target_load = 1;
+int max_parallelization  = 0;
+
 Paths substituters;
 bool useBuildHook = true;
 bool printBuildTrace = false;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d3388e3..2fc6d69 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -67,6 +67,17 @@ extern string thisSystem;
    infinity. */
 extern time_t maxSilentTime;
 
+/* parallel builds:
+ * I don't know yet which settings are best. So allow adjusting both:
+ * target_load and max_parallelization. max_parallelization is typically
+ * a bit higher than the number of cores. If you set it too high load
+ * lead average can increase to much which will prevent make from spawning
+ * extra processes if load dropped again. You want to set both values
+ * approximate to NUM_CORES * 1.5 or such.
+ */
+extern int target_load;
+extern int max_parallelization;
+
 /* The substituters.  There are programs that can somehow realise a
    store path without building, e.g., by downloading it or copying it
    from a CD. */
-- 
1.7.1