[Nix-dev] [PATCH] t/num_cores

[Marc Weber]

pass NUM_CORES to the builder
opt-out possible by
- setting NUM_CORES=1 in a a derivation.
  This intentionally affects the $out hash so that you get a sane
  system when passing NUM_CORES=1 if make install -j4 didn't install everything
  with NUM_CORES > 1
- setting num_cores=1 in Nix config /nix/etc/nix/nix.conf
- passing NUM_CORES=1 to the nix-daemon instance

passing NUM_CORES=`$coreutils/bin/nproc` to nix-daemon is the recommended way
to set NUM_CORES.

Unless you use NUM_CORES aware builder scripts nothing should change.
NUM_CORES aware builder scripts will be available in a top-git branch soon.

TODO: add documentation
---
 src/libmain/shared.cc   |   13 +++++++++++++
 src/libstore/build.cc   |   12 ++++++++++++
 src/libstore/globals.cc |    1 +
 src/libstore/globals.hh |    8 ++++++++
 4 files changed, 34 insertions(+), 0 deletions(-)

diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 3fbec4b..fb02961 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -137,6 +137,19 @@ static void initAndRun(int argc, char * * argv)
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
+    /* initialise num_cores: if it was not set in config try getting value
+     * from environments. Value should have been set by nproc in the upstart
+     * job - I don't want to think about a portable way to get this value.
+     * In fact num_cores can change eg when you pull cores off or online ..
+     * */
+    num_cores = atoi(querySetting("num_cores", "-1").c_str());
+    if (num_cores == -1 && getenv("NUM_CORES")){
+      num_cores = atoi(getenv("NUM_CORES"));
+    }
+    // enusre num_cores >=1 && <= 500
+    if (num_cores > 500) num_cores = 500;
+    if (num_cores <= 0) num_cores = 1;
+
     /* Catch SIGINT. */
     struct sigaction act;
     act.sa_handler = sigintHandler;
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4126406..7bdfa7d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1415,6 +1415,18 @@ void DerivationGoal::startBuilder()
     foreach (StringPairs::iterator, i, drv.env)
         env[i->first] = i->second;
 
+    /* if NUM_CORES was set by derivation it must be "1"
+     * override by global num_cores otherwise which is known to be an int > 0 and < 500
+     */
+    Environment::iterator num_cores_iter = env.find("NUM_CORES");
+    bool invalid_num_cores_value = false;
+    if( num_cores_iter == env.end()
+      || (invalid_num_cores_value = num_cores_iter->second.compare("1") != 0) ) {
+      if (invalid_num_cores_value)
+        trace("overriding invalid NUM_CORES value");
+      env["NUM_CORES"] = (format("%d") % num_cores).str();
+    }
+
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index cc0e44e..adfa0f1 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -25,6 +25,7 @@ unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 time_t maxSilentTime = 0;
+int num_cores = 0;
 Paths substituters;
 bool useBuildHook = true;
 bool printBuildTrace = false;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index d3388e3..a0fb358 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -67,6 +67,14 @@ extern string thisSystem;
    infinity. */
 extern time_t maxSilentTime;
 
+/* 0 = auto which means num_cores is set to the amount of cores the system has
+ * 1 = disable parallel builds which means pass NUM_CORES=1 to the build
+ *     scripts always
+ * n = pass this value to the builder
+ * You can set NUM_CORES=1 in a derivation to prevent a parallel builds
+*/
+extern int num_cores;
+
 /* The substituters.  There are programs that can somehow realise a
    store path without building, e.g., by downloading it or copying it
    from a CD. */
-- 
1.7.1