[Nix-dev] experimental -J NUM_CORES option patch / looking for reviews
Marc Weber
marco-oweber at gmx.de
Wed Jul 16 17:09:14 CEST 2008
Purpose:
When compiling oo, the kernel or ghc it takes much time.
That's where you could use this patch to tell nix to pass NUM_CORES=8 to
the builder (which will not change the hash maybe altering the result
this way).
The builder make phase can be changed to
make -j $NUM_CORES
then.
nix-env -iA -f$ALL_PACAKGES openoffice -J8
will build openoffice utilizing 8 cores then.
On the other hand a lot of times you have dependency chains such as
A -> B -> C -> D where the -j option (maxBuildJobs) can't do much, but -J can.
Problems:
---------------------------
risks: not all Makefiles are pure. thus make -j1 might differ from make -j8
TODO: How to limit access to this feature? Only adminstrators who are
aware of this risk should be able to use it (?)
Sincerly Marc Weber
How to see that it works?
------------------------
>> copy paste shell start <<
mkdir -p /tmp/num_cores_test
cd /tmp/num_cores_test
cat > builder.sh << EOF
#!/bin/sh
echo NUM_CORES \$NUM_CORES
: > \$out
EOF
chmod +x builder.sh
cat > derivation.nix << EOF
derivation{
name="test";
system="x86_64-linux";
builder=./builder.sh;
}
EOF
echo '' >> builder.sh; nix-build -J2 derivation.nix
>> copy paste shell end <<
will create /tmp/num_cores_test/{derivation.nix,builder.sh}
Be aware that nix-daemon must know about this change as well (thus do a nixos-rebuild test)
because I had to pass numCores changing the daemon protocol
============= The patch =================
Note:
----
Before adding patches = [ ./num_cores.patch ]; to nixUnstable
you want to remove the first patch against opt-common.xml because
the utilities building the documentation aren't passed yet which would result
in build failures.
diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml
index 316ad66..f1d8e99 100644
--- a/doc/manual/opt-common.xml
+++ b/doc/manual/opt-common.xml
@@ -101,6 +101,18 @@
exploit I/O latency. </para></listitem>
</varlistentry>
+<varlistentry xml:id="opt-num-cores"><term><option>--num-cores</option></term>
+ <term><option>-J</option></term>
+
+ <listitem><para>Warning: setting NUM_CORES to a value other than default 1 may alter
+ build results as a side effect because NUM_CORES bypasses computing the hash.
+ Use it to speed up testing nix expressions at your own risk. (eg
+ make -j $NUM_CORES) This option doesn't interfer with -j. Thus using -J
+ <literal>3</literal> -j <literal>3</literal> might result in
+ <literal>9</literal> threads. </para><para>This is an experimental feature
+ </para></listitem>
+
+</varlistentry>
<varlistentry xml:id="opt-max-silent-time"><term><option>--max-silent-time</option></term>
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index d6f299d..4b87b92 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -205,6 +205,8 @@ static void initAndRun(int argc, char * * argv)
tryFallback = true;
else if (arg == "--max-jobs" || arg == "-j")
maxBuildJobs = getIntArg(arg, i, args.end());
+ else if (arg == "--num-cores" || arg == "-J")
+ numCores = getIntArg(arg, i, args.end());
else if (arg == "--readonly-mode")
readOnlyMode = true;
else if (arg == "--max-silent-time")
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 1d62472..354f59b 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1565,6 +1565,11 @@ void DerivationGoal::startBuilder()
/* Also set TMPDIR and variants to point to this directory. */
env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir;
+ /* experimental num_cores option only beeing impure when not set to default 1
+ * It allows to make -j $NUM_CORES which *should* result in the same result as -j 1
+ * */
+ if (numCores != 1) env["NUM_CORES"] = int2String(numCores);
+
/* Compatibility hack with Nix <= 0.7: if this is a fixed-output
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
@@ -1757,6 +1762,13 @@ void DerivationGoal::startBuilder()
/* Create the log file and pipe. */
openLogFile();
+
+ /* make the possible source of impurity appear in the log and stdout */
+ if (numCores != 1){
+ format numCoresWarning("Warning: setting NUM_CORES to %1% (other than default 1) may alter build results as a side effect");
+ numCoresWarning % numCores;
+ writeLine(logPipe.writeSide, numCoresWarning.str() );
+ }
/* Fork a child to build the package. Note that while we
currently use forks to run and wait for the children, it
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index eaea4b7..f2d5305 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -22,6 +22,7 @@ bool keepGoing = false;
bool tryFallback = false;
Verbosity buildVerbosity = lvlInfo;
unsigned int maxBuildJobs = 1;
+unsigned int numCores = 1;
bool readOnlyMode = false;
string thisSystem = "unset";
unsigned int maxSilentTime = 0;
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index f650cd1..df11b4c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -55,6 +55,9 @@ extern Verbosity buildVerbosity;
/* Maximum number of parallel build jobs. 0 means unlimited. */
extern unsigned int maxBuildJobs;
+/* experimental passing inpure NUM_CORES builder env var */
+extern unsigned int numCores;
+
/* Read-only mode. Don't copy stuff to the store, don't change the
database. */
extern bool readOnlyMode;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 274196a..dfe4d9c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -180,6 +180,7 @@ void RemoteStore::setOptions()
writeInt(tryFallback, to);
writeInt(verbosity, to);
writeInt(maxBuildJobs, to);
+ writeInt(numCores, to);
writeInt(maxSilentTime, to);
if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
writeInt(useBuildHook, to);
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index d887ee5..780ac29 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -8,7 +8,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x102
+#define PROTOCOL_VERSION 0x103
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 3822de5..506bfd9 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -110,7 +110,9 @@ void run(Strings args)
}
else if (arg == "--attr" || arg == "-A") {
if (i == args.end())
+
throw UsageError("`--attr' requires an argument");
+
attrPaths.push_back(*i++);
}
else if (parseOptionArg(arg, i, args.end(), state, autoArgs))
diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc
index d8d8643..18b6197 100644
--- a/src/nix-worker/nix-worker.cc
+++ b/src/nix-worker/nix-worker.cc
@@ -423,6 +423,7 @@ static void performOp(unsigned int clientVersion,
tryFallback = readInt(from) != 0;
verbosity = (Verbosity) readInt(from);
maxBuildJobs = readInt(from);
+ numCores = readInt(from);
maxSilentTime = readInt(from);
if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
useBuildHook = readInt(from) != 0;
More information about the nix-dev
mailing list