1 : #include <map>
2 : #include <boost/shared_ptr.hpp>
3 : #include <boost/weak_ptr.hpp>
4 : #include <boost/enable_shared_from_this.hpp>
5 :
6 : #include <sys/types.h>
7 : #include <sys/stat.h>
8 : #include <fcntl.h>
9 : #include <unistd.h>
10 : #include <errno.h>
11 :
12 : #include "normalise.hh"
13 : #include "references.hh"
14 : #include "pathlocks.hh"
15 : #include "globals.hh"
16 :
17 :
18 : /* !!! TODO storeExprFromPath shouldn't be used here */
19 :
20 :
21 51 : static string pathNullDevice = "/dev/null";
22 :
23 :
24 : /* Forward definition. */
25 : class Worker;
26 :
27 :
28 : /* A pointer to a goal. */
29 : class Goal;
30 : typedef shared_ptr<Goal> GoalPtr;
31 : typedef weak_ptr<Goal> WeakGoalPtr;
32 :
33 : /* Set of goals. */
34 : typedef set<GoalPtr> Goals;
35 : typedef set<WeakGoalPtr> WeakGoals;
36 :
37 : /* A map of paths to goals (and the other way around). */
38 : typedef map<Path, WeakGoalPtr> WeakGoalMap;
39 :
40 :
41 :
42 : class Goal : public enable_shared_from_this<Goal>
43 : {
44 : protected:
45 :
46 : /* Backlink to the worker. */
47 : Worker & worker;
48 :
49 : /* Goals that this goal is waiting for. */
50 : Goals waitees;
51 :
52 : /* Goals waiting for this one to finish. Must use weak pointers
53 : here to prevent cycles. */
54 : WeakGoals waiters;
55 :
56 : /* Number of goals we are/were waiting for that have failed. */
57 : unsigned int nrFailed;
58 :
59 : /* Whether amDone() has been called. */
60 : bool done;
61 :
62 :
63 344 : Goal(Worker & _worker) : worker(_worker)
64 2816 : {
65 344 : done = false;
66 344 : nrFailed = 0;
67 : }
68 :
69 : virtual ~Goal()
70 688 : {
71 344 : printMsg(lvlVomit, "goal destroyed");
72 344 : }
73 :
74 : public:
75 : virtual void work() = 0;
76 :
77 : virtual string name() = 0;
78 :
79 : void addWaitee(GoalPtr waitee);
80 :
81 : virtual void waiteeDone(GoalPtr waitee, bool success);
82 :
83 : virtual void writeLog(int fd, const unsigned char * buf, size_t count)
84 0 : {
85 0 : abort();
86 : }
87 :
88 : void trace(const format & f);
89 :
90 : protected:
91 : void amDone(bool success = true);
92 : };
93 :
94 :
95 : /* A mapping used to remember for each child process to what goal it
96 : belongs, and a file descriptor for receiving log data. */
97 : struct Child
98 : {
99 : WeakGoalPtr goal;
100 : int fdOutput;
101 : bool inBuildSlot;
102 : };
103 :
104 : typedef map<pid_t, Child> Children;
105 :
106 :
107 : /* The worker class. */
108 : class Worker
109 : {
110 : private:
111 :
112 : /* Note: the worker should only have strong pointers to the
113 : top-level goals. */
114 :
115 : /* The top-level goals of the worker. */
116 698 : Goals topGoals;
117 :
118 : /* Goals that are ready to do some work. */
119 : WeakGoals awake;
120 :
121 : /* Goals waiting for a build slot. */
122 : WeakGoals wantingToBuild;
123 :
124 : /* Child processes currently running. */
125 : Children children;
126 :
127 : /* Number of build slots occupied. Not all child processes
128 : (namely build hooks) count as occupied build slots. */
129 : unsigned int nrChildren;
130 :
131 : /* Maps used to prevent multiple instantiation of a goal for the
132 : same expression / path. */
133 : WeakGoalMap normalisationGoals;
134 : WeakGoalMap realisationGoals;
135 : WeakGoalMap substitutionGoals;
136 :
137 : public:
138 :
139 : Worker();
140 : ~Worker();
141 :
142 : /* Make a goal (with caching). */
143 : GoalPtr makeNormalisationGoal(const Path & nePath);
144 : GoalPtr makeRealisationGoal(const Path & nePath);
145 : GoalPtr makeSubstitutionGoal(const Path & storePath);
146 :
147 : /* Remove a dead goal. */
148 : void removeGoal(GoalPtr goal);
149 :
150 : /* Wake up a goal (i.e., there is something for it to do). */
151 : void wakeUp(GoalPtr goal);
152 :
153 : /* Can we start another child process? */
154 : bool canBuildMore();
155 :
156 : /* Registers / unregisters a running child process. */
157 : void childStarted(GoalPtr goal, pid_t pid, int fdOutput,
158 : bool inBuildSlot);
159 : void childTerminated(pid_t pid, bool wakeSleepers = true);
160 :
161 : /* Add a goal to the set of goals waiting for a build slot. */
162 : void waitForBuildSlot(GoalPtr goal, bool reallyWait = false);
163 :
164 : /* Loop until the specified top-level goal has finished. Returns
165 : true if it has finished succesfully. */
166 : bool run(GoalPtr topGoal);
167 :
168 : /* Wait for input to become available. */
169 : void waitForInput();
170 : };
171 :
172 :
173 : class SubstError : public Error
174 : {
175 : public:
176 2 : SubstError(const format & f) : Error(f) { };
177 : };
178 :
179 :
180 : class BuildError : public Error
181 : {
182 : public:
183 0 : BuildError(const format & f) : Error(f) { };
184 : };
185 :
186 :
187 :
188 : //////////////////////////////////////////////////////////////////////
189 :
190 :
191 : void Goal::addWaitee(GoalPtr waitee)
192 371 : {
193 371 : waitees.insert(waitee);
194 22767 : waitee->waiters.insert(shared_from_this());
195 : }
196 :
197 :
198 : void Goal::waiteeDone(GoalPtr waitee, bool success)
199 357 : {
200 357 : assert(waitees.find(waitee) != waitees.end());
201 357 : waitees.erase(waitee);
202 :
203 357 : if (!success) ++nrFailed;
204 :
205 357 : if (waitees.empty() || (!success && !keepGoing)) {
206 :
207 : /* If we failed and keepGoing is not set, we remove all
208 : remaining waitees. */
209 309 : for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
210 0 : GoalPtr goal = *i;
211 0 : WeakGoals waiters2;
212 0 : for (WeakGoals::iterator j = goal->waiters.begin();
213 : j != goal->waiters.end(); ++j)
214 0 : if (j->lock() != shared_from_this())
215 0 : waiters2.insert(*j);
216 0 : goal->waiters = waiters2;
217 : }
218 309 : waitees.clear();
219 :
220 309 : worker.wakeUp(shared_from_this());
221 : }
222 : }
223 :
224 :
225 : void Goal::amDone(bool success)
226 330 : {
227 330 : trace("done");
228 330 : assert(!done);
229 330 : done = true;
230 701 : for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) {
231 371 : GoalPtr goal = i->lock();
232 371 : if (goal) goal->waiteeDone(shared_from_this(), success);
233 : }
234 330 : waiters.clear();
235 330 : worker.removeGoal(shared_from_this());
236 : }
237 :
238 :
239 : void Goal::trace(const format & f)
240 1779 : {
241 1779 : debug(format("%1%: %2%") % name() % f);
242 : }
243 :
244 :
245 :
246 : //////////////////////////////////////////////////////////////////////
247 :
248 :
249 : /* Common initialisation performed in child processes. */
250 : void commonChildInit(Pipe & logPipe)
251 27 : {
252 : /* Put the child in a separate process group so that it doesn't
253 : receive terminal signals. */
254 27 : if (setpgid(0, 0) == -1)
255 0 : throw SysError(format("setting process group"));
256 :
257 : /* Dup the write side of the logger pipe into stderr. */
258 27 : if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
259 0 : throw SysError("cannot pipe standard error into log file");
260 27 : logPipe.readSide.close();
261 :
262 : /* Dup stderr to stdin. */
263 27 : if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
264 0 : throw SysError("cannot dup stderr into stdout");
265 :
266 : /* Reroute stdin to /dev/null. */
267 27 : int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
268 27 : if (fdDevNull == -1)
269 0 : throw SysError(format("cannot open `%1%'") % pathNullDevice);
270 27 : if (dup2(fdDevNull, STDIN_FILENO) == -1)
271 0 : throw SysError("cannot dup null device into stdin");
272 : }
273 :
274 :
275 : /* Convert a string list to an array of char pointers. Careful: the
276 : string list should outlive the array. */
277 : const char * * strings2CharPtrs(const Strings & ss)
278 43 : {
279 43 : const char * * arr = new const char * [ss.size() + 1];
280 43 : const char * * p = arr;
281 401 : for (Strings::const_iterator i = ss.begin(); i != ss.end(); ++i)
282 358 : *p++ = i->c_str();
283 43 : *p = 0;
284 43 : return arr;
285 : }
286 :
287 :
288 : /* Should only be called after an expression has been normalised. */
289 : Path queryNormalForm(const Path & nePath)
290 123 : {
291 123 : Path nfPath;
292 123 : if (querySuccessor(nePath, nfPath)) return nfPath;
293 : /* If there is no successor, than nePath must be a normal form
294 : itself. */
295 37 : StoreExpr ne = storeExprFromPath(nePath);
296 37 : if (ne.type != StoreExpr::neClosure) abort();
297 37 : return nePath;
298 : }
299 :
300 :
301 :
302 : //////////////////////////////////////////////////////////////////////
303 :
304 :
305 : class NormalisationGoal : public Goal
306 : {
307 : private:
308 : /* The path of the derivation store expression. */
309 : Path nePath;
310 :
311 : /* The store expression stored at nePath. */
312 : StoreExpr expr;
313 :
314 : /* The remainder is state held during the build. */
315 :
316 : /* Locks on the output paths. */
317 : PathLocks outputLocks;
318 :
319 : /* Input paths, with their closure elements. */
320 : ClosureElems inClosures;
321 :
322 : /* Referenceable paths (i.e., input and output paths). */
323 : PathSet allPaths;
324 :
325 : /* The normal forms of the input store expressions. */
326 : PathSet inputNFs;
327 :
328 : /* The successor mappings for the input store expressions. */
329 : map<Path, Path> inputSucs;
330 :
331 : /* The process ID of the builder. */
332 : Pid pid;
333 :
334 : /* The temporary directory. */
335 : Path tmpDir;
336 :
337 : /* File descriptor for the log file. */
338 : AutoCloseFD fdLogFile;
339 :
340 : /* Pipe for the builder's standard output/error. */
341 : Pipe logPipe;
342 :
343 : /* Pipes for talking to the build hook (if any). */
344 : Pipe toHook;
345 : Pipe fromHook;
346 :
347 : typedef void (NormalisationGoal::*GoalState)();
348 : GoalState state;
349 :
350 : public:
351 : NormalisationGoal(const Path & _nePath, Worker & _worker);
352 : ~NormalisationGoal();
353 :
354 : void work();
355 :
356 : private:
357 : /* The states. */
358 : void init();
359 : void haveStoreExpr();
360 : void inputRealised();
361 : void tryToBuild();
362 : void buildDone();
363 :
364 : /* Is the build hook willing to perform the build? */
365 : typedef enum {rpAccept, rpDecline, rpPostpone, rpDone} HookReply;
366 : HookReply tryBuildHook();
367 :
368 : /* Synchronously wait for a build hook to finish. */
369 : void terminateBuildHook();
370 :
371 : /* Acquires locks on the output paths and gathers information
372 : about the build (e.g., the input closures). During this
373 : process its possible that we find out that the build is
374 : unnecessary, in which case we return false (this is not an
375 : error condition!). */
376 : bool prepareBuild();
377 :
378 : /* Start building a derivation. */
379 : void startBuilder();
380 :
381 : /* Must be called after the output paths have become valid (either
382 : due to a successful build or hook, or because they already
383 : were). */
384 : void createClosure();
385 :
386 : /* Open a log file and a pipe to it. */
387 : void openLogFile();
388 :
389 : /* Common initialisation to be performed in child processes (i.e.,
390 : both in builders and in build hooks. */
391 : void initChild();
392 :
393 : /* Delete the temporary directory, if we have one. */
394 : void deleteTmpDir(bool force);
395 :
396 : /* Callback used by the worker to write to the log. */
397 : void writeLog(int fd, const unsigned char * buf, size_t count);
398 :
399 : string name();
400 : };
401 :
402 :
403 : NormalisationGoal::NormalisationGoal(const Path & _nePath, Worker & _worker)
404 63 : : Goal(_worker)
405 63 : {
406 63 : nePath = _nePath;
407 63 : state = &NormalisationGoal::init;
408 : }
409 :
410 :
411 : NormalisationGoal::~NormalisationGoal()
412 126 : {
413 63 : if (pid != -1) worker.childTerminated(pid);
414 :
415 : /* Careful: we should never ever throw an exception from a
416 : destructor. */
417 63 : try {
418 63 : deleteTmpDir(false);
419 0 : } catch (Error & e) {
420 0 : printMsg(lvlError, format("error (ignored): %1%") % e.msg());
421 : }
422 63 : }
423 :
424 :
425 : void NormalisationGoal::work()
426 236 : {
427 236 : (this->*state)();
428 : }
429 :
430 :
431 : void NormalisationGoal::init()
432 63 : {
433 63 : trace("init");
434 :
435 : /* If we already have a successor, then we are done already; don't
436 : add the expression as a goal. */
437 63 : Path nfPath;
438 63 : if (querySuccessor(nePath, nfPath)) {
439 1 : amDone();
440 1 : return;
441 : }
442 :
443 : /* The first thing to do is to make sure that the store expression
444 : exists. If it doesn't, it may be created through a
445 : substitute. */
446 62 : addWaitee(worker.makeSubstitutionGoal(nePath));
447 :
448 62 : state = &NormalisationGoal::haveStoreExpr;
449 : }
450 :
451 :
452 : void NormalisationGoal::haveStoreExpr()
453 62 : {
454 62 : trace("loading store expression");
455 :
456 62 : if (nrFailed != 0) {
457 1 : printMsg(lvlError,
458 : format("cannot normalise missing store expression `%1%'")
459 : % nePath);
460 1 : amDone(false);
461 1 : return;
462 : }
463 :
464 61 : assert(isValidPath(nePath));
465 :
466 : /* Get the store expression. */
467 829 : expr = storeExprFromPath(nePath);
468 :
469 : /* If this is a normal form (i.e., a closure) we are also done. */
470 61 : if (expr.type == StoreExpr::neClosure) {
471 17 : amDone();
472 17 : return;
473 : }
474 44 : assert(expr.type == StoreExpr::neDerivation);
475 :
476 : /* Inputs must be realised before we can build this goal. */
477 134 : for (PathSet::iterator i = expr.derivation.inputs.begin();
478 : i != expr.derivation.inputs.end(); ++i)
479 90 : addWaitee(worker.makeRealisationGoal(*i));
480 :
481 44 : state = &NormalisationGoal::inputRealised;
482 : }
483 :
484 :
485 : void NormalisationGoal::inputRealised()
486 44 : {
487 44 : trace("all inputs realised");
488 :
489 44 : if (nrFailed != 0) {
490 0 : printMsg(lvlError,
491 : format("cannot normalise derivation `%1%': "
492 : "%2% closure element(s) could not be realised")
493 : % nePath % nrFailed);
494 0 : amDone(false);
495 0 : return;
496 : }
497 :
498 : /* Okay, try to build. Note that here we don't wait for a build
499 : slot to become available, since we don't need one if there is a
500 : build hook. */
501 44 : state = &NormalisationGoal::tryToBuild;
502 44 : worker.wakeUp(shared_from_this());
503 : }
504 :
505 :
506 : void NormalisationGoal::tryToBuild()
507 47 : {
508 47 : trace("trying to build");
509 :
510 47 : try {
511 :
512 : /* Is the build hook willing to accept this job? */
513 47 : switch (tryBuildHook()) {
514 : case rpAccept:
515 : /* Yes, it has started doing so. Wait until we get
516 : EOF from the hook. */
517 1 : state = &NormalisationGoal::buildDone;
518 1 : return;
519 : case rpPostpone:
520 : /* Not now; wait until at least one child finishes. */
521 0 : worker.waitForBuildSlot(shared_from_this(), true);
522 0 : return;
523 : case rpDecline:
524 : /* We should do it ourselves. */
525 0 : break;
526 : case rpDone:
527 : /* Somebody else did it (there is a successor now). */
528 0 : amDone();
529 0 : return;
530 : }
531 :
532 : /* Make sure that we are allowed to start a build. */
533 46 : if (!worker.canBuildMore()) {
534 3 : worker.waitForBuildSlot(shared_from_this());
535 3 : return;
536 : }
537 :
538 : /* Acquire locks and such. If we then see that there now is a
539 : successor, we're done. */
540 43 : if (!prepareBuild()) {
541 24 : amDone();
542 24 : return;
543 : }
544 :
545 : /* Okay, we have to build. */
546 19 : startBuilder();
547 :
548 0 : } catch (BuildError & e) {
549 0 : printMsg(lvlError, e.msg());
550 0 : amDone(false);
551 0 : return;
552 : }
553 :
554 : /* This state will be reached when we get EOF on the child's
555 : log pipe. */
556 19 : state = &NormalisationGoal::buildDone;
557 : }
558 :
559 :
560 : void NormalisationGoal::buildDone()
561 20 : {
562 20 : trace("build done");
563 :
564 : /* Since we got an EOF on the logger pipe, the builder is presumed
565 : to have terminated. In fact, the builder could also have
566 : simply have closed its end of the pipe --- just don't do that
567 : :-) */
568 : /* !!! this could block! */
569 20 : pid_t savedPid = pid;
570 20 : int status = pid.wait(true);
571 :
572 : /* So the child is gone now. */
573 20 : worker.childTerminated(savedPid);
574 :
575 : /* Close the read side of the logger pipe. */
576 20 : logPipe.readSide.close();
577 :
578 : /* Close the log file. */
579 20 : fdLogFile.close();
580 :
581 20 : debug(format("builder process for `%1%' finished") % nePath);
582 :
583 : /* Check the exit status. */
584 20 : if (!statusOk(status)) {
585 0 : deleteTmpDir(false);
586 0 : printMsg(lvlError, format("builder for `%1%' %2%")
587 : % nePath % statusToString(status));
588 0 : amDone(false);
589 0 : return;
590 : }
591 :
592 20 : deleteTmpDir(true);
593 :
594 : /* Compute a closure store expression, and register it as our
595 : successor. */
596 20 : try {
597 20 : createClosure();
598 0 : } catch (BuildError & e) {
599 0 : printMsg(lvlError, e.msg());
600 0 : amDone(false);
601 0 : return;
602 : }
603 :
604 20 : amDone();
605 : }
606 :
607 :
608 : static string readLine(int fd)
609 3 : {
610 3 : string s;
611 23 : while (1) {
612 23 : char ch;
613 23 : ssize_t rd = read(fd, &ch, 1);
614 23 : if (rd == -1) {
615 0 : if (errno != EINTR)
616 0 : throw SysError("reading a line");
617 23 : } else if (rd == 0)
618 0 : throw Error("unexpected EOF reading a line");
619 : else {
620 23 : if (ch == '\n') return s;
621 20 : s += ch;
622 : }
623 : }
624 : }
625 :
626 :
627 : static void writeLine(int fd, string s)
628 1 : {
629 1 : s += '\n';
630 1 : writeFull(fd, (const unsigned char *) s.c_str(), s.size());
631 : }
632 :
633 :
634 : /* !!! ugly hack */
635 : static void drain(int fd)
636 4 : {
637 4 : unsigned char buffer[1024];
638 4 : while (1) {
639 4 : ssize_t rd = read(fd, buffer, sizeof buffer);
640 4 : if (rd == -1) {
641 0 : if (errno != EINTR)
642 0 : throw SysError("draining");
643 4 : } else if (rd == 0) break;
644 2 : else writeFull(STDERR_FILENO, buffer, rd);
645 : }
646 : }
647 :
648 :
649 : NormalisationGoal::HookReply NormalisationGoal::tryBuildHook()
650 47 : {
651 47 : Path buildHook = getEnv("NIX_BUILD_HOOK");
652 47 : if (buildHook == "") return rpDecline;
653 3 : buildHook = absPath(buildHook);
654 :
655 : /* Create a directory where we will store files used for
656 : communication between us and the build hook. */
657 3 : tmpDir = createTempDir();
658 :
659 : /* Create the log file and pipe. */
660 3 : openLogFile();
661 :
662 : /* Create the communication pipes. */
663 3 : toHook.create();
664 3 : fromHook.create();
665 :
666 : /* Fork the hook. */
667 3 : pid = fork();
668 6 : switch (pid) {
669 :
670 : case -1:
671 0 : throw SysError("unable to fork");
672 :
673 : case 0:
674 3 : try { /* child */
675 :
676 3 : initChild();
677 :
678 3 : execl(buildHook.c_str(), buildHook.c_str(),
679 : (worker.canBuildMore() ? (string) "1" : "0").c_str(),
680 : thisSystem.c_str(),
681 : expr.derivation.platform.c_str(),
682 : nePath.c_str(), 0);
683 :
684 0 : throw SysError(format("executing `%1%'") % buildHook);
685 :
686 0 : } catch (exception & e) {
687 0 : cerr << format("build error: %1%\n") % e.what();
688 : }
689 0 : _exit(1);
690 : }
691 :
692 : /* parent */
693 3 : logPipe.writeSide.close();
694 3 : worker.childStarted(shared_from_this(),
695 : pid, logPipe.readSide, false);
696 :
697 3 : fromHook.writeSide.close();
698 3 : toHook.readSide.close();
699 :
700 : /* Read the first line of input, which should be a word indicating
701 : whether the hook wishes to perform the build. !!! potential
702 : for deadlock here: we should also read from the child's logger
703 : pipe. */
704 3 : string reply;
705 3 : try {
706 3 : reply = readLine(fromHook.readSide);
707 0 : } catch (Error & e) {
708 0 : drain(logPipe.readSide);
709 0 : throw;
710 : }
711 :
712 3 : debug(format("hook reply is `%1%'") % reply);
713 :
714 3 : if (reply == "decline" || reply == "postpone") {
715 : /* Clean up the child. !!! hacky / should verify */
716 2 : drain(logPipe.readSide);
717 2 : terminateBuildHook();
718 2 : return reply == "decline" ? rpDecline : rpPostpone;
719 : }
720 :
721 1 : else if (reply == "accept") {
722 :
723 : /* Acquire locks and such. If we then see that there now is a
724 : successor, we're done. */
725 1 : if (!prepareBuild()) {
726 : /* Tell the hook to exit. */
727 0 : writeLine(toHook.writeSide, "cancel");
728 0 : terminateBuildHook();
729 0 : return rpDone;
730 : }
731 :
732 1 : printMsg(lvlInfo, format("running hook to build path `%1%'")
733 : % *expr.derivation.outputs.begin());
734 :
735 : /* Write the information that the hook needs to perform the
736 : build, i.e., the set of input paths (including closure
737 : expressions), the set of output paths, and the successor
738 : mappings for the input expressions. */
739 :
740 1 : Path inputListFN = tmpDir + "/inputs";
741 1 : Path outputListFN = tmpDir + "/outputs";
742 1 : Path successorsListFN = tmpDir + "/successors";
743 :
744 1 : string s;
745 2 : for (ClosureElems::iterator i = inClosures.begin();
746 : i != inClosures.end(); ++i)
747 1 : s += i->first + "\n";
748 2 : for (PathSet::iterator i = inputNFs.begin();
749 : i != inputNFs.end(); ++i)
750 1 : s += *i + "\n";
751 1 : writeStringToFile(inputListFN, s);
752 :
753 1 : s = "";
754 2 : for (PathSet::iterator i = expr.derivation.outputs.begin();
755 : i != expr.derivation.outputs.end(); ++i)
756 1 : s += *i + "\n";
757 1 : writeStringToFile(outputListFN, s);
758 :
759 1 : s = "";
760 1 : for (map<Path, Path>::iterator i = inputSucs.begin();
761 : i != inputSucs.end(); ++i)
762 0 : s += i->first + " " + i->second + "\n";
763 1 : writeStringToFile(successorsListFN, s);
764 :
765 : /* Tell the hook to proceed. */
766 1 : writeLine(toHook.writeSide, "okay");
767 :
768 1 : return rpAccept;
769 : }
770 :
771 0 : else throw Error(format("bad hook reply `%1%'") % reply);
772 : }
773 :
774 :
775 : void NormalisationGoal::terminateBuildHook()
776 2 : {
777 : /* !!! drain stdout of hook */
778 2 : debug("terminating build hook");
779 2 : pid_t savedPid = pid;
780 2 : pid.wait(true);
781 2 : worker.childTerminated(savedPid, false);
782 2 : fromHook.readSide.close();
783 2 : toHook.writeSide.close();
784 2 : fdLogFile.close();
785 2 : logPipe.readSide.close();
786 2 : deleteTmpDir(true); /* get rid of the hook's temporary directory */
787 : }
788 :
789 :
790 : bool NormalisationGoal::prepareBuild()
791 44 : {
792 : /* Obtain locks on all output paths. The locks are automatically
793 : released when we exit this function or Nix crashes. */
794 : /* !!! BUG: this could block, which is not allowed. */
795 44 : outputLocks.lockPaths(expr.derivation.outputs);
796 :
797 : /* Now check again whether there is a successor. This is because
798 : another process may have started building in parallel. After
799 : it has finished and released the locks, we can (and should)
800 : reuse its results. (Strictly speaking the first successor
801 : check can be omitted, but that would be less efficient.) Note
802 : that since we now hold the locks on the output paths, no other
803 : process can build this expression, so no further checks are
804 : necessary. */
805 44 : Path nfPath;
806 44 : if (querySuccessor(nePath, nfPath)) {
807 24 : debug(format("skipping build of expression `%1%', someone beat us to it")
808 : % nePath);
809 24 : outputLocks.setDeletion(true);
810 24 : return false;
811 : }
812 :
813 : /* Gather information necessary for computing the closure and/or
814 : running the build hook. */
815 :
816 : /* The outputs are referenceable paths. */
817 40 : for (PathSet::iterator i = expr.derivation.outputs.begin();
818 : i != expr.derivation.outputs.end(); ++i)
819 : {
820 20 : debug(format("building path `%1%'") % *i);
821 20 : allPaths.insert(*i);
822 : }
823 :
824 : /* Get information about the inputs (these all exist now). */
825 56 : for (PathSet::iterator i = expr.derivation.inputs.begin();
826 : i != expr.derivation.inputs.end(); ++i)
827 : {
828 36 : checkInterrupt();
829 36 : Path nePath = *i;
830 36 : Path nfPath = queryNormalForm(nePath);
831 36 : inputNFs.insert(nfPath);
832 36 : if (nfPath != nePath) inputSucs[nePath] = nfPath;
833 : /* !!! nfPath should be a root of the garbage collector while
834 : we are building */
835 36 : StoreExpr ne = storeExprFromPath(nfPath);
836 36 : if (ne.type != StoreExpr::neClosure) abort();
837 72 : for (ClosureElems::iterator j = ne.closure.elems.begin();
838 : j != ne.closure.elems.end(); ++j)
839 : {
840 36 : inClosures[j->first] = j->second;
841 36 : allPaths.insert(j->first);
842 : }
843 : }
844 :
845 : /* We can skip running the builder if all output paths are already
846 : valid. */
847 20 : bool fastBuild = true;
848 20 : for (PathSet::iterator i = expr.derivation.outputs.begin();
849 : i != expr.derivation.outputs.end(); ++i)
850 20 : if (!isValidPath(*i)) {
851 20 : fastBuild = false;
852 20 : break;
853 : }
854 :
855 20 : if (fastBuild) {
856 0 : printMsg(lvlChatty, format("skipping build; output paths already exist"));
857 0 : createClosure();
858 0 : return false;
859 : }
860 :
861 20 : return true;
862 : }
863 :
864 :
865 : void NormalisationGoal::startBuilder()
866 19 : {
867 19 : startNest(nest, lvlInfo,
868 : format("building path `%1%'") % *expr.derivation.outputs.begin());
869 :
870 : /* Right platform? */
871 19 : if (expr.derivation.platform != thisSystem)
872 0 : throw BuildError(
873 : format("a `%1%' is required to build `%3%', but I am a `%2%'")
874 0 : % expr.derivation.platform % thisSystem % nePath);
875 :
876 : /* If any of the outputs already exist but are not registered,
877 : delete them. */
878 38 : for (PathSet::iterator i = expr.derivation.outputs.begin();
879 : i != expr.derivation.outputs.end(); ++i)
880 : {
881 19 : Path path = *i;
882 19 : if (isValidPath(path))
883 0 : throw Error(format("obstructed build: path `%1%' exists") % path);
884 19 : if (pathExists(path)) {
885 0 : debug(format("removing unregistered path `%1%'") % path);
886 0 : deletePath(path);
887 : }
888 : }
889 :
890 : /* Construct the environment passed to the builder. */
891 : typedef map<string, string> Environment;
892 19 : Environment env;
893 :
894 : /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
895 : PATH is not set. We don't want this, so we fill it in with some dummy
896 : value. */
897 19 : env["PATH"] = "/path-not-set";
898 :
899 : /* Set HOME to a non-existing path to prevent certain programs from using
900 : /etc/passwd (or NIS, or whatever) to locate the home directory (for
901 : example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd
902 : if HOME is not set, but they will just assume that the settings file
903 : they are looking for does not exist if HOME is set but points to some
904 : non-existing path. */
905 19 : env["HOME"] = "/homeless-shelter";
906 :
907 : /* Tell the builder where the Nix store is. Usually they
908 : shouldn't care, but this is useful for purity checking (e.g.,
909 : the compiler or linker might only want to accept paths to files
910 : in the store or in the build directory). */
911 19 : env["NIX_STORE"] = nixStore;
912 :
913 : /* Add all bindings specified in the derivation expression. */
914 124 : for (StringPairs::iterator i = expr.derivation.env.begin();
915 : i != expr.derivation.env.end(); ++i)
916 105 : env[i->first] = i->second;
917 :
918 : /* Create a temporary directory where the build will take
919 : place. */
920 19 : tmpDir = createTempDir();
921 :
922 : /* For convenience, set an environment pointing to the top build
923 : directory. */
924 19 : env["NIX_BUILD_TOP"] = tmpDir;
925 :
926 : /* Also set TMPDIR and variants to point to this directory. */
927 19 : env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir;
928 :
929 : /* Run the builder. */
930 19 : printMsg(lvlChatty, format("executing builder `%1%'") %
931 : expr.derivation.builder);
932 :
933 : /* Create the log file and pipe. */
934 19 : openLogFile();
935 :
936 : /* Fork a child to build the package. Note that while we
937 : currently use forks to run and wait for the children, it
938 : shouldn't be hard to use threads for this on systems where
939 : fork() is unavailable or inefficient. */
940 19 : pid = fork();
941 38 : switch (pid) {
942 :
943 : case -1:
944 0 : throw SysError("unable to fork");
945 :
946 : case 0:
947 :
948 : /* Warning: in the child we should absolutely not make any
949 : Berkeley DB calls! */
950 :
951 19 : try { /* child */
952 :
953 19 : initChild();
954 :
955 : /* Fill in the arguments. */
956 19 : Strings args(expr.derivation.args);
957 19 : args.push_front(baseNameOf(expr.derivation.builder));
958 19 : const char * * argArr = strings2CharPtrs(args);
959 :
960 : /* Fill in the environment. */
961 19 : Strings envStrs;
962 276 : for (Environment::const_iterator i = env.begin();
963 : i != env.end(); ++i)
964 257 : envStrs.push_back(i->first + "=" + i->second);
965 19 : const char * * envArr = strings2CharPtrs(envStrs);
966 :
967 : /* Execute the program. This should not return. */
968 19 : execve(expr.derivation.builder.c_str(),
969 : (char * *) argArr, (char * *) envArr);
970 :
971 0 : throw SysError(format("executing `%1%'")
972 : % expr.derivation.builder);
973 :
974 0 : } catch (exception & e) {
975 0 : cerr << format("build error: %1%\n") % e.what();
976 : }
977 0 : _exit(1);
978 : }
979 :
980 : /* parent */
981 19 : pid.setSeparatePG(true);
982 19 : logPipe.writeSide.close();
983 19 : worker.childStarted(shared_from_this(),
984 : pid, logPipe.readSide, true);
985 : }
986 :
987 :
988 : void NormalisationGoal::createClosure()
989 20 : {
990 : /* The resulting closure expression. */
991 20 : StoreExpr nf;
992 20 : nf.type = StoreExpr::neClosure;
993 :
994 20 : startNest(nest, lvlTalkative,
995 : format("computing closure for `%1%'") % nePath);
996 :
997 : /* Check whether the output paths were created, and grep each
998 : output path to determine what other paths it references. Also make all
999 : output paths read-only. */
1000 20 : PathSet usedPaths;
1001 40 : for (PathSet::iterator i = expr.derivation.outputs.begin();
1002 : i != expr.derivation.outputs.end(); ++i)
1003 : {
1004 20 : Path path = *i;
1005 20 : if (!pathExists(path)) {
1006 0 : throw BuildError(
1007 : format("builder for `%1%' failed to produce output path `%2%'")
1008 : % nePath % path);
1009 : }
1010 20 : nf.closure.roots.insert(path);
1011 :
1012 20 : makePathReadOnly(path);
1013 :
1014 : /* For this output path, find the references to other paths contained
1015 : in it. */
1016 20 : startNest(nest2, lvlChatty,
1017 : format("scanning for store references in `%1%'") % path);
1018 20 : Strings refPaths;
1019 20 : if (!pathExists(path + "/nix-support/no-scan")) {
1020 20 : refPaths = filterReferences(path,
1021 : Strings(allPaths.begin(), allPaths.end()));
1022 : }
1023 20 : nest2.close();
1024 :
1025 : /* Construct a closure element for this output path. */
1026 20 : ClosureElem elem;
1027 :
1028 : /* For each path referenced by this output path, add its id to the
1029 : closure element and add the id to the `usedPaths' set (so that the
1030 : elements referenced by *its* closure are added below). */
1031 22 : for (Paths::iterator j = refPaths.begin();
1032 : j != refPaths.end(); ++j)
1033 : {
1034 2 : checkInterrupt();
1035 2 : Path path = *j;
1036 2 : elem.refs.insert(path);
1037 2 : if (inClosures.find(path) != inClosures.end())
1038 2 : usedPaths.insert(path);
1039 0 : else if (expr.derivation.outputs.find(path) ==
1040 : expr.derivation.outputs.end())
1041 0 : abort();
1042 : }
1043 :
1044 20 : nf.closure.elems[path] = elem;
1045 : }
1046 :
1047 : /* Close the closure. That is, for any referenced path, add the paths
1048 : referenced by it. */
1049 20 : PathSet donePaths;
1050 :
1051 22 : while (!usedPaths.empty()) {
1052 2 : checkInterrupt();
1053 2 : PathSet::iterator i = usedPaths.begin();
1054 2 : Path path = *i;
1055 2 : usedPaths.erase(i);
1056 :
1057 2 : if (donePaths.find(path) != donePaths.end()) continue;
1058 2 : donePaths.insert(path);
1059 :
1060 2 : ClosureElems::iterator j = inClosures.find(path);
1061 2 : if (j == inClosures.end()) abort();
1062 :
1063 2 : nf.closure.elems[path] = j->second;
1064 :
1065 2 : for (PathSet::iterator k = j->second.refs.begin();
1066 : k != j->second.refs.end(); k++)
1067 0 : usedPaths.insert(*k);
1068 : }
1069 :
1070 : /* For debugging, print out the referenced and unreferenced paths. */
1071 56 : for (ClosureElems::iterator i = inClosures.begin();
1072 : i != inClosures.end(); ++i)
1073 : {
1074 36 : PathSet::iterator j = donePaths.find(i->first);
1075 36 : if (j == donePaths.end())
1076 34 : debug(format("unreferenced input: `%1%'") % i->first);
1077 : else
1078 2 : debug(format("referenced input: `%1%'") % i->first);
1079 : }
1080 :
1081 : /* Write the normal form. This does not have to occur in the
1082 : transaction below because writing terms is idem-potent. */
1083 20 : ATerm nfTerm = unparseStoreExpr(nf);
1084 20 : Path nfPath = writeTerm(nfTerm, "-s");
1085 :
1086 : /* Register each output path, and register the normal form. This
1087 : is wrapped in one database transaction to ensure that if we
1088 : crash, either everything is registered or nothing is. This is
1089 : for recoverability: unregistered paths in the store can be
1090 : deleted arbitrarily, while registered paths can only be deleted
1091 : by running the garbage collector. */
1092 20 : Transaction txn;
1093 20 : createStoreTransaction(txn);
1094 40 : for (PathSet::iterator i = expr.derivation.outputs.begin();
1095 : i != expr.derivation.outputs.end(); ++i)
1096 20 : registerValidPath(txn, *i);
1097 20 : registerSuccessor(txn, nePath, nfPath);
1098 20 : txn.commit();
1099 :
1100 : /* It is now safe to delete the lock files, since all future
1101 : lockers will see the successor; they will not create new lock
1102 : files with the same names as the old (unlinked) lock files. */
1103 20 : outputLocks.setDeletion(true);
1104 : }
1105 :
1106 :
1107 : void NormalisationGoal::openLogFile()
1108 22 : {
1109 : /* Create a log file. */
1110 22 : Path logFileName = nixLogDir + "/" + baseNameOf(nePath);
1111 22 : fdLogFile = open(logFileName.c_str(),
1112 : O_CREAT | O_WRONLY | O_TRUNC, 0666);
1113 22 : if (fdLogFile == -1)
1114 0 : throw SysError(format("creating log file `%1%'") % logFileName);
1115 :
1116 : /* Create a pipe to get the output of the child. */
1117 22 : logPipe.create();
1118 : }
1119 :
1120 :
1121 : void NormalisationGoal::initChild()
1122 22 : {
1123 22 : commonChildInit(logPipe);
1124 :
1125 22 : if (chdir(tmpDir.c_str()) == -1)
1126 0 : throw SysError(format("changing into `%1%'") % tmpDir);
1127 :
1128 : /* When running a hook, dup the communication pipes. */
1129 22 : bool inHook = fromHook.writeSide.isOpen();
1130 22 : if (inHook) {
1131 3 : fromHook.readSide.close();
1132 3 : if (dup2(fromHook.writeSide, 3) == -1)
1133 0 : throw SysError("dupping from-hook write side");
1134 :
1135 3 : toHook.writeSide.close();
1136 3 : if (dup2(toHook.readSide, 4) == -1)
1137 0 : throw SysError("dupping to-hook read side");
1138 : }
1139 :
1140 : /* Close all other file descriptors. */
1141 22 : int maxFD = 0;
1142 22 : maxFD = sysconf(_SC_OPEN_MAX);
1143 22550 : for (int fd = 0; fd < maxFD; ++fd)
1144 22528 : if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
1145 : && (!inHook || (fd != 3 && fd != 4)))
1146 22456 : close(fd); /* ignore result */
1147 : }
1148 :
1149 :
1150 : void NormalisationGoal::deleteTmpDir(bool force)
1151 85 : {
1152 85 : if (tmpDir != "") {
1153 22 : if (keepFailed && !force)
1154 0 : printMsg(lvlError,
1155 : format("builder for `%1%' failed; keeping build directory `%2%'")
1156 : % nePath % tmpDir);
1157 : else
1158 22 : deletePath(tmpDir);
1159 22 : tmpDir = "";
1160 : }
1161 : }
1162 :
1163 :
1164 : void NormalisationGoal::writeLog(int fd,
1165 : const unsigned char * buf, size_t count)
1166 102 : {
1167 102 : assert(fd == logPipe.readSide);
1168 102 : writeFull(fdLogFile, buf, count);
1169 : }
1170 :
1171 :
1172 : string NormalisationGoal::name()
1173 628 : {
1174 628 : return (format("normalisation of `%1%'") % nePath).str();
1175 : }
1176 :
1177 :
1178 :
1179 : //////////////////////////////////////////////////////////////////////
1180 :
1181 :
1182 : class RealisationGoal : public Goal
1183 : {
1184 : private:
1185 : /* The path of the store expression. */
1186 : Path nePath;
1187 :
1188 : /* The normal form. */
1189 : Path nfPath;
1190 :
1191 : /* Whether we should try to delete a broken successor mapping. */
1192 : bool tryFallback;
1193 :
1194 : /* The store expression stored at nePath. */
1195 : StoreExpr expr;
1196 :
1197 : typedef void (RealisationGoal::*GoalState)();
1198 : GoalState state;
1199 :
1200 : public:
1201 : RealisationGoal(const Path & _nePath, Worker & _worker);
1202 : ~RealisationGoal();
1203 :
1204 : void work();
1205 :
1206 : /* The states. */
1207 : void init();
1208 : void isNormalised();
1209 : void haveStoreExpr();
1210 : void elemFinished();
1211 :
1212 : void fallBack(const format & error);
1213 :
1214 : string name();
1215 : };
1216 :
1217 :
1218 : RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker)
1219 68 : : Goal(_worker)
1220 68 : {
1221 68 : nePath = _nePath;
1222 68 : tryFallback = ::tryFallback;
1223 68 : state = &RealisationGoal::init;
1224 : }
1225 :
1226 :
1227 : RealisationGoal::~RealisationGoal()
1228 136 : {
1229 68 : }
1230 :
1231 :
1232 : void RealisationGoal::work()
1233 265 : {
1234 265 : (this->*state)();
1235 : }
1236 :
1237 :
1238 : void RealisationGoal::init()
1239 69 : {
1240 69 : trace("init");
1241 :
1242 69 : if (querySuccessor(nePath, nfPath)) {
1243 7 : nrFailed = 0;
1244 7 : isNormalised();
1245 7 : return;
1246 : }
1247 :
1248 : /* First normalise the expression (which is a no-op if the
1249 : expression is already a closure). */
1250 62 : addWaitee(worker.makeNormalisationGoal(nePath));
1251 :
1252 : /* Since there is no successor right now, the normalisation goal
1253 : will perform an actual build. So there is no sense in trying a
1254 : fallback if the realisation of the closure fails (it can't
1255 : really fail). */
1256 62 : tryFallback = false;
1257 :
1258 62 : state = &RealisationGoal::isNormalised;
1259 : }
1260 :
1261 :
1262 : void RealisationGoal::isNormalised()
1263 69 : {
1264 69 : trace("has been normalised");
1265 :
1266 69 : if (nrFailed != 0) {
1267 1 : amDone(false);
1268 1 : return;
1269 : }
1270 :
1271 68 : nfPath = queryNormalForm(nePath);
1272 :
1273 : /* Now make sure that the store expression exists. If it doesn't,
1274 : it may be created through a substitute. */
1275 68 : addWaitee(worker.makeSubstitutionGoal(nfPath));
1276 :
1277 68 : state = &RealisationGoal::haveStoreExpr;
1278 : }
1279 :
1280 :
1281 : void RealisationGoal::haveStoreExpr()
1282 68 : {
1283 68 : trace("loading store expression");
1284 :
1285 68 : if (nrFailed != 0) {
1286 1 : fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath);
1287 1 : return;
1288 : }
1289 :
1290 67 : assert(isValidPath(nfPath));
1291 :
1292 : /* Get the store expression. */
1293 67 : expr = storeExprFromPath(nfPath);
1294 :
1295 : /* If this is a normal form (i.e., a closure) we are also done. */
1296 67 : if (expr.type != StoreExpr::neClosure)
1297 0 : throw Error(format("expected closure in `%1%'") % nfPath);
1298 :
1299 : /* Each path in the closure should exist, or should be creatable
1300 : through a substitute. */
1301 136 : for (ClosureElems::const_iterator i = expr.closure.elems.begin();
1302 : i != expr.closure.elems.end(); ++i)
1303 69 : addWaitee(worker.makeSubstitutionGoal(i->first));
1304 :
1305 67 : state = &RealisationGoal::elemFinished;
1306 : }
1307 :
1308 :
1309 : void RealisationGoal::elemFinished()
1310 67 : {
1311 67 : trace("all closure elements present");
1312 :
1313 67 : if (nrFailed != 0) {
1314 0 : fallBack(
1315 : format("cannot realise closure `%1%': "
1316 : "%2% closure element(s) are not present "
1317 : "and could not be substituted")
1318 : % nfPath % nrFailed);
1319 0 : return;
1320 : }
1321 :
1322 67 : amDone();
1323 : }
1324 :
1325 :
1326 : void RealisationGoal::fallBack(const format & error)
1327 1 : {
1328 1 : if (tryFallback && nePath != nfPath) {
1329 1 : printMsg(lvlError, format("%1%; trying to normalise derivation instead")
1330 : % error);
1331 1 : tryFallback = false;
1332 1 : unregisterSuccessor(nePath);
1333 1 : nrFailed = 0;
1334 1 : init();
1335 : } else {
1336 0 : printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error);
1337 0 : amDone(false);
1338 : }
1339 : }
1340 :
1341 :
1342 : string RealisationGoal::name()
1343 579 : {
1344 579 : return (format("realisation of `%1%'") % nePath).str();
1345 : }
1346 :
1347 :
1348 :
1349 : //////////////////////////////////////////////////////////////////////
1350 :
1351 :
1352 : class SubstitutionGoal : public Goal
1353 : {
1354 : private:
1355 : /* The store path that should be realised through a substitute. */
1356 : Path storePath;
1357 :
1358 : /* The remaining substitutes for this path. */
1359 : Substitutes subs;
1360 :
1361 : /* The current substitute. */
1362 : Substitute sub;
1363 :
1364 : /* The normal form of the substitute store expression. */
1365 : Path nfSub;
1366 :
1367 : /* Pipe for the substitute's standard output/error. */
1368 : Pipe logPipe;
1369 :
1370 : /* The process ID of the builder. */
1371 : Pid pid;
1372 :
1373 : /* Lock on the store path. */
1374 : shared_ptr<PathLocks> outputLock;
1375 :
1376 : typedef void (SubstitutionGoal::*GoalState)();
1377 : GoalState state;
1378 :
1379 : public:
1380 : SubstitutionGoal(const Path & _nePath, Worker & _worker);
1381 : ~SubstitutionGoal();
1382 :
1383 : void work();
1384 :
1385 : /* The states. */
1386 : void init();
1387 : void tryNext();
1388 : void exprNormalised();
1389 : void exprRealised();
1390 : void tryToRun();
1391 : void finished();
1392 :
1393 : /* Callback used by the worker to write to the log. */
1394 : void writeLog(int fd, const unsigned char * buf, size_t count);
1395 :
1396 : string name();
1397 : };
1398 :
1399 :
1400 : SubstitutionGoal::SubstitutionGoal(const Path & _storePath, Worker & _worker)
1401 199 : : Goal(_worker)
1402 607 : {
1403 199 : storePath = _storePath;
1404 199 : state = &SubstitutionGoal::init;
1405 : }
1406 :
1407 :
1408 : SubstitutionGoal::~SubstitutionGoal()
1409 398 : {
1410 199 : if (pid != -1) worker.childTerminated(pid);
1411 199 : }
1412 :
1413 :
1414 : void SubstitutionGoal::work()
1415 216 : {
1416 216 : (this->*state)();
1417 : }
1418 :
1419 :
1420 : void SubstitutionGoal::init()
1421 199 : {
1422 199 : trace("init");
1423 :
1424 : /* If the path already exists we're done. */
1425 199 : if (isValidPath(storePath)) {
1426 193 : amDone();
1427 193 : return;
1428 : }
1429 :
1430 : /* Otherwise, get the substitutes. */
1431 6 : subs = querySubstitutes(storePath);
1432 :
1433 : /* Try the first one. */
1434 6 : tryNext();
1435 : }
1436 :
1437 :
1438 : void SubstitutionGoal::tryNext()
1439 8 : {
1440 8 : trace("trying next substitute");
1441 :
1442 8 : if (subs.size() == 0) {
1443 : /* None left. Terminate this goal and let someone else deal
1444 : with it. */
1445 2 : printMsg(lvlError,
1446 : format("path `%1%' is required, but it has no (remaining) substitutes")
1447 : % storePath);
1448 2 : amDone(false);
1449 2 : return;
1450 : }
1451 6 : sub = subs.front();
1452 6 : subs.pop_front();
1453 :
1454 : /* Realise the substitute store expression. */
1455 6 : nrFailed = 0;
1456 6 : addWaitee(worker.makeRealisationGoal(sub.storeExpr));
1457 :
1458 6 : state = &SubstitutionGoal::exprRealised;
1459 : }
1460 :
1461 :
1462 : void SubstitutionGoal::exprRealised()
1463 6 : {
1464 6 : trace("substitute store expression realised");
1465 :
1466 6 : if (nrFailed != 0) {
1467 1 : tryNext();
1468 1 : return;
1469 : }
1470 :
1471 : /* !!! the storeExpr doesn't have to be a derivation, right? */
1472 5 : nfSub = queryNormalForm(sub.storeExpr);
1473 :
1474 5 : state = &SubstitutionGoal::tryToRun;
1475 5 : worker.waitForBuildSlot(shared_from_this());
1476 : }
1477 :
1478 :
1479 : void SubstitutionGoal::tryToRun()
1480 5 : {
1481 5 : trace("trying to run");
1482 :
1483 : /* Make sure that we are allowed to start a build. */
1484 5 : if (!worker.canBuildMore()) {
1485 0 : worker.waitForBuildSlot(shared_from_this());
1486 0 : return;
1487 : }
1488 :
1489 : /* Acquire a lock on the output path. */
1490 5 : PathSet lockPath;
1491 5 : lockPath.insert(storePath);
1492 5 : outputLock = shared_ptr<PathLocks>(new PathLocks);
1493 5 : outputLock->lockPaths(lockPath);
1494 :
1495 : /* Check again whether the path is invalid. */
1496 5 : if (isValidPath(storePath)) {
1497 0 : debug(format("store path `%1%' has become valid") % storePath);
1498 0 : outputLock->setDeletion(true);
1499 0 : amDone();
1500 0 : return;
1501 : }
1502 :
1503 5 : printMsg(lvlInfo,
1504 : format("substituting path `%1%' using substituter `%2%'")
1505 : % storePath % sub.storeExpr);
1506 :
1507 : /* What's the substitute program? */
1508 5 : StoreExpr expr = storeExprFromPath(nfSub);
1509 5 : assert(expr.type == StoreExpr::neClosure);
1510 5 : assert(!expr.closure.roots.empty());
1511 : Path program =
1512 5 : canonPath(*expr.closure.roots.begin() + "/" + sub.program);
1513 :
1514 5 : logPipe.create();
1515 :
1516 : /* Remove the (stale) output path if it exists. */
1517 5 : if (pathExists(storePath))
1518 0 : deletePath(storePath);
1519 :
1520 : /* Fork the substitute program. */
1521 5 : pid = fork();
1522 10 : switch (pid) {
1523 :
1524 : case -1:
1525 0 : throw SysError("unable to fork");
1526 :
1527 : case 0:
1528 5 : try { /* child */
1529 :
1530 5 : logPipe.readSide.close();
1531 :
1532 : /* !!! close other handles */
1533 :
1534 5 : commonChildInit(logPipe);
1535 :
1536 : /* Fill in the arguments. */
1537 5 : Strings args(sub.args);
1538 5 : args.push_front(storePath);
1539 5 : args.push_front(baseNameOf(program));
1540 5 : const char * * argArr = strings2CharPtrs(args);
1541 :
1542 5 : execv(program.c_str(), (char * *) argArr);
1543 :
1544 0 : throw SysError(format("executing `%1%'") % program);
1545 :
1546 0 : } catch (exception & e) {
1547 0 : cerr << format("substitute error: %1%\n") % e.what();
1548 : }
1549 0 : _exit(1);
1550 : }
1551 :
1552 : /* parent */
1553 5 : pid.setSeparatePG(true);
1554 5 : logPipe.writeSide.close();
1555 5 : worker.childStarted(shared_from_this(),
1556 : pid, logPipe.readSide, true);
1557 :
1558 5 : state = &SubstitutionGoal::finished;
1559 : }
1560 :
1561 :
1562 : void SubstitutionGoal::finished()
1563 5 : {
1564 5 : trace("substitute finished");
1565 :
1566 : /* Since we got an EOF on the logger pipe, the substitute is
1567 : presumed to have terminated. */
1568 : /* !!! this could block! */
1569 5 : pid_t savedPid = pid;
1570 5 : int status = pid.wait(true);
1571 :
1572 : /* So the child is gone now. */
1573 5 : worker.childTerminated(savedPid);
1574 :
1575 : /* Close the read side of the logger pipe. */
1576 5 : logPipe.readSide.close();
1577 :
1578 5 : debug(format("substitute for `%1%' finished") % storePath);
1579 :
1580 : /* Check the exit status and the build result. */
1581 5 : try {
1582 :
1583 5 : if (!statusOk(status))
1584 1 : throw SubstError(format("builder for `%1%' %2%")
1585 2 : % storePath % statusToString(status));
1586 :
1587 4 : if (!pathExists(storePath))
1588 0 : throw SubstError(
1589 : format("substitute did not produce path `%1%'")
1590 : % storePath);
1591 :
1592 1 : } catch (SubstError & e) {
1593 :
1594 1 : printMsg(lvlInfo,
1595 : format("substitution of path `%1%' using substituter `%2%' failed: %3%")
1596 : % storePath % sub.storeExpr % e.msg());
1597 :
1598 : /* Try the next substitute. */
1599 1 : state = &SubstitutionGoal::tryNext;
1600 1 : worker.wakeUp(shared_from_this());
1601 1 : return;
1602 : }
1603 :
1604 4 : makePathReadOnly(storePath);
1605 :
1606 4 : Transaction txn;
1607 4 : createStoreTransaction(txn);
1608 4 : registerValidPath(txn, storePath);
1609 4 : txn.commit();
1610 :
1611 4 : outputLock->setDeletion(true);
1612 :
1613 4 : printMsg(lvlChatty,
1614 : format("substitution of path `%1%' succeeded") % storePath);
1615 :
1616 4 : amDone();
1617 : }
1618 :
1619 :
1620 : void SubstitutionGoal::writeLog(int fd,
1621 : const unsigned char * buf, size_t count)
1622 7 : {
1623 7 : assert(fd == logPipe.readSide);
1624 : /* Don't write substitution output to a log file for now. We
1625 : probably should, though. */
1626 : }
1627 :
1628 :
1629 : string SubstitutionGoal::name()
1630 621 : {
1631 621 : return (format("substitution of `%1%'") % storePath).str();
1632 : }
1633 :
1634 :
1635 :
1636 : //////////////////////////////////////////////////////////////////////
1637 :
1638 :
1639 : /* A fake goal used to receive notification of success or failure of
1640 : other goals. */
1641 : class PseudoGoal : public Goal
1642 : {
1643 : private:
1644 : bool success;
1645 :
1646 : public:
1647 14 : PseudoGoal(Worker & _worker) : Goal(_worker)
1648 14 : {
1649 14 : success = true;
1650 : }
1651 :
1652 : void work()
1653 0 : {
1654 0 : abort();
1655 : }
1656 :
1657 : void waiteeDone(GoalPtr waitee, bool success)
1658 14 : {
1659 14 : if (!success) this->success = false;
1660 : }
1661 :
1662 : bool isOkay()
1663 14 : {
1664 14 : return success;
1665 : }
1666 :
1667 : string name()
1668 0 : {
1669 0 : return "pseudo-goal";
1670 : }
1671 : };
1672 :
1673 :
1674 :
1675 : //////////////////////////////////////////////////////////////////////
1676 :
1677 :
1678 : static bool working = false;
1679 :
1680 :
1681 : Worker::Worker()
1682 140 : {
1683 : /* Debugging: prevent recursive workers. */
1684 14 : if (working) abort();
1685 14 : working = true;
1686 14 : nrChildren = 0;
1687 : }
1688 :
1689 :
1690 : Worker::~Worker()
1691 28 : {
1692 14 : working = false;
1693 :
1694 : /* Explicitly get rid of all strong pointers now. After this all
1695 : goals that refer to this worker should be gone. (Otherwise we
1696 : are in trouble, since goals may call childTerminated() etc. in
1697 : their destructors). */
1698 14 : topGoals.clear();
1699 : }
1700 :
1701 :
1702 : template<class T>
1703 : static GoalPtr addGoal(const Path & path,
1704 : Worker & worker, WeakGoalMap & goalMap)
1705 371 : {
1706 371 : GoalPtr goal = goalMap[path].lock();
1707 371 : if (!goal) {
1708 330 : goal = GoalPtr(new T(path, worker));
1709 330 : goalMap[path] = goal;
1710 330 : worker.wakeUp(goal);
1711 : }
1712 330 : return goal;
1713 : }
1714 :
1715 :
1716 : GoalPtr Worker::makeNormalisationGoal(const Path & nePath)
1717 63 : {
1718 63 : return addGoal<NormalisationGoal>(nePath, *this, normalisationGoals);
1719 : }
1720 :
1721 :
1722 : GoalPtr Worker::makeRealisationGoal(const Path & nePath)
1723 109 : {
1724 109 : return addGoal<RealisationGoal>(nePath, *this, realisationGoals);
1725 : }
1726 :
1727 :
1728 : GoalPtr Worker::makeSubstitutionGoal(const Path & storePath)
1729 199 : {
1730 199 : return addGoal<SubstitutionGoal>(storePath, *this, substitutionGoals);
1731 : }
1732 :
1733 :
1734 : static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
1735 990 : {
1736 : /* !!! For now we just let dead goals accumulate. We should
1737 : probably periodically sweep the goalMap to remove dead
1738 : goals. */
1739 : }
1740 :
1741 :
1742 : void Worker::removeGoal(GoalPtr goal)
1743 330 : {
1744 330 : topGoals.erase(goal);
1745 330 : ::removeGoal(goal, normalisationGoals);
1746 330 : ::removeGoal(goal, realisationGoals);
1747 330 : ::removeGoal(goal, substitutionGoals);
1748 : }
1749 :
1750 :
1751 : void Worker::wakeUp(GoalPtr goal)
1752 717 : {
1753 717 : goal->trace("woken up");
1754 717 : awake.insert(goal);
1755 : }
1756 :
1757 :
1758 : bool Worker::canBuildMore()
1759 59 : {
1760 59 : return nrChildren < maxBuildJobs;
1761 : }
1762 :
1763 :
1764 : void Worker::childStarted(GoalPtr goal,
1765 : pid_t pid, int fdOutput, bool inBuildSlot)
1766 27 : {
1767 351 : Child child;
1768 27 : child.goal = goal;
1769 27 : child.fdOutput = fdOutput;
1770 27 : child.inBuildSlot = inBuildSlot;
1771 135 : children[pid] = child;
1772 27 : if (inBuildSlot) nrChildren++;
1773 : }
1774 :
1775 :
1776 : void Worker::childTerminated(pid_t pid, bool wakeSleepers)
1777 27 : {
1778 27 : Children::iterator i = children.find(pid);
1779 27 : assert(i != children.end());
1780 :
1781 27 : if (i->second.inBuildSlot) {
1782 24 : assert(nrChildren > 0);
1783 24 : nrChildren--;
1784 : }
1785 :
1786 27 : children.erase(pid);
1787 :
1788 27 : if (wakeSleepers) {
1789 :
1790 : /* Wake up goals waiting for a build slot. */
1791 28 : for (WeakGoals::iterator i = wantingToBuild.begin();
1792 : i != wantingToBuild.end(); ++i)
1793 : {
1794 3 : GoalPtr goal = i->lock();
1795 3 : if (goal) wakeUp(goal);
1796 : }
1797 :
1798 25 : wantingToBuild.clear();
1799 :
1800 : }
1801 : }
1802 :
1803 :
1804 : void Worker::waitForBuildSlot(GoalPtr goal, bool reallyWait)
1805 8 : {
1806 8 : debug("wait for build slot");
1807 8 : if (reallyWait && children.size() == 0)
1808 0 : throw Error("waiting for a build slot, yet there are no children - "
1809 : "maybe the build hook gave an inappropriate `postpone' reply?");
1810 8 : if (!reallyWait && canBuildMore())
1811 5 : wakeUp(goal); /* we can do it right away */
1812 : else
1813 3 : wantingToBuild.insert(goal);
1814 : }
1815 :
1816 :
1817 : bool Worker::run(GoalPtr topGoal)
1818 14 : {
1819 14 : assert(topGoal);
1820 :
1821 : /* Wrap the specified top-level goal in a pseudo-goal so that we
1822 : can check whether it succeeded. */
1823 42 : shared_ptr<PseudoGoal> pseudo(new PseudoGoal(*this));
1824 14 : pseudo->addWaitee(topGoal);
1825 :
1826 : /* For now, we have only one top-level goal. */
1827 14 : topGoals.insert(topGoal);
1828 :
1829 14 : startNest(nest, lvlDebug, format("entered goal loop"));
1830 :
1831 146 : while (1) {
1832 :
1833 146 : checkInterrupt();
1834 :
1835 : /* Call every wake goal. */
1836 652 : while (!awake.empty()) {
1837 506 : WeakGoals awake2(awake);
1838 506 : awake.clear();
1839 1223 : for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) {
1840 717 : checkInterrupt();
1841 717 : GoalPtr goal = i->lock();
1842 717 : if (goal) goal->work();
1843 : }
1844 : }
1845 :
1846 146 : if (topGoals.empty()) break;
1847 :
1848 : /* !!! not when we're polling */
1849 132 : assert(!children.empty());
1850 :
1851 : /* Wait for input. */
1852 132 : waitForInput();
1853 : }
1854 :
1855 : /* If --keep-going is not set, it's possible that the main goal
1856 : exited while some of its subgoals were still active. But if
1857 : --keep-going *is* set, then they must all be finished now. */
1858 14 : assert(!keepGoing || awake.empty());
1859 14 : assert(!keepGoing || wantingToBuild.empty());
1860 14 : assert(!keepGoing || children.empty());
1861 :
1862 14 : return pseudo->isOkay();
1863 : }
1864 :
1865 :
1866 : void Worker::waitForInput()
1867 132 : {
1868 132 : printMsg(lvlVomit, "waiting for children");
1869 :
1870 : /* Process log output from the children. We also use this to
1871 : detect child termination: if we get EOF on the logger pipe of a
1872 : build, we assume that the builder has terminated. */
1873 :
1874 : /* Use select() to wait for the input side of any logger pipe to
1875 : become `available'. Note that `available' (i.e., non-blocking)
1876 : includes EOF. */
1877 132 : fd_set fds;
1878 132 : FD_ZERO(&fds);
1879 132 : int fdMax = 0;
1880 307 : for (Children::iterator i = children.begin();
1881 : i != children.end(); ++i)
1882 : {
1883 175 : int fd = i->second.fdOutput;
1884 175 : FD_SET(fd, &fds);
1885 175 : if (fd >= fdMax) fdMax = fd + 1;
1886 : }
1887 :
1888 132 : if (select(fdMax, &fds, 0, 0, 0) == -1) {
1889 0 : if (errno == EINTR) return;
1890 0 : throw SysError("waiting for input");
1891 : }
1892 :
1893 : /* Process all available file descriptors. */
1894 307 : for (Children::iterator i = children.begin();
1895 : i != children.end(); ++i)
1896 : {
1897 175 : checkInterrupt();
1898 175 : GoalPtr goal = i->second.goal.lock();
1899 175 : assert(goal);
1900 175 : int fd = i->second.fdOutput;
1901 175 : if (FD_ISSET(fd, &fds)) {
1902 134 : unsigned char buffer[4096];
1903 134 : ssize_t rd = read(fd, buffer, sizeof(buffer));
1904 134 : if (rd == -1) {
1905 0 : if (errno != EINTR)
1906 0 : throw SysError(format("reading from %1%")
1907 : % goal->name());
1908 134 : } else if (rd == 0) {
1909 25 : debug(format("%1%: got EOF") % goal->name());
1910 25 : wakeUp(goal);
1911 : } else {
1912 109 : printMsg(lvlVomit, format("%1%: read %2% bytes")
1913 : % goal->name() % rd);
1914 109 : goal->writeLog(fd, buffer, (size_t) rd);
1915 109 : if (verbosity >= buildVerbosity)
1916 109 : writeFull(STDERR_FILENO, buffer, rd);
1917 : }
1918 : }
1919 : }
1920 : }
1921 :
1922 :
1923 : //////////////////////////////////////////////////////////////////////
1924 :
1925 :
1926 : Path normaliseStoreExpr(const Path & nePath)
1927 1 : {
1928 1 : startNest(nest, lvlDebug, format("normalising `%1%'") % nePath);
1929 :
1930 1 : Worker worker;
1931 1 : if (!worker.run(worker.makeNormalisationGoal(nePath)))
1932 0 : throw Error(format("normalisation of store expression `%1%' failed") % nePath);
1933 :
1934 1 : return queryNormalForm(nePath);
1935 : }
1936 :
1937 :
1938 : Path realiseStoreExpr(const Path & nePath)
1939 13 : {
1940 13 : startNest(nest, lvlDebug, format("realising `%1%'") % nePath);
1941 :
1942 13 : Worker worker;
1943 13 : if (!worker.run(worker.makeRealisationGoal(nePath)))
1944 0 : throw Error(format("realisation of store expression `%1%' failed") % nePath);
1945 :
1946 13 : return queryNormalForm(nePath);
1947 : }
1948 :
1949 :
1950 : void ensurePath(const Path & path)
1951 215 : {
1952 : /* If the path is already valid, we're done. */
1953 215 : if (isValidPath(path)) return;
1954 :
1955 0 : Worker worker;
1956 0 : if (!worker.run(worker.makeSubstitutionGoal(path)))
1957 0 : throw Error(format("path `%1%' does not exist and cannot be created") % path);
1958 51 : }
|
