1 : #include <iostream>
2 : #include <algorithm>
3 :
4 : #include <sys/wait.h>
5 : #include <unistd.h>
6 :
7 : #include "store.hh"
8 : #include "globals.hh"
9 : #include "db.hh"
10 : #include "archive.hh"
11 : #include "pathlocks.hh"
12 :
13 :
14 : /* Nix database. */
15 51 : static Database nixDB;
16 :
17 :
18 : /* Database tables. */
19 :
20 : /* dbValidPaths :: Path -> ()
21 :
22 : The existence of a key $p$ indicates that path $p$ is valid (that
23 : is, produced by a succesful build). */
24 : static TableId dbValidPaths = 0;
25 :
26 : /* dbSuccessors :: Path -> Path
27 :
28 : Each pair $(p_1, p_2)$ in this mapping records the fact that the
29 : Nix expression stored at path $p_1$ has a successor expression
30 : stored at path $p_2$.
31 :
32 : Note that a term $y$ is a successor of $x$ iff there exists a
33 : sequence of rewrite steps that rewrites $x$ into $y$.
34 : */
35 : static TableId dbSuccessors = 0;
36 :
37 : /* dbSuccessorsRev :: Path -> [Path]
38 :
39 : The reverse mapping of dbSuccessors (i.e., it stores the
40 : predecessors of a Nix expression).
41 : */
42 : static TableId dbSuccessorsRev = 0;
43 :
44 : /* dbSubstitutes :: Path -> [(Path, Path, [string])]
45 :
46 : Each pair $(p, subs)$ tells Nix that it can use any of the
47 : substitutes in $subs$ to build path $p$. Each substitute is a
48 : tuple $(storeExpr, program, args)$ (see the type `Substitute' in
49 : `store.hh').
50 :
51 : The main purpose of this is for distributed caching of derivates.
52 : One system can compute a derivate and put it on a website (as a Nix
53 : archive), for instance, and then another system can register a
54 : substitute for that derivate. The substitute in this case might be
55 : a Nix expression that fetches the Nix archive.
56 : */
57 : static TableId dbSubstitutes = 0;
58 :
59 : /* dbSubstitutesRev :: Path -> [Path]
60 :
61 : The reverse mapping of dbSubstitutes; it maps store expressions
62 : back to the paths for which they are substitutes.
63 : */
64 : static TableId dbSubstitutesRev = 0;
65 :
66 :
67 : bool Substitute::operator == (const Substitute & sub)
68 2 : {
69 2 : return storeExpr == sub.storeExpr
70 : && program == sub.program
71 : && args == sub.args;
72 : }
73 :
74 :
75 : void openDB()
76 51 : {
77 51 : if (readOnlyMode) return;
78 39 : try {
79 39 : nixDB.open(nixDBPath);
80 0 : } catch (DbNoPermission & e) {
81 0 : printMsg(lvlTalkative, "cannot access Nix database; continuing anyway");
82 0 : readOnlyMode = true;
83 0 : return;
84 : }
85 39 : dbValidPaths = nixDB.openTable("validpaths");
86 39 : dbSuccessors = nixDB.openTable("successors");
87 39 : dbSuccessorsRev = nixDB.openTable("successors-rev");
88 39 : dbSubstitutes = nixDB.openTable("substitutes");
89 39 : dbSubstitutesRev = nixDB.openTable("substitutes-rev");
90 : }
91 :
92 :
93 : void initDB()
94 1 : {
95 : }
96 :
97 :
98 : void createStoreTransaction(Transaction & txn)
99 34 : {
100 34 : Transaction txn2(nixDB);
101 34 : txn2.moveTo(txn);
102 : }
103 :
104 :
105 : /* Path copying. */
106 :
107 : struct CopySink : DumpSink
108 : {
109 : int fd;
110 : virtual void operator () (const unsigned char * data, unsigned int len)
111 158 : {
112 158 : writeFull(fd, data, len);
113 : }
114 : };
115 :
116 :
117 : struct CopySource : RestoreSource
118 : {
119 : int fd;
120 : virtual void operator () (unsigned char * data, unsigned int len)
121 0 : {
122 0 : readFull(fd, data, len);
123 : }
124 : };
125 :
126 :
127 : void copyPath(const Path & src, const Path & dst)
128 8 : {
129 8 : debug(format("copying `%1%' to `%2%'") % src % dst);
130 :
131 : /* Unfortunately C++ doesn't support coprocedures, so we have no
132 : nice way to chain CopySink and CopySource together. Instead we
133 : fork off a child to run the sink. (Fork-less platforms should
134 : use a thread). */
135 :
136 : /* Create a pipe. */
137 1592 : Pipe pipe;
138 8 : pipe.create();
139 :
140 : /* Fork. */
141 8 : Pid pid;
142 8 : pid = fork();
143 8 : switch (pid) {
144 :
145 : case -1:
146 0 : throw SysError("unable to fork");
147 :
148 : case 0: /* child */
149 0 : try {
150 0 : pipe.writeSide.close();
151 0 : CopySource source;
152 0 : source.fd = pipe.readSide;
153 0 : restorePath(dst, source);
154 0 : _exit(0);
155 0 : } catch (exception & e) {
156 0 : cerr << "error: " << e.what() << endl;
157 : }
158 0 : _exit(1);
159 : }
160 :
161 : /* Parent. */
162 :
163 8 : pipe.readSide.close();
164 :
165 86 : CopySink sink;
166 8 : sink.fd = pipe.writeSide;
167 : {
168 8 : SwitchToOriginalUser sw;
169 8 : dumpPath(src, sink);
170 : }
171 :
172 : /* Wait for the child to finish. */
173 8 : int status = pid.wait(true);
174 8 : if (!statusOk(status))
175 0 : throw Error(format("cannot copy `%1% to `%2%': child %3%")
176 : % src % dst % statusToString(status));
177 : }
178 :
179 :
180 : static bool isInStore(const Path & path)
181 490 : {
182 490 : return path[0] == '/'
183 : && path.compare(0, nixStore.size(), nixStore) == 0
184 : && path.size() >= nixStore.size() + 2
185 : && path[nixStore.size()] == '/'
186 : && path.find('/', nixStore.size() + 1) == Path::npos;
187 : }
188 :
189 :
190 : void assertStorePath(const Path & path)
191 408 : {
192 408 : if (!isInStore(path))
193 0 : throw Error(format("path `%1%' is not in the Nix store") % path);
194 : }
195 :
196 :
197 : static bool isValidPathTxn(const Path & path, const Transaction & txn)
198 806 : {
199 806 : string s;
200 806 : return nixDB.queryString(txn, dbValidPaths, path, s);
201 : }
202 :
203 :
204 : bool isValidPath(const Path & path)
205 783 : {
206 783 : return isValidPathTxn(path, noTxn);
207 : }
208 :
209 :
210 : static bool isUsablePathTxn(const Path & path, const Transaction & txn)
211 23 : {
212 23 : if (isValidPathTxn(path, txn)) return true;
213 3 : Paths subs;
214 3 : nixDB.queryStrings(txn, dbSubstitutes, path, subs);
215 3 : return subs.size() > 0;
216 : }
217 :
218 :
219 : void registerSuccessor(const Transaction & txn,
220 : const Path & srcPath, const Path & sucPath)
221 23 : {
222 23 : assertStorePath(srcPath);
223 23 : assertStorePath(sucPath);
224 :
225 23 : if (!isUsablePathTxn(sucPath, txn)) throw Error(
226 : format("path `%1%' cannot be a successor, since it is not usable")
227 : % sucPath);
228 :
229 23 : Path known;
230 23 : if (nixDB.queryString(txn, dbSuccessors, srcPath, known) &&
231 : known != sucPath)
232 : {
233 0 : throw Error(format(
234 : "the `impossible' happened: expression in path "
235 : "`%1%' appears to have multiple successors "
236 : "(known `%2%', new `%3%'")
237 : % srcPath % known % sucPath);
238 : }
239 :
240 23 : Paths revs;
241 23 : nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
242 23 : if (find(revs.begin(), revs.end(), srcPath) == revs.end())
243 23 : revs.push_back(srcPath);
244 :
245 23 : nixDB.setString(txn, dbSuccessors, srcPath, sucPath);
246 23 : nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
247 : }
248 :
249 :
250 : void unregisterSuccessor(const Path & srcPath)
251 1 : {
252 1 : assertStorePath(srcPath);
253 :
254 1 : Transaction txn(nixDB);
255 :
256 1 : Path sucPath;
257 1 : if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) {
258 0 : txn.abort();
259 0 : return;
260 : }
261 1 : nixDB.delPair(txn, dbSuccessors, srcPath);
262 :
263 1 : Paths revs;
264 1 : nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
265 1 : Paths::iterator i = find(revs.begin(), revs.end(), srcPath);
266 1 : assert(i != revs.end());
267 1 : revs.erase(i);
268 1 : nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
269 :
270 1 : txn.commit();
271 : }
272 :
273 :
274 : bool querySuccessor(const Path & srcPath, Path & sucPath)
275 299 : {
276 299 : return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath);
277 : }
278 :
279 :
280 : Paths queryPredecessors(const Path & sucPath)
281 0 : {
282 0 : Paths revs;
283 0 : nixDB.queryStrings(noTxn, dbSuccessorsRev, sucPath, revs);
284 0 : return revs;
285 : }
286 :
287 :
288 : static Substitutes readSubstitutes(const Transaction & txn,
289 : const Path & srcPath)
290 18 : {
291 18 : Strings ss;
292 18 : nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss);
293 :
294 462 : Substitutes subs;
295 :
296 34 : for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
297 16 : if (i->size() < 4 || (*i)[3] != 0) {
298 : /* Old-style substitute. !!! remove this code
299 : eventually? */
300 16 : break;
301 : }
302 16 : Strings ss2 = unpackStrings(*i);
303 16 : if (ss2.size() != 3) throw Error("malformed substitute");
304 16 : Strings::iterator j = ss2.begin();
305 16 : Substitute sub;
306 16 : sub.storeExpr = *j++;
307 16 : sub.program = *j++;
308 16 : sub.args = unpackStrings(*j++);
309 16 : subs.push_back(sub);
310 : }
311 :
312 18 : return subs;
313 : }
314 :
315 :
316 : static void writeSubstitutes(const Transaction & txn,
317 : const Path & srcPath, const Substitutes & subs)
318 8 : {
319 8 : Strings ss;
320 :
321 17 : for (Substitutes::const_iterator i = subs.begin();
322 : i != subs.end(); ++i)
323 : {
324 9 : Strings ss2;
325 9 : ss2.push_back(i->storeExpr);
326 9 : ss2.push_back(i->program);
327 9 : ss2.push_back(packStrings(i->args));
328 9 : ss.push_back(packStrings(ss2));
329 : }
330 :
331 8 : nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
332 : }
333 :
334 :
335 : typedef map<Path, Paths> SubstitutesRev;
336 :
337 :
338 : void registerSubstitutes(const Transaction & txn,
339 : const SubstitutePairs & subPairs)
340 7 : {
341 21 : SubstitutesRev revMap;
342 :
343 14 : for (SubstitutePairs::const_iterator i = subPairs.begin();
344 : i != subPairs.end(); ++i)
345 : {
346 7 : const Path & srcPath(i->first);
347 7 : const Substitute & sub(i->second);
348 :
349 7 : assertStorePath(srcPath);
350 7 : assertStorePath(sub.storeExpr);
351 :
352 7 : Substitutes subs = readSubstitutes(txn, srcPath);
353 :
354 : /* New substitutes take precedence over old ones. If the
355 : substitute is already present, it's moved to the front. */
356 7 : remove(subs.begin(), subs.end(), sub);
357 7 : subs.push_front(sub);
358 :
359 7 : writeSubstitutes(txn, srcPath, subs);
360 :
361 7 : Paths & revs = revMap[sub.storeExpr];
362 7 : if (revs.empty())
363 7 : nixDB.queryStrings(txn, dbSubstitutesRev, sub.storeExpr, revs);
364 7 : if (find(revs.begin(), revs.end(), srcPath) == revs.end())
365 7 : revs.push_back(srcPath);
366 : }
367 :
368 : /* Re-write the reverse mapping in one go to prevent Theta(n^2)
369 : performance. (This would occur because the data fields of the
370 : `substitutes-rev' table are lists). */
371 14 : for (SubstitutesRev::iterator i = revMap.begin(); i != revMap.end(); ++i)
372 7 : nixDB.setStrings(txn, dbSubstitutesRev, i->first, i->second);
373 : }
374 :
375 :
376 : Substitutes querySubstitutes(const Path & srcPath)
377 6 : {
378 6 : return readSubstitutes(noTxn, srcPath);
379 : }
380 :
381 :
382 : void registerValidPath(const Transaction & txn, const Path & _path)
383 82 : {
384 82 : Path path(canonPath(_path));
385 82 : assertStorePath(path);
386 82 : debug(format("registering path `%1%'") % path);
387 82 : nixDB.setString(txn, dbValidPaths, path, "");
388 : }
389 :
390 :
391 : static void invalidatePath(const Path & path, Transaction & txn)
392 0 : {
393 0 : debug(format("unregistering path `%1%'") % path);
394 :
395 0 : nixDB.delPair(txn, dbValidPaths, path);
396 :
397 : /* Remove any successor mappings to this path (but not *from*
398 : it). */
399 0 : Paths revs;
400 0 : nixDB.queryStrings(txn, dbSuccessorsRev, path, revs);
401 0 : for (Paths::iterator i = revs.begin(); i != revs.end(); ++i)
402 0 : nixDB.delPair(txn, dbSuccessors, *i);
403 0 : nixDB.delPair(txn, dbSuccessorsRev, path);
404 :
405 : /* Remove any substitute mappings to this path. */
406 0 : revs.clear();
407 0 : nixDB.queryStrings(txn, dbSubstitutesRev, path, revs);
408 0 : for (Paths::iterator i = revs.begin(); i != revs.end(); ++i) {
409 0 : Substitutes subs = readSubstitutes(txn, *i), subs2;
410 0 : bool found = false;
411 0 : for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
412 0 : if (j->storeExpr != path)
413 0 : subs2.push_back(*j);
414 : else
415 0 : found = true;
416 : // !!! if (!found) throw Error("integrity error in substitutes mapping");
417 0 : writeSubstitutes(txn, *i, subs);
418 :
419 : /* If path *i now has no substitutes left, and is not valid,
420 : then it too should be invalidated. This is because it may
421 : be a substitute or successor. */
422 0 : if (subs.size() == 0 && !isValidPathTxn(*i, txn))
423 0 : invalidatePath(*i, txn);
424 : }
425 0 : nixDB.delPair(txn, dbSubstitutesRev, path);
426 : }
427 :
428 :
429 : Path addToStore(const Path & _srcPath)
430 23 : {
431 23 : Path srcPath(absPath(_srcPath));
432 23 : debug(format("adding `%1%' to the store") % srcPath);
433 :
434 23 : Hash h;
435 : {
436 23 : SwitchToOriginalUser sw;
437 23 : h = hashPath(srcPath);
438 : }
439 :
440 23 : string baseName = baseNameOf(srcPath);
441 23 : Path dstPath = canonPath(nixStore + "/" + (string) h + "-" + baseName);
442 :
443 23 : if (!readOnlyMode && !isValidPath(dstPath)) {
444 :
445 : /* The first check above is an optimisation to prevent
446 : unnecessary lock acquisition. */
447 :
448 8 : PathSet lockPaths;
449 8 : lockPaths.insert(dstPath);
450 8 : PathLocks outputLock(lockPaths);
451 :
452 8 : if (!isValidPath(dstPath)) {
453 :
454 8 : if (pathExists(dstPath)) deletePath(dstPath);
455 :
456 : /* !!! race: srcPath might change between hashPath() and
457 : here! */
458 :
459 8 : copyPath(srcPath, dstPath);
460 :
461 8 : makePathReadOnly(dstPath);
462 :
463 8 : Transaction txn(nixDB);
464 8 : registerValidPath(txn, dstPath);
465 8 : txn.commit();
466 : }
467 :
468 8 : outputLock.setDeletion(true);
469 : }
470 :
471 23 : return dstPath;
472 : }
473 :
474 :
475 : void addTextToStore(const Path & dstPath, const string & s)
476 50 : {
477 50 : assertStorePath(dstPath);
478 :
479 50 : if (!isValidPath(dstPath)) {
480 :
481 50 : PathSet lockPaths;
482 50 : lockPaths.insert(dstPath);
483 50 : PathLocks outputLock(lockPaths);
484 :
485 50 : if (!isValidPath(dstPath)) {
486 :
487 50 : if (pathExists(dstPath)) deletePath(dstPath);
488 :
489 50 : writeStringToFile(dstPath, s);
490 :
491 50 : makePathReadOnly(dstPath);
492 :
493 50 : Transaction txn(nixDB);
494 50 : registerValidPath(txn, dstPath);
495 50 : txn.commit();
496 : }
497 :
498 50 : outputLock.setDeletion(true);
499 : }
500 : }
501 :
502 :
503 : void deleteFromStore(const Path & _path)
504 0 : {
505 0 : Path path(canonPath(_path));
506 :
507 0 : assertStorePath(path);
508 :
509 0 : Transaction txn(nixDB);
510 0 : invalidatePath(path, txn);
511 0 : txn.commit();
512 :
513 0 : deletePath(path);
514 : }
515 :
516 :
517 : void verifyStore()
518 1 : {
519 1 : Transaction txn(nixDB);
520 :
521 1 : Paths paths;
522 1 : PathSet validPaths;
523 1 : nixDB.enumTable(txn, dbValidPaths, paths);
524 :
525 83 : for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
526 82 : Path path = *i;
527 82 : if (!pathExists(path)) {
528 0 : printMsg(lvlError, format("path `%1%' disappeared") % path);
529 0 : invalidatePath(path, txn);
530 82 : } else if (!isInStore(path)) {
531 0 : printMsg(lvlError, format("path `%1%' is not in the Nix store") % path);
532 0 : invalidatePath(path, txn);
533 : } else
534 82 : validPaths.insert(path);
535 : }
536 :
537 : /* !!! the code below does not allow transitive substitutes.
538 : I.e., if B is a substitute of A, then B must be a valid path.
539 : B cannot itself be invalid but have a substitute. */
540 :
541 : /* "Usable" paths are those that are valid or have a substitute.
542 : These are the paths that are allowed to appear in the
543 : right-hand side of a sute mapping. */
544 1 : PathSet usablePaths(validPaths);
545 :
546 : /* Check that the values of the substitute mappings are valid
547 : paths. */
548 1 : Paths subKeys;
549 1 : nixDB.enumTable(txn, dbSubstitutes, subKeys);
550 6 : for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
551 5 : Substitutes subs = readSubstitutes(txn, *i), subs2;
552 12 : for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
553 7 : if (validPaths.find(j->storeExpr) == validPaths.end())
554 1 : printMsg(lvlError,
555 : format("found substitute mapping to non-existent path `%1%'")
556 : % j->storeExpr);
557 : else
558 6 : subs2.push_back(*j);
559 5 : if (subs.size() != subs2.size())
560 1 : writeSubstitutes(txn, *i, subs2);
561 5 : if (subs2.size() > 0)
562 4 : usablePaths.insert(*i);
563 : }
564 :
565 : /* Check that the keys of the reverse substitute mappings are
566 : valid paths. */
567 1 : Paths rsubKeys;
568 1 : nixDB.enumTable(txn, dbSubstitutesRev, rsubKeys);
569 4 : for (Paths::iterator i = rsubKeys.begin(); i != rsubKeys.end(); ++i) {
570 3 : if (validPaths.find(*i) == validPaths.end()) {
571 1 : printMsg(lvlError,
572 : format("found reverse substitute mapping for non-existent path `%1%'") % *i);
573 1 : nixDB.delPair(txn, dbSubstitutesRev, *i);
574 : }
575 : }
576 :
577 : /* Check that the values of the successor mappings are usable
578 : paths. */
579 1 : Paths sucKeys;
580 1 : nixDB.enumTable(txn, dbSuccessors, sucKeys);
581 23 : for (Paths::iterator i = sucKeys.begin(); i != sucKeys.end(); ++i) {
582 : /* Note that *i itself does not have to be valid, just its
583 : successor. */
584 22 : Path sucPath;
585 22 : if (nixDB.queryString(txn, dbSuccessors, *i, sucPath) &&
586 : usablePaths.find(sucPath) == usablePaths.end())
587 : {
588 0 : printMsg(lvlError,
589 : format("found successor mapping to non-existent path `%1%'") % sucPath);
590 0 : nixDB.delPair(txn, dbSuccessors, *i);
591 : }
592 : }
593 :
594 : /* Check that the keys of the reverse successor mappings are valid
595 : paths. */
596 1 : Paths rsucKeys;
597 1 : nixDB.enumTable(txn, dbSuccessorsRev, rsucKeys);
598 23 : for (Paths::iterator i = rsucKeys.begin(); i != rsucKeys.end(); ++i) {
599 22 : if (usablePaths.find(*i) == usablePaths.end()) {
600 0 : printMsg(lvlError,
601 : format("found reverse successor mapping for non-existent path `%1%'") % *i);
602 0 : nixDB.delPair(txn, dbSuccessorsRev, *i);
603 : }
604 : }
605 :
606 1 : txn.commit();
607 51 : }
|
