1 : #include "store.hh"
2 : #include "util.hh"
3 : #include "globals.hh"
4 : #include "db.hh"
5 : #include "archive.hh"
6 : #include "pathlocks.hh"
7 : #include "gc.hh"
8 : #include "aterm.hh"
9 : #include "derivations-ast.hh"
10 :
11 : #include <iostream>
12 : #include <algorithm>
13 :
14 : #include <sys/types.h>
15 : #include <sys/stat.h>
16 : #include <unistd.h>
17 : #include <utime.h>
18 :
19 : namespace nix {
20 :
21 :
22 : /* Nix database. */
23 345 : static Database nixDB;
24 :
25 :
26 : /* Database tables. */
27 :
28 : /* dbValidPaths :: Path -> ()
29 :
30 : The existence of a key $p$ indicates that path $p$ is valid (that
31 : is, produced by a succesful build). */
32 : static TableId dbValidPaths = 0;
33 :
34 : /* dbReferences :: Path -> [Path]
35 :
36 : This table lists the outgoing file system references for each
37 : output path that has been built by a Nix derivation. These are
38 : found by scanning the path for the hash components of input
39 : paths. */
40 : static TableId dbReferences = 0;
41 :
42 : /* dbReferrers :: Path -> Path
43 :
44 : This table is just the reverse mapping of dbReferences. This table
45 : can have duplicate keys, each corresponding value denoting a single
46 : referrer. */
47 : static TableId dbReferrers = 0;
48 :
49 : /* dbSubstitutes :: Path -> [[Path]]
50 :
51 : Each pair $(p, subs)$ tells Nix that it can use any of the
52 : substitutes in $subs$ to build path $p$. Each substitute defines a
53 : command-line invocation of a program (i.e., the first list element
54 : is the full path to the program, the remaining elements are
55 : arguments).
56 :
57 : The main purpose of this is for distributed caching of derivates.
58 : One system can compute a derivate and put it on a website (as a Nix
59 : archive), for instance, and then another system can register a
60 : substitute for that derivate. The substitute in this case might be
61 : a Nix derivation that fetches the Nix archive.
62 : */
63 : static TableId dbSubstitutes = 0;
64 :
65 : /* dbDerivers :: Path -> [Path]
66 :
67 : This table lists the derivation used to build a path. There can
68 : only be multiple such paths for fixed-output derivations (i.e.,
69 : derivations specifying an expected hash). */
70 : static TableId dbDerivers = 0;
71 :
72 :
73 : bool Substitute::operator == (const Substitute & sub) const
74 11 : {
75 11 : return program == sub.program
76 : && args == sub.args;
77 : }
78 :
79 :
80 : static void upgradeStore07();
81 : static void upgradeStore09();
82 :
83 :
84 : void checkStoreNotSymlink()
85 217 : {
86 217 : if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
87 216 : Path path = nixStore;
88 1512 : struct stat st;
89 1512 : while (path != "/") {
90 1296 : if (lstat(path.c_str(), &st))
91 0 : throw SysError(format("getting status of `%1%'") % path);
92 1296 : if (S_ISLNK(st.st_mode))
93 0 : throw Error(format(
94 : "the path `%1%' is a symlink; "
95 : "this is not allowed for the Nix store and its parent directories")
96 : % path);
97 1296 : path = dirOf(path);
98 : }
99 : }
100 :
101 :
102 : void openDB(bool reserveSpace)
103 263 : {
104 263 : if (readOnlyMode) return;
105 :
106 217 : checkStoreNotSymlink();
107 :
108 217 : try {
109 217 : Path reservedPath = nixDBPath + "/reserved";
110 217 : string s = querySetting("gc-reserved-space", "");
111 217 : int reservedSize;
112 217 : if (!string2Int(s, reservedSize)) reservedSize = 1024 * 1024;
113 217 : if (reserveSpace) {
114 202 : struct stat st;
115 202 : if (stat(reservedPath.c_str(), &st) == -1 ||
116 : st.st_size != reservedSize)
117 13 : writeFile(reservedPath, string(reservedSize, 'X'));
118 : }
119 : else
120 15 : deletePath(reservedPath);
121 0 : } catch (SysError & e) { /* don't care about errors */
122 : }
123 :
124 217 : try {
125 217 : nixDB.open(nixDBPath);
126 0 : } catch (DbNoPermission & e) {
127 0 : printMsg(lvlTalkative, "cannot access Nix database; continuing anyway");
128 0 : readOnlyMode = true;
129 0 : return;
130 : }
131 217 : dbValidPaths = nixDB.openTable("validpaths");
132 217 : dbReferences = nixDB.openTable("references");
133 217 : dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */
134 217 : dbSubstitutes = nixDB.openTable("substitutes");
135 217 : dbDerivers = nixDB.openTable("derivers");
136 :
137 217 : int curSchema = 0;
138 217 : Path schemaFN = nixDBPath + "/schema";
139 217 : if (pathExists(schemaFN)) {
140 213 : string s = readFile(schemaFN);
141 213 : if (!string2Int(s, curSchema))
142 0 : throw Error(format("`%1%' is corrupt") % schemaFN);
143 : }
144 :
145 217 : if (curSchema > nixSchemaVersion)
146 0 : throw Error(format("current Nix store schema is version %1%, but I only support %2%")
147 : % curSchema % nixSchemaVersion);
148 :
149 217 : if (curSchema < nixSchemaVersion) {
150 4 : if (curSchema <= 1)
151 4 : upgradeStore07();
152 4 : if (curSchema == 2)
153 0 : upgradeStore09();
154 4 : writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str());
155 : }
156 : }
157 :
158 :
159 : void initDB()
160 4 : {
161 : }
162 :
163 :
164 : void closeDB()
165 291 : {
166 : /* If the database isn't open, this is a NOP. */
167 291 : nixDB.close();
168 : }
169 :
170 :
171 : void createStoreTransaction(Transaction & txn)
172 84 : {
173 84 : Transaction txn2(nixDB);
174 84 : txn2.moveTo(txn);
175 : }
176 :
177 :
178 : /* Path copying. */
179 :
180 : struct CopySink : DumpSink
181 : {
182 : string s;
183 : virtual void operator () (const unsigned char * data, unsigned int len)
184 900 : {
185 900 : s.append((const char *) data, len);
186 : }
187 : };
188 :
189 :
190 : struct CopySource : RestoreSource
191 : {
192 : string & s;
193 : unsigned int pos;
194 150 : CopySource(string & _s) : s(_s), pos(0) { }
195 : virtual void operator () (unsigned char * data, unsigned int len)
196 900 : {
197 900 : s.copy((char *) data, len, pos);
198 900 : pos += len;
199 900 : assert(pos <= s.size());
200 : }
201 : };
202 :
203 :
204 : void copyPath(const Path & src, const Path & dst)
205 44 : {
206 44 : debug(format("copying `%1%' to `%2%'") % src % dst);
207 :
208 : /* Dump an archive of the path `src' into a string buffer, then
209 : restore the archive to `dst'. This is not a very good method
210 : for very large paths, but `copyPath' is mainly used for small
211 : files. */
212 :
213 700 : CopySink sink;
214 : {
215 44 : SwitchToOriginalUser sw;
216 44 : dumpPath(src, sink);
217 : }
218 :
219 132 : CopySource source(sink.s);
220 44 : restorePath(dst, source);
221 : }
222 :
223 :
224 : bool isInStore(const Path & path)
225 16013 : {
226 16013 : return path[0] == '/'
227 : && string(path, 0, nixStore.size()) == nixStore
228 : && path.size() >= nixStore.size() + 2
229 : && path[nixStore.size()] == '/';
230 : }
231 :
232 :
233 : bool isStorePath(const Path & path)
234 8665 : {
235 8665 : return isInStore(path)
236 : && path.find('/', nixStore.size() + 1) == Path::npos;
237 : }
238 :
239 :
240 : void assertStorePath(const Path & path)
241 8411 : {
242 8411 : if (!isStorePath(path))
243 0 : throw Error(format("path `%1%' is not in the Nix store") % path);
244 : }
245 :
246 :
247 : Path toStorePath(const Path & path)
248 141 : {
249 141 : if (!isInStore(path))
250 0 : throw Error(format("path `%1%' is not in the Nix store") % path);
251 141 : Path::size_type slash = path.find('/', nixStore.size() + 1);
252 141 : if (slash == Path::npos)
253 140 : return path;
254 : else
255 1 : return Path(path, 0, slash);
256 : }
257 :
258 :
259 : void checkStoreName(const string & name)
260 369 : {
261 369 : string validChars = "+-._?=";
262 6698 : for (string::const_iterator i = name.begin(); i != name.end(); ++i)
263 6329 : if (!((*i >= 'A' && *i <= 'Z') ||
264 : (*i >= 'a' && *i <= 'z') ||
265 : (*i >= '0' && *i <= '9') ||
266 : validChars.find(*i) != string::npos))
267 : {
268 0 : throw Error(format("invalid character `%1%' in name `%2%'")
269 : % *i % name);
270 : }
271 : }
272 :
273 :
274 : void canonicalisePathMetaData(const Path & path)
275 2822 : {
276 2822 : checkInterrupt();
277 :
278 2822 : struct stat st;
279 2822 : if (lstat(path.c_str(), &st))
280 0 : throw SysError(format("getting attributes of path `%1%'") % path);
281 :
282 2822 : if (!S_ISLNK(st.st_mode)) {
283 :
284 : /* Mask out all type related bits. */
285 2779 : mode_t mode = st.st_mode & ~S_IFMT;
286 :
287 2779 : if (mode != 0444 && mode != 0555) {
288 2779 : mode = (st.st_mode & S_IFMT)
289 : | 0444
290 : | (st.st_mode & S_IXUSR ? 0111 : 0);
291 2779 : if (chmod(path.c_str(), mode) == -1)
292 0 : throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
293 : }
294 :
295 2779 : if (st.st_uid != getuid() || st.st_gid != getgid()) {
296 0 : if (chown(path.c_str(), getuid(), getgid()) == -1)
297 0 : throw SysError(format("changing owner/group of `%1%' to %2%/%3%")
298 : % path % getuid() % getgid());
299 : }
300 :
301 2779 : if (st.st_mtime != 0) {
302 2779 : struct utimbuf utimbuf;
303 2779 : utimbuf.actime = st.st_atime;
304 2779 : utimbuf.modtime = 0;
305 2779 : if (utime(path.c_str(), &utimbuf) == -1)
306 0 : throw SysError(format("changing modification time of `%1%'") % path);
307 : }
308 :
309 : }
310 :
311 2822 : if (S_ISDIR(st.st_mode)) {
312 67 : Strings names = readDirectory(path);
313 184 : for (Strings::iterator i = names.begin(); i != names.end(); ++i)
314 117 : canonicalisePathMetaData(path + "/" + *i);
315 : }
316 : }
317 :
318 :
319 : bool isValidPathTxn(const Transaction & txn, const Path & path)
320 18040 : {
321 18040 : string s;
322 18040 : return nixDB.queryString(txn, dbValidPaths, path, s);
323 : }
324 :
325 :
326 : bool isValidPath(const Path & path)
327 6765 : {
328 6765 : return isValidPathTxn(noTxn, path);
329 : }
330 :
331 :
332 : static Substitutes readSubstitutes(const Transaction & txn,
333 : const Path & srcPath);
334 :
335 :
336 : static bool isRealisablePath(const Transaction & txn, const Path & path)
337 5916 : {
338 5916 : return isValidPathTxn(txn, path)
339 5956 : || readSubstitutes(txn, path).size() > 0;
340 : }
341 :
342 :
343 : static string addPrefix(const string & prefix, const string & s)
344 5345 : {
345 5345 : return prefix + string(1, (char) 0) + s;
346 : }
347 :
348 :
349 : static string stripPrefix(const string & prefix, const string & s)
350 7 : {
351 7 : if (s.size() <= prefix.size() ||
352 : string(s, 0, prefix.size()) != prefix ||
353 : s[prefix.size()] != 0)
354 0 : throw Error(format("string `%1%' is missing prefix `%2%'")
355 : % s % prefix);
356 7 : return string(s, prefix.size() + 1);
357 : }
358 :
359 :
360 : static PathSet getReferrers(const Transaction & txn, const Path & storePath)
361 2668 : {
362 2668 : PathSet referrers;
363 2668 : Strings keys;
364 2668 : nixDB.enumTable(txn, dbReferrers, keys, storePath + string(1, (char) 0));
365 2675 : for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
366 7 : referrers.insert(stripPrefix(storePath, *i));
367 2668 : return referrers;
368 : }
369 :
370 :
371 : void setReferences(const Transaction & txn, const Path & storePath,
372 : const PathSet & references)
373 5409 : {
374 : /* For unrealisable paths, we can only clear the references. */
375 5409 : if (references.size() > 0 && !isRealisablePath(txn, storePath))
376 0 : throw Error(
377 : format("cannot set references for path `%1%' which is invalid and has no substitutes")
378 : % storePath);
379 :
380 5409 : Paths oldReferences;
381 5409 : nixDB.queryStrings(txn, dbReferences, storePath, oldReferences);
382 :
383 5409 : PathSet oldReferences2(oldReferences.begin(), oldReferences.end());
384 5409 : if (oldReferences2 == references) return;
385 :
386 5198 : nixDB.setStrings(txn, dbReferences, storePath,
387 : Paths(references.begin(), references.end()));
388 :
389 : /* Update the referrers mappings of all new referenced paths. */
390 7883 : for (PathSet::const_iterator i = references.begin();
391 : i != references.end(); ++i)
392 2685 : if (oldReferences2.find(*i) == oldReferences2.end())
393 2685 : nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), "");
394 :
395 : /* Remove referrer mappings from paths that are no longer
396 : references. */
397 7849 : for (Paths::iterator i = oldReferences.begin();
398 : i != oldReferences.end(); ++i)
399 2651 : if (references.find(*i) == references.end())
400 2651 : nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath));
401 : }
402 :
403 :
404 : void queryReferences(const Transaction & txn,
405 : const Path & storePath, PathSet & references)
406 3206 : {
407 3206 : Paths references2;
408 3206 : if (!isRealisablePath(txn, storePath))
409 0 : throw Error(format("path `%1%' is not valid") % storePath);
410 3206 : nixDB.queryStrings(txn, dbReferences, storePath, references2);
411 3206 : references.insert(references2.begin(), references2.end());
412 : }
413 :
414 :
415 : void queryReferrers(const Transaction & txn,
416 : const Path & storePath, PathSet & referrers)
417 2 : {
418 2 : if (!isRealisablePath(txn, storePath))
419 0 : throw Error(format("path `%1%' is not valid") % storePath);
420 2 : PathSet referrers2 = getReferrers(txn, storePath);
421 2 : referrers.insert(referrers2.begin(), referrers2.end());
422 : }
423 :
424 :
425 : void setDeriver(const Transaction & txn, const Path & storePath,
426 : const Path & deriver)
427 2705 : {
428 2705 : assertStorePath(storePath);
429 2705 : if (deriver == "") return;
430 66 : assertStorePath(deriver);
431 66 : if (!isRealisablePath(txn, storePath))
432 0 : throw Error(format("path `%1%' is not valid") % storePath);
433 66 : nixDB.setString(txn, dbDerivers, storePath, deriver);
434 : }
435 :
436 :
437 : Path queryDeriver(const Transaction & txn, const Path & storePath)
438 20 : {
439 20 : if (!isRealisablePath(txn, storePath))
440 0 : throw Error(format("path `%1%' is not valid") % storePath);
441 20 : Path deriver;
442 20 : if (nixDB.queryString(txn, dbDerivers, storePath, deriver))
443 12 : return deriver;
444 : else
445 8 : return "";
446 : }
447 :
448 :
449 : const int substituteVersion = 2;
450 :
451 :
452 : static Substitutes readSubstitutes(const Transaction & txn,
453 : const Path & srcPath)
454 2845 : {
455 2845 : Strings ss;
456 2845 : nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss);
457 :
458 2845 : Substitutes subs;
459 :
460 2916 : for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
461 71 : if (i->size() < 4 || (*i)[3] != 0) {
462 : /* Old-style substitute. !!! remove this code
463 : eventually? */
464 71 : break;
465 : }
466 71 : Strings ss2 = unpackStrings(*i);
467 71 : if (ss2.size() == 0) continue;
468 71 : int version;
469 71 : if (!string2Int(ss2.front(), version)) continue;
470 71 : if (version != substituteVersion) continue;
471 71 : if (ss2.size() != 4) throw Error("malformed substitute");
472 71 : Strings::iterator j = ss2.begin();
473 71 : j++;
474 197 : Substitute sub;
475 71 : sub.deriver = *j++;
476 71 : sub.program = *j++;
477 71 : sub.args = unpackStrings(*j++);
478 71 : subs.push_back(sub);
479 : }
480 :
481 2845 : return subs;
482 : }
483 :
484 :
485 : static void writeSubstitutes(const Transaction & txn,
486 : const Path & srcPath, const Substitutes & subs)
487 31 : {
488 31 : Strings ss;
489 :
490 63 : for (Substitutes::const_iterator i = subs.begin();
491 : i != subs.end(); ++i)
492 : {
493 32 : Strings ss2;
494 32 : ss2.push_back((format("%1%") % substituteVersion).str());
495 32 : ss2.push_back(i->deriver);
496 32 : ss2.push_back(i->program);
497 32 : ss2.push_back(packStrings(i->args));
498 32 : ss.push_back(packStrings(ss2));
499 : }
500 :
501 31 : nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
502 : }
503 :
504 :
505 : void registerSubstitute(const Transaction & txn,
506 : const Path & srcPath, const Substitute & sub)
507 40 : {
508 40 : assertStorePath(srcPath);
509 :
510 40 : Substitutes subs = readSubstitutes(txn, srcPath);
511 :
512 40 : if (find(subs.begin(), subs.end(), sub) != subs.end())
513 9 : return;
514 :
515 : /* New substitutes take precedence over old ones. If the
516 : substitute is already present, it's moved to the front. */
517 31 : remove(subs.begin(), subs.end(), sub);
518 31 : subs.push_front(sub);
519 :
520 31 : writeSubstitutes(txn, srcPath, subs);
521 : }
522 :
523 :
524 : Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath)
525 2771 : {
526 2771 : return readSubstitutes(txn, srcPath);
527 : }
528 :
529 :
530 : static void invalidatePath(Transaction & txn, const Path & path);
531 :
532 :
533 : void clearSubstitutes()
534 1 : {
535 1 : Transaction txn(nixDB);
536 :
537 : /* Iterate over all paths for which there are substitutes. */
538 1 : Paths subKeys;
539 1 : nixDB.enumTable(txn, dbSubstitutes, subKeys);
540 10 : for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
541 :
542 : /* Delete all substitutes for path *i. */
543 9 : nixDB.delPair(txn, dbSubstitutes, *i);
544 :
545 : /* Maintain the cleanup invariant. */
546 9 : if (!isValidPathTxn(txn, *i))
547 1 : invalidatePath(txn, *i);
548 : }
549 :
550 : /* !!! there should be no referrers to any of the invalid
551 : substitutable paths. This should be the case by construction
552 : (the only referrers can be other invalid substitutable paths,
553 : which have all been removed now). */
554 :
555 1 : txn.commit();
556 : }
557 :
558 :
559 : static void setHash(const Transaction & txn, const Path & storePath,
560 : const Hash & hash)
561 2705 : {
562 2705 : assert(hash.type == htSHA256);
563 2705 : nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash));
564 : }
565 :
566 :
567 : static Hash queryHash(const Transaction & txn, const Path & storePath)
568 1 : {
569 1 : string s;
570 1 : nixDB.queryString(txn, dbValidPaths, storePath, s);
571 1 : string::size_type colon = s.find(':');
572 1 : if (colon == string::npos)
573 0 : throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
574 : % s % storePath);
575 1 : HashType ht = parseHashType(string(s, 0, colon));
576 1 : if (ht == htUnknown)
577 0 : throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
578 : % string(s, 0, colon) % storePath);
579 1 : return parseHash(ht, string(s, colon + 1));
580 : }
581 :
582 :
583 : Hash queryPathHash(const Path & path)
584 1 : {
585 1 : if (!isValidPath(path))
586 0 : throw Error(format("path `%1%' is not valid") % path);
587 1 : return queryHash(noTxn, path);
588 : }
589 :
590 :
591 : void registerValidPath(const Transaction & txn,
592 : const Path & path, const Hash & hash, const PathSet & references,
593 : const Path & deriver)
594 204 : {
595 1026 : ValidPathInfo info;
596 204 : info.path = path;
597 204 : info.hash = hash;
598 204 : info.references = references;
599 204 : info.deriver = deriver;
600 478 : ValidPathInfos infos;
601 204 : infos.push_back(info);
602 204 : registerValidPaths(txn, infos);
603 : }
604 :
605 :
606 : void registerValidPaths(const Transaction & txn,
607 : const ValidPathInfos & infos)
608 206 : {
609 206 : PathSet newPaths;
610 2911 : for (ValidPathInfos::const_iterator i = infos.begin();
611 : i != infos.end(); ++i)
612 2705 : newPaths.insert(i->path);
613 :
614 2911 : for (ValidPathInfos::const_iterator i = infos.begin();
615 : i != infos.end(); ++i)
616 : {
617 2705 : assertStorePath(i->path);
618 :
619 2705 : debug(format("registering path `%1%'") % i->path);
620 2705 : setHash(txn, i->path, i->hash);
621 :
622 2705 : setReferences(txn, i->path, i->references);
623 :
624 : /* Check that all referenced paths are also valid (or about to
625 : become valid). */
626 5385 : for (PathSet::iterator j = i->references.begin();
627 : j != i->references.end(); ++j)
628 2680 : if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end())
629 0 : throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
630 : % i->path % *j);
631 :
632 2705 : setDeriver(txn, i->path, i->deriver);
633 : }
634 : }
635 :
636 :
637 : /* Invalidate a path. The caller is responsible for checking that
638 : there are no referrers. */
639 : static void invalidatePath(Transaction & txn, const Path & path)
640 2667 : {
641 2667 : debug(format("unregistering path `%1%'") % path);
642 :
643 : /* Clear the `references' entry for this path, as well as the
644 : inverse `referrers' entries, and the `derivers' entry; but only
645 : if there are no substitutes for this path. This maintains the
646 : cleanup invariant. */
647 2667 : if (querySubstitutes(txn, path).size() == 0) {
648 2664 : setReferences(txn, path, PathSet());
649 2664 : nixDB.delPair(txn, dbDerivers, path);
650 : }
651 :
652 2667 : nixDB.delPair(txn, dbValidPaths, path);
653 : }
654 :
655 :
656 : Path makeStorePath(const string & type,
657 : const Hash & hash, const string & suffix)
658 282 : {
659 : /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
660 : string s = type + ":sha256:" + printHash(hash) + ":"
661 282 : + nixStore + ":" + suffix;
662 :
663 282 : checkStoreName(suffix);
664 :
665 282 : return nixStore + "/"
666 : + printHash32(compressHash(hashString(htSHA256, s), 20))
667 : + "-" + suffix;
668 : }
669 :
670 :
671 : Path makeFixedOutputPath(bool recursive,
672 : string hashAlgo, Hash hash, string name)
673 24 : {
674 : /* !!! copy/paste from primops.cc */
675 : Hash h = hashString(htSHA256, "fixed:out:"
676 : + (recursive ? (string) "r:" : "") + hashAlgo + ":"
677 : + printHash(hash) + ":"
678 24 : + "");
679 24 : return makeStorePath("output:out", h, name);
680 : }
681 :
682 :
683 : static Path _addToStore(bool fixed, bool recursive,
684 : string hashAlgo, const Path & _srcPath)
685 80 : {
686 80 : Path srcPath(absPath(_srcPath));
687 80 : debug(format("adding `%1%' to the store") % srcPath);
688 :
689 80 : Hash h(htSHA256);
690 : {
691 80 : SwitchToOriginalUser sw;
692 80 : h = hashPath(htSHA256, srcPath);
693 : }
694 :
695 79 : string baseName = baseNameOf(srcPath);
696 :
697 79 : Path dstPath;
698 :
699 79 : if (fixed) {
700 :
701 12 : HashType ht(parseHashType(hashAlgo));
702 12 : Hash h2(ht);
703 : {
704 12 : SwitchToOriginalUser sw;
705 12 : h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath);
706 : }
707 :
708 12 : dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
709 : }
710 :
711 67 : else dstPath = makeStorePath("source", h, baseName);
712 :
713 79 : if (!readOnlyMode) addTempRoot(dstPath);
714 :
715 79 : if (!readOnlyMode && !isValidPath(dstPath)) {
716 :
717 : /* The first check above is an optimisation to prevent
718 : unnecessary lock acquisition. */
719 :
720 44 : PathLocks outputLock(singleton<PathSet, Path>(dstPath));
721 :
722 44 : if (!isValidPath(dstPath)) {
723 :
724 44 : if (pathExists(dstPath)) deletePath(dstPath);
725 :
726 44 : copyPath(srcPath, dstPath);
727 :
728 44 : Hash h2 = hashPath(htSHA256, dstPath);
729 44 : if (h != h2)
730 0 : throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
731 : % srcPath % dstPath % printHash(h) % printHash(h2));
732 :
733 44 : canonicalisePathMetaData(dstPath);
734 :
735 44 : Transaction txn(nixDB);
736 44 : registerValidPath(txn, dstPath, h, PathSet(), "");
737 44 : txn.commit();
738 : }
739 :
740 44 : outputLock.setDeletion(true);
741 : }
742 :
743 79 : return dstPath;
744 : }
745 :
746 :
747 : Path addToStore(const Path & srcPath)
748 68 : {
749 68 : return _addToStore(false, false, "", srcPath);
750 : }
751 :
752 :
753 : Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath)
754 12 : {
755 12 : return _addToStore(true, recursive, hashAlgo, srcPath);
756 : }
757 :
758 :
759 : Path addTextToStore(const string & suffix, const string & s,
760 : const PathSet & references)
761 104 : {
762 104 : Hash hash = hashString(htSHA256, s);
763 :
764 104 : Path dstPath = makeStorePath("text", hash, suffix);
765 :
766 104 : if (!readOnlyMode) addTempRoot(dstPath);
767 :
768 104 : if (!readOnlyMode && !isValidPath(dstPath)) {
769 :
770 86 : PathLocks outputLock(singleton<PathSet, Path>(dstPath));
771 :
772 86 : if (!isValidPath(dstPath)) {
773 :
774 86 : if (pathExists(dstPath)) deletePath(dstPath);
775 :
776 86 : writeStringToFile(dstPath, s);
777 :
778 86 : canonicalisePathMetaData(dstPath);
779 :
780 86 : Transaction txn(nixDB);
781 86 : registerValidPath(txn, dstPath,
782 : hashPath(htSHA256, dstPath), references, "");
783 86 : txn.commit();
784 : }
785 :
786 86 : outputLock.setDeletion(true);
787 : }
788 :
789 86 : return dstPath;
790 : }
791 :
792 :
793 : void deleteFromStore(const Path & _path, unsigned long long & bytesFreed)
794 2670 : {
795 2670 : bytesFreed = 0;
796 2670 : Path path(canonPath(_path));
797 :
798 2670 : assertStorePath(path);
799 :
800 2670 : Transaction txn(nixDB);
801 2670 : if (isValidPathTxn(txn, path)) {
802 2666 : PathSet referrers = getReferrers(txn, path);
803 2671 : for (PathSet::iterator i = referrers.begin();
804 : i != referrers.end(); ++i)
805 5 : if (*i != path && isValidPathTxn(txn, *i))
806 0 : throw Error(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i);
807 2666 : invalidatePath(txn, path);
808 : }
809 2670 : txn.commit();
810 :
811 2670 : deletePath(path, bytesFreed);
812 : }
813 :
814 :
815 : void verifyStore(bool checkContents)
816 1 : {
817 1 : Transaction txn(nixDB);
818 :
819 1 : Paths paths;
820 1 : PathSet validPaths;
821 1 : nixDB.enumTable(txn, dbValidPaths, paths);
822 :
823 13 : for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
824 12 : if (!pathExists(*i)) {
825 0 : printMsg(lvlError, format("path `%1%' disappeared") % *i);
826 0 : invalidatePath(txn, *i);
827 12 : } else if (!isStorePath(*i)) {
828 0 : printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i);
829 0 : invalidatePath(txn, *i);
830 : } else {
831 12 : if (checkContents) {
832 0 : debug(format("checking contents of `%1%'") % *i);
833 0 : Hash expected = queryHash(txn, *i);
834 0 : Hash current = hashPath(expected.type, *i);
835 0 : if (current != expected) {
836 0 : printMsg(lvlError, format("path `%1%' was modified! "
837 : "expected hash `%2%', got `%3%'")
838 : % *i % printHash(expected) % printHash(current));
839 : }
840 : }
841 12 : validPaths.insert(*i);
842 : }
843 : }
844 :
845 : /* "Usable" paths are those that are valid or have a
846 : substitute. */
847 1 : PathSet usablePaths(validPaths);
848 :
849 : /* Check that the values of the substitute mappings are valid
850 : paths. */
851 1 : Paths subKeys;
852 1 : nixDB.enumTable(txn, dbSubstitutes, subKeys);
853 4 : for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
854 3 : Substitutes subs = readSubstitutes(txn, *i);
855 3 : if (!isStorePath(*i)) {
856 0 : printMsg(lvlError, format("found substitutes for non-store path `%1%'") % *i);
857 0 : nixDB.delPair(txn, dbSubstitutes, *i);
858 : }
859 3 : else if (subs.size() == 0)
860 0 : nixDB.delPair(txn, dbSubstitutes, *i);
861 : else
862 3 : usablePaths.insert(*i);
863 : }
864 :
865 : /* Check the cleanup invariant: only usable paths can have
866 : `references', `referrers', or `derivers' entries. */
867 :
868 : /* Check the `derivers' table. */
869 1 : Paths deriversKeys;
870 1 : nixDB.enumTable(txn, dbDerivers, deriversKeys);
871 6 : for (Paths::iterator i = deriversKeys.begin();
872 : i != deriversKeys.end(); ++i)
873 : {
874 5 : if (usablePaths.find(*i) == usablePaths.end()) {
875 0 : printMsg(lvlError, format("found deriver entry for unusable path `%1%'")
876 : % *i);
877 0 : nixDB.delPair(txn, dbDerivers, *i);
878 : }
879 : else {
880 5 : Path deriver = queryDeriver(txn, *i);
881 5 : if (!isStorePath(deriver)) {
882 0 : printMsg(lvlError, format("found corrupt deriver `%1%' for `%2%'")
883 : % deriver % *i);
884 0 : nixDB.delPair(txn, dbDerivers, *i);
885 : }
886 : }
887 : }
888 :
889 : /* Check the `references' table. */
890 1 : Paths referencesKeys;
891 1 : nixDB.enumTable(txn, dbReferences, referencesKeys);
892 6 : for (Paths::iterator i = referencesKeys.begin();
893 : i != referencesKeys.end(); ++i)
894 : {
895 5 : if (usablePaths.find(*i) == usablePaths.end()) {
896 0 : printMsg(lvlError, format("found references entry for unusable path `%1%'")
897 : % *i);
898 0 : setReferences(txn, *i, PathSet());
899 : }
900 : else {
901 5 : bool isValid = validPaths.find(*i) != validPaths.end();
902 5 : PathSet references;
903 5 : queryReferences(txn, *i, references);
904 14 : for (PathSet::iterator j = references.begin();
905 : j != references.end(); ++j)
906 : {
907 9 : string dummy;
908 9 : if (!nixDB.queryString(txn, dbReferrers, addPrefix(*j, *i), dummy)) {
909 0 : printMsg(lvlError, format("missing referrer mapping from `%1%' to `%2%'")
910 : % *j % *i);
911 0 : nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), "");
912 : }
913 9 : if (isValid && validPaths.find(*j) == validPaths.end()) {
914 0 : printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
915 : % *i % *j);
916 : }
917 : }
918 : }
919 : }
920 :
921 : #if 0 // !!!
922 : /* Check the `referrers' table. */
923 : Paths referrersKeys;
924 : nixDB.enumTable(txn, dbReferrers, referrersKeys);
925 : for (Paths::iterator i = referrersKeys.begin();
926 : i != referrersKeys.end(); ++i)
927 : {
928 : if (usablePaths.find(*i) == usablePaths.end()) {
929 : printMsg(lvlError, format("found referrers entry for unusable path `%1%'")
930 : % *i);
931 : nixDB.delPair(txn, dbReferrers, *i);
932 : }
933 : else {
934 : PathSet referrers, newReferrers;
935 : queryReferrers(txn, *i, referrers);
936 : for (PathSet::iterator j = referrers.begin();
937 : j != referrers.end(); ++j)
938 : {
939 : Paths references;
940 : if (usablePaths.find(*j) == usablePaths.end()) {
941 : printMsg(lvlError, format("referrer mapping from `%1%' to unusable `%2%'")
942 : % *i % *j);
943 : } else {
944 : nixDB.queryStrings(txn, dbReferences, *j, references);
945 : if (find(references.begin(), references.end(), *i) == references.end()) {
946 : printMsg(lvlError, format("missing reference mapping from `%1%' to `%2%'")
947 : % *j % *i);
948 : /* !!! repair by inserting *i into references */
949 : }
950 : else newReferrers.insert(*j);
951 : }
952 : }
953 : if (referrers != newReferrers)
954 : nixDB.setStrings(txn, dbReferrers, *i,
955 : Paths(newReferrers.begin(), newReferrers.end()));
956 : }
957 : }
958 : #endif
959 :
960 1 : txn.commit();
961 : }
962 :
963 :
964 : /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
965 : static void upgradeStore07()
966 4 : {
967 4 : printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
968 :
969 4 : Transaction txn(nixDB);
970 :
971 4 : Paths validPaths2;
972 4 : nixDB.enumTable(txn, dbValidPaths, validPaths2);
973 4 : PathSet validPaths(validPaths2.begin(), validPaths2.end());
974 :
975 4 : std::cerr << "hashing paths...";
976 4 : int n = 0;
977 4 : for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) {
978 0 : checkInterrupt();
979 0 : string s;
980 0 : nixDB.queryString(txn, dbValidPaths, *i, s);
981 0 : if (s == "") {
982 0 : Hash hash = hashPath(htSHA256, *i);
983 0 : setHash(txn, *i, hash);
984 0 : std::cerr << ".";
985 0 : if (++n % 1000 == 0) {
986 0 : txn.commit();
987 0 : txn.begin(nixDB);
988 : }
989 : }
990 : }
991 4 : std::cerr << std::endl;
992 :
993 4 : txn.commit();
994 :
995 4 : txn.begin(nixDB);
996 :
997 4 : std::cerr << "processing closures...";
998 4 : for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) {
999 0 : checkInterrupt();
1000 0 : if (i->size() > 6 && string(*i, i->size() - 6) == ".store") {
1001 0 : ATerm t = ATreadFromNamedFile(i->c_str());
1002 0 : if (!t) throw Error(format("cannot read aterm from `%1%'") % *i);
1003 :
1004 0 : ATermList roots, elems;
1005 0 : if (!matchOldClosure(t, roots, elems)) continue;
1006 :
1007 0 : for (ATermIterator j(elems); j; ++j) {
1008 :
1009 0 : ATerm path2;
1010 0 : ATermList references2;
1011 0 : if (!matchOldClosureElem(*j, path2, references2)) continue;
1012 :
1013 0 : Path path = aterm2String(path2);
1014 0 : if (validPaths.find(path) == validPaths.end())
1015 : /* Skip this path; it's invalid. This is a normal
1016 : condition (Nix <= 0.7 did not enforce closure
1017 : on closure store expressions). */
1018 0 : continue;
1019 :
1020 0 : PathSet references;
1021 0 : for (ATermIterator k(references2); k; ++k) {
1022 0 : Path reference = aterm2String(*k);
1023 0 : if (validPaths.find(reference) == validPaths.end())
1024 : /* Bad reference. Set it anyway and let the
1025 : user fix it. */
1026 0 : printMsg(lvlError, format("closure `%1%' contains reference from `%2%' "
1027 : "to invalid path `%3%' (run `nix-store --verify')")
1028 : % *i % path % reference);
1029 0 : references.insert(reference);
1030 : }
1031 :
1032 0 : PathSet prevReferences;
1033 0 : queryReferences(txn, path, prevReferences);
1034 0 : if (prevReferences.size() > 0 && references != prevReferences)
1035 0 : printMsg(lvlError, format("warning: conflicting references for `%1%'") % path);
1036 :
1037 0 : if (references != prevReferences)
1038 0 : setReferences(txn, path, references);
1039 : }
1040 :
1041 0 : std::cerr << ".";
1042 : }
1043 : }
1044 4 : std::cerr << std::endl;
1045 :
1046 : /* !!! maybe this transaction is way too big */
1047 4 : txn.commit();
1048 : }
1049 :
1050 :
1051 : /* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >=
1052 : 0.10). The only thing to do here is to upgrade the old `referer'
1053 : table (which causes quadratic complexity in some cases) to the new
1054 : (and properly spelled) `referrer' table. */
1055 : static void upgradeStore09()
1056 0 : {
1057 : /* !!! we should disallow concurrent upgrades */
1058 :
1059 0 : printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
1060 :
1061 0 : if (!pathExists(nixDBPath + "/referers")) return;
1062 :
1063 0 : Transaction txn(nixDB);
1064 :
1065 0 : std::cerr << "converting referers to referrers...";
1066 :
1067 0 : TableId dbReferers = nixDB.openTable("referers"); /* sic! */
1068 :
1069 0 : Paths referersKeys;
1070 0 : nixDB.enumTable(txn, dbReferers, referersKeys);
1071 :
1072 0 : int n = 0;
1073 0 : for (Paths::iterator i = referersKeys.begin();
1074 : i != referersKeys.end(); ++i)
1075 : {
1076 0 : Paths referers;
1077 0 : nixDB.queryStrings(txn, dbReferers, *i, referers);
1078 0 : for (Paths::iterator j = referers.begin();
1079 : j != referers.end(); ++j)
1080 0 : nixDB.setString(txn, dbReferrers, addPrefix(*i, *j), "");
1081 0 : if (++n % 1000 == 0) {
1082 0 : txn.commit();
1083 0 : txn.begin(nixDB);
1084 0 : std::cerr << "|";
1085 : }
1086 0 : std::cerr << ".";
1087 : }
1088 :
1089 0 : txn.commit();
1090 :
1091 0 : std::cerr << std::endl;
1092 :
1093 0 : nixDB.closeTable(dbReferers);
1094 :
1095 0 : nixDB.deleteTable("referers");
1096 : }
1097 :
1098 :
1099 345 : }
|
