1 : #include "gc.hh"
2 : #include "globals.hh"
3 : #include "misc.hh"
4 : #include "pathlocks.hh"
5 : #include "store.hh"
6 : #include "db.hh"
7 : #include "util.hh"
8 :
9 : #include <boost/shared_ptr.hpp>
10 :
11 : #include <sys/types.h>
12 : #include <sys/stat.h>
13 : #include <errno.h>
14 : #include <fcntl.h>
15 : #include <unistd.h>
16 :
17 : #ifdef __CYGWIN__
18 : #include <windows.h>
19 : #include <sys/cygwin.h>
20 : #endif
21 :
22 :
23 : namespace nix {
24 :
25 :
26 690 : static string gcLockName = "gc.lock";
27 690 : static string tempRootsDir = "temproots";
28 690 : static string gcRootsDir = "gcroots";
29 :
30 :
31 : /* Acquire the global GC lock. This is used to prevent new Nix
32 : processes from starting after the temporary root files have been
33 : read. To be precise: when they try to create a new temporary root
34 : file, they will block until the garbage collector has finished /
35 : yielded the GC lock. */
36 : static int openGCLock(LockType lockType)
37 112 : {
38 0 : Path fnGCLock = (format("%1%/%2%")
39 112 : % nixStateDir % gcLockName).str();
40 :
41 112 : debug(format("acquiring global GC lock `%1%'") % fnGCLock);
42 :
43 112 : AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600);
44 112 : if (fdGCLock == -1)
45 0 : throw SysError(format("opening global GC lock `%1%'") % fnGCLock);
46 :
47 112 : if (!lockFile(fdGCLock, lockType, false)) {
48 1 : printMsg(lvlError, format("waiting for the big garbage collector lock..."));
49 1 : lockFile(fdGCLock, lockType, true);
50 : }
51 :
52 : /* !!! Restrict read permission on the GC root. Otherwise any
53 : process that can open the file for reading can DoS the
54 : collector. */
55 :
56 112 : return fdGCLock.borrow();
57 : }
58 :
59 :
60 : void createSymlink(const Path & link, const Path & target, bool careful)
61 19 : {
62 : /* Create directories up to `gcRoot'. */
63 19 : createDirs(dirOf(link));
64 :
65 : /* Remove the old symlink. */
66 19 : if (pathExists(link)) {
67 0 : if (careful && (!isLink(link) || !isInStore(readLink(link))))
68 0 : throw Error(format("cannot create symlink `%1%'; already exists") % link);
69 0 : unlink(link.c_str());
70 : }
71 :
72 : /* And create the new own. */
73 19 : if (symlink(target.c_str(), link.c_str()) == -1)
74 0 : throw SysError(format("symlinking `%1%' to `%2%'")
75 : % link % target);
76 : }
77 :
78 :
79 : Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
80 : bool indirect, bool allowOutsideRootsDir)
81 17 : {
82 17 : Path storePath(canonPath(_storePath));
83 17 : Path gcRoot(canonPath(_gcRoot));
84 17 : assertStorePath(storePath);
85 :
86 : /* Grab the global GC root. This prevents the set of permanent
87 : roots from increasing while a GC is in progress. */
88 17 : AutoCloseFD fdGCLock = openGCLock(ltRead);
89 :
90 17 : if (indirect) {
91 2 : string hash = printHash32(hashString(htSHA1, gcRoot));
92 : Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
93 2 : % nixStateDir % gcRootsDir % hash).str());
94 :
95 : {
96 2 : SwitchToOriginalUser sw;
97 2 : createSymlink(gcRoot, storePath, true);
98 : }
99 2 : createSymlink(realRoot, gcRoot, false);
100 : }
101 :
102 : else {
103 15 : if (!allowOutsideRootsDir) {
104 0 : Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
105 :
106 0 : if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
107 0 : throw Error(format(
108 : "path `%1%' is not a valid garbage collector root; "
109 : "it's not in the directory `%2%'")
110 : % gcRoot % rootsDir);
111 : }
112 :
113 15 : createSymlink(gcRoot, storePath, false);
114 : }
115 :
116 19 : return gcRoot;
117 : }
118 :
119 :
120 : /* The file to which we write our temporary roots. */
121 690 : static Path fnTempRoots;
122 345 : static AutoCloseFD fdTempRoots;
123 :
124 :
125 : void addTempRoot(const Path & path)
126 531 : {
127 : /* Create the temporary roots file for this process. */
128 531 : if (fdTempRoots == -1) {
129 :
130 77 : while (1) {
131 77 : Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
132 77 : createDirs(dir);
133 :
134 77 : fnTempRoots = (format("%1%/%2%")
135 : % dir % getpid()).str();
136 :
137 77 : AutoCloseFD fdGCLock = openGCLock(ltRead);
138 :
139 77 : if (pathExists(fnTempRoots))
140 : /* It *must* be stale, since there can be no two
141 : processes with the same pid. */
142 0 : deletePath(fnTempRoots);
143 :
144 77 : fdTempRoots = openLockFile(fnTempRoots, true);
145 :
146 77 : fdGCLock.close();
147 :
148 : /* Note that on Cygwin a lot of the following complexity
149 : is unnecessary, since we cannot delete open lock
150 : files. If we have the lock file open, then it's valid;
151 : if we can delete it, then it wasn't in use any more.
152 :
153 : Also note that on Cygwin we cannot "upgrade" a lock
154 : from a read lock to a write lock. */
155 :
156 : #ifndef __CYGWIN__
157 77 : debug(format("acquiring read lock on `%1%'") % fnTempRoots);
158 77 : lockFile(fdTempRoots, ltRead, true);
159 :
160 : /* Check whether the garbage collector didn't get in our
161 : way. */
162 77 : struct stat st;
163 77 : if (fstat(fdTempRoots, &st) == -1)
164 0 : throw SysError(format("statting `%1%'") % fnTempRoots);
165 77 : if (st.st_size == 0) break;
166 :
167 : /* The garbage collector deleted this file before we could
168 : get a lock. (It won't delete the file after we get a
169 : lock.) Try again. */
170 :
171 : #else
172 : break;
173 : #endif
174 : }
175 :
176 : }
177 :
178 : /* Upgrade the lock to a write lock. This will cause us to block
179 : if the garbage collector is holding our lock. */
180 531 : debug(format("acquiring write lock on `%1%'") % fnTempRoots);
181 531 : lockFile(fdTempRoots, ltWrite, true);
182 :
183 531 : string s = path + '\0';
184 531 : writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
185 :
186 : #ifndef __CYGWIN__
187 : /* Downgrade to a read lock. */
188 531 : debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
189 531 : lockFile(fdTempRoots, ltRead, true);
190 : #else
191 : debug(format("releasing write lock on `%1%'") % fnTempRoots);
192 : lockFile(fdTempRoots, ltNone, true);
193 : #endif
194 : }
195 :
196 :
197 : void removeTempRoots()
198 340 : {
199 340 : if (fdTempRoots != -1) {
200 77 : fdTempRoots.close();
201 77 : unlink(fnTempRoots.c_str());
202 : }
203 : }
204 :
205 :
206 : typedef boost::shared_ptr<AutoCloseFD> FDPtr;
207 : typedef list<FDPtr> FDs;
208 :
209 :
210 : static void readTempRoots(PathSet & tempRoots, FDs & fds)
211 16 : {
212 : /* Read the `temproots' directory for per-process temporary root
213 : files. */
214 : Strings tempRootFiles = readDirectory(
215 16 : (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
216 :
217 17 : for (Strings::iterator i = tempRootFiles.begin();
218 : i != tempRootFiles.end(); ++i)
219 : {
220 1 : Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
221 :
222 1 : debug(format("reading temporary root file `%1%'") % path);
223 :
224 : #ifdef __CYGWIN__
225 : /* On Cygwin we just try to delete the lock file. */
226 : char win32Path[MAX_PATH];
227 : cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
228 : if (DeleteFile(win32Path)) {
229 : printMsg(lvlError, format("removed stale temporary roots file `%1%'")
230 : % path);
231 : continue;
232 : } else
233 : debug(format("delete of `%1%' failed: %2%") % path % GetLastError());
234 : #endif
235 :
236 5 : FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
237 1 : if (*fd == -1) {
238 : /* It's okay if the file has disappeared. */
239 0 : if (errno == ENOENT) continue;
240 0 : throw SysError(format("opening temporary roots file `%1%'") % path);
241 : }
242 :
243 : /* This should work, but doesn't, for some reason. */
244 : //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
245 : //if (*fd == -1) continue;
246 :
247 : #ifndef __CYGWIN__
248 : /* Try to acquire a write lock without blocking. This can
249 : only succeed if the owning process has died. In that case
250 : we don't care about its temporary roots. */
251 1 : if (lockFile(*fd, ltWrite, false)) {
252 0 : printMsg(lvlError, format("removing stale temporary roots file `%1%'")
253 : % path);
254 0 : unlink(path.c_str());
255 0 : writeFull(*fd, (const unsigned char *) "d", 1);
256 0 : continue;
257 : }
258 : #endif
259 :
260 : /* Acquire a read lock. This will prevent the owning process
261 : from upgrading to a write lock, therefore it will block in
262 : addTempRoot(). */
263 1 : debug(format("waiting for read lock on `%1%'") % path);
264 1 : lockFile(*fd, ltRead, true);
265 :
266 : /* Read the entire file. */
267 1 : string contents = readFile(*fd);
268 :
269 : /* Extract the roots. */
270 1 : string::size_type pos = 0, end;
271 :
272 10 : while ((end = contents.find((char) 0, pos)) != string::npos) {
273 9 : Path root(contents, pos, end - pos);
274 9 : debug(format("got temporary root `%1%'") % root);
275 9 : assertStorePath(root);
276 9 : tempRoots.insert(root);
277 9 : pos = end + 1;
278 : }
279 :
280 1 : fds.push_back(fd); /* keep open */
281 : }
282 : }
283 :
284 :
285 : static void findRoots(const Path & path, bool recurseSymlinks,
286 : PathSet & roots)
287 102 : {
288 102 : struct stat st;
289 102 : if (lstat(path.c_str(), &st) == -1)
290 0 : throw SysError(format("statting `%1%'") % path);
291 :
292 102 : printMsg(lvlVomit, format("looking at `%1%'") % path);
293 :
294 102 : if (S_ISDIR(st.st_mode)) {
295 39 : Strings names = readDirectory(path);
296 104 : for (Strings::iterator i = names.begin(); i != names.end(); ++i)
297 65 : findRoots(path + "/" + *i, recurseSymlinks, roots);
298 : }
299 :
300 63 : else if (S_ISLNK(st.st_mode)) {
301 63 : string target = readLink(path);
302 63 : Path target2 = absPath(target, dirOf(path));
303 :
304 63 : if (isInStore(target2)) {
305 36 : debug(format("found root `%1%' in `%2%'")
306 : % target2 % path);
307 36 : Path target3 = toStorePath(target2);
308 36 : if (isValidPath(target3))
309 36 : roots.insert(target3);
310 : else
311 0 : printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
312 : % path % target3);
313 : }
314 :
315 27 : else if (recurseSymlinks) {
316 21 : if (pathExists(target2))
317 19 : findRoots(target2, false, roots);
318 : else {
319 2 : printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target2);
320 : /* Note that we only delete when recursing, i.e., when
321 : we are still in the `gcroots' tree. We never
322 : delete stuff outside that tree. */
323 2 : unlink(path.c_str());
324 : }
325 : }
326 : }
327 : }
328 :
329 :
330 : static void addAdditionalRoots(PathSet & roots)
331 18 : {
332 : Path rootFinder = getEnv("NIX_ROOT_FINDER",
333 18 : nixLibexecDir + "/nix/find-runtime-roots.pl");
334 :
335 18 : if (rootFinder.empty()) return;
336 :
337 1 : debug(format("executing `%1%' to find additional roots") % rootFinder);
338 :
339 1 : string result = runProgram(rootFinder);
340 :
341 1 : Strings paths = tokenizeString(result, "\n");
342 :
343 7043 : for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) {
344 7042 : if (isInStore(*i)) {
345 1 : Path path = toStorePath(*i);
346 1 : if (roots.find(path) == roots.end() && isValidPath(path)) {
347 1 : debug(format("found additional root `%1%'") % path);
348 1 : roots.insert(path);
349 : }
350 : }
351 : }
352 : }
353 :
354 :
355 : static void dfsVisit(const PathSet & paths, const Path & path,
356 : PathSet & visited, Paths & sorted)
357 5809 : {
358 5809 : if (visited.find(path) != visited.end()) return;
359 2933 : visited.insert(path);
360 :
361 2933 : PathSet references;
362 2933 : if (isValidPath(path))
363 2927 : queryReferences(noTxn, path, references);
364 :
365 5824 : for (PathSet::iterator i = references.begin();
366 : i != references.end(); ++i)
367 : /* Don't traverse into paths that don't exist. That can
368 : happen due to substitutes for non-existent paths. */
369 2891 : if (*i != path && paths.find(*i) != paths.end())
370 2876 : dfsVisit(paths, *i, visited, sorted);
371 :
372 2933 : sorted.push_front(path);
373 : }
374 :
375 :
376 : static Paths topoSort(const PathSet & paths)
377 16 : {
378 16 : Paths sorted;
379 16 : PathSet visited;
380 2949 : for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
381 2933 : dfsVisit(paths, *i, visited, sorted);
382 16 : return sorted;
383 : }
384 :
385 :
386 : void collectGarbage(GCAction action, const PathSet & pathsToDelete,
387 : bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
388 18 : {
389 18 : result.clear();
390 18 : bytesFreed = 0;
391 :
392 : bool gcKeepOutputs =
393 18 : queryBoolSetting("gc-keep-outputs", false);
394 : bool gcKeepDerivations =
395 18 : queryBoolSetting("gc-keep-derivations", true);
396 :
397 : /* Acquire the global GC root. This prevents
398 : a) New roots from being added.
399 : b) Processes from creating new temporary root files. */
400 18 : AutoCloseFD fdGCLock = openGCLock(ltWrite);
401 :
402 : /* Find the roots. Since we've grabbed the GC lock, the set of
403 : permanent roots cannot increase now. */
404 18 : Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
405 18 : PathSet roots;
406 18 : if (!ignoreLiveness)
407 18 : findRoots(rootsDir, true, roots);
408 :
409 : /* Add additional roots returned by the program specified by the
410 : NIX_ROOT_FINDER environment variable. This is typically used
411 : to add running programs to the set of roots (to prevent them
412 : from being garbage collected). */
413 18 : addAdditionalRoots(roots);
414 :
415 18 : if (action == gcReturnRoots) {
416 1 : result = roots;
417 1 : return;
418 : }
419 :
420 : /* Determine the live paths which is just the closure of the
421 : roots under the `references' relation. */
422 17 : PathSet livePaths;
423 45 : for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
424 28 : computeFSClosure(canonPath(*i), livePaths);
425 :
426 17 : if (gcKeepDerivations) {
427 0 : for (PathSet::iterator i = livePaths.begin();
428 : i != livePaths.end(); ++i)
429 : {
430 : /* Note that the deriver need not be valid (e.g., if we
431 : previously ran the collector with `gcKeepDerivations'
432 : turned off). */
433 0 : Path deriver = queryDeriver(noTxn, *i);
434 0 : if (deriver != "" && isValidPath(deriver))
435 0 : computeFSClosure(deriver, livePaths);
436 : }
437 : }
438 :
439 17 : if (gcKeepOutputs) {
440 : /* Hmz, identical to storePathRequisites in nix-store. */
441 0 : for (PathSet::iterator i = livePaths.begin();
442 : i != livePaths.end(); ++i)
443 0 : if (isDerivation(*i)) {
444 0 : Derivation drv = derivationFromPath(*i);
445 0 : for (DerivationOutputs::iterator j = drv.outputs.begin();
446 : j != drv.outputs.end(); ++j)
447 0 : if (isValidPath(j->second.path))
448 0 : computeFSClosure(j->second.path, livePaths);
449 : }
450 : }
451 :
452 17 : if (action == gcReturnLive) {
453 1 : result = livePaths;
454 1 : return;
455 : }
456 :
457 : /* Read the temporary roots. This acquires read locks on all
458 : per-process temporary root files. So after this point no paths
459 : can be added to the set of temporary roots. */
460 16 : PathSet tempRoots;
461 48 : FDs fds;
462 16 : readTempRoots(tempRoots, fds);
463 :
464 : /* Close the temporary roots. Note that we *cannot* do this in
465 : readTempRoots(), because there we may not have all locks yet,
466 : meaning that an invalid path can become valid (and thus add to
467 : the references graph) after we have added it to the closure
468 : (and computeFSClosure() assumes that the presence of a path
469 : means that it has already been closed). */
470 16 : PathSet tempRootsClosed;
471 25 : for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
472 9 : if (isValidPath(*i))
473 8 : computeFSClosure(*i, tempRootsClosed);
474 : else
475 1 : tempRootsClosed.insert(*i);
476 :
477 : /* For testing - see tests/gc-concurrent.sh. */
478 16 : if (getenv("NIX_DEBUG_GC_WAIT"))
479 1 : sleep(2);
480 :
481 : /* After this point the set of roots or temporary roots cannot
482 : increase, since we hold locks on everything. So everything
483 : that is not currently in in `livePaths' or `tempRootsClosed'
484 : can be deleted. */
485 :
486 : /* Read the Nix store directory to find all currently existing
487 : paths. */
488 16 : PathSet storePathSet;
489 16 : if (action != gcDeleteSpecific) {
490 13 : Paths entries = readDirectory(nixStore);
491 2943 : for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
492 2930 : storePathSet.insert(canonPath(nixStore + "/" + *i));
493 : } else {
494 6 : for (PathSet::iterator i = pathsToDelete.begin();
495 : i != pathsToDelete.end(); ++i)
496 : {
497 3 : assertStorePath(*i);
498 3 : storePathSet.insert(*i);
499 : }
500 : }
501 :
502 : /* Topologically sort them under the `referrers' relation. That
503 : is, a < b iff a is in referrers(b). This gives us the order in
504 : which things can be deleted safely. */
505 : /* !!! when we have multiple output paths per derivation, this
506 : will not work anymore because we get cycles. */
507 16 : Paths storePaths = topoSort(storePathSet);
508 :
509 : /* Try to delete store paths in the topologically sorted order. */
510 2947 : for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
511 :
512 2933 : debug(format("considering deletion of `%1%'") % *i);
513 :
514 2933 : if (livePaths.find(*i) != livePaths.end()) {
515 68 : if (action == gcDeleteSpecific)
516 2 : throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
517 66 : debug(format("live path `%1%'") % *i);
518 6 : continue;
519 : }
520 :
521 2865 : if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
522 5 : debug(format("temporary root `%1%'") % *i);
523 5 : continue;
524 : }
525 :
526 2860 : debug(format("dead path `%1%'") % *i);
527 2860 : result.insert(*i);
528 :
529 : /* If just returning the set of dead paths, we also return the
530 : space that would be freed if we deleted them. */
531 2860 : if (action == gcReturnDead) {
532 189 : struct stat st;
533 189 : if (lstat(i->c_str(), &st) == -1)
534 0 : st.st_size = 0;
535 189 : bytesFreed += st.st_size;
536 : }
537 :
538 2860 : if (action == gcDeleteDead || action == gcDeleteSpecific) {
539 :
540 : #ifndef __CYGWIN__
541 2671 : AutoCloseFD fdLock;
542 :
543 : /* Only delete a lock file if we can acquire a write lock
544 : on it. That means that it's either stale, or the
545 : process that created it hasn't locked it yet. In the
546 : latter case the other process will detect that we
547 : deleted the lock, and retry (see pathlocks.cc). */
548 2671 : if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
549 3 : fdLock = openLockFile(*i, false);
550 3 : if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
551 1 : debug(format("skipping active lock `%1%'") % *i);
552 1 : continue;
553 : }
554 : }
555 : #endif
556 :
557 2670 : printMsg(lvlInfo, format("deleting `%1%'") % *i);
558 :
559 : /* Okay, it's safe to delete. */
560 2670 : unsigned long long freed;
561 2670 : deleteFromStore(*i, freed);
562 2670 : bytesFreed += freed;
563 :
564 : #ifndef __CYGWIN__
565 2670 : if (fdLock != -1)
566 : /* Write token to stale (deleted) lock file. */
567 2 : writeFull(fdLock, (const unsigned char *) "d", 1);
568 : #endif
569 : }
570 : }
571 : }
572 :
573 :
574 345 : }
|
