💾 Archived View for vk3.wtf › gemlog › 2021-08-25.gmi captured on 2021-12-05 at 23:47:19. Gemini links have been rewritten to link to archived content
⬅️ Previous capture (2021-12-03)
-=-=-=-=-=-=-
After a build has completed, Nix sometimes needs to rewrite hashes in the result, e.g.,:
rewriting hashes in '/nix/store/2sp9hma57rj2f0drhyikab993jkqa3px-bionix-query-tmb-csv'; cross fingers
This is currently done in ram, which causes issues when using BioNix as some of these outputs are too large to fit in ram, such as BAM files. I patched Nix to instead do the hash rewriting on disk via a memory map:
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dd932cee9..c1e9f8ac3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3705,18 +3705,45 @@ void DerivationGoal::registerOutputs() .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) }); + + /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); - deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); - restorePath(actualPath, source); + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/rewrite"; + { + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if(!fd) throw SysError("creating temporary file '%s'", tmpFile); + FdSink sink(fd.get()); + dumpPath(actualPath, sink); + deletePath(actualPath); + } + + { + AutoCloseFD fd = open(tmpFile.c_str(), O_RDWR); + if(!fd) throw SysError("Opening temporary file '%s'", tmpFile); + struct stat stat_buf; + if(fstat(fd.get(), &stat_buf) == -1) throw SysError("fstat: '%s'", tmpFile); + void *ptr = mmap(NULL, stat_buf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0); + if(!ptr) throw SysError("mmap: '%s'", tmpFile); + for(auto & i : outputRewrites) { + if(i.first == i.second) continue; + void *j; + while((j = memmem(ptr, stat_buf.st_size, i.first.c_str(), i.first.size()))) + memcpy(j, i.second.c_str(), i.first.size()); + } + munmap(ptr, stat_buf.st_size); + } + + { + AutoCloseFD fd = open(tmpFile.c_str(), O_RDONLY); + if(!fd) throw SysError("Opening temporary file '%s'", tmpFile); + FdSource source(fd.get()); + restorePath(actualPath, source); + } rewritten = true; }