💾 Archived View for vk3.wtf › gemlog › 2021-08-25.gmi captured on 2021-12-05 at 23:47:19. Gemini links have been rewritten to link to archived content

View Raw

More Information

⬅️ Previous capture (2021-12-03)

-=-=-=-=-=-=-

Nix: cross fingers

After a build has completed, Nix sometimes needs to rewrite hashes in the result, e.g.,:

rewriting hashes in '/nix/store/2sp9hma57rj2f0drhyikab993jkqa3px-bionix-query-tmb-csv'; cross fingers

This is currently done in ram, which causes issues when using BioNix as some of these outputs are too large to fit in ram, such as BAM files. I patched Nix to instead do the hash rewriting on disk via a memory map:

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dd932cee9..c1e9f8ac3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3705,18 +3705,45 @@ void DerivationGoal::registerOutputs()
                 .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
             });
 
+            
+
             /* Canonicalise first.  This ensures that the path we're
                rewriting doesn't contain a hard link to /etc/shadow or
                something like that. */
             canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
 
-            /* FIXME: this is in-memory. */
-            StringSink sink;
-            dumpPath(actualPath, sink);
-            deletePath(actualPath);
-            sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-            StringSource source(*sink.s);
-            restorePath(actualPath, source);
+            AutoDelete tmpDir(createTempDir(), true);
+            Path tmpFile = (Path) tmpDir + "/rewrite";
+            {
+              AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+              if(!fd) throw SysError("creating temporary file '%s'", tmpFile);
+              FdSink sink(fd.get());
+              dumpPath(actualPath, sink);
+              deletePath(actualPath);
+            }
+
+            {
+              AutoCloseFD fd = open(tmpFile.c_str(), O_RDWR);
+              if(!fd) throw SysError("Opening temporary file '%s'", tmpFile);
+              struct stat stat_buf;
+              if(fstat(fd.get(), &stat_buf) == -1) throw SysError("fstat: '%s'", tmpFile);
+              void *ptr = mmap(NULL, stat_buf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0);
+              if(!ptr) throw SysError("mmap: '%s'", tmpFile);
+              for(auto & i : outputRewrites) {
+                if(i.first == i.second) continue;
+                void *j;
+                while((j = memmem(ptr, stat_buf.st_size, i.first.c_str(), i.first.size())))
+                  memcpy(j, i.second.c_str(), i.first.size());
+              }
+              munmap(ptr, stat_buf.st_size);
+            }
+
+            {
+              AutoCloseFD fd = open(tmpFile.c_str(), O_RDONLY);
+              if(!fd) throw SysError("Opening temporary file '%s'", tmpFile);
+              FdSource source(fd.get());
+              restorePath(actualPath, source);
+            }
 
             rewritten = true;
         }