Skip to content

Instantly share code, notes, and snippets.

@r2r-dev
Last active April 5, 2022 16:07
Show Gist options
  • Save r2r-dev/a9e57799c95529f945ad193de3ccc688 to your computer and use it in GitHub Desktop.
Save r2r-dev/a9e57799c95529f945ad193de3ccc688 to your computer and use it in GitHub Desktop.
quick and dirty lazy /nix/store

Create a lazy /nix/store filesystem using nixfs.py

  1. Grab static nix from https://hydra.nixos.org/build/170454219/download/1/nix and place in the same dir as py script
  2. Install fusepy (on nix: nix-shell -p python3Packages.fusepy)
  3. Ensure your /etc/fuse.conf contains user_allow_other
  4. Create workdirs: mkdir -p $(pwd)/{fakenix,workdir} && mkdir -p $(pwd)/fakenix/nix/store
  5. Mount our fs: python nixfs.py $(pwd)/fakenix $(pwd)/workdir
  6. Run some container image with workdir mounted: docker run -v $(pwd)/workdir/nix:/nix ubuntu:latest
  7. Try running some command, for example: /nix/store/pqpa3glx3iqd0cavslmr0lfkzgq1iias-cowsay-3.03+dfsg2/bin/cowsay thefuck?

Serve /nix/store and download required paths on the fly using pynarfs.py:

  1. Grab static nix from https://hydra.nixos.org/build/170454219/download/1/nix and place in the same dir as py script
  2. Install fusepy (on nix: nix-shell -p python3Packages.fusepy)
  3. Ensure your /etc/fuse.conf contains user_allow_other
  4. Create workdirs: mkdir -p $(pwd)/workdir/nix/store
  5. Run server: python pynarfs.py /nix/store $(pwd)/workdir
  6. Try running some command locally, for example: $(pwd)/workdir/nix/store/pqpa3glx3iqd0cavslmr0lfkzgq1iias-cowsay-3.03+dfsg2/bin/cowsay thefuck?
  7. Try fetching some path via http: curl http://localhost:8000/nix/store/x86psm12d79g6lpn685kzg6igv5vs12w-lsd-0.16.0/bin/lsd
#!/usr/bin/env python
from __future__ import with_statement
import os
import sys
import errno
from fuse import FUSE, FuseOSError, Operations
class Passthrough(Operations):
def __init__(self, root):
self.root = root
def _full_path(self, partial):
if partial.startswith("/"):
partial = partial[1:]
path = os.path.join(self.root, partial)
if partial.startswith("nix/store/") and not os.path.exists(self.root + "/" + "/".join(partial.split("/")[0:3])):
os.system(".//nix copy --to " + self.root + " --from https://cache.nixos.org /" + partial + " --extra-experimental-features nix-command")
return path
def access(self, path, mode):
full_path = self._full_path(path)
if not os.access(full_path, mode):
raise FuseOSError(errno.EACCES)
def chmod(self, path, mode):
full_path = self._full_path(path)
return os.chmod(full_path, mode)
def chown(self, path, uid, gid):
full_path = self._full_path(path)
return os.chown(full_path, uid, gid)
def getattr(self, path, fh=None):
full_path = self._full_path(path)
st = os.lstat(full_path)
return dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime',
'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
def readdir(self, path, fh):
full_path = self._full_path(path)
dirents = ['.', '..']
if os.path.isdir(full_path):
dirents.extend(os.listdir(full_path))
for r in dirents:
yield r
def readlink(self, path):
pathname = os.readlink(self._full_path(path))
if pathname.startswith("/"):
# Path name is absolute, sanitize it.
return os.path.relpath(pathname, self.root)
else:
return pathname
def mknod(self, path, mode, dev):
return os.mknod(self._full_path(path), mode, dev)
def rmdir(self, path):
full_path = self._full_path(path)
return os.rmdir(full_path)
def mkdir(self, path, mode):
return os.mkdir(self._full_path(path), mode)
def statfs(self, path):
full_path = self._full_path(path)
stv = os.statvfs(full_path)
return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
def unlink(self, path):
return os.unlink(self._full_path(path))
def symlink(self, name, target):
return os.symlink(target, self._full_path(name))
def rename(self, old, new):
return os.rename(self._full_path(old), self._full_path(new))
def link(self, target, name):
return os.link(self._full_path(name), self._full_path(target))
def utimens(self, path, times=None):
return os.utime(self._full_path(path), times)
def open(self, path, flags):
full_path = self._full_path(path)
return os.open(full_path, flags)
def create(self, path, mode, fi=None):
full_path = self._full_path(path)
return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def read(self, path, length, offset, fh):
os.lseek(fh, offset, os.SEEK_SET)
return os.read(fh, length)
def write(self, path, buf, offset, fh):
os.lseek(fh, offset, os.SEEK_SET)
return os.write(fh, buf)
def truncate(self, path, length, fh=None):
full_path = self._full_path(path)
with open(full_path, 'r+') as f:
f.truncate(length)
def flush(self, path, fh):
return os.fsync(fh)
def release(self, path, fh):
return os.close(fh)
def fsync(self, path, fdatasync, fh):
return self.flush(path, fh)
def main(mountpoint, root):
FUSE(Passthrough(root), mountpoint, nothreads=True, foreground=True, **{'allow_other': True})
if __name__ == '__main__':
main(sys.argv[2], sys.argv[1])
#!/usr/bin/env python
from __future__ import with_statement
import atexit
import os
import sys
import errno
import multiprocessing
import http.server
import socketserver
import functools
from fuse import FUSE, FuseOSError, Operations
def serve():
try:
PORT = 8000
DIRECTORY = "workdir"
Handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=DIRECTORY)
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print("serving at port", PORT)
httpd.serve_forever()
finally:
atexit._run_exitfuncs()
class Passthrough(Operations):
def __init__(self, root):
self.root = root
def _full_path(self, partial):
if partial.startswith("/"):
partial = partial[1:]
path = os.path.join(self.root, partial)
if not os.path.exists("/".join(path.split("/")[:4])):
os.system("nix copy --from https://cache.nixos.org " + path)
return path
def access(self, path, mode):
full_path = self._full_path(path)
if not os.access(full_path, mode):
raise FuseOSError(errno.EACCES)
def chmod(self, path, mode):
full_path = self._full_path(path)
return os.chmod(full_path, mode)
def chown(self, path, uid, gid):
full_path = self._full_path(path)
return os.chown(full_path, uid, gid)
def getattr(self, path, fh=None):
full_path = self._full_path(path)
st = os.lstat(full_path)
return dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime',
'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
def readdir(self, path, fh):
full_path = self._full_path(path)
dirents = ['.', '..']
if os.path.isdir(full_path):
dirents.extend(os.listdir(full_path))
for r in dirents:
yield r
def readlink(self, path):
pathname = os.readlink(self._full_path(path))
if pathname.startswith("/"):
# Path name is absolute, sanitize it.
return os.path.relpath(pathname, self.root)
else:
return pathname
def mknod(self, path, mode, dev):
return os.mknod(self._full_path(path), mode, dev)
def rmdir(self, path):
full_path = self._full_path(path)
return os.rmdir(full_path)
def mkdir(self, path, mode):
return os.mkdir(self._full_path(path), mode)
def statfs(self, path):
full_path = self._full_path(path)
stv = os.statvfs(full_path)
return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
def unlink(self, path):
return os.unlink(self._full_path(path))
def symlink(self, name, target):
return os.symlink(target, self._full_path(name))
def rename(self, old, new):
return os.rename(self._full_path(old), self._full_path(new))
def link(self, target, name):
return os.link(self._full_path(name), self._full_path(target))
def utimens(self, path, times=None):
return os.utime(self._full_path(path), times)
def open(self, path, flags):
full_path = self._full_path(path)
return os.open(full_path, flags)
def create(self, path, mode, fi=None):
full_path = self._full_path(path)
return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
def read(self, path, length, offset, fh):
os.lseek(fh, offset, os.SEEK_SET)
return os.read(fh, length)
def write(self, path, buf, offset, fh):
os.lseek(fh, offset, os.SEEK_SET)
return os.write(fh, buf)
def truncate(self, path, length, fh=None):
full_path = self._full_path(path)
with open(full_path, 'r+') as f:
f.truncate(length)
def flush(self, path, fh):
return os.fsync(fh)
def release(self, path, fh):
return os.close(fh)
def fsync(self, path, fdatasync, fh):
return self.flush(path, fh)
def main(mountpoint, root):
try:
FUSE(Passthrough(root), os.path.join(mountpoint, "nix", "store"), nothreads=True, foreground=True, **{'allow_other': True})
finally:
atexit._run_exitfuncs()
if __name__ == '__main__':
fuse = multiprocessing.Process(target=main, args=(sys.argv[2], sys.argv[1],))
serve = multiprocessing.Process(target=serve)
fuse.start()
serve.start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment