go filesystem api design walkthrough

fsys.md

io package

Core interfaces

type Reader interface {
	Read(p []byte) (n int, err error)
}

type Writer interface {
	Write(p []byte) (n int, err error)
}

type Closer interface {
	Close() error
}

Composite interfaces

type ReadWriter interface {
	Reader
	Writer
}

type ReadWriteCloser interface {
	Reader
	Writer
	Closer
}

Utilities

func Copy(dst Writer, src Reader) (written int64, err error)
func ReadAll(r Reader) ([]byte, error)
func MultiWriter(writers ...Writer) Writer
func TeeReader(r Reader, w Writer) Reader

os package

os.File implementation

type File struct {
	// contains filtered or unexported fields
}

func (f *File) Read(b []byte) (n int, err error)
func (f *File) Write(b []byte) (n int, err error)
func (f *File) Close() error

func (f *File) Stat() (FileInfo, error)
func (f *File) Chmod(mode FileMode) error
func (f *File) Chown(uid, gid int) error

func (f *File) ReadDir(n int) ([]DirEntry, error)

func (f *File) Seek(offset int64, whence int) (ret int64, err error)
func (f *File) Sync() error
// ...

Filesystem operations

// files
func Create(name string) (*File, error)
func Open(name string) (*File, error)
func OpenFile(name string, flag int, perm FileMode) (*File, error)

// attributes
func Stat(name string) (FileInfo, error)
func Chmod(name string, mode FileMode) error
func Chown(name string, uid, gid int) error
func Chtimes(name string, atime time.Time, mtime time.Time) error

// names
func Mkdir(name string, perm FileMode) error
func MkdirAll(name string, perm FileMode) error
func Remove(name string) error
func RemoveAll(name string) error
func Link(oldname, newname string) error
func Rename(oldpath, newpath string) error

// convenience
func ReadDir(name string) ([]DirEntry, error)
func ReadFile(name string) ([]byte, error)
func WriteFile(name string, data []byte, perm FileMode) error

// more ...

os.FileInfo interface

type FileInfo interface {
	Name() string       // base name of the file
	Size() int64        // length in bytes for regular files; system-dependent for others
	Mode() FileMode     // file mode bits
	ModTime() time.Time // modification time
	IsDir() bool        // abbreviation for Mode().IsDir()
	Sys() any           // underlying data source (can return nil)
}

afero package

Filesystem and File interfaces

type Fs interface {
  // files
  Create(name string) (File, error)
  Open(name string) (File, error)
  OpenFile(name string, flag int, perm os.FileMode) (File, error)

  // attributes
  Stat(name string) (os.FileInfo, error)
  Chmod(name string, mode os.FileMode) error
  Chown(name string, uid, gid int) error
  Chtimes(name string, atime time.Time, mtime time.Time) error

  // names
  Mkdir(name string, perm os.FileMode) error
  MkdirAll(path string, perm os.FileMode) error
  Remove(name string) error
  RemoveAll(path string) error
  Rename(oldname, newname string) error
  
  // more ...
}

type File interface {
	io.Closer
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Writer
	io.WriterAt

	Name() string
	Readdir(count int) ([]os.FileInfo, error)
	Readdirnames(n int) ([]string, error)
	Stat() (os.FileInfo, error)
	Sync() error
	Truncate(size int64) error
	WriteString(s string) (ret int, err error)
}

Utilities

DirExists(path string) (bool, error)
Exists(path string) (bool, error)
FileContainsBytes(filename string, subslice []byte) (bool, error)
GetTempDir(subPath string) string
IsDir(path string) (bool, error)
IsEmpty(path string) (bool, error)
ReadDir(dirname string) ([]os.FileInfo, error)
ReadFile(filename string) ([]byte, error)
SafeWriteReader(path string, r io.Reader) (err error)
TempDir(dir, prefix string) (name string, err error)
TempFile(dir, prefix string) (f File, err error)
Walk(root string, walkFn filepath.WalkFunc) error
WriteFile(filename string, data []byte, perm os.FileMode) error
WriteReader(path string, r io.Reader) (err error)

Filesystem implementations

// core storage implementations
type OsFs struct{} 	// OsFs is a Fs implementation that uses functions provided by the os package.
type MemMapFs struct {} // MemMapFs is a Fs implementation that is backed by in-memory map structures.

// wrappers
type ReadOnlyFs struct {}	// ReadOnlyFs wraps Fs value with write methods that return an error.
type BasePathFs struct {}	// BasePathFs wraps Fs value and prefixes a base path to operations.

// composite
type CacheOnReadFs struct {}	// CacheOnReadFs wraps an Fs value and caches any read files to another Fs.
type CopyOnWriteFs struct {}	// CopyOnWriteFs unions a base Fs value and another Fs that gets all writes.

// others
// * tarfs package
// * zipfs package
// * cloud store filesystems (Google Cloud Storage, S3, etc)
// * more exotic/experimental (JSON file,...)

io/fs package

Core interfaces

type FS interface {
	Open(name string) (File, error)
}

type File interface {
	Stat() (FileInfo, error)
	Read([]byte) (int, error)
	Close() error
}

Interface extensions

type StatFS interface {
	FS
	Stat(name string) (FileInfo, error)
}

Using extensions via type assertion

fsys := os.DirFS("/root")
statfs, ok := fsys.(StatFS)
if ok {
  _, err := fsys.Stat("foobar")
  if errors.Is(err, fs.ErrNotExist) {
    log.Fatal("foobar does not exist")
  }
}

Utility function

func Stat(fsys FS, name string) (FileInfo, error)

Using extensions via utility

fsys := os.DirFS("/root")
_, err = fs.Stat(fsys, "foobar")
if errors.Is(err, fs.ErrNotExist) {
  log.Fatal("foobar does not exist")
}

Toolkit fs package

Current core interface

type MutableFS interface {
  StatFS
  
  Create(name string) (File, error)
  OpenFile(name string, flag int, perm FileMode) (File, error)
  
  Chmod(name string, mode FileMode) error
  Chown(name string, uid, gid int) error
  Chtimes(name string, atime time.Time, mtime time.Time) error
	
  Mkdir(name string, perm FileMode) error	
  MkdirAll(path string, perm FileMode) error
  Remove(name string) error
  RemoveAll(path string) error
  Rename(oldname, newname string) error
}

New design for Toolkit fs package

Break down into specific interfaces with utility functions

type OpenFileFS interface {
  FS
  OpenFile(name string, flag int, perm FileMode) (File, error)
}

func OpenFile(fsys FS, name string) (File, error)

type MkdirFS interface {
  FS
  Mkdir(name string, perm FileMode) error	
}

func Mkdir(fsys FS, name string, perm FileMode) error	

type MkdirAllFS interface {
  FS
  MkdirAll(path string, perm FileMode) error	
}

func MkdirAll(fsys FS, path string, perm FileMode) error	

// etc

Do everything via utility functions

fsys := os.DirFS("/root")

if err := fs.MkdirAll(fsys, "new/subdir", 0755); err != nil {
  log.Fatal(err)
}

Potential "full API" wrapper type

type PosixFS struct {
  FS
}

func (fsys PosixFS) OpenFile(name string, flag int, perm FileMode) (File, error)
func (fsys PosixFS) Mkdir(name string, perm FileMode) error	
func (fsys PosixFS) MkdirAll(name string, perm FileMode) error	

// etc

Is it even worth it?

fsys := os.DirFS("/root")
pfs := PosixFS{fsys}

if err := pfs.MkdirAll("new/subdir", 0755); err != nil {
  log.Fatal(err)
}

Toolkit Filesystem Watch API

Existing API:

type WatchFS interface {
	fs.FS
	Watch(name string, cfg *Config) (*Watch, error)
}

type Config struct {
	Recursive bool
	EventMask uint
	Ignores   []string
	Handler   func(Event)
}

type Watch struct {}

func (w *Watch) Iter() <-chan Event
func (w *Watch) Close()

type Event struct {
	Type    EventType
	Path    string
	OldPath string
	Err     error
	
	fs.FileInfo
}

type EventType uint

const (
	EventError EventType = 1 << iota
	EventCreate
	EventWrite
	EventRemove
	EventRename
	EventChmod
	EventMove
)

New API:

• replace Watch with iterator. batch?
• use glob instead of excludes
• no event callback, only iterator
• combine event mask and recursive into single flags arg
• simplify event? at least make interface
• WatchFile utility. wrap if fsys doesn't support.
• WaitForEvent utility. returns when selected events happen. use context
• WaitForContent utility. returns when file content matches. use context