usrmerge prototype of golang tool for wolfi

README.md

usrmerge tool

This is function for a usrMerge tool.

The goal is to call usrSbinMergeRoot with a target directory and it should apply a usrSbinMerge to the contents of that directory.

Plan to utilize

The plan for making usrMerge in wolfi was:

1. setup things such that this tool can be called from a pipeline

   uses: usrmerge-tool

2. get a list of all packages/subpackages that ship things in usr/bin or usr/sbin or bin/

3. update all those origin packages inserting the usrmerge pipeline

testing

For the brave, you can create a root dir 'rootd/' and put things in it and then

go build .
./usrmerge rootd/

And see if it did what you wanted.

For the not-so-brave, use the test in apply_test to do the above.

dificulties

The difficulty here is that go isnt really easy to utilize in a pipeline. In order to do that we have to create a package of this thing.

I did this in go rather than shell for 2 reasons:

1. because doing this in shell would be painful, and testing would be more painful.
2. to push the envelope on an idea that I want to pursue for a pipelene-tools package.

The idea of a pipeline-tools package is that we would maintain an upstream project ("wolfi-pipeline-tools") that had tools that are easily utilized inside pipelines, like this one. That upstream project would be released with tags, updated by automation into wolfi and new builds would use it.

We could use golang or other languages in that upstream package and make nicer pipelines than we're able to do with bash inline in a yaml file.

apply.go

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"strings"
)

// resolveUnderRoot resolves dest relative to source and ensures the resulting path stays within root.
// returns full cleaned paths to source and dest
// - If source is not absolute, it is treated as relative to root.
// - If dest is absolute, it is reinterpreted as relative to root (its leading separator is removed).
// - Any upward traversals (e.g. "../") that would escape root are clamped.
func resolveUnderRoot(source, dest, root string) (string, string, error) {
	// Ensure root is an absolute, clean path.
	root, err := filepath.Abs(filepath.Clean(root))
	if err != nil {
		return "", "", fmt.Errorf("failed to resolve root path: %w", err)
	}

	// If source is not absolute, treat it as relative to root.
	if !filepath.IsAbs(source) {
		source = filepath.Join(root, source)
	}
	source = filepath.Clean(source)

	var candidate string
	if filepath.IsAbs(dest) {
		// If dest is absolute, strip the leading separator and treat it as relative to root.
		dest = filepath.Clean(dest)
		dest = strings.TrimPrefix(dest, string(filepath.Separator))
		candidate = filepath.Join(root, dest)
	} else {
		// Otherwise, dest is relative to source.
		candidate = filepath.Join(filepath.Dir(source), dest)
	}
	candidate = filepath.Clean(candidate)

	// At this point, candidate might be outside of root if dest contained "../" segments.
	// We compute the relative path from root to candidate.
	rel, err := filepath.Rel(root, candidate)
	if err != nil {
		return "", "", fmt.Errorf("unable to compute relative path: %w", err)
	}

	// Split the relative path into its segments.
	parts := strings.Split(rel, string(filepath.Separator))
	// Remove any leading ".." segments, effectively clamping the upward moves.
	safeParts := []string{}
	for _, part := range parts {
		if part == ".." {
			// Skip any upward traversal that would leave root.
			continue
		}
		// Also ignore empty parts (which may occur if rel is ".")
		if part != "" && part != "." {
			safeParts = append(safeParts, part)
		}
	}

	// Reconstruct the final path as root joined with the safe relative parts.
	finalPath := filepath.Join(append([]string{root}, safeParts...)...)
	finalPath = filepath.Clean(finalPath)

	// Ensure finalPath is absolute.
	finalPath, err = filepath.Abs(finalPath)
	if err != nil {
		return "", "", fmt.Errorf("unable to obtain absolute path: %w", err)
	}

	return source, finalPath, nil
}

func newLinkDest(curSrc, curDest, rootPath, newDirPath string, equivs []string) (string, error) {

	rootPath, err := filepath.Abs(filepath.Clean(rootPath))
	if err != nil {
		return "", fmt.Errorf("failed to resolve root path: %w", err)
	}

	_, dest, err := resolveUnderRoot(curSrc, curDest, rootPath)
	if err != nil {
		return "", err
	}

	if dest == rootPath {
		r, err := filepath.Rel(newDirPath, "")
		if err != nil {
			return "", err
		}
		return filepath.Clean(r), nil
	}

	// src and dest are both under rootPath
	relDest := strings.TrimPrefix(dest, rootPath+string(filepath.Separator))
	relDestDir := filepath.Dir(relDest)
	// fmt.Printf("reldest=%s newDirPath=%s relDestDir=%s\n", relDest, newDirPath, relDestDir)
	if relDestDir == newDirPath || slices.Contains(equivs, relDestDir) {
		return filepath.Base(curDest), nil
	}

	r, err := filepath.Rel(newDirPath, filepath.Join(relDestDir, filepath.Base(curDest)))
	if err != nil {
		return "", err
	}
	return r, nil
}

// move
//
//	bin/* into usr/bin/*
//	sbin/* into usr/sbin/*
func usrSbinMergeRoot(rootPath string) error {
	var err error
	moves := []struct{ src, dest string }{
		{"bin", "usr/bin"},
		{"sbin", "usr/bin"},
		{"usr/sbin", "usr/bin"},
	}
	equivs := []string{"usr/sbin", "sbin", "bin"}

	rpIn := rootPath
	if rootPath, err = filepath.Abs(rootPath); err != nil {
		return fmt.Errorf("failed to find absolute path to %s", rpIn)
	}
	rootPath = filepath.Clean(rootPath)

	for _, m := range moves {
		dest := filepath.Join(rootPath, m.dest)
		src := filepath.Join(rootPath, m.src)
		srcInfo, err := os.Lstat(src)
		if err != nil {
			if os.IsNotExist(err) {
				continue
			}
			return err
		} else if srcInfo.Mode()&os.ModeSymlink != 0 {
			srcDest, err := os.Readlink(src)
			if err != nil {
				return fmt.Errorf("failed Readlink(%s): %v", src, err)
			}
			if srcDest == m.dest {
				continue
			}
			return fmt.Errorf("%s existed as a symlink not to %s. it pointed to %s", m.src, m.dest, srcDest)
		}

		if err := os.MkdirAll(dest, 0755); err != nil && !os.IsExist(err) {
			return err
		}

		entries, err := os.ReadDir(src)
		if err != nil {
			return fmt.Errorf("Failed to open %s for reading: %v", src, err)
		}

		for _, dirEnt := range entries {
			fInfo, err := dirEnt.Info()
			if err != nil {
				return fmt.Errorf("failed reading file info for %s: %v", dirEnt.Name(), err)
			}

			fpSrc := filepath.Join(src, dirEnt.Name())
			fpDest := filepath.Join(dest, dirEnt.Name())

			if fInfo.Mode()&os.ModeSymlink == 0 {
				if err := os.Rename(fpSrc, fpDest); err != nil {
					return fmt.Errorf("failed renaming %s -> %s: %v",
						filepath.Join(m.src, dirEnt.Name()), filepath.Join(m.dest, dirEnt.Name()), err)
				}
			} else {
				curTarget, err := os.Readlink(fpSrc)
				if err != nil {
					return fmt.Errorf("failed reading link for %s: %v", fpSrc, err)
				}

				newDest, err := newLinkDest(fpSrc, curTarget, rootPath, "usr/bin", equivs)
				if err != nil {
					return err
				}

				if err := os.Symlink(newDest, fpDest); err != nil {
					return err
				}

				if err := os.Remove(fpSrc); err != nil {
					return err
				}
			}
		}

		if err := os.Remove(src); err != nil {
			filepath.WalkDir(src, func(path string, _ os.DirEntry, err error) error { fmt.Printf("%s\n", path); return nil })
			return fmt.Errorf("failed to remove %s after moves: %v", m.src, err)
		}

		entries, err = os.ReadDir(dest)
		if len(entries) == 0 {
			if err := os.Remove(dest); err != nil {
				return fmt.Errorf("failed removing empty usr/bin (%s) dir: %v", dest, err)
			}
		}

		usr := filepath.Join(rootPath, "usr")
		entries, err = os.ReadDir(usr)
		if len(entries) == 0 {
			if err := os.Remove(usr); err != nil {
				return fmt.Errorf("failed removing empty usr dir: %v", err)
			}
		}
	}

	return nil
}

func usrMergeDest(curPath, curDest, rootPath string) (string, error) {
	return newLinkDest(curPath, curDest, rootPath, "usr/bin", []string{"bin"})
}

apply_test.go

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"syscall"
	"testing"
)

func TestUsrMergeDest(t *testing.T) {
	type test struct {
		curPath  string
		curDest  string
		expected string
		err      error
	}

	tests := []test{
		{"bin/f1", "f2", "f2", nil},
		{"bin/f1", "../bin/f2", "f2", nil},
		{"bin/f1", "/usr/bin/f2", "f2", nil},
		{"bin/f1", "../../bin/f2", "f2", nil},
		{"bin/f1", "../usr/bin/f2", "f2", nil},
		{"bin/f1", "/etc/alternatives/f2", "../../etc/alternatives/f2", nil},
		{"bin/f1", "../../../wark/../usr/bin/f2", "f2", nil},
		{"bin/f1", "../opt/bin/f1", "../../opt/bin/f1", nil},
		{"usr/bin/f1", "/opt/bin/f1", "../../opt/bin/f1", nil},
		{"usr/bin/f1", "../bin/f2", "f2", nil},
		{"sbin/f1", "../bin/f2", "f2", nil},
		{"bin/f1", "../usr/local/bin/f1", "../local/bin/f1", nil},
		{"bin/f1", "/usr/local/bin/f1", "../local/bin/f1", nil},
	}

	for _, v := range tests {
		newDest, errFound := usrMergeDest(v.curPath, v.curDest, "/home/melange/output")
		if errFound != nil {
			if errFound == v.err {
				continue
			}
			if v.err == nil {
				t.Errorf("Unexpected non-nil err value received. %v -> (%s, %v)", v, newDest, errFound)
			} else if v.err != errFound {
				t.Errorf("Unexpected err value received. %v -> (%s, %v)", v, newDest, errFound)
			}
			continue
		} else {
			if newDest != v.expected {
				t.Errorf("Expected %s, found %s. %v", v.expected, newDest, v)
			}
		}
	}
}

var testTrees = []struct{ input, expected []FSEntry }{
	{
		input: []FSEntry{
			{Path: "bin", Type: "dir"},
		},
		expected: []FSEntry{},
	},
	{
		input: []FSEntry{
			{Path: "bin", Type: "dir"},
			{Path: "bin/busybox", Type: "file"},
			{Path: "bin/sh", Type: "slink", Target: "busybox"},
			{Path: "sbin", Type: "dir"},
			{Path: "sbin/chroot", Type: "slink", Target: "../bin/busybox"},
		},
		expected: []FSEntry{
			{Path: "usr", Type: "dir"},
			{Path: "usr/bin", Type: "dir"},
			{Path: "usr/bin/busybox", Type: "file"},
			{Path: "usr/bin/sh", Type: "slink", Target: "busybox"},
			{Path: "usr/bin/chroot", Type: "slink", Target: "busybox"},
		},
	},
}

func TestMerge(t *testing.T) {
	for _, v := range testTrees {
		sort.Slice(v.expected, func(i, j int) bool { return v.expected[i].Path < v.expected[j].Path })
		tmpd, err := os.MkdirTemp("", "usrmergetest")
		if err != nil {
			t.Errorf("failed creating tmpdir")
			continue
		}

		defer os.RemoveAll(tmpd)

		if err = populateFromDescription(tmpd, v.input); err != nil {
			t.Errorf("failed to populate dir: %v", err)
			continue
		}

		err = usrSbinMergeRoot(tmpd)
		if err != nil {
			t.Errorf("filed merge %v", err)
			continue
		}

		err = compareFileSystem(tmpd, FSDesc{v.expected})
		if err != nil {
			t.Errorf("Different results than expected: %v", err)
		}
	}
}

type FSEntry struct {
	Path    string `yaml:"path"`
	Type    string `yaml:"type"`
	Content string `yaml:"content,omitempty"`
	Target  string `yaml:"target,omitempty"`
}

type FSDesc struct {
	Entries []FSEntry `yaml:"ents"`
}

func createEntry(baseDir string, entry FSEntry) error {
	fullPath := filepath.Join(baseDir, entry.Path)
	switch entry.Type {
	case "dir":
		return os.MkdirAll(fullPath, 0755)
	case "file":
		return ioutil.WriteFile(fullPath, []byte(entry.Content), 0644)
	case "slink":
		return os.Symlink(entry.Target, fullPath)
	case "hlink":
		return os.Link(filepath.Join(baseDir, entry.Target), fullPath)
	default:
		return fmt.Errorf("unsupported type: %s", entry.Type)
	}
}

func populateFromDescription(baseDir string, fsDescs []FSEntry) error {
	for _, entry := range fsDescs {
		if err := createEntry(baseDir, entry); err != nil {
			return err
		}
	}
	return nil
}

func scanDirectory(baseDir string) (FSDesc, error) {
	var fsDesc FSDesc = FSDesc{Entries: []FSEntry{}}
	inodeMap := make(map[uint64]string)

	var paths []string
	filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
		if err == nil {
			paths = append(paths, path)
		}
		return nil
	})

	sort.Strings(paths)

	for _, path := range paths {
		if path == baseDir {
			continue
		}
		info, err := os.Lstat(path)
		if err != nil {
			return fsDesc, err
		}

		relPath, _ := filepath.Rel(baseDir, path)
		if relPath == "." {
			continue
		}

		entry := FSEntry{Path: relPath}
		switch {
		case info.IsDir():
			entry.Type = "dir"
		case info.Mode()&os.ModeSymlink != 0:
			target, _ := os.Readlink(path)
			entry.Type = "slink"
			entry.Target = target
		case info.Mode().IsRegular():
			var stat syscall.Stat_t
			if err := syscall.Stat(path, &stat); err == nil {
				inode := stat.Ino
				if existingPath, found := inodeMap[inode]; found {
					entry.Type = "hlink"
					entry.Target = existingPath
				} else {
					inodeMap[inode] = relPath
					entry.Type = "file"
					content, _ := ioutil.ReadFile(path)
					entry.Content = string(content)
				}
			}
		}
		fsDesc.Entries = append(fsDesc.Entries, entry)
	}

	return fsDesc, nil
}

func compareFileSystem(baseDir string, expectedFS FSDesc) error {
	foundFSD, err := scanDirectory(baseDir)
	if err != nil {
		return err
	}

	expectedJSON, _ := json.MarshalIndent(expectedFS, "", "  ")
	actualJSON, _ := json.MarshalIndent(foundFSD, "", "  ")

	if !bytes.Equal(expectedJSON, actualJSON) {
		fmt.Println("Filesystem does not match expected structure!")
		fmt.Println("Expected:")
		fmt.Println(string(expectedJSON))
		fmt.Println("Actual:")
		fmt.Println(string(actualJSON))
		return fmt.Errorf("mismatch detected")
	}

	return nil
}

go.mod

module github.com/smoser/usrmerge

go 1.23.1

go.sum

main.go

package main

import (
	"fmt"
	"os"
)

func main() {
	fmt.Printf("hi, got:%v\n", os.Args)

	if len(os.Args) != 2 {
		fmt.Printf("Must give only 1 directory to work on")
		os.Exit(1)
	}

	destdir := os.Args[1]
	err := usrSbinMergeRoot(destdir)
	if err != nil {
		panic(err)
	}
	fmt.Printf("merged %s", destdir)
	return
}

does this handle broken links fine?

It does not care if they are dangling or not. The intent is just to make sure the destination is adjusted for the move.

…

On Tue, Feb 4, 2025 at 7:30 PM Dimitri John Ledkov ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ does this handle broken links fine? — Reply to this email directly, view it on GitHub <https://gist.github.com/smoser/c542d3b2e67caff433e4eb0737c899c6#gistcomment-5423858> or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAAETMINI63BCJZMFCF5AZL2OFLQTBFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTGNRRG43DIOJQU52HE2LHM5SXFJTDOJSWC5DF> . You are receiving this email because you authored the thread. Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub> .