Reproduction of "Element not found" error when hot attaching HCS endpoint to container

element-not-found-repro.go

package main

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	mathrand "math/rand"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/Microsoft/hcsshim"
)

const numContainers = 50

func main() {
	rootfsPath, present := os.LookupEnv("ROOTFS_PATH")
	if !present {
		log.Fatal("Must set ROOTFS_PATH env var")
	}
	networkName := "nat"
	if envNnetworkName, present := os.LookupEnv("NETWORK_NAME"); present {
		networkName = envNnetworkName
	}
	imageStore := `C:\windows\temp\image-store`
	driverInfo := hcsshim.DriverInfo{
		HomeDir: imageStore,
		Flavour: 1,
	}

	count := 0
	mathrand.Seed(time.Now().UnixNano())
	for {
		var wg sync.WaitGroup
		var foundErr error
		var mutex sync.Mutex
		for i := 0; i < numContainers; i++ {
			wg.Add(1)
			go func() {
				if err := createContainer(rootfsPath, networkName, imageStore, driverInfo, &mutex); err != nil {
					foundErr = err
				}
				wg.Done()
			}()
		}
		wg.Wait()
		if foundErr != nil {
			log.Fatalf("Error creating container: %s", foundErr.Error())
		}
		count++
		log.Printf("Successfully ran %d times\n", count)
		time.Sleep(5 * time.Second)
	}
}

func createContainer(rootfsPath, networkName, imageStore string, driverInfo hcsshim.DriverInfo, mutex *sync.Mutex) error {
	containerId := randomContainerId()

	parentLayerChain, err := ioutil.ReadFile(filepath.Join(rootfsPath, "layerchain.json"))
	if err != nil {
		return err
	}

	var parentLayers []string
	if err := json.Unmarshal(parentLayerChain, &parentLayers); err != nil {
		return err
	}
	sandboxLayers := append([]string{rootfsPath}, parentLayers...)

	err = createImage(rootfsPath, containerId, driverInfo, sandboxLayers, mutex)
	if err != nil {
		return err
	}
	defer deleteImage(containerId, driverInfo)

	volumePath, err := hcsshim.GetLayerMountPath(driverInfo, containerId)
	if err != nil {
		return err
	}

	var layerInfos []hcsshim.Layer
	for _, layerPath := range sandboxLayers {
		layerId := filepath.Base(layerPath)
		layerGuid, err := hcsshim.NameToGuid(layerId)
		if err != nil {
			return err
		}

		layerInfos = append(layerInfos, hcsshim.Layer{
			ID:   layerGuid.ToString(),
			Path: layerPath,
		})
	}

	containerConfig := hcsshim.ContainerConfig{
		SystemType:      "Container",
		Name:            containerId,
		VolumePath:      volumePath,
		Owner:           "owner",
		LayerFolderPath: filepath.Join(imageStore, containerId),
		Layers:          layerInfos,
	}

	container, err := hcsshim.CreateContainer(containerId, &containerConfig)
	if err != nil {
		return err
	}
	defer container.Close()
	defer func() {
		if err := container.Terminate(); err != nil {
			if hcsshim.IsPending(err) {
				if err := container.WaitTimeout(time.Minute); err != nil {
					log.Printf("hcsContainer.WaitTimeout error after Terminate for container %s: %s\n", containerId, err.Error())
				}
			} else {
				log.Printf("hcsContainer.Terminate error for container %s: %s\n", containerId, err.Error())
			}
		}
		log.Printf("Destroyed %s\n", containerId)
	}()
	log.Printf("Created %s\n", containerId)

	if err := container.Start(); err != nil {
		return err
	}

	network, err := hcsshim.GetHNSNetworkByName(networkName)
	if err != nil {
		return err
	}

	endpoint := &hcsshim.HNSEndpoint{
		VirtualNetwork: network.Id,
		Name:           containerId,
	}

	createdEndpoint, err := endpoint.Create()
	if err != nil {
		return err
	}
	defer endpoint.Delete()

	if err := hcsshim.HotAttachEndpoint(containerId, createdEndpoint.Id); err != nil {
		err = fmt.Errorf("Error hot attaching %s to %s: %s", createdEndpoint.Id, containerId, err.Error())
		log.Printf("*** %s ***\n", err.Error())
		return err
	}
	log.Printf("Hot attached %s to %s\n", createdEndpoint.Id, containerId)
	defer hcsshim.HotDetachEndpoint(containerId, createdEndpoint.Id)

	return nil
}

func createImage(rootfsPath, containerId string, driverInfo hcsshim.DriverInfo, sandboxLayers []string, mutex *sync.Mutex) error {
	mutex.Lock()
	defer mutex.Unlock()
	if err := hcsshim.CreateSandboxLayer(driverInfo, containerId, rootfsPath, sandboxLayers); err != nil {
		return err
	}

	if err := hcsshim.ActivateLayer(driverInfo, containerId); err != nil {
		return err
	}

	if err := hcsshim.PrepareLayer(driverInfo, containerId, sandboxLayers); err != nil {
		return err
	}
	return nil
}

func deleteImage(containerId string, driverInfo hcsshim.DriverInfo) {
	hcsshim.UnprepareLayer(driverInfo, containerId)
	hcsshim.DeactivateLayer(driverInfo, containerId)
	hcsshim.DestroyLayer(driverInfo, containerId)
}

func randomContainerId() string {
	r, _ := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	return fmt.Sprintf("container-%d", r.Int64())
}