Use new types for hyperv isolated containers

Signed-off-by: Amit Barve <ambarve@microsoft.com>

cmd/containerd-shim-runhcs-v1/pod.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/uvm"
@@ -122,22 +123,15 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 				return nil, err
 			}
 		case *uvm.OptionsWCOW:
+			var layerFolders []string
+			if s.Windows != nil {
+				layerFolders = s.Windows.LayerFolders
+			}
 			wopts := (opts).(*uvm.OptionsWCOW)
-
-			// In order for the UVM sandbox.vhdx not to collide with the actual
-			// nested Argon sandbox.vhdx we append the \vm folder to the last
-			// entry in the list.
-			layersLen := len(s.Windows.LayerFolders)
-			layers := make([]string, layersLen)
-			copy(layers, s.Windows.LayerFolders)
-
-			vmPath := filepath.Join(layers[layersLen-1], "vm")
-			err := os.MkdirAll(vmPath, 0)
+			wopts.BootFiles, err = layers.ParseWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders)
 			if err != nil {
 				return nil, err
 			}
-			layers[layersLen-1] = vmPath
-			wopts.LayerFolders = layers
 
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -123,14 +123,6 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		}
 	}
 
-	var layerFolders []string
-	if spec.Windows != nil {
-		layerFolders = spec.Windows.LayerFolders
-	}
-	if err := validateRootfsAndLayers(req.Rootfs, layerFolders); err != nil {
-		return nil, err
-	}
-
 	if len(req.Rootfs) > 0 {
 		// write the rootfs to a file so that it can be used for proper cleanup during shim
 		// delete. We can't write to the config.json as it is read-only for shim.

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -83,23 +83,15 @@ func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.Creat
 				return nil, err
 			}
 		case *uvm.OptionsWCOW:
+			var layerFolders []string
+			if s.Windows != nil {
+				layerFolders = s.Windows.LayerFolders
+			}
 			wopts := (opts).(*uvm.OptionsWCOW)
-
-			// In order for the UVM sandbox.vhdx not to collide with the actual
-			// nested Argon sandbox.vhdx we append the \vm folder to the last
-			// entry in the list.
-			layersLen := len(s.Windows.LayerFolders)
-			layers := make([]string, layersLen)
-			copy(layers, s.Windows.LayerFolders)
-
-			vmPath := filepath.Join(layers[layersLen-1], "vm")
-			err := os.MkdirAll(vmPath, 0)
+			wopts.BootFiles, err = layers.ParseWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders)
 			if err != nil {
 				return nil, err
 			}
-			layers[layersLen-1] = vmPath
-			wopts.LayerFolders = layers
-
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {
 				return nil, err

internal/hcsoci/hcsdoc_wcow.go

@@ -305,7 +305,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter
 
 	// Strip off the top-most RW/scratch layer as that's passed in separately to HCS for v1
 	// TODO(ambarve) Understand how this path is exactly used and fix it.
-	v1.LayerFolderPath = coi.Spec.Windows.LayerFolders[len(coi.Spec.Windows.LayerFolders)-1]
+	// v1.LayerFolderPath = coi.Spec.Windows.LayerFolders[len(coi.Spec.Windows.LayerFolders)-1]
 
 	if coi.isV2Argon() || coi.isV1Argon() {
 		// Argon v1 or v2.

internal/layers/layers_wcow.go

@@ -19,12 +19,14 @@ import (
 	"golang.org/x/sys/windows"
 
 	"github.com/Microsoft/hcsshim/computestorage"
+	"github.com/Microsoft/hcsshim/internal/copyfile"
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/hcserror"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/resources"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/internal/uvm/scsi"
+	"github.com/Microsoft/hcsshim/internal/uvmfolder"
 	"github.com/Microsoft/hcsshim/internal/wclayer"
 	cimlayer "github.com/Microsoft/hcsshim/internal/wclayer/cim"
 )
@@ -168,6 +170,11 @@ func parseForkedCimMount(containerID string, m *types.Mount) (*wcowHostForkedCIM
 // them for mounting.  If `vm` is nil, these are considered to be process isolate container layers, and if
 // `vm` value is set these are considered to be hyperv isolated container layers.
 func ParseWCOWLayers(containerID string, vm *uvm.UtilityVM, rootfs []*types.Mount, layerFolders []string) (WCOWLayers, error) {
+
+	if err := validateRootfsAndLayers(rootfs, layerFolders); err != nil {
+		return nil, err
+	}
+
 	if len(layerFolders) > 0 {
 		if vm == nil {
 			// If layerFolders are being passed it MUST be WCIFS based layers
@@ -177,8 +184,10 @@ func ParseWCOWLayers(containerID string, vm *uvm.UtilityVM, rootfs []*types.Moun
 	}
 
 	m := rootfs[0]
+	//TODO(ambare): Move snapshotter layer types, mount parsing logic etc inside the containerd
+	// snapshotter code.
 	switch m.Type {
-	case "windows":
+	case "windows-layer":
 		scratchLayer, parentLayers, err := parseLegacyRootfsMount(m)
 		if err != nil {
 			return nil, err
@@ -316,7 +325,6 @@ func (l *wcowHostWCIFSLayers) Mount(ctx context.Context) (_ *MountedWCOWLayers,
 			return nil, nil, err
 		}
 	}
-
 	return &MountedWCOWLayers{
 			RootFS:            mountPath,
 			MountedLayerPaths: layersWithID,
@@ -542,3 +550,61 @@ func ensureScratchVHD(ctx context.Context, scratchFolder string, layerFolders []
 	}
 	return nil
 }
+
+// Parse the layer data to get the UVM boot files from the layers.
+func ParseWCOWUVMBootFilesFromLayers(ctx context.Context, rootfs []*types.Mount, layerFolders []string) (*uvm.WCOWBootFiles, error) {
+	var parentLayers []string
+	var scratchLayer string
+	var err error
+
+	if err = validateRootfsAndLayers(rootfs, layerFolders); err != nil {
+		return nil, err
+	}
+
+	if len(layerFolders) > 0 {
+		parentLayers = layerFolders[:len(layerFolders)-1]
+		scratchLayer = layerFolders[len(layerFolders)-1]
+	} else {
+		m := rootfs[0]
+		//TODO(ambare): Move snapshotter layer types, mount parsing logic etc inside the containerd
+		// snapshotter code.
+		switch m.Type {
+		case "windows-layer":
+			scratchLayer, parentLayers, err = parseLegacyRootfsMount(m)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("mount type %s is not supported for UVM boot", m.Type)
+		}
+	}
+
+	uvmFolder, err := uvmfolder.LocateUVMFolder(ctx, parentLayers)
+	if err != nil {
+		return nil, fmt.Errorf("failed to locate utility VM folder from layer folders: %w", err)
+	}
+
+	// In order for the UVM sandbox.vhdx not to collide with the actual
+	// nested Argon sandbox.vhdx we append the \vm folder to the last
+	// entry in the list.
+	scratchLayer = filepath.Join(scratchLayer, "vm")
+	scratchVHDPath := filepath.Join(scratchLayer, "sandbox.vhdx")
+	if err = os.MkdirAll(scratchLayer, 0777); err != nil {
+		return nil, err
+	}
+
+	if _, err = os.Stat(scratchVHDPath); os.IsNotExist(err) {
+		sourceScratch := filepath.Join(uvmFolder, `UtilityVM\SystemTemplate.vhdx`)
+		if err := copyfile.CopyFile(ctx, sourceScratch, scratchVHDPath, true); err != nil {
+			return nil, err
+		}
+	}
+	return &uvm.WCOWBootFiles{
+		OSFilesPath:           filepath.Join(uvmFolder, `UtilityVM\Files`),
+		OSRelativeBootDirPath: `\EFI\Microsoft\Boot`,
+		// In order for the UVM sandbox.vhdx not to collide with the actual
+		// nested Argon sandbox.vhdx we append the \vm folder to the last
+		// entry in the list.
+		ScratchVHDPath: scratchVHDPath,
+	}, nil
+}

internal/tools/uvmboot/wcow.go

@@ -15,6 +15,7 @@ import (
 	"github.com/urfave/cli"
 
 	"github.com/Microsoft/hcsshim/internal/cmd"
+	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 )
@@ -55,19 +56,19 @@ var wcowCommand = cli.Command{
 		runMany(c, func(id string) error {
 			options := uvm.NewDefaultOptionsWCOW(id, "")
 			setGlobalOptions(c, options.Options)
-			var layers []string
+			var layerFolders []string
 			if wcowImage != "" {
 				layer, err := filepath.Abs(wcowImage)
 				if err != nil {
 					return err
 				}
-				layers = []string{layer}
+				layerFolders = []string{layer}
 			} else {
 				if wcowDockerImage == "" {
 					wcowDockerImage = "mcr.microsoft.com/windows/nanoserver:1809"
 				}
 				var err error
-				layers, err = getLayers(wcowDockerImage)
+				layerFolders, err = getLayers(wcowDockerImage)
 				if err != nil {
 					return err
 				}
@@ -77,7 +78,11 @@ var wcowCommand = cli.Command{
 				return err
 			}
 			defer os.RemoveAll(tempDir)
-			options.LayerFolders = append(layers, tempDir)
+			layerFolders = append(layerFolders, tempDir)
+			options.BootFiles, err = layers.ParseWCOWUVMBootFilesFromLayers(context.TODO(), nil, layerFolders)
+			if err != nil {
+				return err
+			}
 			vm, err := uvm.CreateWCOW(context.TODO(), options)
 			if err != nil {
 				return err

internal/uvm/create.go

@@ -133,9 +133,6 @@ func verifyOptions(ctx context.Context, options interface{}) error {
 		if opts.EnableDeferredCommit && !opts.AllowOvercommit {
 			return errors.New("EnableDeferredCommit is not supported on physically backed VMs")
 		}
-		if len(opts.LayerFolders) < 2 {
-			return errors.New("at least 2 LayerFolders must be supplied")
-		}
 		if opts.SCSIControllerCount != 1 {
 			return errors.New("exactly 1 SCSI controller is required for WCOW")
 		}

internal/uvm/create_test.go

@@ -4,7 +4,6 @@ package uvm
 
 import (
 	"context"
-	"fmt"
 	"testing"
 )
 
@@ -20,12 +19,3 @@ func TestCreateBadBootFilesPath(t *testing.T) {
 		t.Fatal(err)
 	}
 }
-
-func TestCreateWCOWBadLayerFolders(t *testing.T) {
-	opts := NewDefaultOptionsWCOW(t.Name(), "")
-	_, err := CreateWCOW(context.Background(), opts)
-	errMsg := fmt.Sprintf("%s: %s", errBadUVMOpts, "at least 2 LayerFolders must be supplied")
-	if err == nil || err.Error() != errMsg {
-		t.Fatal(err)
-	}
-}

internal/uvm/create_wcow.go

@@ -23,17 +23,18 @@ import (
 	"github.com/Microsoft/hcsshim/internal/schemaversion"
 	"github.com/Microsoft/hcsshim/internal/security"
 	"github.com/Microsoft/hcsshim/internal/uvm/scsi"
-	"github.com/Microsoft/hcsshim/internal/uvmfolder"
 	"github.com/Microsoft/hcsshim/internal/wclayer"
-	"github.com/Microsoft/hcsshim/internal/wcow"
 	"github.com/Microsoft/hcsshim/osversion"
 )
 
 // OptionsWCOW are the set of options passed to CreateWCOW() to create a utility vm.
 type OptionsWCOW struct {
 	*Options
 
-	LayerFolders []string // Set of folders for base layers and scratch. Ordered from top most read-only through base read-only layer, followed by scratch
+	// Deprecated: LayerFolders is a deprecated field and kept only for legacy purposes. Use `BootFiles`
+	LayerFolders []string
+
+	BootFiles *WCOWBootFiles
 
 	// NoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM
 	NoDirectMap bool
@@ -46,7 +47,7 @@ type OptionsWCOW struct {
 }
 
 // NewDefaultOptionsWCOW creates the default options for a bootable version of
-// WCOW. The caller `MUST` set the `LayerFolders` path on the returned value.
+// WCOW. The caller `MUST` set the `BootFiles` on the returned value.
 //
 // `id` the ID of the compute system. If not passed will generate a new GUID.
 //
@@ -73,7 +74,7 @@ func (uvm *UtilityVM) startExternalGcsListener(ctx context.Context) error {
 	return nil
 }
 
-func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uvmFolder string) (*hcsschema.ComputeSystem, error) {
+func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW) (*hcsschema.ComputeSystem, error) {
 	processorTopology, err := processorinfo.HostProcessorInfo(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get host processor information: %w", err)
@@ -94,7 +95,7 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv
 		Shares: []hcsschema.VirtualSmbShare{
 			{
 				Name:    "os",
-				Path:    filepath.Join(uvmFolder, `UtilityVM\Files`),
+				Path:    opts.BootFiles.OSFilesPath,
 				Options: vsmbOpts,
 			},
 		},
@@ -174,7 +175,7 @@ func prepareConfigDoc(ctx context.Context, uvm *UtilityVM, opts *OptionsWCOW, uv
 			Chipset: &hcsschema.Chipset{
 				Uefi: &hcsschema.Uefi{
 					BootThis: &hcsschema.UefiBootEntry{
-						DevicePath: `\EFI\Microsoft\Boot\bootmgfw.efi`,
+						DevicePath: filepath.Join(opts.BootFiles.OSRelativeBootDirPath, "bootmgfw.efi"),
 						DeviceType: "VmbFs",
 					},
 				},
@@ -280,42 +281,13 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
 		return nil, errors.Wrap(err, errBadUVMOpts.Error())
 	}
 
-	uvmFolder, err := uvmfolder.LocateUVMFolder(ctx, opts.LayerFolders)
-	if err != nil {
-		return nil, fmt.Errorf("failed to locate utility VM folder from layer folders: %w", err)
-	}
-
-	// TODO: BUGBUG Remove this. @jhowardmsft
-	//       It should be the responsibility of the caller to do the creation and population.
-	//       - Update runhcs too (vm.go).
-	//       - Remove comment in function header
-	//       - Update tests that rely on this current behavior.
-	// Create the RW scratch in the top-most layer folder, creating the folder if it doesn't already exist.
-	scratchFolder := opts.LayerFolders[len(opts.LayerFolders)-1]
-
-	// Create the directory if it doesn't exist
-	if _, err := os.Stat(scratchFolder); os.IsNotExist(err) {
-		if err := os.MkdirAll(scratchFolder, 0777); err != nil {
-			return nil, fmt.Errorf("failed to create utility VM scratch folder: %w", err)
-		}
-	}
-
-	doc, err := prepareConfigDoc(ctx, uvm, opts, uvmFolder)
+	doc, err := prepareConfigDoc(ctx, uvm, opts)
 	if err != nil {
 		return nil, fmt.Errorf("error in preparing config doc: %w", err)
 	}
 
-	// Create sandbox.vhdx in the scratch folder based on the template, granting the correct permissions to it
-	scratchPath := filepath.Join(scratchFolder, "sandbox.vhdx")
-	if _, err := os.Stat(scratchPath); os.IsNotExist(err) {
-		if err := wcow.CreateUVMScratch(ctx, uvmFolder, scratchFolder, uvm.id); err != nil {
-			return nil, fmt.Errorf("failed to create scratch: %w", err)
-		}
-	} else {
-		// Sandbox.vhdx exists, just need to grant vm access to it.
-		if err := wclayer.GrantVmAccess(ctx, uvm.id, scratchPath); err != nil {
-			return nil, errors.Wrap(err, "failed to grant vm access to scratch")
-		}
+	if err := wclayer.GrantVmAccess(ctx, uvm.id, opts.BootFiles.ScratchVHDPath); err != nil {
+		return nil, errors.Wrap(err, "failed to grant vm access to scratch")
 	}
 
 	doc.VirtualMachine.Devices.Scsi = map[string]hcsschema.Scsi{}
@@ -327,7 +299,7 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error
 
 	doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{
 
-		Path:  scratchPath,
+		Path:  opts.BootFiles.ScratchVHDPath,
 		Type_: "VirtualDisk",
 	}
 

internal/uvm/types.go

@@ -159,4 +159,6 @@ type WCOWBootFiles struct {
 	// Path of the boot directory relative to the `OSFilesPath`. This boot directory MUST
 	// contain the BCD & bootmgfw.efi files.
 	OSRelativeBootDirPath string
+	// Path for the scratch VHD of thef UVM
+	ScratchVHDPath string
 }