//go:build windows package uvm import ( "context" "encoding/base64" "fmt" "maps" "net" "os" "path/filepath" "runtime" "strings" "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/ot" "github.com/Microsoft/hcsshim/internal/processorinfo" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/security" "github.com/Microsoft/hcsshim/internal/uvm/scsi" "github.com/Microsoft/hcsshim/internal/vm/vmutils" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/securitypolicy" "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" ) // General information about how this works at a high level. // // The purpose is to start an LCOW Utility VM or UVM using the Host Compute Service, an API to create and manipulate running virtual machines // HCS takes json descriptions of the work to be done. // // When a pod (there is a one to one mapping of pod to UVM) is to be created various annotations and defaults are combined into an options object which is // passed to CreateLCOW (see below) where the options are transformed into a json document to be presented to the HCS VM creation code. // // There are two paths in CreateLCOW to creating the json document. The most flexible case is makeLCOWDoc which is used where no specialist hardware security // applies, then there is makeLCOWSecurityDoc which is used in the case of AMD SEV-SNP memory encryption and integrity protection. There is quite // a lot of difference between the two paths, for example the regular path has options about the type of kernel and initrd binary whereas the AMD SEV-SNP // path has only one file but there are many other detail differences, so the code is split for clarity. // // makeLCOW*Doc returns an instance of hcsschema.ComputeSystem. That is then serialised to the json string provided to the flat C api. A similar scheme is used // for later adjustments, for example adding a newtwork adpator. // // Examples of the eventual json are inline as comments by these two functions to show the eventual effect of the code. // // Note that the schema files, ie the Go objects that represent the json, are generated outside of the local build process. type PreferredRootFSType int const ( PreferredRootFSTypeInitRd PreferredRootFSType = iota PreferredRootFSTypeVHD PreferredRootFSTypeNA ) type ConfidentialLCOWOptions struct { *ConfidentialCommonOptions UseGuestStateFile bool // Use a vmgs file that contains a kernel and initrd, required for SNP DmVerityRootFsVhd string // The VHD file (bound to the vmgs file via embedded dmverity hash data file) to load. DmVerityMode bool // override to be able to turn off dmverity for debugging DmVerityCreateArgs string // set dm-verity args when booting with verity in non-SNP mode } // OptionsLCOW are the set of options passed to CreateLCOW() to create a utility vm. type OptionsLCOW struct { *Options *ConfidentialLCOWOptions // Folder in which kernel and root file system reside. Defaults to \Program Files\Linux Containers. // // It is preferred to use [UpdateBootFilesPath] to change this value and update associated fields. BootFilesPath string KernelFile string // Filename under `BootFilesPath` for the kernel. Defaults to `kernel` KernelDirect bool // Skip UEFI and boot directly to `kernel` RootFSFile string // Filename under `BootFilesPath` for the UVMs root file system. Defaults to `InitrdFile` KernelBootOptions string // Additional boot options for the kernel UseGuestConnection bool // Whether the HCS should connect to the UVM's GCS. Defaults to true ExecCommandLine string // The command line to exec from init. Defaults to GCS ForwardStdout bool // Whether stdout will be forwarded from the executed program. Defaults to false ForwardStderr bool // Whether stderr will be forwarded from the executed program. Defaults to true OutputHandlerCreator vmutils.OutputHandlerCreator `json:"-"` // Creates an [OutputHandler] that controls how output received over HVSocket from the UVM is handled. Defaults to parsing output as logrus messages VPMemDeviceCount uint32 // Number of VPMem devices. Defaults to `DefaultVPMEMCount`. Limit at 128. If booting UVM from VHD, device 0 is taken. VPMemSizeBytes uint64 // Size of the VPMem devices. Defaults to `DefaultVPMemSizeBytes`. VPMemNoMultiMapping bool // Disables LCOW layer multi mapping PreferredRootFSType PreferredRootFSType // If `KernelFile` is `InitrdFile` use `PreferredRootFSTypeInitRd`. If `KernelFile` is `VhdFile` use `PreferredRootFSTypeVHD` EnableColdDiscardHint bool // Whether the HCS should use cold discard hints. Defaults to false VPCIEnabled bool // Whether the kernel should enable pci EnableScratchEncryption bool // Whether the scratch should be encrypted DisableTimeSyncService bool // Disables the time synchronization service HclEnabled *bool // Whether to enable the host compatibility layer ExtraVSockPorts []uint32 // Extra vsock ports to allow AssignedDevices []VPCIDeviceID // AssignedDevices are devices to add on pod boot PolicyBasedRouting bool // Whether we should use policy based routing when configuring net interfaces in guest WritableOverlayDirs bool // Whether init should create writable overlay mounts for /var and /etc } // NewDefaultOptionsLCOW creates the default options for a bootable version of // LCOW. // // `id` the ID of the compute system. If not passed will generate a new GUID. // // `owner` the owner of the compute system. If not passed will use the // executable files name. func NewDefaultOptionsLCOW(id, owner string) *OptionsLCOW { // Use KernelDirect boot by default on all builds that support it. kernelDirectSupported := osversion.Build() >= 18286 var vPmemCount uint32 = vmutils.DefaultVPMEMCount if runtime.GOARCH == "arm64" { // Todo: Add a conditional check for osversion once KernelDirect // becomes available on ARM64, and enable it for supported versions. // This is used by create-scratch and cannot be overriden by annotations kernelDirectSupported = false vPmemCount = 0 } opts := &OptionsLCOW{ Options: newDefaultOptions(id, owner), KernelFile: vmutils.KernelFile, KernelDirect: kernelDirectSupported, RootFSFile: vmutils.InitrdFile, KernelBootOptions: "", UseGuestConnection: true, ExecCommandLine: fmt.Sprintf("/bin/gcs -v4 -log-format json -loglevel %s", logrus.StandardLogger().Level.String()), ForwardStdout: false, ForwardStderr: true, OutputHandlerCreator: vmutils.ParseGCSLogrus, VPMemDeviceCount: vPmemCount, VPMemSizeBytes: vmutils.DefaultVPMemSizeBytes, VPMemNoMultiMapping: osversion.Get().Build < osversion.V19H1, PreferredRootFSType: PreferredRootFSTypeInitRd, EnableColdDiscardHint: false, VPCIEnabled: false, EnableScratchEncryption: false, DisableTimeSyncService: false, ConfidentialLCOWOptions: &ConfidentialLCOWOptions{ ConfidentialCommonOptions: &ConfidentialCommonOptions{ SecurityPolicyEnabled: false, UVMReferenceInfoFile: vmutils.DefaultUVMReferenceInfoFile, UVMHashEnvelopeReferenceInfoFile: vmutils.DefaultUVMHashEnvelopeReferenceInfoFile, }, }, } opts.UpdateBootFilesPath(context.TODO(), vmutils.DefaultLCOWOSBootFilesPath()) return opts } // UpdateBootFilesPath updates the LCOW BootFilesPath field and associated settings. // Specifically, if [VhdFile] is found in path, RootFS is updated, and, if KernelDirect is set, // KernelFile is also updated if [UncompressedKernelFile] is found in path. // // This is a nop if the current BootFilesPath is equal to path (case-insensitive). func (opts *OptionsLCOW) UpdateBootFilesPath(ctx context.Context, path string) { if p, err := filepath.Abs(path); err == nil { path = p } else { // if its a filesystem issue, we'll error out elsewhere when we try to access the boot files // otherwise, it might be transient, or a Go issue, so log and move on log.G(ctx).WithFields(logrus.Fields{ logfields.Path: p, logrus.ErrorKey: err, }).Warning("could not make boot files path absolute") } if strings.EqualFold(opts.BootFilesPath, path) { // Windows is case-insensitive, so compare paths that way too return } opts.BootFilesPath = path if _, err := os.Stat(filepath.Join(opts.BootFilesPath, vmutils.VhdFile)); err == nil { // We have a rootfs.vhd in the boot files path. Use it over an initrd.img opts.RootFSFile = vmutils.VhdFile opts.PreferredRootFSType = PreferredRootFSTypeVHD log.G(ctx).WithFields(logrus.Fields{ logfields.UVMID: opts.ID, vmutils.VhdFile: filepath.Join(opts.BootFilesPath, vmutils.VhdFile), }).Debug("updated LCOW root filesystem to " + vmutils.VhdFile) } if opts.KernelDirect { // KernelDirect supports uncompressed kernel if the kernel is present. // Default to uncompressed if on box. NOTE: If `kernel` is already // uncompressed and simply named 'kernel' it will still be used // uncompressed automatically. if _, err := os.Stat(filepath.Join(opts.BootFilesPath, vmutils.UncompressedKernelFile)); err == nil { opts.KernelFile = vmutils.UncompressedKernelFile log.G(ctx).WithFields(logrus.Fields{ logfields.UVMID: opts.ID, vmutils.UncompressedKernelFile: filepath.Join(opts.BootFilesPath, vmutils.UncompressedKernelFile), }).Debug("updated LCOW kernel file to " + vmutils.UncompressedKernelFile) } } } // Get an acceptable number of processors given option and actual constraints. func fetchProcessor(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (*hcsschema.VirtualMachineProcessor, error) { processorTopology, err := processorinfo.HostProcessorInfo(ctx) if err != nil { return nil, fmt.Errorf("failed to get host processor information: %w", err) } // To maintain compatibility with Docker we need to automatically downgrade // a user CPU count if the setting is not possible. uvm.processorCount = vmutils.NormalizeProcessorCount(ctx, uvm.id, opts.ProcessorCount, processorTopology) processor := &hcsschema.VirtualMachineProcessor{ Count: uint32(uvm.processorCount), Limit: uint64(opts.ProcessorLimit), Weight: uint64(opts.ProcessorWeight), } // We can set a cpu group for the VM at creation time in recent builds. if opts.CPUGroupID != "" { if osversion.Build() < osversion.V21H1 { return nil, vmutils.ErrCPUGroupCreateNotSupported } processor.CpuGroup = &hcsschema.CpuGroup{Id: opts.CPUGroupID} } return processor, nil } /* Example JSON document produced once the hcsschema.ComputeSytem returned by makeLCOWSecurityDoc is serialised: { "Owner": "containerd-shim-runhcs-v1.exe", "SchemaVersion": { "Major": 2, "Minor": 5 }, "ShouldTerminateOnLastHandleClosed": true, "VirtualMachine": { "Chipset": { "Uefi": { "ApplySecureBootTemplate": "Apply", "SecureBootTemplateId": "1734c6e8-3154-4dda-ba5f-a874cc483422" } }, "ComputeTopology": { "Memory": { "SizeInMB": 1024 }, "Processor": { "Count": 2 } }, "Devices": { "Scsi" : { "0" : {} }, "HvSocket": { "HvSocketConfig": { "DefaultBindSecurityDescriptor": "D:P(A;;FA;;;WD)", "DefaultConnectSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)", "ServiceTable" : { "00000808-facb-11e6-bd58-64006a7986d3" : { "AllowWildcardBinds" : true, "BindSecurityDescriptor": "D:P(A;;FA;;;WD)", "ConnectSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)" }, "0000006d-facb-11e6-bd58-64006a7986d3" : { "AllowWildcardBinds" : true, "BindSecurityDescriptor": "D:P(A;;FA;;;WD)", "ConnectSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)" }, "00000001-facb-11e6-bd58-64006a7986d3" : { "AllowWildcardBinds" : true, "BindSecurityDescriptor": "D:P(A;;FA;;;WD)", "ConnectSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)" }, "40000000-facb-11e6-bd58-64006a7986d3" : { "AllowWildcardBinds" : true, "BindSecurityDescriptor": "D:P(A;;FA;;;WD)", "ConnectSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)" } } } }, "Plan9": {} }, "GuestState": { "GuestStateFilePath": "d:\\ken\\aug27\\gcsinitnew.vmgs", "GuestStateFileType": "FileMode", "ForceTransientState": true }, "SecuritySettings": { "Isolation": { "IsolationType": "SecureNestedPaging", "LaunchData": "kBifgKNijdHjxdSUshmavrNofo2B01LiIi1cr8R4ytI=", "HclEnabled": false } }, "Version": { "Major": 254, "Minor": 0 } } } */ // A large part of difference between the SNP case and the usual kernel+option+initrd case is to do with booting // from a VMGS file. The VMGS part may be used other than with SNP so is split out here. // Make a hcsschema.ComputeSytem with the parts that target booting from a VMGS file. func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcsschema.ComputeSystem, err error) { // Raise an error if instructed to use a particular sort of rootfs. if opts.PreferredRootFSType != PreferredRootFSTypeNA { return nil, errors.New("specifying a PreferredRootFSType is incompatible with SNP mode") } // The kernel and minimal initrd are combined into a single vmgs file. vmgsTemplatePath := filepath.Join(opts.BootFilesPath, opts.GuestStateFilePath) if _, err := os.Stat(vmgsTemplatePath); os.IsNotExist(err) { return nil, fmt.Errorf("the GuestState vmgs file '%s' was not found", vmgsTemplatePath) } // The root file system comes from the dmverity vhd file which is mounted by the initrd in the vmgs file. dmVerityRootfsTemplatePath := filepath.Join(opts.BootFilesPath, opts.DmVerityRootFsVhd) if _, err := os.Stat(dmVerityRootfsTemplatePath); os.IsNotExist(err) { return nil, fmt.Errorf("the DM Verity VHD file '%s' was not found", dmVerityRootfsTemplatePath) } var processor *hcsschema.VirtualMachineProcessor processor, err = fetchProcessor(ctx, opts, uvm) if err != nil { return nil, err } vmgsFileFullPath := filepath.Join(opts.BundleDirectory, opts.GuestStateFilePath) if err := copyfile.CopyFile(ctx, vmgsTemplatePath, vmgsFileFullPath, true); err != nil { return nil, fmt.Errorf("failed to copy VMGS template file: %w", err) } defer func() { if err != nil { os.Remove(vmgsFileFullPath) } }() dmVerityRootFsFullPath := filepath.Join(opts.BundleDirectory, vmutils.DefaultDmVerityRootfsVhd) if err := copyfile.CopyFile(ctx, dmVerityRootfsTemplatePath, dmVerityRootFsFullPath, true); err != nil { return nil, fmt.Errorf("failed to copy DM Verity rootfs template file: %w", err) } defer func() { if err != nil { os.Remove(dmVerityRootFsFullPath) } }() for _, filename := range []string{ vmgsFileFullPath, dmVerityRootFsFullPath, } { if err := security.GrantVmGroupAccessWithMask(filename, security.AccessMaskAll); err != nil { return nil, fmt.Errorf("failed to grant VM group access ALL: %w", err) } } // Align the requested memory size. memorySizeInMB := vmutils.NormalizeMemorySize(ctx, uvm.id, opts.MemorySizeInMB) doc := &hcsschema.ComputeSystem{ Owner: uvm.owner, SchemaVersion: schemaversion.SchemaV25(), ShouldTerminateOnLastHandleClosed: true, VirtualMachine: &hcsschema.VirtualMachine{ StopOnReset: true, Chipset: &hcsschema.Chipset{}, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.VirtualMachineMemory{ SizeInMB: memorySizeInMB, AllowOvercommit: opts.AllowOvercommit, EnableDeferredCommit: opts.EnableDeferredCommit, EnableColdDiscardHint: opts.EnableColdDiscardHint, LowMMIOGapInMB: opts.LowMMIOGapInMB, HighMMIOBaseInMB: opts.HighMMIOBaseInMB, HighMMIOGapInMB: opts.HighMMIOGapInMB, }, Processor: processor, }, Devices: &hcsschema.Devices{ HvSocket: &hcsschema.HvSocket2{ HvSocketConfig: &hcsschema.HvSocketSystemConfig{ // Allow administrators and SYSTEM to bind to vsock sockets // so that we can create a GCS log socket. DefaultBindSecurityDescriptor: "D:P(A;;FA;;;WD)", // Differs for SNP DefaultConnectSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)", ServiceTable: make(map[string]hcsschema.HvSocketServiceConfig), }, }, Plan9: &hcsschema.Plan9{}, }, }, } maps.Copy(doc.VirtualMachine.Devices.HvSocket.HvSocketConfig.ServiceTable, opts.AdditionalHyperVConfig) // Set permissions for the VSock ports: // entropyVsockPort - 1 is the entropy port, // linuxLogVsockPort - 109 used by vsockexec to log stdout/stderr logging, // 0x40000000 + 1 (LinuxGcsVsockPort + 1) is the bridge (see guestconnectiuon.go) hvSockets := []uint32{vmutils.LinuxEntropyVsockPort, vmutils.LinuxLogVsockPort, prot.LinuxGcsVsockPort, prot.LinuxGcsVsockPort + 1} hvSockets = append(hvSockets, opts.ExtraVSockPorts...) for _, whichSocket := range hvSockets { key := winio.VsockServiceID(whichSocket).String() doc.VirtualMachine.Devices.HvSocket.HvSocketConfig.ServiceTable[key] = hcsschema.HvSocketServiceConfig{ AllowWildcardBinds: true, BindSecurityDescriptor: "D:P(A;;FA;;;WD)", ConnectSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)", } } // Handle StorageQoS if set if opts.StorageQoSBandwidthMaximum > 0 || opts.StorageQoSIopsMaximum > 0 { doc.VirtualMachine.StorageQoS = &hcsschema.StorageQoS{ IopsMaximum: opts.StorageQoSIopsMaximum, BandwidthMaximum: opts.StorageQoSBandwidthMaximum, } } if uvm.scsiControllerCount > 0 { logrus.Debug("makeLCOWVMGSDoc configuring scsi devices") doc.VirtualMachine.Devices.Scsi = map[string]hcsschema.Scsi{} for i := 0; i < int(uvm.scsiControllerCount); i++ { doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[i]] = hcsschema.Scsi{ Attachments: make(map[string]hcsschema.Attachment), } } if opts.DmVerityMode { logrus.Debug("makeLCOWVMGSDoc DmVerityMode true") scsiController0 := guestrequest.ScsiControllerGuids[0] doc.VirtualMachine.Devices.Scsi[scsiController0].Attachments["0"] = hcsschema.Attachment{ Type_: "VirtualDisk", Path: dmVerityRootFsFullPath, ReadOnly: true, } uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 0}) } } // Required by HCS for the isolated boot scheme, see also https://docs.microsoft.com/en-us/windows-server/virtualization/hyper-v/learn-more/generation-2-virtual-machine-security-settings-for-hyper-v // A complete explanation of the why's and wherefores of starting an encrypted, isolated VM are beond the scope of these comments. doc.VirtualMachine.Chipset.Uefi = &hcsschema.Uefi{ ApplySecureBootTemplate: "Apply", SecureBootTemplateId: "1734c6e8-3154-4dda-ba5f-a874cc483422", // aka MicrosoftWindowsSecureBootTemplateGUID equivalent to "Microsoft Windows" template from Get-VMHost | select SecureBootTemplates, } // Point at the file that contains the linux kernel and initrd images. doc.VirtualMachine.GuestState = &hcsschema.GuestState{ GuestStateFilePath: vmgsFileFullPath, GuestStateFileType: "FileMode", ForceTransientState: true, // tell HCS that this is just the source of the images, not ongoing state } return doc, nil } // Programatically make the hcsschema.ComputeSystem document for the SNP case. // This is done prior to json seriaisation and sending to the HCS layer to actually do the work of creating the VM. // Many details are quite different (see the typical JSON examples), in particular it boots from a VMGS file // which contains both the kernel and initrd as well as kernel boot options. func makeLCOWSecurityDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcsschema.ComputeSystem, err error) { doc, vmgsErr := makeLCOWVMGSDoc(ctx, opts, uvm) if vmgsErr != nil { return nil, vmgsErr } // Part of the protocol to ensure that the rules in the user's Security Policy are // respected is to provide a hash of the policy to the hardware. This is immutable // and can be used to check that the policy used by opengcs is the required one as // a condition of releasing secrets to the container. policyDigest, err := securitypolicy.NewSecurityPolicyDigest(opts.SecurityPolicy) if err != nil { return nil, err } // HCS API expect a base64 encoded string as LaunchData. Internally it // decodes it to bytes. SEV later returns the decoded byte blob as HostData // field of the report. hostData := base64.StdEncoding.EncodeToString(policyDigest) // Put the measurement into the LaunchData field of the HCS creation command. // This will end-up in HOST_DATA of SNP_LAUNCH_FINISH command the and ATTESTATION_REPORT // retrieved by the guest later. doc.VirtualMachine.SecuritySettings = &hcsschema.SecuritySettings{ EnableTpm: false, Isolation: &hcsschema.IsolationSettings{ IsolationType: "SecureNestedPaging", LaunchData: hostData, // HclEnabled: true, /* Not available in schema 2.5 - REQUIRED when using BlockStorage in 2.6 */ HclEnabled: opts.HclEnabled, }, } return doc, nil } /* Example JSON document produced once the hcsschema.ComputeSytem returned by makeLCOWDoc is serialised. Note that the boot scheme is entirely different. { "Owner": "containerd-shim-runhcs-v1.exe", "SchemaVersion": { "Major": 2, "Minor": 1 }, "VirtualMachine": { "StopOnReset": true, "Chipset": { "LinuxKernelDirect": { "KernelFilePath": "C:\\ContainerPlat\\LinuxBootFiles\\vmlinux", "InitRdPath": "C:\\ContainerPlat\\LinuxBootFiles\\initrd.img", "KernelCmdLine": " 8250_core.nr_uarts=0 panic=-1 quiet pci=off nr_cpus=2 brd.rd_nr=0 pmtmr=0 -- -e 1 /bin/vsockexec -e 109 /bin/gcs -v4 -log-format json -loglevel debug" } }, "ComputeTopology": { "Memory": { "SizeInMB": 1024, "AllowOvercommit": true }, "Processor": { "Count": 2 } }, "Devices": { "Scsi": { "0": {} }, "HvSocket": { "HvSocketConfig": { "DefaultBindSecurityDescriptor": "D:P(A;;FA;;;SY)(A;;FA;;;BA)" } }, "Plan9": {} } }, "ShouldTerminateOnLastHandleClosed": true } */ // Make the ComputeSystem document object that will be serialized to json to be presented to the HCS api. func MakeLCOWDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ *hcsschema.ComputeSystem, err error) { if logrus.IsLevelEnabled(logrus.TraceLevel) { log.G(ctx).WithField("options", log.Format(ctx, opts)).Trace("makeLCOWDoc") } kernelFullPath := filepath.Join(opts.BootFilesPath, opts.KernelFile) if _, err := os.Stat(kernelFullPath); os.IsNotExist(err) { return nil, fmt.Errorf("kernel: '%s' not found", kernelFullPath) } rootfsFullPath := filepath.Join(opts.BootFilesPath, opts.RootFSFile) if _, err := os.Stat(rootfsFullPath); os.IsNotExist(err) { return nil, fmt.Errorf("boot file: '%s' not found", rootfsFullPath) } var processor *hcsschema.VirtualMachineProcessor processor, err = fetchProcessor(ctx, opts, uvm) // must happen after the file existence tests above. if err != nil { return nil, err } numa, numaProcessors, err := vmutils.PrepareVNumaTopology(ctx, &vmutils.NumaConfig{ MaxProcessorsPerNumaNode: opts.MaxProcessorsPerNumaNode, MaxMemorySizePerNumaNode: opts.MaxMemorySizePerNumaNode, PreferredPhysicalNumaNodes: opts.PreferredPhysicalNumaNodes, NumaMappedPhysicalNodes: opts.NumaMappedPhysicalNodes, NumaProcessorCounts: opts.NumaProcessorCounts, NumaMemoryBlocksCounts: opts.NumaMemoryBlocksCounts, }) if err != nil { return nil, err } // Align the requested memory size. memorySizeInMB := vmutils.NormalizeMemorySize(ctx, uvm.id, opts.MemorySizeInMB) if numa != nil { if opts.AllowOvercommit { return nil, fmt.Errorf("vNUMA supports only Physical memory backing type") } if err := vmutils.ValidateNumaForVM(numa, processor.Count, memorySizeInMB); err != nil { return nil, fmt.Errorf("failed to validate vNUMA settings: %w", err) } } if numaProcessors != nil { processor.NumaProcessorsSettings = numaProcessors } doc := &hcsschema.ComputeSystem{ Owner: uvm.owner, SchemaVersion: schemaversion.SchemaV21(), ShouldTerminateOnLastHandleClosed: true, VirtualMachine: &hcsschema.VirtualMachine{ StopOnReset: true, Chipset: &hcsschema.Chipset{}, ComputeTopology: &hcsschema.Topology{ Memory: &hcsschema.VirtualMachineMemory{ SizeInMB: memorySizeInMB, AllowOvercommit: opts.AllowOvercommit, EnableDeferredCommit: opts.EnableDeferredCommit, EnableColdDiscardHint: opts.EnableColdDiscardHint, LowMMIOGapInMB: opts.LowMMIOGapInMB, HighMMIOBaseInMB: opts.HighMMIOBaseInMB, HighMMIOGapInMB: opts.HighMMIOGapInMB, }, Processor: processor, Numa: numa, }, Devices: &hcsschema.Devices{ HvSocket: &hcsschema.HvSocket2{ HvSocketConfig: &hcsschema.HvSocketSystemConfig{ // Allow administrators and SYSTEM to bind to vsock sockets // so that we can create a GCS log socket. DefaultBindSecurityDescriptor: "D:P(A;;FA;;;SY)(A;;FA;;;BA)", ServiceTable: make(map[string]hcsschema.HvSocketServiceConfig), }, }, Plan9: &hcsschema.Plan9{}, }, }, } // Expose ACPI information into UVM if numa != nil || numaProcessors != nil { firmwareFallbackMeasured := hcsschema.VirtualSlitType_FIRMWARE_FALLBACK_MEASURED doc.VirtualMachine.ComputeTopology.Memory.SlitType = &firmwareFallbackMeasured } if opts.ResourcePartitionID != nil { log.G(ctx).WithField("resource-partition-id", opts.ResourcePartitionID.String()).Debug("setting resource partition ID") doc.VirtualMachine.ResourcePartitionId = opts.ResourcePartitionID.String() } // Add optional devices that were specified on the UVM spec if len(opts.AssignedDevices) > 0 { if doc.VirtualMachine.Devices.VirtualPci == nil { doc.VirtualMachine.Devices.VirtualPci = make(map[string]hcsschema.VirtualPciDevice) } for _, d := range opts.AssignedDevices { // we don't need to hold the modify lock here because the UVM has // not yet been created. existingDevice := uvm.vpciDevices[d] if existingDevice != nil { return nil, fmt.Errorf("device %s with index %d is specified multiple times", d.deviceInstanceID, d.virtualFunctionIndex) } vmbusGUID, err := guid.NewV4() if err != nil { return nil, err } var propagateAffinity *bool T := true if osversion.Get().Build >= osversion.V25H1Server && (numa != nil || numaProcessors != nil) { propagateAffinity = &T } doc.VirtualMachine.Devices.VirtualPci[vmbusGUID.String()] = hcsschema.VirtualPciDevice{ Functions: []hcsschema.VirtualPciFunction{ { DeviceInstancePath: d.deviceInstanceID, VirtualFunction: d.virtualFunctionIndex, }, }, PropagateNumaAffinity: propagateAffinity, } device := &VPCIDevice{ vm: uvm, VMBusGUID: vmbusGUID.String(), deviceInstanceID: d.deviceInstanceID, virtualFunctionIndex: d.virtualFunctionIndex, refCount: 1, } uvm.vpciDevices[d] = device } } maps.Copy(doc.VirtualMachine.Devices.HvSocket.HvSocketConfig.ServiceTable, opts.AdditionalHyperVConfig) // Handle StorageQoS if set if opts.StorageQoSBandwidthMaximum > 0 || opts.StorageQoSIopsMaximum > 0 { doc.VirtualMachine.StorageQoS = &hcsschema.StorageQoS{ IopsMaximum: opts.StorageQoSIopsMaximum, BandwidthMaximum: opts.StorageQoSBandwidthMaximum, } } if uvm.scsiControllerCount > 0 { doc.VirtualMachine.Devices.Scsi = map[string]hcsschema.Scsi{} for i := 0; i < int(uvm.scsiControllerCount); i++ { doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[i]] = hcsschema.Scsi{ Attachments: make(map[string]hcsschema.Attachment), } } } if uvm.vpmemMaxCount > 0 { doc.VirtualMachine.Devices.VirtualPMem = &hcsschema.VirtualPMemController{ MaximumCount: uvm.vpmemMaxCount, MaximumSizeBytes: uvm.vpmemMaxSizeBytes, } } var kernelArgs string switch opts.PreferredRootFSType { case PreferredRootFSTypeInitRd: if !opts.KernelDirect { kernelArgs = "initrd=/" + opts.RootFSFile } case PreferredRootFSTypeVHD: if uvm.vpmemMaxCount > 0 { // Support for VPMem VHD(X) booting rather than initrd.. kernelArgs = "root=/dev/pmem0 ro rootwait init=/init" imageFormat := "Vhd1" if strings.ToLower(filepath.Ext(opts.RootFSFile)) == "vhdx" { imageFormat = "Vhdx" } doc.VirtualMachine.Devices.VirtualPMem.Devices = map[string]hcsschema.VirtualPMemDevice{ "0": { HostPath: rootfsFullPath, ReadOnly: true, ImageFormat: imageFormat, }, } if uvm.vpmemMultiMapping { pmem := newPackedVPMemDevice() pmem.maxMappedDeviceCount = 1 st, stErr := os.Stat(rootfsFullPath) if stErr != nil { return nil, errors.Wrapf(stErr, "failed to stat rootfs: %q", rootfsFullPath) } devSize := pageAlign(uint64(st.Size())) memReg, pErr := pmem.Allocate(devSize) if pErr != nil { return nil, errors.Wrap(pErr, "failed to allocate memory for rootfs") } defer func() { if err != nil { if err = pmem.Release(memReg); err != nil { log.G(ctx).WithError(err).Debug("failed to release memory region") } } }() dev := newVPMemMappedDevice(opts.RootFSFile, "/", devSize, memReg) if err := pmem.mapVHDLayer(ctx, dev); err != nil { return nil, errors.Wrapf(err, "failed to save internal state for a multi-mapped rootfs device") } uvm.vpmemDevicesMultiMapped[0] = pmem } else { dev := newDefaultVPMemInfo(opts.RootFSFile, "/") uvm.vpmemDevicesDefault[0] = dev } } else { kernelArgs = "root=/dev/sda ro rootwait init=/init" if opts.DmVerityMode { if len(opts.DmVerityCreateArgs) == 0 { return nil, errors.New("DmVerityCreateArgs must be set when DmVerityMode is true and not booting from a vmgs file.") } kernelArgs = fmt.Sprintf("root=/dev/dm-0 dm-mod.create=%q init=/init", opts.DmVerityCreateArgs) } doc.VirtualMachine.Devices.Scsi[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{ Type_: "VirtualDisk", Path: rootfsFullPath, ReadOnly: true, } uvm.reservedSCSISlots = append(uvm.reservedSCSISlots, scsi.Slot{Controller: 0, LUN: 0}) } } // Explicitly disable virtio_vsock_init, to make sure that we use hv_sock transport. For kernels built without // virtio-vsock this is a no-op. kernelArgs += " initcall_blacklist=virtio_vsock_init" vmDebugging := false if opts.ConsolePipe != "" { vmDebugging = true if runtime.GOARCH == "arm64" { kernelArgs += " console=ttyAMA0,115200" } else { kernelArgs += " 8250_core.nr_uarts=1 8250_core.skip_txen_test=1 console=ttyS0,115200" } doc.VirtualMachine.Devices.ComPorts = map[string]hcsschema.ComPort{ "0": { // Which is actually COM1 NamedPipe: opts.ConsolePipe, }, } } else { if runtime.GOARCH != "arm64" { kernelArgs += " 8250_core.nr_uarts=0" } } if opts.EnableGraphicsConsole { vmDebugging = true kernelArgs += " console=tty" doc.VirtualMachine.Devices.Keyboard = &hcsschema.Keyboard{} doc.VirtualMachine.Devices.EnhancedModeVideo = &hcsschema.EnhancedModeVideo{} doc.VirtualMachine.Devices.VideoMonitor = &hcsschema.VideoMonitor{} } if !vmDebugging { // Terminate the VM if there is a kernel panic. kernelArgs += " panic=-1 quiet" } // Add Kernel Boot options if opts.KernelBootOptions != "" { kernelArgs += " " + opts.KernelBootOptions } if !opts.VPCIEnabled { kernelArgs += ` pci=off` } // Inject initial entropy over vsock during init launch. entropyArgs := fmt.Sprintf("-e %d", vmutils.LinuxEntropyVsockPort) // With default options, run GCS with stderr pointing to the vsock port // created below in order to forward guest logs to logrus. execCmdArgs := "/bin/vsockexec" if opts.ForwardStdout { execCmdArgs += fmt.Sprintf(" -o %d", vmutils.LinuxLogVsockPort) } if opts.ForwardStderr { execCmdArgs += fmt.Sprintf(" -e %d", vmutils.LinuxLogVsockPort) } if opts.DisableTimeSyncService { opts.ExecCommandLine = fmt.Sprintf("%s -disable-time-sync", opts.ExecCommandLine) } // Scrubbing is enabled by default in GCS. Only pass the flag when scrubbing // has been explicitly disabled on the host to inform GCS to turn it off. if !log.IsScrubbingEnabled() { opts.ExecCommandLine += " -scrub-logs=false" } execCmdArgs += " " + opts.ExecCommandLine if opts.ProcessDumpLocation != "" { execCmdArgs += " -core-dump-location " + opts.ProcessDumpLocation } initArgs := entropyArgs if opts.WritableOverlayDirs { switch opts.PreferredRootFSType { case PreferredRootFSTypeInitRd: log.G(ctx).Warn("ignoring `WritableOverlayDirs` option since rootfs is already writable") case PreferredRootFSTypeVHD: initArgs += " -w" } } if vmDebugging { // Launch a shell on the console. initArgs += ` sh -c "` + execCmdArgs + ` & exec sh"` } else { initArgs += " " + execCmdArgs } kernelArgs += fmt.Sprintf(" nr_cpus=%d", opts.ProcessorCount) kernelArgs += ` brd.rd_nr=0 pmtmr=0 -- ` + initArgs if !opts.KernelDirect { doc.VirtualMachine.Chipset.Uefi = &hcsschema.Uefi{ BootThis: &hcsschema.UefiBootEntry{ DevicePath: `\` + opts.KernelFile, DeviceType: "VmbFs", VmbFsRootPath: opts.BootFilesPath, OptionalData: kernelArgs, }, } } else { doc.VirtualMachine.Chipset.LinuxKernelDirect = &hcsschema.LinuxKernelDirect{ KernelFilePath: kernelFullPath, KernelCmdLine: kernelArgs, } if opts.PreferredRootFSType == PreferredRootFSTypeInitRd { doc.VirtualMachine.Chipset.LinuxKernelDirect.InitRdPath = rootfsFullPath } } return doc, nil } // CreateLCOW creates an HCS compute system representing a utility VM. It // consumes a set of options derived from various defaults and options // expressed as annotations. func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error) { ctx, span := ot.StartSpan(ctx, "uvm::CreateLCOW") defer span.End() defer func() { ot.SetSpanStatus(span, err) }() if opts.ID == "" { g, err := guid.NewV4() if err != nil { return nil, err } opts.ID = g.String() } span.SetAttributes(attribute.String(logfields.UVMID, opts.ID)) if logrus.IsLevelEnabled(logrus.DebugLevel) { log.G(ctx).WithField("options", log.Format(ctx, opts)).Debug("uvm::CreateLCOW options") } // We don't serialize OutputHandlerCreator so if it is missing we need to put it back to the default. if opts.OutputHandlerCreator == nil { opts.OutputHandlerCreator = vmutils.ParseGCSLogrus } uvm := &UtilityVM{ id: opts.ID, owner: opts.Owner, operatingSystem: "linux", scsiControllerCount: opts.SCSIControllerCount, vpmemMaxCount: opts.VPMemDeviceCount, vpmemMaxSizeBytes: opts.VPMemSizeBytes, vpciDevices: make(map[VPCIDeviceID]*VPCIDevice), physicallyBacked: !opts.AllowOvercommit, devicesPhysicallyBacked: opts.FullyPhysicallyBacked, createOpts: opts, vpmemMultiMapping: !opts.VPMemNoMultiMapping, encryptScratch: opts.EnableScratchEncryption, noWritableFileShares: opts.NoWritableFileShares, policyBasedRouting: opts.PolicyBasedRouting, } defer func() { if err != nil { uvm.Close() } }() // vpmemMaxCount has been set to 0 which means we are going to need multiple SCSI controllers // to support lots of layers. if osversion.Build() >= osversion.RS5 && uvm.vpmemMaxCount == 0 { uvm.scsiControllerCount = 4 } if err = VerifyOptions(ctx, opts); err != nil { return nil, errors.Wrap(err, errBadUVMOpts.Error()) } // HCS config for SNP isolated vm is quite different to the usual case var doc *hcsschema.ComputeSystem if opts.GuestStateFilePath != "" { doc, err = makeLCOWSecurityDoc(ctx, opts, uvm) if logrus.IsLevelEnabled(logrus.TraceLevel) { log.G(ctx).WithFields(logrus.Fields{ "doc": log.Format(ctx, doc), logrus.ErrorKey: err, }).Trace("create_lcow::CreateLCOW makeLCOWSecurityDoc result") } } else { doc, err = MakeLCOWDoc(ctx, opts, uvm) if logrus.IsLevelEnabled(logrus.TraceLevel) { log.G(ctx).WithFields(logrus.Fields{ "doc": log.Format(ctx, doc), logrus.ErrorKey: err, }).Trace("create_lcow::CreateLCOW makeLCOWDoc result") } } if err != nil { return nil, err } if err = uvm.create(ctx, doc); err != nil { return nil, fmt.Errorf("error while creating the compute system: %w", err) } if logrus.IsLevelEnabled(logrus.TraceLevel) { log.G(ctx).WithField("uvm", log.Format(ctx, uvm)).Trace("create_lcow::CreateLCOW uvm.create result") } // Create a socket to inject entropy during boot. uvm.entropyListener, err = uvm.listenVsock(vmutils.LinuxEntropyVsockPort) if err != nil { return nil, err } // Create a socket that the executed program can send to. This is usually // used by GCS to send log data. if opts.ForwardStdout || opts.ForwardStderr { uvm.outputHandler = opts.OutputHandlerCreator(opts.ID) uvm.outputProcessingDone = make(chan struct{}) uvm.outputListener, err = uvm.listenVsock(vmutils.LinuxLogVsockPort) if err != nil { return nil, err } } if opts.UseGuestConnection { log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge") l, err := uvm.listenVsock(prot.LinuxGcsVsockPort) if err != nil { return nil, err } uvm.gcListener = l } uvm.ncProxyClientAddress = opts.NetworkConfigProxy return uvm, nil } func (uvm *UtilityVM) listenVsock(port uint32) (net.Listener, error) { return winio.ListenHvsock(&winio.HvsockAddr{ VMID: uvm.runtimeID, ServiceID: winio.VsockServiceID(port), }) }