//go:build linux // +build linux package hcsv2 import ( "bufio" "bytes" "context" "encoding/json" "fmt" "io" "os" "os/exec" "path" "path/filepath" "regexp" "slices" "strings" "sync" "syscall" "time" cgroup1stats "github.com/containerd/cgroups/v3/cgroup1/stats" "github.com/mattn/go-shellwords" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "golang.org/x/sys/unix" "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/debug" "github.com/Microsoft/hcsshim/internal/guest/cgroup" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime" specGuest "github.com/Microsoft/hcsshim/internal/guest/spec" "github.com/Microsoft/hcsshim/internal/guest/stdio" "github.com/Microsoft/hcsshim/internal/guest/storage" "github.com/Microsoft/hcsshim/internal/guest/storage/overlay" "github.com/Microsoft/hcsshim/internal/guest/storage/pci" "github.com/Microsoft/hcsshim/internal/guest/storage/plan9" "github.com/Microsoft/hcsshim/internal/guest/storage/pmem" "github.com/Microsoft/hcsshim/internal/guest/storage/scsi" "github.com/Microsoft/hcsshim/internal/guest/transport" "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/ot" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/verity" "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/pkg/securitypolicy" ) // UVMContainerID is the ContainerID that will be sent on any prot.MessageBase // for V2 where the specific message is targeted at the UVM itself. const UVMContainerID = "00000000-0000-0000-0000-000000000000" // Prevent path traversal via malformed container / sandbox IDs. Container IDs // can be either UVMContainerID, or a 64 character hex string. This is also used // to check that sandbox IDs (which is also used in paths) are valid, which has // the same format. const validContainerIDRegexRaw = `[0-9a-fA-F]{64}` var validContainerIDRegex = regexp.MustCompile("^" + validContainerIDRegexRaw + "$") // idType just changes the error message func checkValidContainerID(id string, idType string) error { if id == UVMContainerID || validContainerIDRegex.MatchString(id) { return nil } return errors.Errorf("invalid %s id: %s (must match %s)", idType, id, validContainerIDRegex.String()) } // Cgroup path formats for pod and container cgroups within the UVM. const ( // podCgroupPathFmt is the cgroup path for a pod (sandbox or standalone): /pods/{sandboxID}. podCgroupPathFmt = "/pods/%s" // containerCgroupPathFmt is the cgroup path for a workload container nested under its pod: /pods/{sandboxID}/{containerID}. containerCgroupPathFmt = "/pods/%s/%s" ) // pod tracks pod-level state within the UVM. type pod struct { sandboxID string cgroupControl cgroup.Manager containers map[string]bool } // Host is the structure tracking all UVM host state including all containers // and processes. type Host struct { // containersMutex guards both containers and pods maps. containersMutex sync.Mutex containers map[string]*Container pods map[string]*pod cgroupMemoryMutex sync.Mutex podsCgroup cgroupUpdater rootMemReserveBytes uint64 lastAppliedCgroupMemLimit int64 externalProcessesMutex sync.Mutex externalProcesses map[int]*externalProcess // sandboxRoots maps sandboxID to the resolved sandbox root directory. // Populated via registerSandboxRoot during sandbox creation using // the host-provided OCIBundlePath as source of truth. // Lock ordering: containersMutex -> sandboxRootsMutex (never reverse). sandboxRootsMutex sync.RWMutex sandboxRoots map[string]string rtime runtime.Runtime vsock transport.Transport devNullTransport transport.Transport // state required for the security policy enforcement securityOptions *securitypolicy.SecurityOptions // hostMounts keeps the state of currently mounted devices and file systems, // which is used for GCS hardening. It is only used for confidential // containers, and is initialized in SetConfidentialUVMOptions. If this is // nil, we do not do add any special restrictions on mounts / unmounts. hostMounts *hostMounts // uvmError contains a permanent flag to indicate that further mounts, // unmounts and container creation / deletion should not be allowed. This // is set when, because of a failure during an unmount operation, we end up // in a state where the policy enforcer's state is out of sync with what we // have actually done, but we cannot safely revert its state. // // Only consulted in confidential mode (see Host.checkState). uvmError uvmConsistencyError } type cgroupUpdater interface { Update(resources *specs.LinuxResources) error } func calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) { if totalMemoryBytes <= rootMemReserveBytes { return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes) } limit := totalMemoryBytes - rootMemReserveBytes const maxInt64 = uint64(1<<63 - 1) if limit > maxInt64 { return 0, errors.Errorf("pods cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64) } return int64(limit), nil } func currentTotalMemoryBytes() (uint64, error) { sinfo := syscall.Sysinfo_t{} if err := syscall.Sysinfo(&sinfo); err != nil { return 0, errors.Wrap(err, "failed to get sys info") } totalMemoryBytes := uint64(sinfo.Totalram) unit := uint64(sinfo.Unit) if unit != 0 { const maxUint64 = ^uint64(0) if totalMemoryBytes > maxUint64/unit { return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit) } totalMemoryBytes *= unit } return totalMemoryBytes, nil } func setCgroupMemoryLimit(podsCgroup cgroupUpdater, totalMemoryBytes, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) { limit, err := calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes) if err != nil { return lastAppliedLimit, false, err } if limit == lastAppliedLimit { return limit, false, nil } resources := &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: &limit}} if err := podsCgroup.Update(resources); err != nil { return lastAppliedLimit, false, errors.Wrap(err, "failed to update pods cgroup memory limit") } return limit, true, nil } // InitializeCgroupMemoryLimitUpdater configures the cgroup updated when the UVM memory changes. func (h *Host) InitializeCgroupMemoryLimitUpdater(podsCgroup cgroup.Manager, rootMemReserveBytes uint64, initialLimit int64) { h.cgroupMemoryMutex.Lock() defer h.cgroupMemoryMutex.Unlock() h.podsCgroup = podsCgroup h.rootMemReserveBytes = rootMemReserveBytes h.lastAppliedCgroupMemLimit = initialLimit } // UpdateCgroupMemoryLimits refreshes the pods cgroup limit from the UVM's current memory size. func (h *Host) UpdateCgroupMemoryLimits() (int64, bool, error) { h.cgroupMemoryMutex.Lock() defer h.cgroupMemoryMutex.Unlock() if h.podsCgroup == nil { return h.lastAppliedCgroupMemLimit, false, errors.New("cgroup memory limit updater is not initialized") } totalMemoryBytes, err := currentTotalMemoryBytes() if err != nil { return h.lastAppliedCgroupMemLimit, false, err } limit, updated, err := setCgroupMemoryLimit(h.podsCgroup, totalMemoryBytes, h.rootMemReserveBytes, h.lastAppliedCgroupMemLimit) if err != nil { return h.lastAppliedCgroupMemLimit, false, err } if updated { h.lastAppliedCgroupMemLimit = limit } return limit, updated, nil } type uvmConsistencyError struct { mu sync.Mutex // The error describing why the UVM has entered an inconsistent state. If // this is nil, there is no error and Check() returns nil. cause error } // Mark that the UVM has entered an inconsistent state, and store the cause if // it is not already set. func (u *uvmConsistencyError) Set(cause error) { u.mu.Lock() defer u.mu.Unlock() if u.cause == nil { u.cause = cause } } // Check returns a non-nil error if the UVM has been marked inconsistent. func (u *uvmConsistencyError) Check() error { u.mu.Lock() defer u.mu.Unlock() if u.cause == nil { return nil } return fmt.Errorf( "mount, unmount, container creation and deletion have been disabled in this UVM due to a previous error: %w", u.cause, ) } func NewHost(rtime runtime.Runtime, vsock transport.Transport, initialEnforcer securitypolicy.SecurityPolicyEnforcer, logWriter io.Writer) *Host { securityPolicyOptions := securitypolicy.NewSecurityOptions( initialEnforcer, false, "", "", logWriter, ) return &Host{ containers: make(map[string]*Container), externalProcesses: make(map[int]*externalProcess), pods: make(map[string]*pod), sandboxRoots: make(map[string]string), rtime: rtime, vsock: vsock, devNullTransport: &transport.DevNullTransport{}, hostMounts: nil, securityOptions: securityPolicyOptions, uvmError: uvmConsistencyError{}, } } // registerSandboxRoot stores the resolved sandbox root directory for a given sandbox ID. // For virtual pods, it derives the shared root from OCIBundlePath's parent directory. func (h *Host) registerSandboxRoot(sandboxID, ociBundlePath, virtualPodID string) (string, error) { var sandboxRoot string if virtualPodID != "" { // Validate virtualPodID to prevent path traversal. cleanID := filepath.Clean(virtualPodID) if filepath.IsAbs(cleanID) || strings.Contains(cleanID, "..") { return "", errors.Errorf("invalid virtual pod ID %q: path traversal attempt", virtualPodID) } sandboxRoot = filepath.Join(filepath.Dir(ociBundlePath), "virtual-pods", cleanID) } else { sandboxRoot = ociBundlePath } h.sandboxRootsMutex.Lock() defer h.sandboxRootsMutex.Unlock() h.sandboxRoots[sandboxID] = sandboxRoot logrus.WithFields(logrus.Fields{ "sandboxID": sandboxID, "sandboxRoot": sandboxRoot, }).Debug("registered sandbox root") return sandboxRoot, nil } // resolveSandboxRoot returns the resolved sandbox root for the given sandbox ID. // Falls back to legacy path derivation if no mapping exists. func (h *Host) resolveSandboxRoot(sandboxID string) string { h.sandboxRootsMutex.RLock() root, ok := h.sandboxRoots[sandboxID] h.sandboxRootsMutex.RUnlock() if ok { return root } // Fallback to legacy derivation for backwards compatibility. // TODO: remove fallback after shim v1 sunset fallback := specGuest.SandboxRootDir(sandboxID) logrus.WithFields(logrus.Fields{ "sandboxID": sandboxID, "fallback": fallback, }).Warn("sandbox root not found in mapping, falling back to legacy path derivation") return fallback } // unregisterSandboxRoot removes the sandbox root mapping for a given sandbox ID. func (h *Host) unregisterSandboxRoot(sandboxID string) { h.sandboxRootsMutex.Lock() defer h.sandboxRootsMutex.Unlock() delete(h.sandboxRoots, sandboxID) } func (h *Host) SecurityPolicyEnforcer() securitypolicy.SecurityPolicyEnforcer { return h.securityOptions.PolicyEnforcer } func (h *Host) SecurityOptions() *securitypolicy.SecurityOptions { return h.securityOptions } func (h *Host) Transport() transport.Transport { return h.vsock } func (h *Host) RemoveContainer(id string) { h.containersMutex.Lock() defer h.containersMutex.Unlock() c, ok := h.containers[id] if !ok { return } criType, isCRI := c.spec.Annotations[annotations.KubernetesContainerType] // Do NOT call RemoveNetworkNamespace for virtual pod sandbox containers. // The host-driven teardown path (TearDownNetworking → RemoveNetNS → removeNIC) // removes adapters first and then the namespace. Calling it here would fail // with "contains adapters" because the host hasn't removed them yet. virtualPodID := c.spec.Annotations[annotations.VirtualPodID] isVirtualPodSandbox := virtualPodID != "" && id == virtualPodID if !isVirtualPodSandbox && (!isCRI || criType == "sandbox") { if err := RemoveNetworkNamespace(context.Background(), id); err != nil { logrus.WithError(err).WithField(logfields.ContainerID, id).Warn("failed to remove network namespace") } } delete(h.containers, id) // Clean up pod tracking. For standalone containers, sandboxID == id but // no pod entry exists (createPodInUVM is only called for CRI sandboxes), // so the lookup returns false and the block is skipped. if pod, exists := h.pods[c.sandboxID]; exists { delete(pod.containers, id) // When the sandbox container itself is removed, tear down the pod. if id == c.sandboxID { if pod.cgroupControl != nil { if err := pod.cgroupControl.Delete(); err != nil { logrus.WithFields(logrus.Fields{ "sandboxID": c.sandboxID, }).WithError(err).Warn("failed to delete pod cgroup") } } delete(h.pods, c.sandboxID) } } // Clean up the sandbox root mapping for sandbox containers. if c.isSandbox { h.unregisterSandboxRoot(id) } } func (h *Host) GetInitializedContainer(id string) (*Container, error) { h.containersMutex.Lock() defer h.containersMutex.Unlock() c, ok := h.containers[id] if !ok { return nil, gcserr.NewHresultError(gcserr.HrVmcomputeSystemNotFound) } if c.getStatus() == containerCreating { return nil, fmt.Errorf("container is still being created: %w", gcserr.NewHresultError(gcserr.HrVmcomputeInvalidState)) } return c, nil } func (h *Host) AddContainer(id string, c *Container) error { h.containersMutex.Lock() defer h.containersMutex.Unlock() if _, ok := h.containers[id]; ok { return gcserr.NewHresultError(gcserr.HrVmcomputeSystemAlreadyExists) } h.containers[id] = c return nil } // setupSandboxMountsPath creates the sandboxMounts directory from a resolved root. func setupSandboxMountsPath(sandboxRoot string) (err error) { mountPath := specGuest.SandboxMountsDirFromRoot(sandboxRoot) if err := os.MkdirAll(mountPath, 0755); err != nil { return errors.Wrapf(err, "failed to create sandboxMounts dir at %v", mountPath) } defer func() { if err != nil { _ = os.RemoveAll(mountPath) } }() return storage.MountRShared(mountPath) } // setupSandboxTmpfsMountsPath creates the sandbox tmpfs mounts directory from a resolved root. func setupSandboxTmpfsMountsPath(sandboxRoot string) (err error) { tmpfsDir := specGuest.SandboxTmpfsMountsDirFromRoot(sandboxRoot) if err := os.MkdirAll(tmpfsDir, 0755); err != nil { return errors.Wrapf(err, "failed to create sandbox tmpfs mounts dir at %v", tmpfsDir) } defer func() { if err != nil { _ = os.RemoveAll(tmpfsDir) } }() // mount a tmpfs at the tmpfsDir // this ensures that the tmpfsDir is a mount point and not just a directory // we don't care if it is already mounted, so ignore EBUSY if err := unix.Mount("tmpfs", tmpfsDir, "tmpfs", 0, ""); err != nil && !errors.Is(err, unix.EBUSY) { return errors.Wrapf(err, "failed to mount tmpfs at %s", tmpfsDir) } //TODO: should tmpfs be mounted as noexec? return storage.MountRShared(tmpfsDir) } // setupSandboxHugePageMountsPath creates the hugepages mounts directory from a resolved root. func setupSandboxHugePageMountsPath(sandboxRoot string) error { mountPath := specGuest.SandboxHugePagesMountsDirFromRoot(sandboxRoot) if err := os.MkdirAll(mountPath, 0755); err != nil { return errors.Wrapf(err, "failed to create hugepage mounts dir at %v", mountPath) } return storage.MountRShared(mountPath) } // setupSandboxLogDir creates the directory to house all redirected stdio logs from a resolved root. func setupSandboxLogDir(sandboxRoot string) error { mountPath := specGuest.SandboxLogsDirFromRoot(sandboxRoot) if err := mkdirAllModePerm(mountPath); err != nil { return errors.Wrapf(err, "failed to create sandbox logs dir at %v", mountPath) } return nil } // TODO: unify workload and standalone logic for non-sandbox features (e.g., block devices, huge pages, uVM mounts) // TODO(go1.24): use [os.Root] instead of `!strings.HasPrefix(, )` // Returns whether this host has a security policy set, i.e. if it's running // confidential containers. func (h *Host) HasSecurityPolicy() bool { return len(h.securityOptions.PolicyEnforcer.EncodedSecurityPolicy()) > 0 } // For confidential containers, make sure that the host can't use unexpected // bundle paths / scratch dir / rootfs func checkContainerSettings(sandboxID, containerID string, settings *prot.VMHostedContainerSettingsV2) error { if settings.OCISpecification == nil { return errors.Errorf("OCISpecification is nil") } if settings.OCISpecification.Root == nil { return errors.Errorf("OCISpecification.Root is nil") } // matches with CreateContainer / createLinuxContainerDocument in internal/hcsoci containerRootInUVM := path.Join(guestpath.LCOWRootPrefixInUVM, containerID) if settings.OCIBundlePath != containerRootInUVM { return errors.Errorf("OCIBundlePath %q must equal expected %q", settings.OCIBundlePath, containerRootInUVM) } expectedContainerRootfs := path.Join(containerRootInUVM, guestpath.RootfsPath) if settings.OCISpecification.Root.Path != expectedContainerRootfs { return errors.Errorf("OCISpecification.Root.Path %q must equal expected %q", settings.OCISpecification.Root.Path, expectedContainerRootfs) } // matches with MountLCOWLayers scratchDirPath := settings.ScratchDirPath expectedScratchDirPathNonShared := path.Join(containerRootInUVM, guestpath.ScratchDir, containerID) expectedScratchDirPathShared := path.Join(guestpath.LCOWRootPrefixInUVM, sandboxID, guestpath.ScratchDir, containerID) if scratchDirPath != expectedScratchDirPathNonShared && scratchDirPath != expectedScratchDirPathShared { return errors.Errorf("ScratchDirPath %q must be either %q or %q", scratchDirPath, expectedScratchDirPathNonShared, expectedScratchDirPathShared) } if settings.OCISpecification.Hooks != nil { return errors.Errorf("OCISpecification.Hooks must be nil.") } return nil } // checkState returns an error if the UVM has entered an inconsistent state from // which it cannot safely recover. Only enforced for confidential containers. func (h *Host) checkState() error { if h.HasSecurityPolicy() { return h.uvmError.Check() } return nil } // setUVMInconsistent records that the UVM has entered an inconsistent state and // logs the cause. The flag is only consulted in confidential mode (see // checkState), so this is a no-op for non-confidential hosts. func (h *Host) setUVMInconsistent(cause error) { if !h.HasSecurityPolicy() { return } h.uvmError.Set(cause) log.G(context.Background()).WithFields(logrus.Fields{ "cause": cause, }).Error("Host marked inconsistent. All further mounts/unmounts, container creation and deletion will fail.") } func checkExists(path string) (bool, error) { if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { return false, nil } return false, errors.Wrapf(err, "failed to determine if path '%s' exists", path) } return true, nil } func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VMHostedContainerSettingsV2) (_ *Container, err error) { if err = h.checkState(); err != nil { return nil, err } criType, isCRI := settings.OCISpecification.Annotations[annotations.KubernetesContainerType] // Check for virtual pod annotation virtualPodID := settings.OCISpecification.Annotations[annotations.VirtualPodID] isVirtualPod := virtualPodID != "" if h.HasSecurityPolicy() { if err = checkValidContainerID(id, "container"); err != nil { return nil, err } if virtualPodID != "" { if err = checkValidContainerID(virtualPodID, "virtual pod"); err != nil { return nil, err } } } // Special handling for virtual pod sandbox containers: // The first container in a virtual pod (containerID == virtualPodID) should be treated as a sandbox // even if the CRI annotation might indicate otherwise due to host-side UVM setup differences if isVirtualPod && id == virtualPodID { criType = "sandbox" isCRI = true logrus.WithFields(logrus.Fields{ logfields.ContainerID: id, logfields.VirtualSandboxID: virtualPodID, "originalCriType": settings.OCISpecification.Annotations[annotations.KubernetesContainerType], }).Info("Virtual pod first container detected - treating as sandbox container") } c := &Container{ id: id, vsock: h.vsock, spec: settings.OCISpecification, ociBundlePath: settings.OCIBundlePath, isSandbox: criType == "sandbox", exitType: prot.NtUnexpectedExit, processes: make(map[uint32]*containerProcess), scratchDirPath: settings.ScratchDirPath, } c.setStatus(containerCreating) // Resolve sandboxID early so all downstream code uses the correct value. // For sandbox containers, sandboxID == id (or virtualPodID for virtual pods). // For workload containers, sandboxID comes from the KubernetesSandboxID annotation. // For standalone containers, sandboxID == id. sandboxID := id if isVirtualPod { sandboxID = virtualPodID } else if criType == "container" { sandboxID = settings.OCISpecification.Annotations[annotations.KubernetesSandboxID] } c.sandboxID = sandboxID if err := h.AddContainer(id, c); err != nil { return nil, err } defer func() { if err != nil { h.RemoveContainer(id) } }() // Take a backup of the devices array before we populate it with any devices // found by GCS, in order to pass to the policy enforcer later. // // In specGuest.ApplyAnnotationsToSpec, if this is a privileged container, // we will add devices found in the GCS namespace's /dev. Regardless of // privileged or not, we also always include /dev/sev-guest. Since the // policy already lets the user enforce whether the container should be // privileged or not, and the sev-guest device is always added for a // confidential container, we do not need the policy enforcer to check these // devices we dynamically add again. extraLinuxDevices := slices.Clone(settings.OCISpecification.Linux.Devices) var namespaceID string if isCRI { switch criType { case "sandbox": namespaceID = specGuest.GetNetworkNamespaceID(settings.OCISpecification) // Resolve the sandbox root from OCIBundlePath. sandboxRoot, err := h.registerSandboxRoot(id, settings.OCIBundlePath, virtualPodID) if err != nil { return nil, err } c.sandboxRoot = sandboxRoot err = setupSandboxContainerSpec(ctx, id, sandboxRoot, settings.OCISpecification) if err != nil { return nil, err } defer func() { if err != nil { _ = os.RemoveAll(settings.OCIBundlePath) } }() if err = setupSandboxMountsPath(sandboxRoot); err != nil { return nil, err } if err = setupSandboxTmpfsMountsPath(sandboxRoot); err != nil { return nil, err } if err = setupSandboxHugePageMountsPath(sandboxRoot); err != nil { return nil, err } if err = setupSandboxLogDir(sandboxRoot); err != nil { return nil, err } if err := securitypolicy.ExtendPolicyWithNetworkingMounts(sandboxRoot, h.securityOptions.PolicyEnforcer, settings.OCISpecification); err != nil { return nil, err } if err := h.createPodInUVM(sandboxID, settings.OCISpecification); err != nil { return nil, err } case "container": if h.HasSecurityPolicy() { if err = checkValidContainerID(sandboxID, "sandbox"); err != nil { return nil, err } } if sandboxID == "" { return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sandboxID) } if err = h.createContainerInPod(sandboxID, id); err != nil { return nil, err } sandboxRoot := h.resolveSandboxRoot(sandboxID) c.sandboxRoot = sandboxRoot if err = setupWorkloadContainerSpec(ctx, sandboxID, id, sandboxRoot, settings.OCISpecification, settings.OCIBundlePath); err != nil { return nil, err } // Add SEV device when security policy is not empty, except when privileged annotation is // set to "true", in which case all UVMs devices are added. if h.HasSecurityPolicy() && !oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWPrivileged, false) { if err := specGuest.AddDevSev(ctx, settings.OCISpecification); err != nil { log.G(ctx).WithError(err).Debug("failed to add SEV device") } } defer func() { if err != nil { _ = os.RemoveAll(settings.OCIBundlePath) } }() if err := securitypolicy.ExtendPolicyWithNetworkingMounts(sandboxRoot, h.securityOptions.PolicyEnforcer, settings.OCISpecification); err != nil { return nil, err } default: return nil, errors.Errorf("unsupported 'io.kubernetes.cri.container-type': '%s'", criType) } } else { // Standalone container: no pod entry is created. namespaceID = specGuest.GetNetworkNamespaceID(settings.OCISpecification) // Standalone uses OCIBundlePath directly as its root. c.sandboxRoot = settings.OCIBundlePath if err := setupStandaloneContainerSpec(ctx, id, settings.OCIBundlePath, settings.OCISpecification); err != nil { return nil, err } defer func() { if err != nil { _ = os.RemoveAll(settings.OCIBundlePath) } }() if err := securitypolicy.ExtendPolicyWithNetworkingMounts(c.sandboxRoot, h.securityOptions.PolicyEnforcer, settings.OCISpecification); err != nil { return nil, err } } // don't specialize tee logs (both files and mounts) just for workload containers // add log directory mount before enforcing (mount) policy if logDirMount := settings.OCISpecification.Annotations[annotations.LCOWTeeLogDirMount]; logDirMount != "" { settings.OCISpecification.Mounts = append(settings.OCISpecification.Mounts, specs.Mount{ Destination: logDirMount, Type: "bind", Source: specGuest.SandboxLogsDirFromRoot(c.sandboxRoot), Options: []string{"bind"}, }) } if h.HasSecurityPolicy() { if err = checkContainerSettings(sandboxID, id, settings); err != nil { return nil, err } } user, groups, umask, err := h.securityOptions.PolicyEnforcer.GetUserInfo(settings.OCISpecification.Process, settings.OCISpecification.Root.Path) if err != nil { return nil, err } seccomp, err := securitypolicy.MeasureSeccompProfile(settings.OCISpecification.Linux.Seccomp) if err != nil { return nil, err } privileged := isPrivilegedContainerCreationRequest(ctx, settings.OCISpecification) noNewPrivileges := settings.OCISpecification.Process.NoNewPrivileges opts := &securitypolicy.CreateContainerOptions{ SandboxID: sandboxID, Privileged: &privileged, NoNewPrivileges: &noNewPrivileges, Groups: groups, Umask: umask, Capabilities: settings.OCISpecification.Process.Capabilities, SeccompProfileSHA256: seccomp, IsSandboxContainer: c.isSandbox, LinuxDevices: extraLinuxDevices, } envToKeep, capsToKeep, allowStdio, err := h.securityOptions.PolicyEnforcer.EnforceCreateContainerPolicyV2( ctx, id, settings.OCISpecification.Process.Args, settings.OCISpecification.Process.Env, settings.OCISpecification.Process.Cwd, settings.OCISpecification.Mounts, user, opts, ) if err != nil { return nil, errors.Wrapf(err, "container creation denied due to policy") } if !allowStdio { // stdio access isn't allow for this container. Switch to the /dev/null // transport that will eat all input/ouput. c.vsock = h.devNullTransport } // delay creating the directory to house the container's stdio until after we've verified // policy on log settings. // TODO: is using allowStdio appropriate here, since longs aren't leaving the uVM? if logPath := settings.OCISpecification.Annotations[annotations.LCOWTeeLogPath]; logPath != "" { if !allowStdio { return nil, errors.Errorf("teeing container stdio to log path %q denied due to policy not allowing stdio access", logPath) } logsDir := specGuest.SandboxLogsDirFromRoot(c.sandboxRoot) c.logPath = filepath.Join(logsDir, logPath) // verify the logpath is still under the correct directory if !strings.HasPrefix(c.logPath, logsDir+"/") { return nil, errors.Errorf("log path %v is not within sandbox's log dir", c.logPath) } dir := filepath.Dir(c.logPath) log.G(ctx).WithFields(logrus.Fields{ logfields.Path: dir, logfields.ContainerID: id, }).Debug("creating container log file parent directory in uVM") if err := mkdirAllModePerm(dir); err != nil { return nil, errors.Wrapf(err, "failed to create log file parent directory: %s", dir) } } if envToKeep != nil { settings.OCISpecification.Process.Env = []string(envToKeep) } if capsToKeep != nil { settings.OCISpecification.Process.Capabilities = capsToKeep } if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) { if _, err := h.securityOptions.WriteSecurityContextDir(settings.OCISpecification); err != nil { return nil, fmt.Errorf("failed to write security context dir: %w", err) } } // Determine hostNetwork mode. For sandbox/standalone containers, check their // own annotations. For workload containers, inherit from the sandbox so that // only pod.snp.json needs the LCOWHostNetwork annotation — container.json // does not need it (and the value is ignored if present). var hostNetwork bool if criType == "sandbox" || !isCRI { hostNetwork = oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWHostNetwork, false) } else if criType == "container" { h.containersMutex.Lock() if sandbox, ok := h.containers[sandboxID]; ok { hostNetwork = oci.ParseAnnotationsBool(ctx, sandbox.spec.Annotations, annotations.LCOWHostNetwork, false) } h.containersMutex.Unlock() } if hostNetwork { if err := h.securityOptions.PolicyEnforcer.EnforceHostNetworkPolicy(ctx); err != nil { return nil, err } } if hostNetwork && settings.OCISpecification.Linux != nil { filtered := settings.OCISpecification.Linux.Namespaces[:0] for _, ns := range settings.OCISpecification.Linux.Namespaces { if ns.Type != specs.NetworkNamespace { filtered = append(filtered, ns) } } settings.OCISpecification.Linux.Namespaces = filtered log.G(ctx).WithField("cid", id).Info("Host network namespace enabled: removed network namespace from OCI spec") } // Create the BundlePath if err := os.MkdirAll(settings.OCIBundlePath, 0700); err != nil { return nil, errors.Wrapf(err, "failed to create OCIBundlePath: '%s'", settings.OCIBundlePath) } if err := writeSpecToFile(ctx, path.Join(settings.OCIBundlePath, "config.json"), settings.OCISpecification); err != nil { return nil, err } con, err := h.rtime.CreateContainer(sandboxID, id, settings.OCIBundlePath, nil) if err != nil { return nil, errors.Wrapf(err, "failed to create container") } init, err := con.GetInitProcess() if err != nil { return nil, errors.Wrapf(err, "failed to get container init process") } c.container = con c.initProcess = newProcess(c, settings.OCISpecification.Process, init, uint32(c.container.Pid()), true) // Sandbox or standalone, set up networks. if criType == "sandbox" || !isCRI { ns, err := getNetworkNamespace(namespaceID) // skip network activity for sandbox containers marked with skip uvm networking annotation if isCRI && err != nil && !strings.EqualFold(settings.OCISpecification.Annotations[annotations.SkipPodNetworking], "true") { return nil, err } // standalone is not required to have a networking namespace setup if ns != nil { if hostNetwork { // Host network mode: configure adapters in-place in the init // network namespace (do NOT move them). This keeps eth0 visible // to all containers sharing the init netns while still setting // up IP addresses, routes, and gateways. if err := ns.ConfigureInInitNS(ctx); err != nil { return nil, err } } else { // Normal mode: move adapters into the sandbox's network namespace. if err := ns.AssignContainerPid(ctx, c.container.Pid()); err != nil { return nil, err } if err := ns.Sync(ctx); err != nil { return nil, err } } } } c.setStatus(containerCreated) return c, nil } func writeSpecToFile(ctx context.Context, configFile string, spec *specs.Spec) error { f, err := os.Create(configFile) if err != nil { return errors.Wrapf(err, "failed to create config.json at: '%s'", configFile) } defer f.Close() writer := bufio.NewWriter(f) // capture what we write to the config file in a byte buffer so we can log it later var w io.Writer = writer buf := &bytes.Buffer{} if logrus.IsLevelEnabled(logrus.TraceLevel) { w = io.MultiWriter(writer, buf) } enc := json.NewEncoder(w) enc.SetEscapeHTML(false) // not embedding JSON into HTML, so no need to escape if err := enc.Encode(spec); err != nil { return errors.Wrapf(err, "failed to write OCISpecification to config.json at: '%s'", configFile) } if err := writer.Flush(); err != nil { return errors.Wrapf(err, "failed to flush writer for config.json at: '%s'", configFile) } if logrus.IsLevelEnabled(logrus.TraceLevel) { entry := log.G(ctx).WithField(logfields.Path, configFile) if b, err := log.ScrubOCISpec(buf.Bytes()); err != nil { entry.WithError(err).Warning("could not scrub OCI spec written to config.json") } else { log.G(ctx).WithField( "config", string(bytes.TrimSpace(b)), ).Trace("wrote OCI spec to config.json") } } return nil } // Returns whether there is a running container that is currently using the // given overlay (as its rootfs). func (h *Host) IsOverlayInUse(overlayPath string) bool { h.containersMutex.Lock() defer h.containersMutex.Unlock() for _, c := range h.containers { if c.getStatus() != containerRunning { continue } if c.spec.Root.Path == overlayPath { return true } } return false } func (h *Host) modifyHostSettings(ctx context.Context, containerID string, req *guestrequest.ModificationRequest) (retErr error) { if h.HasSecurityPolicy() { if err := checkValidContainerID(containerID, "container"); err != nil { return err } } switch req.ResourceType { case guestresource.ResourceTypeSCSIDevice: return modifySCSIDevice(ctx, req.RequestType, req.Settings.(*guestresource.SCSIDevice)) case guestresource.ResourceTypeMappedVirtualDisk: if err := h.checkState(); err != nil { return err } mvd := req.Settings.(*guestresource.LCOWMappedVirtualDisk) // find the actual controller number on the bus and update the incoming request. var cNum uint8 cNum, err := scsi.ActualControllerNumber(ctx, mvd.Controller) if err != nil { return err } mvd.Controller = cNum return h.modifyMappedVirtualDisk(ctx, req.RequestType, mvd) case guestresource.ResourceTypeMappedDirectory: if err := h.checkState(); err != nil { return err } return h.modifyMappedDirectory(ctx, h.vsock, req.RequestType, req.Settings.(*guestresource.LCOWMappedDirectory)) case guestresource.ResourceTypeVPMemDevice: if err := h.checkState(); err != nil { return err } return h.modifyMappedVPMemDevice(ctx, req.RequestType, req.Settings.(*guestresource.LCOWMappedVPMemDevice)) case guestresource.ResourceTypeCombinedLayers: if err := h.checkState(); err != nil { return err } return h.modifyCombinedLayers(ctx, req.RequestType, req.Settings.(*guestresource.LCOWCombinedLayers)) case guestresource.ResourceTypeNetwork: return modifyNetwork(ctx, req.RequestType, req.Settings.(*guestresource.LCOWNetworkAdapter)) case guestresource.ResourceTypeVPCIDevice: return modifyMappedVPCIDevice(ctx, req.RequestType, req.Settings.(*guestresource.LCOWMappedVPCIDevice)) case guestresource.ResourceTypeContainerConstraints: c, err := h.GetInitializedContainer(containerID) if err != nil { return err } return c.modifyContainerConstraints(ctx, req.RequestType, req.Settings.(*guestresource.LCOWContainerConstraints)) case guestresource.ResourceTypeSecurityPolicy: r, ok := req.Settings.(*guestresource.ConfidentialOptions) if !ok { return errors.New("the request's settings are not of type ConfidentialOptions") } err := h.securityOptions.SetConfidentialOptions(ctx, r.EnforcerType, r.EncodedSecurityPolicy, r.EncodedUVMReference, r.EncodedUVMHashEnvelopeReference) if err != nil { return err } // Start tracking mounts and restricting unmounts on confidential containers. // As long as we started off with the ClosedDoorSecurityPolicyEnforcer, no // mounts should have been allowed until this point. if h.HasSecurityPolicy() { log.G(ctx).Debug("hostMounts initialized") h.hostMounts = newHostMounts() } return nil case guestresource.ResourceTypePolicyFragment: r, ok := req.Settings.(*guestresource.SecurityPolicyFragment) if !ok { return errors.New("the request settings are not of type SecurityPolicyFragment") } return h.securityOptions.InjectFragment(ctx, r) case guestresource.ResourceTypePodCgroupMemoryLimit: switch req.RequestType { case guestrequest.RequestTypeUpdate: _, _, err := h.UpdateCgroupMemoryLimits() return err default: return newInvalidRequestTypeError(req.RequestType) } default: return errors.Errorf("the ResourceType %q is not supported for UVM", req.ResourceType) } } func (h *Host) modifyContainerSettings(ctx context.Context, containerID string, req *guestrequest.ModificationRequest) error { if h.HasSecurityPolicy() { if err := checkValidContainerID(containerID, "container"); err != nil { return err } } c, err := h.GetInitializedContainer(containerID) if err != nil { return err } switch req.ResourceType { case guestresource.ResourceTypeContainerConstraints: return c.modifyContainerConstraints(ctx, req.RequestType, req.Settings.(*guestresource.LCOWContainerConstraints)) default: return errors.Errorf("the ResourceType \"%s\" is not supported for containers", req.ResourceType) } } func (h *Host) ModifySettings(ctx context.Context, containerID string, req *guestrequest.ModificationRequest) error { if containerID == UVMContainerID { return h.modifyHostSettings(ctx, containerID, req) } return h.modifyContainerSettings(ctx, containerID, req) } // Shutdown terminates this UVM. This is a destructive call and will destroy all // state that has not been cleaned before calling this function. func (*Host) Shutdown() { _ = syscall.Reboot(syscall.LINUX_REBOOT_CMD_POWER_OFF) } // Called to shutdown a container func (h *Host) ShutdownContainer(ctx context.Context, containerID string, graceful bool) error { c, err := h.GetInitializedContainer(containerID) if err != nil { return err } err = h.securityOptions.PolicyEnforcer.EnforceShutdownContainerPolicy(ctx, containerID) if err != nil { return err } signal := unix.SIGTERM if !graceful { signal = unix.SIGKILL } return c.Kill(ctx, signal) } func (h *Host) SignalContainerProcess(ctx context.Context, containerID string, processID uint32, signal syscall.Signal) error { c, err := h.GetInitializedContainer(containerID) if err != nil { return err } p, err := c.GetProcess(processID) if err != nil { return err } signalingInitProcess := processID == c.initProcess.pid startupArgList := p.(*containerProcess).spec.Args err = h.securityOptions.PolicyEnforcer.EnforceSignalContainerProcessPolicy(ctx, containerID, signal, signalingInitProcess, startupArgList) if err != nil { return err } return p.Kill(ctx, signal) } func (h *Host) ExecProcess(ctx context.Context, containerID string, params prot.ProcessParameters, conSettings stdio.ConnectionSettings) (_ int, err error) { var pid int var c *Container if params.IsExternal || containerID == UVMContainerID { var envToKeep securitypolicy.EnvList var allowStdioAccess bool envToKeep, allowStdioAccess, err = h.securityOptions.PolicyEnforcer.EnforceExecExternalProcessPolicy( ctx, params.CommandArgs, processParamEnvToOCIEnv(params.Environment), params.WorkingDirectory, ) if err != nil { return pid, errors.Wrapf(err, "exec is denied due to policy") } // It makes no sense to allow access if stdio access is denied and the // process requires a terminal. if params.EmulateConsole && !allowStdioAccess { return pid, errors.New("exec of process that requires terminal access denied due to policy not allowing stdio access") } if envToKeep != nil { params.Environment = processOCIEnvToParam(envToKeep) } var tport = h.vsock if !allowStdioAccess { tport = h.devNullTransport } pid, err = h.runExternalProcess(ctx, params, conSettings, tport) } else if c, err = h.GetInitializedContainer(containerID); err == nil { // We found a V2 container. Treat this as a V2 process. if params.OCIProcess == nil { // We've already done policy enforcement for creating a container so // there's no policy enforcement to do for starting pid, err = c.Start(ctx, conSettings) } else { // Windows uses a different field for command, there's no enforcement // around this yet for Windows so this is Linux specific at the moment. var envToKeep securitypolicy.EnvList var capsToKeep *specs.LinuxCapabilities var user securitypolicy.IDName var groups []securitypolicy.IDName var umask string var allowStdioAccess bool user, groups, umask, err = h.securityOptions.PolicyEnforcer.GetUserInfo(params.OCIProcess, c.spec.Root.Path) if err != nil { return 0, err } envToKeep, capsToKeep, allowStdioAccess, err = h.securityOptions.PolicyEnforcer.EnforceExecInContainerPolicy( ctx, containerID, params.OCIProcess.Args, params.OCIProcess.Env, params.OCIProcess.Cwd, params.OCIProcess.NoNewPrivileges, user, groups, umask, params.OCIProcess.Capabilities, ) if err != nil { return pid, errors.Wrapf(err, "exec in container denied due to policy") } // It makes no sense to allow access if stdio access is denied and the // process requires a terminal. if params.OCIProcess.Terminal && !allowStdioAccess { return pid, errors.New("exec in container of process that requires terminal access denied due to policy not allowing stdio access") } if envToKeep != nil { params.OCIProcess.Env = envToKeep } if capsToKeep != nil { params.OCIProcess.Capabilities = capsToKeep } pid, err = c.ExecProcess(ctx, params.OCIProcess, conSettings) } } return pid, err } func (h *Host) GetExternalProcess(pid int) (Process, error) { h.externalProcessesMutex.Lock() defer h.externalProcessesMutex.Unlock() p, ok := h.externalProcesses[pid] if !ok { return nil, gcserr.NewHresultError(gcserr.HrErrNotFound) } return p, nil } func (h *Host) GetProperties(ctx context.Context, containerID string, query prot.PropertyQuery) (*prot.PropertiesV2, error) { err := h.securityOptions.PolicyEnforcer.EnforceGetPropertiesPolicy(ctx) if err != nil { return nil, errors.Wrapf(err, "get properties denied due to policy") } c, err := h.GetInitializedContainer(containerID) if err != nil { return nil, err } properties := &prot.PropertiesV2{} for _, requestedProperty := range query.PropertyTypes { switch requestedProperty { case prot.PtProcessList: pids, err := c.GetAllProcessPids(ctx) if err != nil { return nil, err } properties.ProcessList = make([]prot.ProcessDetails, len(pids)) for i, pid := range pids { if specGuest.OutOfUint32Bounds(pid) { return nil, errors.Errorf("PID (%d) exceeds uint32 bounds", pid) } properties.ProcessList[i].ProcessID = uint32(pid) } case prot.PtStatistics: cgroupMetrics, err := c.GetStats(ctx) if err != nil { return nil, err } // zero out [Blkio] sections, since: // 1. (Az)CRI (currently) only looks at the CPU and memory sections; and // 2. it can get very large for containers with many layers if cgroupMetrics.GetBlkio() != nil { cgroupMetrics.Blkio.Reset() } // also preemptively zero out [Rdma] and [Network], since they could also grow untenable large if cgroupMetrics.GetRdma() != nil { cgroupMetrics.Rdma.Reset() } if len(cgroupMetrics.GetNetwork()) > 0 { cgroupMetrics.Network = []*cgroup1stats.NetworkStat{} } if logrus.IsLevelEnabled(logrus.TraceLevel) { log.G(ctx).WithField("stats", log.Format(ctx, cgroupMetrics)).Trace("queried cgroup statistics") } properties.Metrics = cgroupMetrics default: log.G(ctx).WithField("propertyType", requestedProperty).Warn("unknown or empty property type") } } return properties, nil } func (h *Host) GetStacks(ctx context.Context) (string, error) { err := h.securityOptions.PolicyEnforcer.EnforceDumpStacksPolicy(ctx) if err != nil { return "", errors.Wrapf(err, "dump stacks denied due to policy") } return debug.DumpStacks(), nil } // RunExternalProcess runs a process in the utility VM. func (h *Host) runExternalProcess( ctx context.Context, params prot.ProcessParameters, conSettings stdio.ConnectionSettings, tport transport.Transport, ) (_ int, err error) { var stdioSet *stdio.ConnectionSet stdioSet, err = stdio.Connect(tport, conSettings) if err != nil { return -1, err } defer func() { if err != nil { stdioSet.Close() } }() args := params.CommandArgs if len(args) == 0 { args, err = processParamCommandLineToOCIArgs(params.CommandLine) if err != nil { return -1, err } } cmd := exec.Command(args[0], args[1:]...) cmd.Dir = params.WorkingDirectory cmd.Env = processParamEnvToOCIEnv(params.Environment) var relay *stdio.TtyRelay if params.EmulateConsole { // Allocate a console for the process. var ( master *os.File consolePath string ) master, consolePath, err = stdio.NewConsole() if err != nil { return -1, errors.Wrap(err, "failed to create console for external process") } defer func() { if err != nil { master.Close() } }() var console *os.File console, err = os.OpenFile(consolePath, os.O_RDWR|syscall.O_NOCTTY, 0777) if err != nil { return -1, errors.Wrap(err, "failed to open console file for external process") } defer console.Close() relay = stdio.NewTtyRelay(stdioSet, master) cmd.Stdin = console cmd.Stdout = console cmd.Stderr = console // Make the child process a session leader and adopt the pty as // the controlling terminal. cmd.SysProcAttr = &syscall.SysProcAttr{ Setsid: true, Setctty: true, Ctty: syscall.Stdin, } } else { var fileSet *stdio.FileSet fileSet, err = stdioSet.Files() if err != nil { return -1, errors.Wrap(err, "failed to set cmd stdio") } defer fileSet.Close() defer stdioSet.Close() cmd.Stdin = fileSet.In cmd.Stdout = fileSet.Out cmd.Stderr = fileSet.Err } onRemove := func(pid int) { h.externalProcessesMutex.Lock() delete(h.externalProcesses, pid) h.externalProcessesMutex.Unlock() } p, err := newExternalProcess(ctx, cmd, relay, onRemove) if err != nil { return -1, err } h.externalProcessesMutex.Lock() h.externalProcesses[p.Pid()] = p h.externalProcessesMutex.Unlock() return p.Pid(), nil } func newInvalidRequestTypeError(rt guestrequest.RequestType) error { return errors.Errorf("the RequestType %q is not supported", rt) } func modifySCSIDevice( ctx context.Context, rt guestrequest.RequestType, msd *guestresource.SCSIDevice, ) error { switch rt { case guestrequest.RequestTypeRemove: cNum, err := scsi.ActualControllerNumber(ctx, msd.Controller) if err != nil { return err } return scsi.UnplugDevice(ctx, cNum, msd.Lun) default: return newInvalidRequestTypeError(rt) } } func (h *Host) modifyMappedVirtualDisk( ctx context.Context, rt guestrequest.RequestType, mvd *guestresource.LCOWMappedVirtualDisk, ) (err error) { ctx, span := ot.StartSpan(ctx, "gcs::Host::modifyMappedVirtualDisk") defer span.End() defer func() { ot.SetSpanStatus(span, err) }() span.SetAttributes( attribute.String("requestType", string(rt)), attribute.Bool("hasHostMounts", h.hostMounts != nil), attribute.Int64("controller", int64(mvd.Controller)), attribute.Int64("lun", int64(mvd.Lun)), attribute.Int64("partition", int64(mvd.Partition)), attribute.Bool("readOnly", mvd.ReadOnly), attribute.String("mountPath", mvd.MountPath), ) var verityInfo *guestresource.DeviceVerityInfo securityPolicy := h.securityOptions.PolicyEnforcer devPath, err := scsi.GetDevicePath(ctx, mvd.Controller, mvd.Lun, mvd.Partition) if err != nil { return err } span.SetAttributes(attribute.String("devicePath", devPath)) if mvd.ReadOnly { // The only time the policy is empty, and we want it to be empty // is when no policy is provided, and we default to open door // policy. In any other case, e.g. explicit open door or any // other rego policy we would like to mount layers with verity. if h.HasSecurityPolicy() { verityInfo, err = verity.ReadVeritySuperBlock(ctx, devPath) if err != nil { return err } if mvd.Filesystem != "" && mvd.Filesystem != "ext4" { return errors.Errorf("filesystem must be ext4 for read-only scsi mounts") } } } // For confidential containers, we revert the policy metadata via the // transaction rollback mechanism on both mount and unmount errors, but if // we've actually called Unmount and it fails we permanently block further // device operations by marking the UVM state as inconsistent. return securityPolicy.WithMetadataRollback(func() error { switch rt { case guestrequest.RequestTypeAdd: mountCtx, cancel := context.WithTimeout(ctx, time.Second*5) defer cancel() if mvd.MountPath != "" { if h.HasSecurityPolicy() { // The only option we allow if there is policy enforcement is // "ro", and it must be present iff the request is readonly. expectedOptions := []string{} if mvd.ReadOnly { expectedOptions = []string{"ro"} } if !slices.Equal(mvd.Options, expectedOptions) { return errors.Errorf( "mounting scsi device controller %d lun %d onto %s with mount options %q denied by policy: expected %q (mvd.ReadOnly=%t)", mvd.Controller, mvd.Lun, mvd.MountPath, strings.Join(mvd.Options, ","), strings.Join(expectedOptions, ","), mvd.ReadOnly, ) } } if mvd.BlockDev { if err = securityPolicy.EnforceMountBlockDevicePolicy(ctx, mvd.MountPath); err != nil { return errors.Wrapf(err, "creating blockdev symlink at %s (-> scsi controller %d lun %d) denied by policy", mvd.MountPath, mvd.Controller, mvd.Lun) } } else if mvd.ReadOnly { var deviceHash string if verityInfo != nil { deviceHash = verityInfo.RootDigest } err = securityPolicy.EnforceDeviceMountPolicy(ctx, mvd.MountPath, deviceHash) if err != nil { return errors.Wrapf(err, "mounting scsi device controller %d lun %d onto %s denied by policy", mvd.Controller, mvd.Lun, mvd.MountPath) } if h.hostMounts != nil { h.hostMounts.Lock() defer h.hostMounts.Unlock() err = h.hostMounts.AddRODevice(mvd.MountPath, devPath) if err != nil { return err } // Note: "When a function returns, its deferred calls are // executed in last-in-first-out order." - so we are safe to // call RemoveRODevice in this defer. defer func() { if err != nil { _ = h.hostMounts.RemoveRODevice(mvd.MountPath, devPath) } }() } } else { err = securityPolicy.EnforceRWDeviceMountPolicy(ctx, mvd.MountPath, mvd.Encrypted, mvd.EnsureFilesystem, mvd.Filesystem) if err != nil { return errors.Wrapf(err, "mounting scsi device controller %d lun %d onto %s denied by policy", mvd.Controller, mvd.Lun, mvd.MountPath) } if h.hostMounts != nil { h.hostMounts.Lock() defer h.hostMounts.Unlock() err = h.hostMounts.AddRWDevice(mvd.MountPath, devPath, mvd.Encrypted) if err != nil { return err } defer func() { if err != nil { _ = h.hostMounts.RemoveRWDevice(mvd.MountPath, devPath, mvd.Encrypted) } }() } } config := &scsi.Config{ Encrypted: mvd.Encrypted, VerityInfo: verityInfo, EnsureFilesystem: mvd.EnsureFilesystem, Filesystem: mvd.Filesystem, BlockDev: mvd.BlockDev, } // Since we're rolling back the policy metadata on failure, we // need to ensure that we have reverted all the side effects // from this failed mount attempt, otherwise the Rego metadata // is technically still inconsistent with reality. Mount cleans // up the created directory and dm devices on failure, so we're // good. return scsi.Mount(mountCtx, mvd.Controller, mvd.Lun, mvd.Partition, mvd.MountPath, mvd.ReadOnly, mvd.Options, config) } return nil case guestrequest.RequestTypeRemove: if mvd.MountPath != "" { if mvd.BlockDev { if err = securityPolicy.EnforceUnmountBlockDevicePolicy(ctx, mvd.MountPath); err != nil { return fmt.Errorf("removing blockdev symlink at %s (-> scsi controller %d lun %d) denied by policy: %w", mvd.MountPath, mvd.Controller, mvd.Lun, err) } } else if mvd.ReadOnly { if err = securityPolicy.EnforceDeviceUnmountPolicy(ctx, mvd.MountPath); err != nil { return fmt.Errorf("unmounting scsi device at %s denied by policy: %w", mvd.MountPath, err) } if h.hostMounts != nil { h.hostMounts.Lock() defer h.hostMounts.Unlock() if err = h.hostMounts.RemoveRODevice(mvd.MountPath, devPath); err != nil { return err } defer func() { if err != nil { _ = h.hostMounts.AddRODevice(mvd.MountPath, devPath) } }() } } else { if err = securityPolicy.EnforceRWDeviceUnmountPolicy(ctx, mvd.MountPath); err != nil { return fmt.Errorf("unmounting scsi device at %s denied by policy: %w", mvd.MountPath, err) } if h.hostMounts != nil { h.hostMounts.Lock() defer h.hostMounts.Unlock() if err = h.hostMounts.RemoveRWDevice(mvd.MountPath, devPath, mvd.Encrypted); err != nil { return err } defer func() { if err != nil { _ = h.hostMounts.AddRWDevice(mvd.MountPath, devPath, mvd.Encrypted) } }() } } // Check that the directory actually exists first, and if it // does not then we just refuse to do anything, without closing // the dm device or marking the UVM inconsistent. Policy // metadata is still reverted to reflect the fact that we have // not done anything. // // Note: we should not do this check before calling the policy // enforcer (which we have done above), otherwise we will // inadvertently allow the host to find out whether an arbitrary // path (which may point to sensitive data within a container // rootfs) exists or not if h.HasSecurityPolicy() { exists, err := checkExists(mvd.MountPath) if err != nil { return err } if !exists { return errors.Errorf("unmounting scsi device at %s failed: directory does not exist", mvd.MountPath) } } config := &scsi.Config{ Encrypted: mvd.Encrypted, VerityInfo: verityInfo, EnsureFilesystem: mvd.EnsureFilesystem, Filesystem: mvd.Filesystem, BlockDev: mvd.BlockDev, } err = scsi.Unmount(ctx, mvd.Controller, mvd.Lun, mvd.Partition, mvd.MountPath, config) if err != nil { h.setUVMInconsistent( fmt.Errorf("unmounting scsi device at %s failed: %w", mvd.MountPath, err), ) return err } } return nil default: return newInvalidRequestTypeError(rt) } }) } func (h *Host) modifyMappedDirectory( ctx context.Context, vsock transport.Transport, rt guestrequest.RequestType, md *guestresource.LCOWMappedDirectory, ) (err error) { securityPolicy := h.securityOptions.PolicyEnforcer // For confidential containers, we revert the policy metadata via the // transaction rollback mechanism on both mount and unmount errors, but if // we've actually called Unmount and it fails we permanently block further // device operations. return securityPolicy.WithMetadataRollback(func() error { switch rt { case guestrequest.RequestTypeAdd: err = securityPolicy.EnforcePlan9MountPolicy(ctx, md.MountPath) if err != nil { return errors.Wrapf(err, "mounting plan9 device at %s denied by policy", md.MountPath) } if h.HasSecurityPolicy() { if err = plan9.ValidateShareName(md.ShareName); err != nil { return err } } // Similar to the reasoning in modifyMappedVirtualDisk, since we're // rolling back the policy metadata, plan9.Mount here must clean up // everything if it fails, which it does do. return plan9.Mount(ctx, vsock, md.MountPath, md.ShareName, uint32(md.Port), md.ReadOnly) case guestrequest.RequestTypeRemove: err = securityPolicy.EnforcePlan9UnmountPolicy(ctx, md.MountPath) if err != nil { return errors.Wrapf(err, "unmounting plan9 device at %s denied by policy", md.MountPath) } // Note: storage.UnmountPath is nop if path does not exist. err = storage.UnmountPath(ctx, md.MountPath, true) if err != nil { h.setUVMInconsistent( fmt.Errorf("unmounting plan9 device at %s failed: %w", md.MountPath, err), ) return err } return nil default: return newInvalidRequestTypeError(rt) } }) } func (h *Host) modifyMappedVPMemDevice(ctx context.Context, rt guestrequest.RequestType, vpd *guestresource.LCOWMappedVPMemDevice, ) (err error) { var verityInfo *guestresource.DeviceVerityInfo securityPolicy := h.securityOptions.PolicyEnforcer var deviceHash string if h.HasSecurityPolicy() { if vpd.MappingInfo != nil { return fmt.Errorf("multi mapping is not supported with verity") } verityInfo, err = verity.ReadVeritySuperBlock(ctx, pmem.GetDevicePath(vpd.DeviceNumber)) if err != nil { return err } deviceHash = verityInfo.RootDigest } // For confidential containers, we revert the policy metadata via the // transaction rollback mechanism on both mount and unmount errors, but if // we've actually called Unmount and it fails we permanently block further // device operations. return securityPolicy.WithMetadataRollback(func() error { switch rt { case guestrequest.RequestTypeAdd: err = securityPolicy.EnforceDeviceMountPolicy(ctx, vpd.MountPath, deviceHash) if err != nil { return errors.Wrapf(err, "mounting pmem device %d onto %s denied by policy", vpd.DeviceNumber, vpd.MountPath) } // Similar to the reasoning in modifyMappedVirtualDisk, since we're // rolling back the policy metadata, pmem.Mount here must clean up // everything if it fails, which it does do. return pmem.Mount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, verityInfo) case guestrequest.RequestTypeRemove: if err = securityPolicy.EnforceDeviceUnmountPolicy(ctx, vpd.MountPath); err != nil { return errors.Wrapf(err, "unmounting pmem device from %s denied by policy", vpd.MountPath) } // Check that the directory actually exists first, and if it does not // then we just refuse to do anything, without closing the dm-linear or // dm-verity device or marking the UVM inconsistent. // // Similar to the reasoning in modifyMappedVirtualDisk, we should not do // this check before calling the policy enforcer. if h.HasSecurityPolicy() { exists, err := checkExists(vpd.MountPath) if err != nil { return err } if !exists { return errors.Errorf("unmounting pmem device at %s failed: directory does not exist", vpd.MountPath) } } err = pmem.Unmount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, verityInfo) if err != nil { h.setUVMInconsistent( fmt.Errorf("unmounting pmem device at %s failed: %w", vpd.MountPath, err), ) return err } return nil default: return newInvalidRequestTypeError(rt) } }) } func modifyMappedVPCIDevice(ctx context.Context, rt guestrequest.RequestType, vpciDev *guestresource.LCOWMappedVPCIDevice) error { switch rt { case guestrequest.RequestTypeAdd: return pci.WaitForPCIDeviceFromVMBusGUID(ctx, vpciDev.VMBusGUID) default: return newInvalidRequestTypeError(rt) } } func (h *Host) modifyCombinedLayers( ctx context.Context, rt guestrequest.RequestType, cl *guestresource.LCOWCombinedLayers, ) (err error) { ctx, span := ot.StartSpan(ctx, "gcs::Host::modifyCombinedLayers") defer span.End() defer func() { ot.SetSpanStatus(span, err) }() span.SetAttributes( attribute.String("requestType", string(rt)), attribute.Bool("hasHostMounts", h.hostMounts != nil), attribute.String("containerRootPath", cl.ContainerRootPath), attribute.String("scratchPath", cl.ScratchPath), ) securityPolicy := h.securityOptions.PolicyEnforcer containerID := cl.ContainerID // For confidential containers, we revert the policy metadata via the // transaction rollback mechanism on both mount and unmount errors, but if // we've actually called Unmount and it fails we permanently block further // device operations. return securityPolicy.WithMetadataRollback(func() error { if h.hostMounts != nil { // We will need this in multiple places, let's take the lock once here. h.hostMounts.Lock() defer h.hostMounts.Unlock() } switch rt { case guestrequest.RequestTypeAdd: if h.HasSecurityPolicy() { if err := checkValidContainerID(containerID, "container"); err != nil { return err } // We check this regardless of what the policy says, as long as we're in // confidential mode. This matches with checkContainerSettings called for // container creation request. expectedContainerRootfs := path.Join(guestpath.LCOWRootPrefixInUVM, containerID, guestpath.RootfsPath) if cl.ContainerRootPath != expectedContainerRootfs { return fmt.Errorf("combined layers target %q does not match expected path %q", cl.ContainerRootPath, expectedContainerRootfs) } if cl.ScratchPath != "" { // At this point, we do not know what the sandbox ID would be yet, so we // have to allow anything reasonable. scratchDirRegexStr := fmt.Sprintf( "^%s/%s/%s/%s$", guestpath.LCOWRootPrefixInUVM, validContainerIDRegexRaw, guestpath.ScratchDir, containerID, ) scratchDirRegex := regexp.MustCompile(scratchDirRegexStr) if !scratchDirRegex.MatchString(cl.ScratchPath) { return fmt.Errorf("scratch path %q must match regex %q", cl.ScratchPath, scratchDirRegexStr) } } } layerPaths := make([]string, len(cl.Layers)) for i, layer := range cl.Layers { layerPaths[i] = layer.Path } var upperdirPath string var workdirPath string readonly := false if cl.ScratchPath == "" { // The user did not pass a scratch path. Mount overlay as readonly. readonly = true } else { upperdirPath = filepath.Join(cl.ScratchPath, "upper") workdirPath = filepath.Join(cl.ScratchPath, "work") scratchEncrypted := false if h.hostMounts != nil { scratchEncrypted = h.hostMounts.IsEncrypted(cl.ScratchPath) } if err := securityPolicy.EnforceScratchMountPolicy(ctx, cl.ScratchPath, scratchEncrypted); err != nil { return fmt.Errorf("scratch mounting denied by policy: %w", err) } } if err = securityPolicy.EnforceOverlayMountPolicy(ctx, containerID, layerPaths, cl.ContainerRootPath); err != nil { return fmt.Errorf("overlay creation denied by policy: %w", err) } if h.hostMounts != nil { if err = h.hostMounts.AddOverlay(cl.ContainerRootPath, layerPaths, cl.ScratchPath); err != nil { return err } defer func() { if err != nil { _, _ = h.hostMounts.RemoveOverlay(cl.ContainerRootPath) } }() } // The sandbox root (the bundle dir) must be disk-backed so that // everything created under it - sandbox:// mounts, logs, etc. - lives // on disk rather than the UVM tmpfs. When the shim already mounts the // scratch disk at the bundle path, the scratch dir is nested under the // bundle and there is nothing to do. Otherwise (the scratch is mounted // elsewhere and the bundle would be tmpfs) bind a scratch directory // onto the bundle. This is decided purely from the paths, so V1 and V2 // are handled by the same rule. bundleDir := filepath.Dir(cl.ContainerRootPath) bindBundle := bundleNeedsScratchBind(bundleDir, cl.ScratchPath) if bindBundle { if bindErr := bindBundleToScratch(cl.ScratchPath, bundleDir); bindErr != nil { return bindErr } } // Correctness for policy transaction rollback: // MountLayer does two things - mkdir, then mount. On mount failure, the // target directory is cleaned up. Therefore we're clean in terms of // side effects. mountErr := overlay.MountLayer(ctx, layerPaths, upperdirPath, workdirPath, cl.ContainerRootPath, readonly) if mountErr != nil && bindBundle { _ = storage.UnmountPath(ctx, bundleDir, false) } return mountErr case guestrequest.RequestTypeRemove: // cl.ContainerID is not set on remove requests, but rego checks that we can // only umount previously mounted targets anyway if err = securityPolicy.EnforceOverlayUnmountPolicy(ctx, cl.ContainerRootPath); err != nil { return errors.Wrap(err, "overlay removal denied by policy") } // Check that no running container is using this overlay as its rootfs. if h.HasSecurityPolicy() && h.IsOverlayInUse(cl.ContainerRootPath) { return fmt.Errorf("overlay %q is in use by a running container", cl.ContainerRootPath) } if h.hostMounts != nil { var undoRemoveOverlay func() if undoRemoveOverlay, err = h.hostMounts.RemoveOverlay(cl.ContainerRootPath); err != nil { return err } defer func() { if err != nil && undoRemoveOverlay != nil { undoRemoveOverlay() } }() } // Note: storage.UnmountPath is a no-op if the path does not exist. err = storage.UnmountPath(ctx, cl.ContainerRootPath, true) if err != nil { h.setUVMInconsistent( fmt.Errorf("unmounting overlay at %s failed: %w", cl.ContainerRootPath, err), ) return err } return nil default: return newInvalidRequestTypeError(rt) } }) } // bundleNeedsScratchBind reports whether the container's bundle directory needs // to be bind-mounted onto its scratch disk to be disk-backed. It returns false // when there is no scratch, or when the scratch dir is already nested under the // bundle dir (i.e. the shim mounted the scratch disk at the bundle path, so the // bundle is already on disk). It returns true when the scratch is mounted // elsewhere and the bundle would otherwise live on the UVM tmpfs. The decision // is derived only from the paths, so it applies identically to both shims. func bundleNeedsScratchBind(bundleDir, scratchDir string) bool { if scratchDir == "" { return false } rel, err := filepath.Rel(bundleDir, scratchDir) if err != nil { // Unrelated paths: the scratch is not under the bundle, so it needs a bind. return true } // A bind is needed only when scratchDir is not nested under bundleDir, i.e. // when the relative path escapes the bundle (is ".." or starts with "../"). return rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) } // bindBundleToScratch backs a container bundle directory with a directory on the // container's scratch disk, so the sandbox root - and everything created under // it (sandbox:// mounts, logs, etc.) - is disk-backed rather than tmpfs-backed. // The bind is removed during container delete, before the scratch disk is // unmapped. func bindBundleToScratch(scratchPath, bundleDir string) error { src := filepath.Join(scratchPath, "bundle") if err := os.MkdirAll(src, 0755); err != nil { return fmt.Errorf("create bundle backing dir %s: %w", src, err) } if err := os.MkdirAll(bundleDir, 0755); err != nil { return fmt.Errorf("create bundle dir %s: %w", bundleDir, err) } if err := unix.Mount(src, bundleDir, "", unix.MS_BIND, ""); err != nil { return fmt.Errorf("bind bundle dir %s to %s: %w", src, bundleDir, err) } return nil } func modifyNetwork(ctx context.Context, rt guestrequest.RequestType, na *guestresource.LCOWNetworkAdapter) (err error) { switch rt { case guestrequest.RequestTypeAdd: ns := GetOrAddNetworkNamespace(na.NamespaceID) if err := ns.AddAdapter(ctx, na); err != nil { return err } // This code doesnt know if the namespace was already added to the // container or not so it must always call `Sync`. return ns.Sync(ctx) case guestrequest.RequestTypeRemove: ns, err := getNetworkNamespace(na.NamespaceID) if err != nil { log.G(ctx).WithFields(logrus.Fields{ logfields.NamespaceID: na.NamespaceID, "adapterID": na.ID, }).WithError(err).Warn("namespace not found for adapter removal, skipping") return nil } if err := ns.RemoveAdapter(ctx, na.ID); err != nil { return err } return nil default: return newInvalidRequestTypeError(rt) } } // processParamCommandLineToOCIArgs converts a CommandLine field from // ProcessParameters (a space separate argument string) into an array of string // arguments which can be used by an oci.Process. func processParamCommandLineToOCIArgs(commandLine string) ([]string, error) { args, err := shellwords.Parse(commandLine) if err != nil { return nil, errors.Wrapf(err, "failed to parse command line string \"%s\"", commandLine) } return args, nil } // processParamEnvToOCIEnv converts an Environment field from ProcessParameters // (a map from environment variable to value) into an array of environment // variable assignments (where each is in the form "=") which // can be used by an oci.Process. func processParamEnvToOCIEnv(environment map[string]string) []string { environmentList := make([]string, 0, len(environment)) for k, v := range environment { // TODO: Do we need to escape things like quotation marks in // environment variable values? environmentList = append(environmentList, fmt.Sprintf("%s=%s", k, v)) } return environmentList } // processOCIEnvToParam is the inverse of processParamEnvToOCIEnv func processOCIEnvToParam(envs []string) map[string]string { paramEnv := make(map[string]string, len(envs)) for _, env := range envs { parts := strings.SplitN(env, "=", 2) paramEnv[parts[0]] = parts[1] } return paramEnv } // isPrivilegedContainerCreationRequest returns if a given container // creation request would create a privileged container func isPrivilegedContainerCreationRequest(ctx context.Context, spec *specs.Spec) bool { return oci.ParseAnnotationsBool(ctx, spec.Annotations, annotations.LCOWPrivileged, false) } // createPodInUVM allocates a cgroup for a pod and registers it in the host. func (h *Host) createPodInUVM(sid string, pSpec *specs.Spec) error { cgroupPath := fmt.Sprintf(podCgroupPathFmt, sid) resources := &specs.LinuxResources{} if pSpec != nil && pSpec.Linux != nil && pSpec.Linux.Resources != nil { resources = pSpec.Linux.Resources } h.containersMutex.Lock() defer h.containersMutex.Unlock() if _, exists := h.pods[sid]; exists { return fmt.Errorf("pod %s already exists", sid) } cgroupControl, err := cgroup.NewManager(cgroupPath, resources) if err != nil { return fmt.Errorf("failed to create cgroup for pod %s: %w", sid, err) } h.pods[sid] = &pod{ sandboxID: sid, cgroupControl: cgroupControl, // Register the sandbox container with its pod. containers: map[string]bool{sid: true}, } logrus.WithFields(logrus.Fields{ "sandboxID": sid, "cgroupPath": cgroupPath, }).Info("pod created in UVM") return nil } func (h *Host) createContainerInPod(sandboxID string, containerID string) error { h.containersMutex.Lock() defer h.containersMutex.Unlock() // Find the pod corresponding to the sandboxID. pod, podExists := h.pods[sandboxID] if !podExists { return fmt.Errorf("pod %s does not exist for workload container %s", sandboxID, containerID) } // Validate that the container was not already registered. if _, exists := pod.containers[containerID]; exists { return fmt.Errorf("container %s already registered with pod %s", containerID, sandboxID) } // Register container with the pod. pod.containers[containerID] = true return nil } func (h *Host) DeleteContainerState(ctx context.Context, containerID string) error { if h.HasSecurityPolicy() { if err := checkValidContainerID(containerID, "container"); err != nil { return err } } if err := h.checkState(); err != nil { return err } c, err := h.GetInitializedContainer(containerID) if err != nil { return err } if h.HasSecurityPolicy() { if c.getStatus() == containerRunning { return errors.Errorf("Denied deleting state of a running container %q", containerID) } overlay := c.spec.Root.Path h.hostMounts.Lock() defer h.hostMounts.Unlock() if h.hostMounts.HasOverlayMountedAt(overlay) { return errors.Errorf("Denied deleting state of a container with a overlay mount still active") } } // remove container state regardless of delete's success defer h.RemoveContainer(containerID) if err = c.Delete(ctx); err != nil { return err } return nil }