platform: add SlimVM platform

SlimVM is a lightweight VMX-based platform for gVisor that accelerates
sentry system calls by eliminating unnecessary user/kernel context
switches.

In a traditional gVisor + KVM setup, every guest syscall triggers a
full VM exit to userspace, then re-enters the kernel via a regular
syscall:

  gr0 -> hr0 (vm_exit) -> hr3 (ioctl return) -> hr0 (re-invoke syscall)

SlimVM eliminates this overhead. When the guest executes a VMCALL, the
VM exits into VMX root mode where SlimVM directly invokes the host
kernel's syscall handler, then immediately resumes the guest:

  gr0 -> hr0 (vmcall + direct function call)

SlimVM requires a companion kernel module that provides the /dev/slimvm
device interface. See https://github.com/antgroup/slimvm for the
kernel module and full documentation.

Co-developed-by: Chenggang <chenggang.qcg@antfin.com>
Co-developed-by: Robin Luk <lubin.lu@antgroup.com>
Co-developed-by: Tiwei Bie <tiwei.btw@antgroup.com>
Co-developed-by: Jianfeng Tan <henry.tjf@antgroup.com>
Co-developed-by: liushi.ls <liushi.ls@antfin.com>
Co-developed-by: Yong He <chenglang.hy@antgroup.com>
Co-developed-by: Aaron Lu <ziqian.lzq@antfin.com>
Co-developed-by: chris.zn <chris.zn@alibaba-inc.com>
Co-developed-by: Lai Jiangshan <jiangshan.ljs@antfin.com>
Co-developed-by: Lingfu <yupeng.chenyp@alibaba-inc.com>
Co-developed-by: Min Le <lemin.lm@antgroup.com>
Co-developed-by: Quan Xu <wutu.xq@alibaba-inc.com>
Co-developed-by: Zhang Haoyu <zhanghaoyu.zhy@alibaba-inc.com>
Co-developed-by: Dawei Shen <shendawei.sdw@antgroup.com>
Signed-off-by: Tianyu Zhou <albert.zty@antgroup.com>

Author: tianyuzhou95

pkg/hostarch/hostarch_arm64.go

@@ -29,9 +29,15 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// JumboPageSize is the 1GB jumbo page size.
+	JumboPageSize = 1 << JumboPageShift
+
 	// CacheLineSize is the size of the cache line.
 	CacheLineSize = 1 << CacheLineShift
 
+	// JumboPageShift is the binary log of jumbo page whose size is 1GB.
+	JumboPageShift = 30
+
 	// CacheLineShift is the binary log of the cache line size.
 	CacheLineShift = 6
 )

pkg/hostarch/hostarch_x86.go

@@ -26,6 +26,9 @@ const (
 	// HugePageSize is the system huge page size.
 	HugePageSize = 1 << HugePageShift
 
+	// JumboPageSize is the 1GB jumbo page size.
+	JumboPageSize = 1 << JumboPageShift
+
 	// CacheLineSize is the size of the cache line.
 	CacheLineSize = 1 << CacheLineShift
 
@@ -35,6 +38,9 @@ const (
 	// HugePageShift is the binary log of the system huge page size.
 	HugePageShift = 21
 
+	// JumboPageShift is the binary log of jumbo page whose size is 1GB.
+	JumboPageShift = 30
+
 	// CacheLineShift is the binary log of the cache line size.
 	CacheLineShift = 6
 )

pkg/hostarch/sizes_util.go

@@ -11,6 +11,7 @@ const (
 	PageMask      = PageSize - 1
 	HugePageMask  = HugePageSize - 1
 	CacheLineMask = CacheLineSize - 1
+	JumboPageMask = ^uintptr(JumboPageSize - 1)
 )
 
 type bytecount interface {

pkg/ring0/aarch64.go

@@ -102,5 +102,6 @@ const (
 const (
 	Syscall                 Vector = El0SyncSVC
 	PageFault               Vector = El0SyncDa
+	OOMException            Vector = El0Fiq
 	VirtualizationException Vector = El0ErrBounce
 )

pkg/ring0/defs.go

@@ -20,6 +20,18 @@ import (
 	"gvisor.dev/gvisor/pkg/sentry/arch/fpu"
 )
 
+const (
+	// CPUIntel is Intel CPU.
+	CPUIntel uint64 = iota
+
+	// CPUAMD is AMD (and compatible) CPU.
+	CPUAMD
+)
+
+var (
+	CPUVendor uint64
+)
+
 // Kernel is a global kernel object.
 //
 // This contains global state, shared by multiple CPUs.

pkg/ring0/defs_amd64.go

@@ -108,6 +108,9 @@ type kernelEntry struct {
 	// kernelCR3 is the cr3 used for sentry kernel.
 	kernelCR3 uintptr
 
+	// whether enable VMCALL
+	enableVMCALL uint64
+
 	// gdt is the CPU's descriptor table.
 	gdt descriptorTable
 
@@ -180,6 +183,14 @@ func (c *CPU) FaultAddr() uintptr {
 	return c.faultAddr
 }
 
+func (c *CPU) EnableVMCALL() {
+	c.enableVMCALL = 1
+}
+
+func (c *CPU) DisableVMCALL() {
+	c.enableVMCALL = 0
+}
+
 // SwitchArchOpts are embedded in SwitchOpts.
 type SwitchArchOpts struct {
 	// UserPCID indicates that the application PCID to be used on switch,

pkg/ring0/entry_amd64.go

@@ -117,6 +117,7 @@ func machineCheck()
 func simdFloatingPointException()
 func virtualizationException()
 func securityException()
+func oomException()
 func syscallInt80()
 
 // These returns the start address of the functions above.
@@ -145,6 +146,7 @@ func addrOfMachineCheck() uintptr
 func addrOfSimdFloatingPointException() uintptr
 func addrOfVirtualizationException() uintptr
 func addrOfSecurityException() uintptr
+func addrOfOOMException() uintptr
 func addrOfSyscallInt80() uintptr
 
 // Exception handler index.
@@ -170,5 +172,6 @@ var handlers = map[Vector]uintptr{
 	SIMDFloatingPointException: addrOfSimdFloatingPointException(),
 	VirtualizationException:    addrOfVirtualizationException(),
 	SecurityException:          addrOfSecurityException(),
+	OOMException:               addrOfOOMException(),
 	SyscallInt80:               addrOfSyscallInt80(),
 }

pkg/ring0/entry_amd64.s

@@ -33,6 +33,7 @@
 #define ENTRY_STACK_TOP  264 // +checkoffset . kernelEntry.stackTop
 #define ENTRY_CPU_SELF   272 // +checkoffset . kernelEntry.cpuSelf
 #define ENTRY_KERNEL_CR3 280 // +checkoffset . kernelEntry.kernelCR3
+#define ENTRY_ENABLE_VMCALL 288 // +checkoffset . kernelEntry.enableVMCALL
 
 // Bits.
 #define _RFLAGS_IF    512  // +checkconst . _RFLAGS_IF
@@ -61,9 +62,16 @@
 #define SIMDFloatingPointException 19 // +checkconst . SIMDFloatingPointException
 #define VirtualizationException    20 // +checkconst . VirtualizationException
 #define SecurityException          30 // +checkconst . SecurityException
+#define OOMException            32 // +checkconst . OOMException
 #define SyscallInt80               128 // +checkconst . SyscallInt80
 #define Syscall                    256 // +checkconst . Syscall
 
+#define SyscallExit       60         // +checkconst . SyscallExit
+#define SyscallExitGroup  231        // +checkconst . SyscallExitGroup
+#define SyscallRedPill    4294967295 // +checkconst . SyscallRedPill
+
+#define CPUIntel        0 // +checkconst . CPUIntel
+
 #define PTRACE_R15      0   // +checkoffset linux PtraceRegs.R15
 #define PTRACE_R14      8   // +checkoffset linux PtraceRegs.R14
 #define PTRACE_R13      16  // +checkoffset linux PtraceRegs.R13
@@ -160,6 +168,15 @@
 #define LOAD_KERNEL_STACK(entry) \
 	MOVQ ENTRY_STACK_TOP(entry), SP;
 
+// VMCALL do vmcall/vmmcal instruction
+#define VMCALL() \
+	CMPQ ·CPUVendor(SB), $CPUIntel; \
+	JE 2(PC); \
+	JMP 5(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
+	BYTE $0x0F; BYTE $0x01; BYTE $0xC1; \
+	JMP 4(PC); \ // vmmcall and vmcall will be treated as 3 independent instructions
+	BYTE $0x0F; BYTE $0x01; BYTE $0xD9;
+
 // ADDR_OF_FUNC defines a function named 'name' that returns the address of
 // 'symbol'.
 #define ADDR_OF_FUNC(name, symbol) \
@@ -488,6 +505,45 @@ sysenter_skip_gs:
 	RET
 
 kernel:
+	// Handle any syscalls from GR0 in HR3 when EnableVMCALL is false.
+	// Currently there are 2 use cases:
+	// 1. Using KVM platform.
+	// 2. Upgrading SlimVM platform. This is one such method to return M to
+	//    user mode (HR3) for upgrading platform.
+	CMPQ ENTRY_ENABLE_VMCALL(GS), $0
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallRedPill
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallExit
+	JE hr3_do_syscall
+
+	CMPQ AX, $SyscallExitGroup
+	JE hr3_do_syscall
+
+vmcall:
+	// handle syscall from GR0 in host kernel
+	// copy from "handle system calls from G0" part of __dune_syscall in libdune/dune.S
+	PUSHQ R11
+	POPFQ
+
+	CMPQ AX, $158 // arch_prctl syscall
+	JNE 3(PC)
+	CMPQ DI, $0x1002 //ARCH_SET_FS
+	JE arch_prctl_vmcall
+
+	VMCALL()
+	JMP *CX
+
+arch_prctl_vmcall:
+	VMCALL()
+	CMPQ AX, $0
+	JNE 2(PC)
+	MOVQ SI, CPU_REGISTERS+PTRACE_FS_BASE(GS)
+	JMP *CX
+
+hr3_do_syscall:
 	// We can't restore the original stack, but we can access the registers
 	// in the CPU state directly. No need for temporary juggling.
 	MOVQ AX,  ENTRY_SCRATCH0(GS)
@@ -705,4 +761,5 @@ EXCEPTION_WITHOUT_ERROR(MachineCheck, ·machineCheck(SB), ·addrOfMachineCheck(S
 EXCEPTION_WITHOUT_ERROR(SIMDFloatingPointException, ·simdFloatingPointException(SB), ·addrOfSimdFloatingPointException(SB))
 EXCEPTION_WITHOUT_ERROR(VirtualizationException, ·virtualizationException(SB), ·addrOfVirtualizationException(SB))
 EXCEPTION_WITH_ERROR(SecurityException, ·securityException(SB), ·addrOfSecurityException(SB))
+EXCEPTION_WITH_ERROR(OOMException, ·oomException(SB), ·addrOfOOMException(SB))
 EXCEPTION_WITHOUT_ERROR(SyscallInt80, ·syscallInt80(SB), ·addrOfSyscallInt80(SB))

pkg/ring0/entry_arm64.s

@@ -64,6 +64,7 @@
 #define El0ErrNMI               34 // +checkconst . El0ErrNMI
 #define PageFault               23 // +checkconst . PageFault
 #define Syscall                 22 // +checkconst . Syscall
+#define OOMException         10 // +checkconst . OOMException
 #define VirtualizationException 35 // +checkconst . VirtualizationException
 
 #define PTRACE_REGS     0 // +checkoffset linux PtraceRegs.Regs
@@ -765,7 +766,10 @@ TEXT ·El0_irq(SB),NOSPLIT,$0
 	B ·Shutdown(SB)
 
 TEXT ·El0_fiq(SB),NOSPLIT,$0
-	B ·Shutdown(SB)
+	KERNEL_ENTRY_FROM_EL0
+	MOVD $0x8400000a, R8
+	HVC $0
+	EXCEPTION_EL0(OOMException)
 
 TEXT ·El0_error(SB),NOSPLIT,$0
 	KERNEL_ENTRY_FROM_EL0

pkg/ring0/kernel_amd64.go

@@ -324,3 +324,30 @@ func startGo(c *CPU) {
 func ReadCR2() uintptr {
 	return readCR2()
 }
+
+//go:noinline
+//go:nosplit
+func (c *CPU) PrefaultIDT() uint32 {
+	return c.kernel.globalIDT[0].bits[0] + c.kernel.globalIDT[_NR_INTERRUPTS-1].bits[3]
+}
+
+// SetCPUIDFaulting sets CPUID faulting per the boolean value.
+//
+// True is returned if faulting could be set.
+//
+//go:nosplit
+func SetCPUIDFaulting(on bool) bool {
+	// Per the SDM (Vol 3, Table 2-43), PLATFORM_INFO bit 31 denotes support
+	// for CPUID faulting, and we enable and disable via the MISC_FEATURES MSR.
+	if rdmsr(_MSR_PLATFORM_INFO)&_PLATFORM_INFO_CPUID_FAULT != 0 {
+		features := rdmsr(_MSR_MISC_FEATURES)
+		if on {
+			features |= _MISC_FEATURE_CPUID_TRAP
+		} else {
+			features &^= _MISC_FEATURE_CPUID_TRAP
+		}
+		wrmsr(_MSR_MISC_FEATURES, features)
+		return true // Setting successful.
+	}
+	return false
+}