Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __KVM_X86_VMX_INSN_H #define __KVM_X86_VMX_INSN_H #include <linux/nospec.h> #include <asm/kvm_host.h> #include <asm/vmx.h> #include "evmcs.h" #include "vmcs.h" #define __ex(x) __kvm_handle_fault_on_reboot(x) asmlinkage void vmread_error(unsigned long field, bool fault); void vmwrite_error(unsigned long field, unsigned long value); void vmclear_error(struct vmcs *vmcs, u64 phys_addr); void vmptrld_error(struct vmcs *vmcs, u64 phys_addr); void invvpid_error(unsigned long ext, u16 vpid, gva_t gva); void invept_error(unsigned long ext, u64 eptp, gpa_t gpa); static __always_inline void vmcs_check16(unsigned long field) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, "16-bit accessor invalid for 64-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, "16-bit accessor invalid for 64-bit high field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, "16-bit accessor invalid for 32-bit high field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, "16-bit accessor invalid for natural width field"); } static __always_inline void vmcs_check32(unsigned long field) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, "32-bit accessor invalid for 16-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, "32-bit accessor invalid for natural width field"); } static __always_inline void vmcs_check64(unsigned long field) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, "64-bit accessor invalid for 16-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, "64-bit accessor invalid for 64-bit high field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, "64-bit accessor invalid for 32-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000, "64-bit accessor invalid for natural width field"); } static __always_inline void vmcs_checkl(unsigned long field) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0, "Natural width accessor invalid for 16-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000, "Natural width accessor invalid for 64-bit field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001, "Natural width accessor invalid for 64-bit high field"); BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000, "Natural width accessor invalid for 32-bit field"); } static __always_inline unsigned long __vmcs_readl(unsigned long field) { unsigned long value; asm volatile("1: vmread %2, %1\n\t" ".byte 0x3e\n\t" /* branch taken hint */ "ja 3f\n\t" "mov %2, %%" _ASM_ARG1 "\n\t" "xor %%" _ASM_ARG2 ", %%" _ASM_ARG2 "\n\t" "2: call vmread_error\n\t" "xor %k1, %k1\n\t" "3:\n\t" ".pushsection .fixup, \"ax\"\n\t" "4: mov %2, %%" _ASM_ARG1 "\n\t" "mov $1, %%" _ASM_ARG2 "\n\t" "jmp 2b\n\t" ".popsection\n\t" _ASM_EXTABLE(1b, 4b) : ASM_CALL_CONSTRAINT, "=r"(value) : "r"(field) : "cc"); return value; } static __always_inline u16 vmcs_read16(unsigned long field) { vmcs_check16(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_read16(field); return __vmcs_readl(field); } static __always_inline u32 vmcs_read32(unsigned long field) { vmcs_check32(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_read32(field); return __vmcs_readl(field); } static __always_inline u64 vmcs_read64(unsigned long field) { vmcs_check64(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_read64(field); #ifdef CONFIG_X86_64 return __vmcs_readl(field); #else return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32); #endif } static __always_inline unsigned long vmcs_readl(unsigned long field) { vmcs_checkl(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_read64(field); return __vmcs_readl(field); } #define vmx_asm1(insn, op1, error_args...) \ do { \ asm_volatile_goto("1: " __stringify(insn) " %0\n\t" \ ".byte 0x2e\n\t" /* branch not taken hint */ \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ : : op1 : "cc" : error, fault); \ return; \ error: \ insn##_error(error_args); \ return; \ fault: \ kvm_spurious_fault(); \ } while (0) #define vmx_asm2(insn, op1, op2, error_args...) \ do { \ asm_volatile_goto("1: " __stringify(insn) " %1, %0\n\t" \ ".byte 0x2e\n\t" /* branch not taken hint */ \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ : : op1, op2 : "cc" : error, fault); \ return; \ error: \ insn##_error(error_args); \ return; \ fault: \ kvm_spurious_fault(); \ } while (0) static __always_inline void __vmcs_writel(unsigned long field, unsigned long value) { vmx_asm2(vmwrite, "r"(field), "rm"(value), field, value); } static __always_inline void vmcs_write16(unsigned long field, u16 value) { vmcs_check16(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write16(field, value); __vmcs_writel(field, value); } static __always_inline void vmcs_write32(unsigned long field, u32 value) { vmcs_check32(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write32(field, value); __vmcs_writel(field, value); } static __always_inline void vmcs_write64(unsigned long field, u64 value) { vmcs_check64(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write64(field, value); __vmcs_writel(field, value); #ifndef CONFIG_X86_64 __vmcs_writel(field+1, value >> 32); #endif } static __always_inline void vmcs_writel(unsigned long field, unsigned long value) { vmcs_checkl(field); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write64(field, value); __vmcs_writel(field, value); } static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, "vmcs_clear_bits does not support 64-bit fields"); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write32(field, evmcs_read32(field) & ~mask); __vmcs_writel(field, __vmcs_readl(field) & ~mask); } static __always_inline void vmcs_set_bits(unsigned long field, u32 mask) { BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000, "vmcs_set_bits does not support 64-bit fields"); if (static_branch_unlikely(&enable_evmcs)) return evmcs_write32(field, evmcs_read32(field) | mask); __vmcs_writel(field, __vmcs_readl(field) | mask); } static inline void vmcs_clear(struct vmcs *vmcs) { u64 phys_addr = __pa(vmcs); vmx_asm1(vmclear, "m"(phys_addr), vmcs, phys_addr); } static inline void vmcs_load(struct vmcs *vmcs) { u64 phys_addr = __pa(vmcs); if (static_branch_unlikely(&enable_evmcs)) return evmcs_load(phys_addr); vmx_asm1(vmptrld, "m"(phys_addr), vmcs, phys_addr); } static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva) { struct { u64 vpid : 16; u64 rsvd : 48; u64 gva; } operand = { vpid, 0, gva }; vmx_asm2(invvpid, "r"(ext), "m"(operand), ext, vpid, gva); } static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa) { struct { u64 eptp, gpa; } operand = {eptp, gpa}; vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa); } static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr) { if (vpid == 0) return true; if (cpu_has_vmx_invvpid_individual_addr()) { __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); return true; } return false; } static inline void vpid_sync_vcpu_single(int vpid) { if (vpid == 0) return; if (cpu_has_vmx_invvpid_single()) __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); } static inline void vpid_sync_vcpu_global(void) { if (cpu_has_vmx_invvpid_global()) __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); } static inline void vpid_sync_context(int vpid) { if (cpu_has_vmx_invvpid_single()) vpid_sync_vcpu_single(vpid); else vpid_sync_vcpu_global(); } static inline void ept_sync_global(void) { __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0); } static inline void ept_sync_context(u64 eptp) { if (cpu_has_vmx_invept_context()) __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0); else ept_sync_global(); } #endif /* __KVM_X86_VMX_INSN_H */ |