mpx.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * mpx.c - Memory Protection eXtensions
  4. *
  5. * Copyright (c) 2014, Intel Corporation.
  6. * Qiaowei Ren <qiaowei.ren@intel.com>
  7. * Dave Hansen <dave.hansen@intel.com>
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/mm_types.h>
  12. #include <linux/syscalls.h>
  13. #include <linux/sched/sysctl.h>
  14. #include <asm/insn.h>
  15. #include <asm/insn-eval.h>
  16. #include <asm/mman.h>
  17. #include <asm/mmu_context.h>
  18. #include <asm/mpx.h>
  19. #include <asm/processor.h>
  20. #include <asm/fpu/internal.h>
  21. #define CREATE_TRACE_POINTS
  22. #include <asm/trace/mpx.h>
  23. static inline unsigned long mpx_bd_size_bytes(struct mm_struct *mm)
  24. {
  25. if (is_64bit_mm(mm))
  26. return MPX_BD_SIZE_BYTES_64;
  27. else
  28. return MPX_BD_SIZE_BYTES_32;
  29. }
  30. static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm)
  31. {
  32. if (is_64bit_mm(mm))
  33. return MPX_BT_SIZE_BYTES_64;
  34. else
  35. return MPX_BT_SIZE_BYTES_32;
  36. }
  37. /*
  38. * This is really a simplified "vm_mmap". it only handles MPX
  39. * bounds tables (the bounds directory is user-allocated).
  40. */
  41. static unsigned long mpx_mmap(unsigned long len)
  42. {
  43. struct mm_struct *mm = current->mm;
  44. unsigned long addr, populate;
  45. /* Only bounds table can be allocated here */
  46. if (len != mpx_bt_size_bytes(mm))
  47. return -EINVAL;
  48. down_write(&mm->mmap_sem);
  49. addr = do_mmap(NULL, 0, len, PROT_READ | PROT_WRITE,
  50. MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate, NULL);
  51. up_write(&mm->mmap_sem);
  52. if (populate)
  53. mm_populate(addr, populate);
  54. return addr;
  55. }
  56. static int mpx_insn_decode(struct insn *insn,
  57. struct pt_regs *regs)
  58. {
  59. unsigned char buf[MAX_INSN_SIZE];
  60. int x86_64 = !test_thread_flag(TIF_IA32);
  61. int not_copied;
  62. int nr_copied;
  63. not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf));
  64. nr_copied = sizeof(buf) - not_copied;
  65. /*
  66. * The decoder _should_ fail nicely if we pass it a short buffer.
  67. * But, let's not depend on that implementation detail. If we
  68. * did not get anything, just error out now.
  69. */
  70. if (!nr_copied)
  71. return -EFAULT;
  72. insn_init(insn, buf, nr_copied, x86_64);
  73. insn_get_length(insn);
  74. /*
  75. * copy_from_user() tries to get as many bytes as we could see in
  76. * the largest possible instruction. If the instruction we are
  77. * after is shorter than that _and_ we attempt to copy from
  78. * something unreadable, we might get a short read. This is OK
  79. * as long as the read did not stop in the middle of the
  80. * instruction. Check to see if we got a partial instruction.
  81. */
  82. if (nr_copied < insn->length)
  83. return -EFAULT;
  84. insn_get_opcode(insn);
  85. /*
  86. * We only _really_ need to decode bndcl/bndcn/bndcu
  87. * Error out on anything else.
  88. */
  89. if (insn->opcode.bytes[0] != 0x0f)
  90. goto bad_opcode;
  91. if ((insn->opcode.bytes[1] != 0x1a) &&
  92. (insn->opcode.bytes[1] != 0x1b))
  93. goto bad_opcode;
  94. return 0;
  95. bad_opcode:
  96. return -EINVAL;
  97. }
  98. /*
  99. * If a bounds overflow occurs then a #BR is generated. This
  100. * function decodes MPX instructions to get violation address
  101. * and set this address into extended struct siginfo.
  102. *
  103. * Note that this is not a super precise way of doing this.
  104. * Userspace could have, by the time we get here, written
  105. * anything it wants in to the instructions. We can not
  106. * trust anything about it. They might not be valid
  107. * instructions or might encode invalid registers, etc...
  108. *
  109. * The caller is expected to kfree() the returned siginfo_t.
  110. */
  111. siginfo_t *mpx_generate_siginfo(struct pt_regs *regs)
  112. {
  113. const struct mpx_bndreg_state *bndregs;
  114. const struct mpx_bndreg *bndreg;
  115. siginfo_t *info = NULL;
  116. struct insn insn;
  117. uint8_t bndregno;
  118. int err;
  119. err = mpx_insn_decode(&insn, regs);
  120. if (err)
  121. goto err_out;
  122. /*
  123. * We know at this point that we are only dealing with
  124. * MPX instructions.
  125. */
  126. insn_get_modrm(&insn);
  127. bndregno = X86_MODRM_REG(insn.modrm.value);
  128. if (bndregno > 3) {
  129. err = -EINVAL;
  130. goto err_out;
  131. }
  132. /* get bndregs field from current task's xsave area */
  133. bndregs = get_xsave_field_ptr(XFEATURE_MASK_BNDREGS);
  134. if (!bndregs) {
  135. err = -EINVAL;
  136. goto err_out;
  137. }
  138. /* now go select the individual register in the set of 4 */
  139. bndreg = &bndregs->bndreg[bndregno];
  140. info = kzalloc(sizeof(*info), GFP_KERNEL);
  141. if (!info) {
  142. err = -ENOMEM;
  143. goto err_out;
  144. }
  145. /*
  146. * The registers are always 64-bit, but the upper 32
  147. * bits are ignored in 32-bit mode. Also, note that the
  148. * upper bounds are architecturally represented in 1's
  149. * complement form.
  150. *
  151. * The 'unsigned long' cast is because the compiler
  152. * complains when casting from integers to different-size
  153. * pointers.
  154. */
  155. info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound;
  156. info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound;
  157. info->si_addr_lsb = 0;
  158. info->si_signo = SIGSEGV;
  159. info->si_errno = 0;
  160. info->si_code = SEGV_BNDERR;
  161. info->si_addr = insn_get_addr_ref(&insn, regs);
  162. /*
  163. * We were not able to extract an address from the instruction,
  164. * probably because there was something invalid in it.
  165. */
  166. if (info->si_addr == (void __user *)-1) {
  167. err = -EINVAL;
  168. goto err_out;
  169. }
  170. trace_mpx_bounds_register_exception(info->si_addr, bndreg);
  171. return info;
  172. err_out:
  173. /* info might be NULL, but kfree() handles that */
  174. kfree(info);
  175. return ERR_PTR(err);
  176. }
  177. static __user void *mpx_get_bounds_dir(void)
  178. {
  179. const struct mpx_bndcsr *bndcsr;
  180. if (!cpu_feature_enabled(X86_FEATURE_MPX))
  181. return MPX_INVALID_BOUNDS_DIR;
  182. /*
  183. * The bounds directory pointer is stored in a register
  184. * only accessible if we first do an xsave.
  185. */
  186. bndcsr = get_xsave_field_ptr(XFEATURE_MASK_BNDCSR);
  187. if (!bndcsr)
  188. return MPX_INVALID_BOUNDS_DIR;
  189. /*
  190. * Make sure the register looks valid by checking the
  191. * enable bit.
  192. */
  193. if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG))
  194. return MPX_INVALID_BOUNDS_DIR;
  195. /*
  196. * Lastly, mask off the low bits used for configuration
  197. * flags, and return the address of the bounds table.
  198. */
  199. return (void __user *)(unsigned long)
  200. (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK);
  201. }
  202. int mpx_enable_management(void)
  203. {
  204. void __user *bd_base = MPX_INVALID_BOUNDS_DIR;
  205. struct mm_struct *mm = current->mm;
  206. int ret = 0;
  207. /*
  208. * runtime in the userspace will be responsible for allocation of
  209. * the bounds directory. Then, it will save the base of the bounds
  210. * directory into XSAVE/XRSTOR Save Area and enable MPX through
  211. * XRSTOR instruction.
  212. *
  213. * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is
  214. * expected to be relatively expensive. Storing the bounds
  215. * directory here means that we do not have to do xsave in the
  216. * unmap path; we can just use mm->context.bd_addr instead.
  217. */
  218. bd_base = mpx_get_bounds_dir();
  219. down_write(&mm->mmap_sem);
  220. /* MPX doesn't support addresses above 47 bits yet. */
  221. if (find_vma(mm, DEFAULT_MAP_WINDOW)) {
  222. pr_warn_once("%s (%d): MPX cannot handle addresses "
  223. "above 47-bits. Disabling.",
  224. current->comm, current->pid);
  225. ret = -ENXIO;
  226. goto out;
  227. }
  228. mm->context.bd_addr = bd_base;
  229. if (mm->context.bd_addr == MPX_INVALID_BOUNDS_DIR)
  230. ret = -ENXIO;
  231. out:
  232. up_write(&mm->mmap_sem);
  233. return ret;
  234. }
  235. int mpx_disable_management(void)
  236. {
  237. struct mm_struct *mm = current->mm;
  238. if (!cpu_feature_enabled(X86_FEATURE_MPX))
  239. return -ENXIO;
  240. down_write(&mm->mmap_sem);
  241. mm->context.bd_addr = MPX_INVALID_BOUNDS_DIR;
  242. up_write(&mm->mmap_sem);
  243. return 0;
  244. }
  245. static int mpx_cmpxchg_bd_entry(struct mm_struct *mm,
  246. unsigned long *curval,
  247. unsigned long __user *addr,
  248. unsigned long old_val, unsigned long new_val)
  249. {
  250. int ret;
  251. /*
  252. * user_atomic_cmpxchg_inatomic() actually uses sizeof()
  253. * the pointer that we pass to it to figure out how much
  254. * data to cmpxchg. We have to be careful here not to
  255. * pass a pointer to a 64-bit data type when we only want
  256. * a 32-bit copy.
  257. */
  258. if (is_64bit_mm(mm)) {
  259. ret = user_atomic_cmpxchg_inatomic(curval,
  260. addr, old_val, new_val);
  261. } else {
  262. u32 uninitialized_var(curval_32);
  263. u32 old_val_32 = old_val;
  264. u32 new_val_32 = new_val;
  265. u32 __user *addr_32 = (u32 __user *)addr;
  266. ret = user_atomic_cmpxchg_inatomic(&curval_32,
  267. addr_32, old_val_32, new_val_32);
  268. *curval = curval_32;
  269. }
  270. return ret;
  271. }
  272. /*
  273. * With 32-bit mode, a bounds directory is 4MB, and the size of each
  274. * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB,
  275. * and the size of each bounds table is 4MB.
  276. */
  277. static int allocate_bt(struct mm_struct *mm, long __user *bd_entry)
  278. {
  279. unsigned long expected_old_val = 0;
  280. unsigned long actual_old_val = 0;
  281. unsigned long bt_addr;
  282. unsigned long bd_new_entry;
  283. int ret = 0;
  284. /*
  285. * Carve the virtual space out of userspace for the new
  286. * bounds table:
  287. */
  288. bt_addr = mpx_mmap(mpx_bt_size_bytes(mm));
  289. if (IS_ERR((void *)bt_addr))
  290. return PTR_ERR((void *)bt_addr);
  291. /*
  292. * Set the valid flag (kinda like _PAGE_PRESENT in a pte)
  293. */
  294. bd_new_entry = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
  295. /*
  296. * Go poke the address of the new bounds table in to the
  297. * bounds directory entry out in userspace memory. Note:
  298. * we may race with another CPU instantiating the same table.
  299. * In that case the cmpxchg will see an unexpected
  300. * 'actual_old_val'.
  301. *
  302. * This can fault, but that's OK because we do not hold
  303. * mmap_sem at this point, unlike some of the other part
  304. * of the MPX code that have to pagefault_disable().
  305. */
  306. ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, bd_entry,
  307. expected_old_val, bd_new_entry);
  308. if (ret)
  309. goto out_unmap;
  310. /*
  311. * The user_atomic_cmpxchg_inatomic() will only return nonzero
  312. * for faults, *not* if the cmpxchg itself fails. Now we must
  313. * verify that the cmpxchg itself completed successfully.
  314. */
  315. /*
  316. * We expected an empty 'expected_old_val', but instead found
  317. * an apparently valid entry. Assume we raced with another
  318. * thread to instantiate this table and desclare succecss.
  319. */
  320. if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) {
  321. ret = 0;
  322. goto out_unmap;
  323. }
  324. /*
  325. * We found a non-empty bd_entry but it did not have the
  326. * VALID_FLAG set. Return an error which will result in
  327. * a SEGV since this probably means that somebody scribbled
  328. * some invalid data in to a bounds table.
  329. */
  330. if (expected_old_val != actual_old_val) {
  331. ret = -EINVAL;
  332. goto out_unmap;
  333. }
  334. trace_mpx_new_bounds_table(bt_addr);
  335. return 0;
  336. out_unmap:
  337. vm_munmap(bt_addr, mpx_bt_size_bytes(mm));
  338. return ret;
  339. }
  340. /*
  341. * When a BNDSTX instruction attempts to save bounds to a bounds
  342. * table, it will first attempt to look up the table in the
  343. * first-level bounds directory. If it does not find a table in
  344. * the directory, a #BR is generated and we get here in order to
  345. * allocate a new table.
  346. *
  347. * With 32-bit mode, the size of BD is 4MB, and the size of each
  348. * bound table is 16KB. With 64-bit mode, the size of BD is 2GB,
  349. * and the size of each bound table is 4MB.
  350. */
  351. static int do_mpx_bt_fault(void)
  352. {
  353. unsigned long bd_entry, bd_base;
  354. const struct mpx_bndcsr *bndcsr;
  355. struct mm_struct *mm = current->mm;
  356. bndcsr = get_xsave_field_ptr(XFEATURE_MASK_BNDCSR);
  357. if (!bndcsr)
  358. return -EINVAL;
  359. /*
  360. * Mask off the preserve and enable bits
  361. */
  362. bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK;
  363. /*
  364. * The hardware provides the address of the missing or invalid
  365. * entry via BNDSTATUS, so we don't have to go look it up.
  366. */
  367. bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK;
  368. /*
  369. * Make sure the directory entry is within where we think
  370. * the directory is.
  371. */
  372. if ((bd_entry < bd_base) ||
  373. (bd_entry >= bd_base + mpx_bd_size_bytes(mm)))
  374. return -EINVAL;
  375. return allocate_bt(mm, (long __user *)bd_entry);
  376. }
  377. int mpx_handle_bd_fault(void)
  378. {
  379. /*
  380. * Userspace never asked us to manage the bounds tables,
  381. * so refuse to help.
  382. */
  383. if (!kernel_managing_mpx_tables(current->mm))
  384. return -EINVAL;
  385. return do_mpx_bt_fault();
  386. }
  387. /*
  388. * A thin wrapper around get_user_pages(). Returns 0 if the
  389. * fault was resolved or -errno if not.
  390. */
  391. static int mpx_resolve_fault(long __user *addr, int write)
  392. {
  393. long gup_ret;
  394. int nr_pages = 1;
  395. gup_ret = get_user_pages((unsigned long)addr, nr_pages,
  396. write ? FOLL_WRITE : 0, NULL, NULL);
  397. /*
  398. * get_user_pages() returns number of pages gotten.
  399. * 0 means we failed to fault in and get anything,
  400. * probably because 'addr' is bad.
  401. */
  402. if (!gup_ret)
  403. return -EFAULT;
  404. /* Other error, return it */
  405. if (gup_ret < 0)
  406. return gup_ret;
  407. /* must have gup'd a page and gup_ret>0, success */
  408. return 0;
  409. }
  410. static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct *mm,
  411. unsigned long bd_entry)
  412. {
  413. unsigned long bt_addr = bd_entry;
  414. int align_to_bytes;
  415. /*
  416. * Bit 0 in a bt_entry is always the valid bit.
  417. */
  418. bt_addr &= ~MPX_BD_ENTRY_VALID_FLAG;
  419. /*
  420. * Tables are naturally aligned at 8-byte boundaries
  421. * on 64-bit and 4-byte boundaries on 32-bit. The
  422. * documentation makes it appear that the low bits
  423. * are ignored by the hardware, so we do the same.
  424. */
  425. if (is_64bit_mm(mm))
  426. align_to_bytes = 8;
  427. else
  428. align_to_bytes = 4;
  429. bt_addr &= ~(align_to_bytes-1);
  430. return bt_addr;
  431. }
  432. /*
  433. * We only want to do a 4-byte get_user() on 32-bit. Otherwise,
  434. * we might run off the end of the bounds table if we are on
  435. * a 64-bit kernel and try to get 8 bytes.
  436. */
  437. static int get_user_bd_entry(struct mm_struct *mm, unsigned long *bd_entry_ret,
  438. long __user *bd_entry_ptr)
  439. {
  440. u32 bd_entry_32;
  441. int ret;
  442. if (is_64bit_mm(mm))
  443. return get_user(*bd_entry_ret, bd_entry_ptr);
  444. /*
  445. * Note that get_user() uses the type of the *pointer* to
  446. * establish the size of the get, not the destination.
  447. */
  448. ret = get_user(bd_entry_32, (u32 __user *)bd_entry_ptr);
  449. *bd_entry_ret = bd_entry_32;
  450. return ret;
  451. }
  452. /*
  453. * Get the base of bounds tables pointed by specific bounds
  454. * directory entry.
  455. */
  456. static int get_bt_addr(struct mm_struct *mm,
  457. long __user *bd_entry_ptr,
  458. unsigned long *bt_addr_result)
  459. {
  460. int ret;
  461. int valid_bit;
  462. unsigned long bd_entry;
  463. unsigned long bt_addr;
  464. if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr)))
  465. return -EFAULT;
  466. while (1) {
  467. int need_write = 0;
  468. pagefault_disable();
  469. ret = get_user_bd_entry(mm, &bd_entry, bd_entry_ptr);
  470. pagefault_enable();
  471. if (!ret)
  472. break;
  473. if (ret == -EFAULT)
  474. ret = mpx_resolve_fault(bd_entry_ptr, need_write);
  475. /*
  476. * If we could not resolve the fault, consider it
  477. * userspace's fault and error out.
  478. */
  479. if (ret)
  480. return ret;
  481. }
  482. valid_bit = bd_entry & MPX_BD_ENTRY_VALID_FLAG;
  483. bt_addr = mpx_bd_entry_to_bt_addr(mm, bd_entry);
  484. /*
  485. * When the kernel is managing bounds tables, a bounds directory
  486. * entry will either have a valid address (plus the valid bit)
  487. * *OR* be completely empty. If we see a !valid entry *and* some
  488. * data in the address field, we know something is wrong. This
  489. * -EINVAL return will cause a SIGSEGV.
  490. */
  491. if (!valid_bit && bt_addr)
  492. return -EINVAL;
  493. /*
  494. * Do we have an completely zeroed bt entry? That is OK. It
  495. * just means there was no bounds table for this memory. Make
  496. * sure to distinguish this from -EINVAL, which will cause
  497. * a SEGV.
  498. */
  499. if (!valid_bit)
  500. return -ENOENT;
  501. *bt_addr_result = bt_addr;
  502. return 0;
  503. }
  504. static inline int bt_entry_size_bytes(struct mm_struct *mm)
  505. {
  506. if (is_64bit_mm(mm))
  507. return MPX_BT_ENTRY_BYTES_64;
  508. else
  509. return MPX_BT_ENTRY_BYTES_32;
  510. }
  511. /*
  512. * Take a virtual address and turns it in to the offset in bytes
  513. * inside of the bounds table where the bounds table entry
  514. * controlling 'addr' can be found.
  515. */
  516. static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct *mm,
  517. unsigned long addr)
  518. {
  519. unsigned long bt_table_nr_entries;
  520. unsigned long offset = addr;
  521. if (is_64bit_mm(mm)) {
  522. /* Bottom 3 bits are ignored on 64-bit */
  523. offset >>= 3;
  524. bt_table_nr_entries = MPX_BT_NR_ENTRIES_64;
  525. } else {
  526. /* Bottom 2 bits are ignored on 32-bit */
  527. offset >>= 2;
  528. bt_table_nr_entries = MPX_BT_NR_ENTRIES_32;
  529. }
  530. /*
  531. * We know the size of the table in to which we are
  532. * indexing, and we have eliminated all the low bits
  533. * which are ignored for indexing.
  534. *
  535. * Mask out all the high bits which we do not need
  536. * to index in to the table. Note that the tables
  537. * are always powers of two so this gives us a proper
  538. * mask.
  539. */
  540. offset &= (bt_table_nr_entries-1);
  541. /*
  542. * We now have an entry offset in terms of *entries* in
  543. * the table. We need to scale it back up to bytes.
  544. */
  545. offset *= bt_entry_size_bytes(mm);
  546. return offset;
  547. }
  548. /*
  549. * How much virtual address space does a single bounds
  550. * directory entry cover?
  551. *
  552. * Note, we need a long long because 4GB doesn't fit in
  553. * to a long on 32-bit.
  554. */
  555. static inline unsigned long bd_entry_virt_space(struct mm_struct *mm)
  556. {
  557. unsigned long long virt_space;
  558. unsigned long long GB = (1ULL << 30);
  559. /*
  560. * This covers 32-bit emulation as well as 32-bit kernels
  561. * running on 64-bit hardware.
  562. */
  563. if (!is_64bit_mm(mm))
  564. return (4ULL * GB) / MPX_BD_NR_ENTRIES_32;
  565. /*
  566. * 'x86_virt_bits' returns what the hardware is capable
  567. * of, and returns the full >32-bit address space when
  568. * running 32-bit kernels on 64-bit hardware.
  569. */
  570. virt_space = (1ULL << boot_cpu_data.x86_virt_bits);
  571. return virt_space / MPX_BD_NR_ENTRIES_64;
  572. }
  573. /*
  574. * Free the backing physical pages of bounds table 'bt_addr'.
  575. * Assume start...end is within that bounds table.
  576. */
  577. static noinline int zap_bt_entries_mapping(struct mm_struct *mm,
  578. unsigned long bt_addr,
  579. unsigned long start_mapping, unsigned long end_mapping)
  580. {
  581. struct vm_area_struct *vma;
  582. unsigned long addr, len;
  583. unsigned long start;
  584. unsigned long end;
  585. /*
  586. * if we 'end' on a boundary, the offset will be 0 which
  587. * is not what we want. Back it up a byte to get the
  588. * last bt entry. Then once we have the entry itself,
  589. * move 'end' back up by the table entry size.
  590. */
  591. start = bt_addr + mpx_get_bt_entry_offset_bytes(mm, start_mapping);
  592. end = bt_addr + mpx_get_bt_entry_offset_bytes(mm, end_mapping - 1);
  593. /*
  594. * Move end back up by one entry. Among other things
  595. * this ensures that it remains page-aligned and does
  596. * not screw up zap_page_range()
  597. */
  598. end += bt_entry_size_bytes(mm);
  599. /*
  600. * Find the first overlapping vma. If vma->vm_start > start, there
  601. * will be a hole in the bounds table. This -EINVAL return will
  602. * cause a SIGSEGV.
  603. */
  604. vma = find_vma(mm, start);
  605. if (!vma || vma->vm_start > start)
  606. return -EINVAL;
  607. /*
  608. * A NUMA policy on a VM_MPX VMA could cause this bounds table to
  609. * be split. So we need to look across the entire 'start -> end'
  610. * range of this bounds table, find all of the VM_MPX VMAs, and
  611. * zap only those.
  612. */
  613. addr = start;
  614. while (vma && vma->vm_start < end) {
  615. /*
  616. * We followed a bounds directory entry down
  617. * here. If we find a non-MPX VMA, that's bad,
  618. * so stop immediately and return an error. This
  619. * probably results in a SIGSEGV.
  620. */
  621. if (!(vma->vm_flags & VM_MPX))
  622. return -EINVAL;
  623. len = min(vma->vm_end, end) - addr;
  624. zap_page_range(vma, addr, len);
  625. trace_mpx_unmap_zap(addr, addr+len);
  626. vma = vma->vm_next;
  627. addr = vma->vm_start;
  628. }
  629. return 0;
  630. }
  631. static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm,
  632. unsigned long addr)
  633. {
  634. /*
  635. * There are several ways to derive the bd offsets. We
  636. * use the following approach here:
  637. * 1. We know the size of the virtual address space
  638. * 2. We know the number of entries in a bounds table
  639. * 3. We know that each entry covers a fixed amount of
  640. * virtual address space.
  641. * So, we can just divide the virtual address by the
  642. * virtual space used by one entry to determine which
  643. * entry "controls" the given virtual address.
  644. */
  645. if (is_64bit_mm(mm)) {
  646. int bd_entry_size = 8; /* 64-bit pointer */
  647. /*
  648. * Take the 64-bit addressing hole in to account.
  649. */
  650. addr &= ((1UL << boot_cpu_data.x86_virt_bits) - 1);
  651. return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
  652. } else {
  653. int bd_entry_size = 4; /* 32-bit pointer */
  654. /*
  655. * 32-bit has no hole so this case needs no mask
  656. */
  657. return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
  658. }
  659. /*
  660. * The two return calls above are exact copies. If we
  661. * pull out a single copy and put it in here, gcc won't
  662. * realize that we're doing a power-of-2 divide and use
  663. * shifts. It uses a real divide. If we put them up
  664. * there, it manages to figure it out (gcc 4.8.3).
  665. */
  666. }
  667. static int unmap_entire_bt(struct mm_struct *mm,
  668. long __user *bd_entry, unsigned long bt_addr)
  669. {
  670. unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
  671. unsigned long uninitialized_var(actual_old_val);
  672. int ret;
  673. while (1) {
  674. int need_write = 1;
  675. unsigned long cleared_bd_entry = 0;
  676. pagefault_disable();
  677. ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val,
  678. bd_entry, expected_old_val, cleared_bd_entry);
  679. pagefault_enable();
  680. if (!ret)
  681. break;
  682. if (ret == -EFAULT)
  683. ret = mpx_resolve_fault(bd_entry, need_write);
  684. /*
  685. * If we could not resolve the fault, consider it
  686. * userspace's fault and error out.
  687. */
  688. if (ret)
  689. return ret;
  690. }
  691. /*
  692. * The cmpxchg was performed, check the results.
  693. */
  694. if (actual_old_val != expected_old_val) {
  695. /*
  696. * Someone else raced with us to unmap the table.
  697. * That is OK, since we were both trying to do
  698. * the same thing. Declare success.
  699. */
  700. if (!actual_old_val)
  701. return 0;
  702. /*
  703. * Something messed with the bounds directory
  704. * entry. We hold mmap_sem for read or write
  705. * here, so it could not be a _new_ bounds table
  706. * that someone just allocated. Something is
  707. * wrong, so pass up the error and SIGSEGV.
  708. */
  709. return -EINVAL;
  710. }
  711. /*
  712. * Note, we are likely being called under do_munmap() already. To
  713. * avoid recursion, do_munmap() will check whether it comes
  714. * from one bounds table through VM_MPX flag.
  715. */
  716. return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm), NULL);
  717. }
  718. static int try_unmap_single_bt(struct mm_struct *mm,
  719. unsigned long start, unsigned long end)
  720. {
  721. struct vm_area_struct *next;
  722. struct vm_area_struct *prev;
  723. /*
  724. * "bta" == Bounds Table Area: the area controlled by the
  725. * bounds table that we are unmapping.
  726. */
  727. unsigned long bta_start_vaddr = start & ~(bd_entry_virt_space(mm)-1);
  728. unsigned long bta_end_vaddr = bta_start_vaddr + bd_entry_virt_space(mm);
  729. unsigned long uninitialized_var(bt_addr);
  730. void __user *bde_vaddr;
  731. int ret;
  732. /*
  733. * We already unlinked the VMAs from the mm's rbtree so 'start'
  734. * is guaranteed to be in a hole. This gets us the first VMA
  735. * before the hole in to 'prev' and the next VMA after the hole
  736. * in to 'next'.
  737. */
  738. next = find_vma_prev(mm, start, &prev);
  739. /*
  740. * Do not count other MPX bounds table VMAs as neighbors.
  741. * Although theoretically possible, we do not allow bounds
  742. * tables for bounds tables so our heads do not explode.
  743. * If we count them as neighbors here, we may end up with
  744. * lots of tables even though we have no actual table
  745. * entries in use.
  746. */
  747. while (next && (next->vm_flags & VM_MPX))
  748. next = next->vm_next;
  749. while (prev && (prev->vm_flags & VM_MPX))
  750. prev = prev->vm_prev;
  751. /*
  752. * We know 'start' and 'end' lie within an area controlled
  753. * by a single bounds table. See if there are any other
  754. * VMAs controlled by that bounds table. If there are not
  755. * then we can "expand" the are we are unmapping to possibly
  756. * cover the entire table.
  757. */
  758. next = find_vma_prev(mm, start, &prev);
  759. if ((!prev || prev->vm_end <= bta_start_vaddr) &&
  760. (!next || next->vm_start >= bta_end_vaddr)) {
  761. /*
  762. * No neighbor VMAs controlled by same bounds
  763. * table. Try to unmap the whole thing
  764. */
  765. start = bta_start_vaddr;
  766. end = bta_end_vaddr;
  767. }
  768. bde_vaddr = mm->context.bd_addr + mpx_get_bd_entry_offset(mm, start);
  769. ret = get_bt_addr(mm, bde_vaddr, &bt_addr);
  770. /*
  771. * No bounds table there, so nothing to unmap.
  772. */
  773. if (ret == -ENOENT) {
  774. ret = 0;
  775. return 0;
  776. }
  777. if (ret)
  778. return ret;
  779. /*
  780. * We are unmapping an entire table. Either because the
  781. * unmap that started this whole process was large enough
  782. * to cover an entire table, or that the unmap was small
  783. * but was the area covered by a bounds table.
  784. */
  785. if ((start == bta_start_vaddr) &&
  786. (end == bta_end_vaddr))
  787. return unmap_entire_bt(mm, bde_vaddr, bt_addr);
  788. return zap_bt_entries_mapping(mm, bt_addr, start, end);
  789. }
  790. static int mpx_unmap_tables(struct mm_struct *mm,
  791. unsigned long start, unsigned long end)
  792. {
  793. unsigned long one_unmap_start;
  794. trace_mpx_unmap_search(start, end);
  795. one_unmap_start = start;
  796. while (one_unmap_start < end) {
  797. int ret;
  798. unsigned long next_unmap_start = ALIGN(one_unmap_start+1,
  799. bd_entry_virt_space(mm));
  800. unsigned long one_unmap_end = end;
  801. /*
  802. * if the end is beyond the current bounds table,
  803. * move it back so we only deal with a single one
  804. * at a time
  805. */
  806. if (one_unmap_end > next_unmap_start)
  807. one_unmap_end = next_unmap_start;
  808. ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end);
  809. if (ret)
  810. return ret;
  811. one_unmap_start = next_unmap_start;
  812. }
  813. return 0;
  814. }
  815. /*
  816. * Free unused bounds tables covered in a virtual address region being
  817. * munmap()ed. Assume end > start.
  818. *
  819. * This function will be called by do_munmap(), and the VMAs covering
  820. * the virtual address region start...end have already been split if
  821. * necessary, and the 'vma' is the first vma in this range (start -> end).
  822. */
  823. void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
  824. unsigned long start, unsigned long end)
  825. {
  826. int ret;
  827. /*
  828. * Refuse to do anything unless userspace has asked
  829. * the kernel to help manage the bounds tables,
  830. */
  831. if (!kernel_managing_mpx_tables(current->mm))
  832. return;
  833. /*
  834. * This will look across the entire 'start -> end' range,
  835. * and find all of the non-VM_MPX VMAs.
  836. *
  837. * To avoid recursion, if a VM_MPX vma is found in the range
  838. * (start->end), we will not continue follow-up work. This
  839. * recursion represents having bounds tables for bounds tables,
  840. * which should not occur normally. Being strict about it here
  841. * helps ensure that we do not have an exploitable stack overflow.
  842. */
  843. do {
  844. if (vma->vm_flags & VM_MPX)
  845. return;
  846. vma = vma->vm_next;
  847. } while (vma && vma->vm_start < end);
  848. ret = mpx_unmap_tables(mm, start, end);
  849. if (ret)
  850. force_sig(SIGSEGV, current);
  851. }
  852. /* MPX cannot handle addresses above 47 bits yet. */
  853. unsigned long mpx_unmapped_area_check(unsigned long addr, unsigned long len,
  854. unsigned long flags)
  855. {
  856. if (!kernel_managing_mpx_tables(current->mm))
  857. return addr;
  858. if (addr + len <= DEFAULT_MAP_WINDOW)
  859. return addr;
  860. if (flags & MAP_FIXED)
  861. return -ENOMEM;
  862. /*
  863. * Requested len is larger than the whole area we're allowed to map in.
  864. * Resetting hinting address wouldn't do much good -- fail early.
  865. */
  866. if (len > DEFAULT_MAP_WINDOW)
  867. return -ENOMEM;
  868. /* Look for unmap area within DEFAULT_MAP_WINDOW */
  869. return 0;
  870. }