pgalloc.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Page table allocation functions
  4. *
  5. * Copyright IBM Corp. 2016
  6. * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
  7. */
  8. #include <linux/sysctl.h>
  9. #include <linux/slab.h>
  10. #include <linux/mm.h>
  11. #include <asm/mmu_context.h>
  12. #include <asm/page-states.h>
  13. #include <asm/pgalloc.h>
  14. #include <asm/gmap.h>
  15. #include <asm/tlb.h>
  16. #include <asm/tlbflush.h>
  17. #ifdef CONFIG_PGSTE
  18. int page_table_allocate_pgste = 0;
  19. EXPORT_SYMBOL(page_table_allocate_pgste);
  20. static struct ctl_table page_table_sysctl[] = {
  21. {
  22. .procname = "allocate_pgste",
  23. .data = &page_table_allocate_pgste,
  24. .maxlen = sizeof(int),
  25. .mode = S_IRUGO | S_IWUSR,
  26. .proc_handler = proc_dointvec_minmax,
  27. .extra1 = SYSCTL_ZERO,
  28. .extra2 = SYSCTL_ONE,
  29. },
  30. };
  31. static int __init page_table_register_sysctl(void)
  32. {
  33. return register_sysctl("vm", page_table_sysctl) ? 0 : -ENOMEM;
  34. }
  35. __initcall(page_table_register_sysctl);
  36. #endif /* CONFIG_PGSTE */
  37. unsigned long *crst_table_alloc(struct mm_struct *mm)
  38. {
  39. gfp_t gfp = GFP_KERNEL_ACCOUNT;
  40. struct ptdesc *ptdesc;
  41. unsigned long *table;
  42. if (mm == &init_mm)
  43. gfp &= ~__GFP_ACCOUNT;
  44. ptdesc = pagetable_alloc(gfp, CRST_ALLOC_ORDER);
  45. if (!ptdesc)
  46. return NULL;
  47. table = ptdesc_to_virt(ptdesc);
  48. __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);
  49. return table;
  50. }
  51. void crst_table_free(struct mm_struct *mm, unsigned long *table)
  52. {
  53. if (!table)
  54. return;
  55. pagetable_free(virt_to_ptdesc(table));
  56. }
  57. static void __crst_table_upgrade(void *arg)
  58. {
  59. struct mm_struct *mm = arg;
  60. /* change all active ASCEs to avoid the creation of new TLBs */
  61. if (current->active_mm == mm) {
  62. get_lowcore()->user_asce.val = mm->context.asce;
  63. local_ctl_load(7, &get_lowcore()->user_asce);
  64. }
  65. __tlb_flush_local();
  66. }
  67. int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
  68. {
  69. unsigned long *pgd = NULL, *p4d = NULL, *__pgd;
  70. unsigned long asce_limit = mm->context.asce_limit;
  71. /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */
  72. VM_BUG_ON(asce_limit < _REGION2_SIZE);
  73. if (end <= asce_limit)
  74. return 0;
  75. if (asce_limit == _REGION2_SIZE) {
  76. p4d = crst_table_alloc(mm);
  77. if (unlikely(!p4d))
  78. goto err_p4d;
  79. crst_table_init(p4d, _REGION2_ENTRY_EMPTY);
  80. }
  81. if (end > _REGION1_SIZE) {
  82. pgd = crst_table_alloc(mm);
  83. if (unlikely(!pgd))
  84. goto err_pgd;
  85. crst_table_init(pgd, _REGION1_ENTRY_EMPTY);
  86. }
  87. spin_lock_bh(&mm->page_table_lock);
  88. /*
  89. * This routine gets called with mmap_lock lock held and there is
  90. * no reason to optimize for the case of otherwise. However, if
  91. * that would ever change, the below check will let us know.
  92. */
  93. VM_BUG_ON(asce_limit != mm->context.asce_limit);
  94. if (p4d) {
  95. __pgd = (unsigned long *) mm->pgd;
  96. p4d_populate(mm, (p4d_t *) p4d, (pud_t *) __pgd);
  97. mm->pgd = (pgd_t *) p4d;
  98. mm->context.asce_limit = _REGION1_SIZE;
  99. mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
  100. _ASCE_USER_BITS | _ASCE_TYPE_REGION2;
  101. mm_inc_nr_puds(mm);
  102. }
  103. if (pgd) {
  104. __pgd = (unsigned long *) mm->pgd;
  105. pgd_populate(mm, (pgd_t *) pgd, (p4d_t *) __pgd);
  106. mm->pgd = (pgd_t *) pgd;
  107. mm->context.asce_limit = TASK_SIZE_MAX;
  108. mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
  109. _ASCE_USER_BITS | _ASCE_TYPE_REGION1;
  110. }
  111. spin_unlock_bh(&mm->page_table_lock);
  112. on_each_cpu(__crst_table_upgrade, mm, 0);
  113. return 0;
  114. err_pgd:
  115. crst_table_free(mm, p4d);
  116. err_p4d:
  117. return -ENOMEM;
  118. }
  119. #ifdef CONFIG_PGSTE
  120. struct ptdesc *page_table_alloc_pgste(struct mm_struct *mm)
  121. {
  122. struct ptdesc *ptdesc;
  123. u64 *table;
  124. ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0);
  125. if (ptdesc) {
  126. table = (u64 *)ptdesc_to_virt(ptdesc);
  127. __arch_set_page_dat(table, 1);
  128. memset64(table, _PAGE_INVALID, PTRS_PER_PTE);
  129. memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
  130. }
  131. return ptdesc;
  132. }
  133. void page_table_free_pgste(struct ptdesc *ptdesc)
  134. {
  135. pagetable_free(ptdesc);
  136. }
  137. #endif /* CONFIG_PGSTE */
  138. unsigned long *page_table_alloc(struct mm_struct *mm)
  139. {
  140. gfp_t gfp = GFP_KERNEL_ACCOUNT;
  141. struct ptdesc *ptdesc;
  142. unsigned long *table;
  143. if (mm == &init_mm)
  144. gfp &= ~__GFP_ACCOUNT;
  145. ptdesc = pagetable_alloc(gfp, 0);
  146. if (!ptdesc)
  147. return NULL;
  148. if (!pagetable_pte_ctor(ptdesc)) {
  149. pagetable_free(ptdesc);
  150. return NULL;
  151. }
  152. table = ptdesc_to_virt(ptdesc);
  153. __arch_set_page_dat(table, 1);
  154. /* pt_list is used by gmap only */
  155. INIT_LIST_HEAD(&ptdesc->pt_list);
  156. memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
  157. memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
  158. return table;
  159. }
  160. static void pagetable_pte_dtor_free(struct ptdesc *ptdesc)
  161. {
  162. pagetable_pte_dtor(ptdesc);
  163. pagetable_free(ptdesc);
  164. }
  165. void page_table_free(struct mm_struct *mm, unsigned long *table)
  166. {
  167. struct ptdesc *ptdesc = virt_to_ptdesc(table);
  168. pagetable_pte_dtor_free(ptdesc);
  169. }
  170. void __tlb_remove_table(void *table)
  171. {
  172. struct ptdesc *ptdesc = virt_to_ptdesc(table);
  173. struct page *page = ptdesc_page(ptdesc);
  174. if (compound_order(page) == CRST_ALLOC_ORDER) {
  175. /* pmd, pud, or p4d */
  176. pagetable_free(ptdesc);
  177. return;
  178. }
  179. pagetable_pte_dtor_free(ptdesc);
  180. }
  181. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  182. static void pte_free_now(struct rcu_head *head)
  183. {
  184. struct ptdesc *ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
  185. pagetable_pte_dtor_free(ptdesc);
  186. }
  187. void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
  188. {
  189. struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
  190. call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
  191. }
  192. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  193. /*
  194. * Base infrastructure required to generate basic asces, region, segment,
  195. * and page tables that do not make use of enhanced features like EDAT1.
  196. */
  197. static struct kmem_cache *base_pgt_cache;
  198. static unsigned long *base_pgt_alloc(void)
  199. {
  200. unsigned long *table;
  201. table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);
  202. if (table)
  203. memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
  204. return table;
  205. }
  206. static void base_pgt_free(unsigned long *table)
  207. {
  208. kmem_cache_free(base_pgt_cache, table);
  209. }
  210. static unsigned long *base_crst_alloc(unsigned long val)
  211. {
  212. unsigned long *table;
  213. struct ptdesc *ptdesc;
  214. ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER);
  215. if (!ptdesc)
  216. return NULL;
  217. table = ptdesc_address(ptdesc);
  218. crst_table_init(table, val);
  219. return table;
  220. }
  221. static void base_crst_free(unsigned long *table)
  222. {
  223. if (!table)
  224. return;
  225. pagetable_free(virt_to_ptdesc(table));
  226. }
  227. #define BASE_ADDR_END_FUNC(NAME, SIZE) \
  228. static inline unsigned long base_##NAME##_addr_end(unsigned long addr, \
  229. unsigned long end) \
  230. { \
  231. unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1); \
  232. \
  233. return (next - 1) < (end - 1) ? next : end; \
  234. }
  235. BASE_ADDR_END_FUNC(page, _PAGE_SIZE)
  236. BASE_ADDR_END_FUNC(segment, _SEGMENT_SIZE)
  237. BASE_ADDR_END_FUNC(region3, _REGION3_SIZE)
  238. BASE_ADDR_END_FUNC(region2, _REGION2_SIZE)
  239. BASE_ADDR_END_FUNC(region1, _REGION1_SIZE)
  240. static inline unsigned long base_lra(unsigned long address)
  241. {
  242. unsigned long real;
  243. asm volatile(
  244. " lra %0,0(%1)\n"
  245. : "=d" (real) : "a" (address) : "cc");
  246. return real;
  247. }
  248. static int base_page_walk(unsigned long *origin, unsigned long addr,
  249. unsigned long end, int alloc)
  250. {
  251. unsigned long *pte, next;
  252. if (!alloc)
  253. return 0;
  254. pte = origin;
  255. pte += (addr & _PAGE_INDEX) >> _PAGE_SHIFT;
  256. do {
  257. next = base_page_addr_end(addr, end);
  258. *pte = base_lra(addr);
  259. } while (pte++, addr = next, addr < end);
  260. return 0;
  261. }
  262. static int base_segment_walk(unsigned long *origin, unsigned long addr,
  263. unsigned long end, int alloc)
  264. {
  265. unsigned long *ste, next, *table;
  266. int rc;
  267. ste = origin;
  268. ste += (addr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
  269. do {
  270. next = base_segment_addr_end(addr, end);
  271. if (*ste & _SEGMENT_ENTRY_INVALID) {
  272. if (!alloc)
  273. continue;
  274. table = base_pgt_alloc();
  275. if (!table)
  276. return -ENOMEM;
  277. *ste = __pa(table) | _SEGMENT_ENTRY;
  278. }
  279. table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);
  280. rc = base_page_walk(table, addr, next, alloc);
  281. if (rc)
  282. return rc;
  283. if (!alloc)
  284. base_pgt_free(table);
  285. cond_resched();
  286. } while (ste++, addr = next, addr < end);
  287. return 0;
  288. }
  289. static int base_region3_walk(unsigned long *origin, unsigned long addr,
  290. unsigned long end, int alloc)
  291. {
  292. unsigned long *rtte, next, *table;
  293. int rc;
  294. rtte = origin;
  295. rtte += (addr & _REGION3_INDEX) >> _REGION3_SHIFT;
  296. do {
  297. next = base_region3_addr_end(addr, end);
  298. if (*rtte & _REGION_ENTRY_INVALID) {
  299. if (!alloc)
  300. continue;
  301. table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
  302. if (!table)
  303. return -ENOMEM;
  304. *rtte = __pa(table) | _REGION3_ENTRY;
  305. }
  306. table = __va(*rtte & _REGION_ENTRY_ORIGIN);
  307. rc = base_segment_walk(table, addr, next, alloc);
  308. if (rc)
  309. return rc;
  310. if (!alloc)
  311. base_crst_free(table);
  312. } while (rtte++, addr = next, addr < end);
  313. return 0;
  314. }
  315. static int base_region2_walk(unsigned long *origin, unsigned long addr,
  316. unsigned long end, int alloc)
  317. {
  318. unsigned long *rste, next, *table;
  319. int rc;
  320. rste = origin;
  321. rste += (addr & _REGION2_INDEX) >> _REGION2_SHIFT;
  322. do {
  323. next = base_region2_addr_end(addr, end);
  324. if (*rste & _REGION_ENTRY_INVALID) {
  325. if (!alloc)
  326. continue;
  327. table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
  328. if (!table)
  329. return -ENOMEM;
  330. *rste = __pa(table) | _REGION2_ENTRY;
  331. }
  332. table = __va(*rste & _REGION_ENTRY_ORIGIN);
  333. rc = base_region3_walk(table, addr, next, alloc);
  334. if (rc)
  335. return rc;
  336. if (!alloc)
  337. base_crst_free(table);
  338. } while (rste++, addr = next, addr < end);
  339. return 0;
  340. }
  341. static int base_region1_walk(unsigned long *origin, unsigned long addr,
  342. unsigned long end, int alloc)
  343. {
  344. unsigned long *rfte, next, *table;
  345. int rc;
  346. rfte = origin;
  347. rfte += (addr & _REGION1_INDEX) >> _REGION1_SHIFT;
  348. do {
  349. next = base_region1_addr_end(addr, end);
  350. if (*rfte & _REGION_ENTRY_INVALID) {
  351. if (!alloc)
  352. continue;
  353. table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
  354. if (!table)
  355. return -ENOMEM;
  356. *rfte = __pa(table) | _REGION1_ENTRY;
  357. }
  358. table = __va(*rfte & _REGION_ENTRY_ORIGIN);
  359. rc = base_region2_walk(table, addr, next, alloc);
  360. if (rc)
  361. return rc;
  362. if (!alloc)
  363. base_crst_free(table);
  364. } while (rfte++, addr = next, addr < end);
  365. return 0;
  366. }
  367. /**
  368. * base_asce_free - free asce and tables returned from base_asce_alloc()
  369. * @asce: asce to be freed
  370. *
  371. * Frees all region, segment, and page tables that were allocated with a
  372. * corresponding base_asce_alloc() call.
  373. */
  374. void base_asce_free(unsigned long asce)
  375. {
  376. unsigned long *table = __va(asce & _ASCE_ORIGIN);
  377. if (!asce)
  378. return;
  379. switch (asce & _ASCE_TYPE_MASK) {
  380. case _ASCE_TYPE_SEGMENT:
  381. base_segment_walk(table, 0, _REGION3_SIZE, 0);
  382. break;
  383. case _ASCE_TYPE_REGION3:
  384. base_region3_walk(table, 0, _REGION2_SIZE, 0);
  385. break;
  386. case _ASCE_TYPE_REGION2:
  387. base_region2_walk(table, 0, _REGION1_SIZE, 0);
  388. break;
  389. case _ASCE_TYPE_REGION1:
  390. base_region1_walk(table, 0, TASK_SIZE_MAX, 0);
  391. break;
  392. }
  393. base_crst_free(table);
  394. }
  395. static int base_pgt_cache_init(void)
  396. {
  397. static DEFINE_MUTEX(base_pgt_cache_mutex);
  398. unsigned long sz = _PAGE_TABLE_SIZE;
  399. if (base_pgt_cache)
  400. return 0;
  401. mutex_lock(&base_pgt_cache_mutex);
  402. if (!base_pgt_cache)
  403. base_pgt_cache = kmem_cache_create("base_pgt", sz, sz, 0, NULL);
  404. mutex_unlock(&base_pgt_cache_mutex);
  405. return base_pgt_cache ? 0 : -ENOMEM;
  406. }
  407. /**
  408. * base_asce_alloc - create kernel mapping without enhanced DAT features
  409. * @addr: virtual start address of kernel mapping
  410. * @num_pages: number of consecutive pages
  411. *
  412. * Generate an asce, including all required region, segment and page tables,
  413. * that can be used to access the virtual kernel mapping. The difference is
  414. * that the returned asce does not make use of any enhanced DAT features like
  415. * e.g. large pages. This is required for some I/O functions that pass an
  416. * asce, like e.g. some service call requests.
  417. *
  418. * Note: the returned asce may NEVER be attached to any cpu. It may only be
  419. * used for I/O requests. tlb entries that might result because the
  420. * asce was attached to a cpu won't be cleared.
  421. */
  422. unsigned long base_asce_alloc(unsigned long addr, unsigned long num_pages)
  423. {
  424. unsigned long asce, *table, end;
  425. int rc;
  426. if (base_pgt_cache_init())
  427. return 0;
  428. end = addr + num_pages * PAGE_SIZE;
  429. if (end <= _REGION3_SIZE) {
  430. table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
  431. if (!table)
  432. return 0;
  433. rc = base_segment_walk(table, addr, end, 1);
  434. asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;
  435. } else if (end <= _REGION2_SIZE) {
  436. table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
  437. if (!table)
  438. return 0;
  439. rc = base_region3_walk(table, addr, end, 1);
  440. asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
  441. } else if (end <= _REGION1_SIZE) {
  442. table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
  443. if (!table)
  444. return 0;
  445. rc = base_region2_walk(table, addr, end, 1);
  446. asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
  447. } else {
  448. table = base_crst_alloc(_REGION1_ENTRY_EMPTY);
  449. if (!table)
  450. return 0;
  451. rc = base_region1_walk(table, addr, end, 1);
  452. asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;
  453. }
  454. if (rc) {
  455. base_asce_free(asce);
  456. asce = 0;
  457. }
  458. return asce;
  459. }