kaslr.c 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * This file implements KASLR memory randomization for x86_64. It randomizes
  4. * the virtual address space of kernel memory regions (physical memory
  5. * mapping, vmalloc & vmemmap) for x86_64. This security feature mitigates
  6. * exploits relying on predictable kernel addresses.
  7. *
  8. * Entropy is generated using the KASLR early boot functions now shared in
  9. * the lib directory (originally written by Kees Cook). Randomization is
  10. * done on PGD & P4D/PUD page table levels to increase possible addresses.
  11. * The physical memory mapping code was adapted to support P4D/PUD level
  12. * virtual addresses. This implementation on the best configuration provides
  13. * 30,000 possible virtual addresses in average for each memory region.
  14. * An additional low memory page is used to ensure each CPU can start with
  15. * a PGD aligned virtual address (for realmode).
  16. *
  17. * The order of each memory region is not changed. The feature looks at
  18. * the available space for the regions based on different configuration
  19. * options and randomizes the base and space between each. The size of the
  20. * physical memory mapping is the available physical memory.
  21. */
  22. #include <linux/kernel.h>
  23. #include <linux/init.h>
  24. #include <linux/random.h>
  25. #include <asm/pgalloc.h>
  26. #include <asm/pgtable.h>
  27. #include <asm/setup.h>
  28. #include <asm/kaslr.h>
  29. #include "mm_internal.h"
  30. #define TB_SHIFT 40
  31. /*
  32. * The end address could depend on more configuration options to make the
  33. * highest amount of space for randomization available, but that's too hard
  34. * to keep straight and caused issues already.
  35. */
  36. static const unsigned long vaddr_end = CPU_ENTRY_AREA_BASE;
  37. /*
  38. * Memory regions randomized by KASLR (except modules that use a separate logic
  39. * earlier during boot). The list is ordered based on virtual addresses. This
  40. * order is kept after randomization.
  41. */
  42. static __initdata struct kaslr_memory_region {
  43. unsigned long *base;
  44. unsigned long size_tb;
  45. } kaslr_regions[] = {
  46. { &page_offset_base, 0 },
  47. { &vmalloc_base, 0 },
  48. { &vmemmap_base, 0 },
  49. };
  50. /* Get size in bytes used by the memory region */
  51. static inline unsigned long get_padding(struct kaslr_memory_region *region)
  52. {
  53. return (region->size_tb << TB_SHIFT);
  54. }
  55. /*
  56. * Apply no randomization if KASLR was disabled at boot or if KASAN
  57. * is enabled. KASAN shadow mappings rely on regions being PGD aligned.
  58. */
  59. static inline bool kaslr_memory_enabled(void)
  60. {
  61. return kaslr_enabled() && !IS_ENABLED(CONFIG_KASAN);
  62. }
  63. /* Initialize base and padding for each memory region randomized with KASLR */
  64. void __init kernel_randomize_memory(void)
  65. {
  66. size_t i;
  67. unsigned long vaddr_start, vaddr;
  68. unsigned long rand, memory_tb;
  69. struct rnd_state rand_state;
  70. unsigned long remain_entropy;
  71. unsigned long vmemmap_size;
  72. vaddr_start = pgtable_l5_enabled() ? __PAGE_OFFSET_BASE_L5 : __PAGE_OFFSET_BASE_L4;
  73. vaddr = vaddr_start;
  74. /*
  75. * These BUILD_BUG_ON checks ensure the memory layout is consistent
  76. * with the vaddr_start/vaddr_end variables. These checks are very
  77. * limited....
  78. */
  79. BUILD_BUG_ON(vaddr_start >= vaddr_end);
  80. BUILD_BUG_ON(vaddr_end != CPU_ENTRY_AREA_BASE);
  81. BUILD_BUG_ON(vaddr_end > __START_KERNEL_map);
  82. if (!kaslr_memory_enabled())
  83. return;
  84. kaslr_regions[0].size_tb = 1 << (MAX_PHYSMEM_BITS - TB_SHIFT);
  85. kaslr_regions[1].size_tb = VMALLOC_SIZE_TB;
  86. /*
  87. * Update Physical memory mapping to available and
  88. * add padding if needed (especially for memory hotplug support).
  89. */
  90. BUG_ON(kaslr_regions[0].base != &page_offset_base);
  91. memory_tb = DIV_ROUND_UP(max_pfn << PAGE_SHIFT, 1UL << TB_SHIFT) +
  92. CONFIG_RANDOMIZE_MEMORY_PHYSICAL_PADDING;
  93. /* Adapt phyiscal memory region size based on available memory */
  94. if (memory_tb < kaslr_regions[0].size_tb)
  95. kaslr_regions[0].size_tb = memory_tb;
  96. /*
  97. * Calculate the vmemmap region size in TBs, aligned to a TB
  98. * boundary.
  99. */
  100. vmemmap_size = (kaslr_regions[0].size_tb << (TB_SHIFT - PAGE_SHIFT)) *
  101. sizeof(struct page);
  102. kaslr_regions[2].size_tb = DIV_ROUND_UP(vmemmap_size, 1UL << TB_SHIFT);
  103. /* Calculate entropy available between regions */
  104. remain_entropy = vaddr_end - vaddr_start;
  105. for (i = 0; i < ARRAY_SIZE(kaslr_regions); i++)
  106. remain_entropy -= get_padding(&kaslr_regions[i]);
  107. prandom_seed_state(&rand_state, kaslr_get_random_long("Memory"));
  108. for (i = 0; i < ARRAY_SIZE(kaslr_regions); i++) {
  109. unsigned long entropy;
  110. /*
  111. * Select a random virtual address using the extra entropy
  112. * available.
  113. */
  114. entropy = remain_entropy / (ARRAY_SIZE(kaslr_regions) - i);
  115. prandom_bytes_state(&rand_state, &rand, sizeof(rand));
  116. if (pgtable_l5_enabled())
  117. entropy = (rand % (entropy + 1)) & P4D_MASK;
  118. else
  119. entropy = (rand % (entropy + 1)) & PUD_MASK;
  120. vaddr += entropy;
  121. *kaslr_regions[i].base = vaddr;
  122. /*
  123. * Jump the region and add a minimum padding based on
  124. * randomization alignment.
  125. */
  126. vaddr += get_padding(&kaslr_regions[i]);
  127. if (pgtable_l5_enabled())
  128. vaddr = round_up(vaddr + 1, P4D_SIZE);
  129. else
  130. vaddr = round_up(vaddr + 1, PUD_SIZE);
  131. remain_entropy -= entropy;
  132. }
  133. }
  134. static void __meminit init_trampoline_pud(void)
  135. {
  136. unsigned long paddr, paddr_next;
  137. pgd_t *pgd;
  138. pud_t *pud_page, *pud_page_tramp;
  139. int i;
  140. pud_page_tramp = alloc_low_page();
  141. paddr = 0;
  142. pgd = pgd_offset_k((unsigned long)__va(paddr));
  143. pud_page = (pud_t *) pgd_page_vaddr(*pgd);
  144. for (i = pud_index(paddr); i < PTRS_PER_PUD; i++, paddr = paddr_next) {
  145. pud_t *pud, *pud_tramp;
  146. unsigned long vaddr = (unsigned long)__va(paddr);
  147. pud_tramp = pud_page_tramp + pud_index(paddr);
  148. pud = pud_page + pud_index(vaddr);
  149. paddr_next = (paddr & PUD_MASK) + PUD_SIZE;
  150. *pud_tramp = *pud;
  151. }
  152. set_pgd(&trampoline_pgd_entry,
  153. __pgd(_KERNPG_TABLE | __pa(pud_page_tramp)));
  154. }
  155. static void __meminit init_trampoline_p4d(void)
  156. {
  157. unsigned long paddr, paddr_next;
  158. pgd_t *pgd;
  159. p4d_t *p4d_page, *p4d_page_tramp;
  160. int i;
  161. p4d_page_tramp = alloc_low_page();
  162. paddr = 0;
  163. pgd = pgd_offset_k((unsigned long)__va(paddr));
  164. p4d_page = (p4d_t *) pgd_page_vaddr(*pgd);
  165. for (i = p4d_index(paddr); i < PTRS_PER_P4D; i++, paddr = paddr_next) {
  166. p4d_t *p4d, *p4d_tramp;
  167. unsigned long vaddr = (unsigned long)__va(paddr);
  168. p4d_tramp = p4d_page_tramp + p4d_index(paddr);
  169. p4d = p4d_page + p4d_index(vaddr);
  170. paddr_next = (paddr & P4D_MASK) + P4D_SIZE;
  171. *p4d_tramp = *p4d;
  172. }
  173. set_pgd(&trampoline_pgd_entry,
  174. __pgd(_KERNPG_TABLE | __pa(p4d_page_tramp)));
  175. }
  176. /*
  177. * Create PGD aligned trampoline table to allow real mode initialization
  178. * of additional CPUs. Consume only 1 low memory page.
  179. */
  180. void __meminit init_trampoline(void)
  181. {
  182. if (!kaslr_memory_enabled()) {
  183. init_trampoline_default();
  184. return;
  185. }
  186. if (pgtable_l5_enabled())
  187. init_trampoline_p4d();
  188. else
  189. init_trampoline_pud();
  190. }