jump_label.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806
  1. /*
  2. * jump label support
  3. *
  4. * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
  5. * Copyright (C) 2011 Peter Zijlstra
  6. *
  7. */
  8. #include <linux/memory.h>
  9. #include <linux/uaccess.h>
  10. #include <linux/module.h>
  11. #include <linux/list.h>
  12. #include <linux/slab.h>
  13. #include <linux/sort.h>
  14. #include <linux/err.h>
  15. #include <linux/static_key.h>
  16. #include <linux/jump_label_ratelimit.h>
  17. #include <linux/bug.h>
  18. #include <linux/cpu.h>
  19. #include <asm/sections.h>
  20. /* mutex to protect coming/going of the the jump_label table */
  21. static DEFINE_MUTEX(jump_label_mutex);
  22. void jump_label_lock(void)
  23. {
  24. mutex_lock(&jump_label_mutex);
  25. }
  26. void jump_label_unlock(void)
  27. {
  28. mutex_unlock(&jump_label_mutex);
  29. }
  30. static int jump_label_cmp(const void *a, const void *b)
  31. {
  32. const struct jump_entry *jea = a;
  33. const struct jump_entry *jeb = b;
  34. if (jea->key < jeb->key)
  35. return -1;
  36. if (jea->key > jeb->key)
  37. return 1;
  38. return 0;
  39. }
  40. static void
  41. jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
  42. {
  43. unsigned long size;
  44. size = (((unsigned long)stop - (unsigned long)start)
  45. / sizeof(struct jump_entry));
  46. sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
  47. }
  48. static void jump_label_update(struct static_key *key);
  49. /*
  50. * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
  51. * The use of 'atomic_read()' requires atomic.h and its problematic for some
  52. * kernel headers such as kernel.h and others. Since static_key_count() is not
  53. * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
  54. * to have it be a function here. Similarly, for 'static_key_enable()' and
  55. * 'static_key_disable()', which require bug.h. This should allow jump_label.h
  56. * to be included from most/all places for CONFIG_JUMP_LABEL.
  57. */
  58. int static_key_count(struct static_key *key)
  59. {
  60. /*
  61. * -1 means the first static_key_slow_inc() is in progress.
  62. * static_key_enabled() must return true, so return 1 here.
  63. */
  64. int n = atomic_read(&key->enabled);
  65. return n >= 0 ? n : 1;
  66. }
  67. EXPORT_SYMBOL_GPL(static_key_count);
  68. void static_key_slow_inc_cpuslocked(struct static_key *key)
  69. {
  70. int v, v1;
  71. STATIC_KEY_CHECK_USE(key);
  72. lockdep_assert_cpus_held();
  73. /*
  74. * Careful if we get concurrent static_key_slow_inc() calls;
  75. * later calls must wait for the first one to _finish_ the
  76. * jump_label_update() process. At the same time, however,
  77. * the jump_label_update() call below wants to see
  78. * static_key_enabled(&key) for jumps to be updated properly.
  79. *
  80. * So give a special meaning to negative key->enabled: it sends
  81. * static_key_slow_inc() down the slow path, and it is non-zero
  82. * so it counts as "enabled" in jump_label_update(). Note that
  83. * atomic_inc_unless_negative() checks >= 0, so roll our own.
  84. */
  85. for (v = atomic_read(&key->enabled); v > 0; v = v1) {
  86. v1 = atomic_cmpxchg(&key->enabled, v, v + 1);
  87. if (likely(v1 == v))
  88. return;
  89. }
  90. jump_label_lock();
  91. if (atomic_read(&key->enabled) == 0) {
  92. atomic_set(&key->enabled, -1);
  93. jump_label_update(key);
  94. /*
  95. * Ensure that if the above cmpxchg loop observes our positive
  96. * value, it must also observe all the text changes.
  97. */
  98. atomic_set_release(&key->enabled, 1);
  99. } else {
  100. atomic_inc(&key->enabled);
  101. }
  102. jump_label_unlock();
  103. }
  104. void static_key_slow_inc(struct static_key *key)
  105. {
  106. cpus_read_lock();
  107. static_key_slow_inc_cpuslocked(key);
  108. cpus_read_unlock();
  109. }
  110. EXPORT_SYMBOL_GPL(static_key_slow_inc);
  111. void static_key_enable_cpuslocked(struct static_key *key)
  112. {
  113. STATIC_KEY_CHECK_USE(key);
  114. lockdep_assert_cpus_held();
  115. if (atomic_read(&key->enabled) > 0) {
  116. WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
  117. return;
  118. }
  119. jump_label_lock();
  120. if (atomic_read(&key->enabled) == 0) {
  121. atomic_set(&key->enabled, -1);
  122. jump_label_update(key);
  123. /*
  124. * See static_key_slow_inc().
  125. */
  126. atomic_set_release(&key->enabled, 1);
  127. }
  128. jump_label_unlock();
  129. }
  130. EXPORT_SYMBOL_GPL(static_key_enable_cpuslocked);
  131. void static_key_enable(struct static_key *key)
  132. {
  133. cpus_read_lock();
  134. static_key_enable_cpuslocked(key);
  135. cpus_read_unlock();
  136. }
  137. EXPORT_SYMBOL_GPL(static_key_enable);
  138. void static_key_disable_cpuslocked(struct static_key *key)
  139. {
  140. STATIC_KEY_CHECK_USE(key);
  141. lockdep_assert_cpus_held();
  142. if (atomic_read(&key->enabled) != 1) {
  143. WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
  144. return;
  145. }
  146. jump_label_lock();
  147. if (atomic_cmpxchg(&key->enabled, 1, 0))
  148. jump_label_update(key);
  149. jump_label_unlock();
  150. }
  151. EXPORT_SYMBOL_GPL(static_key_disable_cpuslocked);
  152. void static_key_disable(struct static_key *key)
  153. {
  154. cpus_read_lock();
  155. static_key_disable_cpuslocked(key);
  156. cpus_read_unlock();
  157. }
  158. EXPORT_SYMBOL_GPL(static_key_disable);
  159. static void __static_key_slow_dec_cpuslocked(struct static_key *key,
  160. unsigned long rate_limit,
  161. struct delayed_work *work)
  162. {
  163. int val;
  164. lockdep_assert_cpus_held();
  165. /*
  166. * The negative count check is valid even when a negative
  167. * key->enabled is in use by static_key_slow_inc(); a
  168. * __static_key_slow_dec() before the first static_key_slow_inc()
  169. * returns is unbalanced, because all other static_key_slow_inc()
  170. * instances block while the update is in progress.
  171. */
  172. val = atomic_fetch_add_unless(&key->enabled, -1, 1);
  173. if (val != 1) {
  174. WARN(val < 0, "jump label: negative count!\n");
  175. return;
  176. }
  177. jump_label_lock();
  178. if (atomic_dec_and_test(&key->enabled)) {
  179. if (rate_limit) {
  180. atomic_inc(&key->enabled);
  181. schedule_delayed_work(work, rate_limit);
  182. } else {
  183. jump_label_update(key);
  184. }
  185. }
  186. jump_label_unlock();
  187. }
  188. static void __static_key_slow_dec(struct static_key *key,
  189. unsigned long rate_limit,
  190. struct delayed_work *work)
  191. {
  192. cpus_read_lock();
  193. __static_key_slow_dec_cpuslocked(key, rate_limit, work);
  194. cpus_read_unlock();
  195. }
  196. static void jump_label_update_timeout(struct work_struct *work)
  197. {
  198. struct static_key_deferred *key =
  199. container_of(work, struct static_key_deferred, work.work);
  200. __static_key_slow_dec(&key->key, 0, NULL);
  201. }
  202. void static_key_slow_dec(struct static_key *key)
  203. {
  204. STATIC_KEY_CHECK_USE(key);
  205. __static_key_slow_dec(key, 0, NULL);
  206. }
  207. EXPORT_SYMBOL_GPL(static_key_slow_dec);
  208. void static_key_slow_dec_cpuslocked(struct static_key *key)
  209. {
  210. STATIC_KEY_CHECK_USE(key);
  211. __static_key_slow_dec_cpuslocked(key, 0, NULL);
  212. }
  213. void static_key_slow_dec_deferred(struct static_key_deferred *key)
  214. {
  215. STATIC_KEY_CHECK_USE(key);
  216. __static_key_slow_dec(&key->key, key->timeout, &key->work);
  217. }
  218. EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
  219. void static_key_deferred_flush(struct static_key_deferred *key)
  220. {
  221. STATIC_KEY_CHECK_USE(key);
  222. flush_delayed_work(&key->work);
  223. }
  224. EXPORT_SYMBOL_GPL(static_key_deferred_flush);
  225. void jump_label_rate_limit(struct static_key_deferred *key,
  226. unsigned long rl)
  227. {
  228. STATIC_KEY_CHECK_USE(key);
  229. key->timeout = rl;
  230. INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
  231. }
  232. EXPORT_SYMBOL_GPL(jump_label_rate_limit);
  233. static int addr_conflict(struct jump_entry *entry, void *start, void *end)
  234. {
  235. if (entry->code <= (unsigned long)end &&
  236. entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
  237. return 1;
  238. return 0;
  239. }
  240. static int __jump_label_text_reserved(struct jump_entry *iter_start,
  241. struct jump_entry *iter_stop, void *start, void *end)
  242. {
  243. struct jump_entry *iter;
  244. iter = iter_start;
  245. while (iter < iter_stop) {
  246. if (addr_conflict(iter, start, end))
  247. return 1;
  248. iter++;
  249. }
  250. return 0;
  251. }
  252. /*
  253. * Update code which is definitely not currently executing.
  254. * Architectures which need heavyweight synchronization to modify
  255. * running code can override this to make the non-live update case
  256. * cheaper.
  257. */
  258. void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
  259. enum jump_label_type type)
  260. {
  261. arch_jump_label_transform(entry, type);
  262. }
  263. static inline struct jump_entry *static_key_entries(struct static_key *key)
  264. {
  265. WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);
  266. return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);
  267. }
  268. static inline bool static_key_type(struct static_key *key)
  269. {
  270. return key->type & JUMP_TYPE_TRUE;
  271. }
  272. static inline bool static_key_linked(struct static_key *key)
  273. {
  274. return key->type & JUMP_TYPE_LINKED;
  275. }
  276. static inline void static_key_clear_linked(struct static_key *key)
  277. {
  278. key->type &= ~JUMP_TYPE_LINKED;
  279. }
  280. static inline void static_key_set_linked(struct static_key *key)
  281. {
  282. key->type |= JUMP_TYPE_LINKED;
  283. }
  284. static inline struct static_key *jump_entry_key(struct jump_entry *entry)
  285. {
  286. return (struct static_key *)((unsigned long)entry->key & ~1UL);
  287. }
  288. static bool jump_entry_branch(struct jump_entry *entry)
  289. {
  290. return (unsigned long)entry->key & 1UL;
  291. }
  292. /***
  293. * A 'struct static_key' uses a union such that it either points directly
  294. * to a table of 'struct jump_entry' or to a linked list of modules which in
  295. * turn point to 'struct jump_entry' tables.
  296. *
  297. * The two lower bits of the pointer are used to keep track of which pointer
  298. * type is in use and to store the initial branch direction, we use an access
  299. * function which preserves these bits.
  300. */
  301. static void static_key_set_entries(struct static_key *key,
  302. struct jump_entry *entries)
  303. {
  304. unsigned long type;
  305. WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
  306. type = key->type & JUMP_TYPE_MASK;
  307. key->entries = entries;
  308. key->type |= type;
  309. }
  310. static enum jump_label_type jump_label_type(struct jump_entry *entry)
  311. {
  312. struct static_key *key = jump_entry_key(entry);
  313. bool enabled = static_key_enabled(key);
  314. bool branch = jump_entry_branch(entry);
  315. /* See the comment in linux/jump_label.h */
  316. return enabled ^ branch;
  317. }
  318. static void __jump_label_update(struct static_key *key,
  319. struct jump_entry *entry,
  320. struct jump_entry *stop)
  321. {
  322. for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
  323. /*
  324. * An entry->code of 0 indicates an entry which has been
  325. * disabled because it was in an init text area.
  326. */
  327. if (entry->code) {
  328. if (kernel_text_address(entry->code))
  329. arch_jump_label_transform(entry, jump_label_type(entry));
  330. else
  331. WARN_ONCE(1, "can't patch jump_label at %pS",
  332. (void *)(unsigned long)entry->code);
  333. }
  334. }
  335. }
  336. void __init jump_label_init(void)
  337. {
  338. struct jump_entry *iter_start = __start___jump_table;
  339. struct jump_entry *iter_stop = __stop___jump_table;
  340. struct static_key *key = NULL;
  341. struct jump_entry *iter;
  342. /*
  343. * Since we are initializing the static_key.enabled field with
  344. * with the 'raw' int values (to avoid pulling in atomic.h) in
  345. * jump_label.h, let's make sure that is safe. There are only two
  346. * cases to check since we initialize to 0 or 1.
  347. */
  348. BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
  349. BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
  350. if (static_key_initialized)
  351. return;
  352. cpus_read_lock();
  353. jump_label_lock();
  354. jump_label_sort_entries(iter_start, iter_stop);
  355. for (iter = iter_start; iter < iter_stop; iter++) {
  356. struct static_key *iterk;
  357. /* rewrite NOPs */
  358. if (jump_label_type(iter) == JUMP_LABEL_NOP)
  359. arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
  360. iterk = jump_entry_key(iter);
  361. if (iterk == key)
  362. continue;
  363. key = iterk;
  364. static_key_set_entries(key, iter);
  365. }
  366. static_key_initialized = true;
  367. jump_label_unlock();
  368. cpus_read_unlock();
  369. }
  370. /* Disable any jump label entries in __init/__exit code */
  371. void __init jump_label_invalidate_initmem(void)
  372. {
  373. struct jump_entry *iter_start = __start___jump_table;
  374. struct jump_entry *iter_stop = __stop___jump_table;
  375. struct jump_entry *iter;
  376. for (iter = iter_start; iter < iter_stop; iter++) {
  377. if (init_section_contains((void *)(unsigned long)iter->code, 1))
  378. iter->code = 0;
  379. }
  380. }
  381. #ifdef CONFIG_MODULES
  382. static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
  383. {
  384. struct static_key *key = jump_entry_key(entry);
  385. bool type = static_key_type(key);
  386. bool branch = jump_entry_branch(entry);
  387. /* See the comment in linux/jump_label.h */
  388. return type ^ branch;
  389. }
  390. struct static_key_mod {
  391. struct static_key_mod *next;
  392. struct jump_entry *entries;
  393. struct module *mod;
  394. };
  395. static inline struct static_key_mod *static_key_mod(struct static_key *key)
  396. {
  397. WARN_ON_ONCE(!(key->type & JUMP_TYPE_LINKED));
  398. return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
  399. }
  400. /***
  401. * key->type and key->next are the same via union.
  402. * This sets key->next and preserves the type bits.
  403. *
  404. * See additional comments above static_key_set_entries().
  405. */
  406. static void static_key_set_mod(struct static_key *key,
  407. struct static_key_mod *mod)
  408. {
  409. unsigned long type;
  410. WARN_ON_ONCE((unsigned long)mod & JUMP_TYPE_MASK);
  411. type = key->type & JUMP_TYPE_MASK;
  412. key->next = mod;
  413. key->type |= type;
  414. }
  415. static int __jump_label_mod_text_reserved(void *start, void *end)
  416. {
  417. struct module *mod;
  418. preempt_disable();
  419. mod = __module_text_address((unsigned long)start);
  420. WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
  421. preempt_enable();
  422. if (!mod)
  423. return 0;
  424. return __jump_label_text_reserved(mod->jump_entries,
  425. mod->jump_entries + mod->num_jump_entries,
  426. start, end);
  427. }
  428. static void __jump_label_mod_update(struct static_key *key)
  429. {
  430. struct static_key_mod *mod;
  431. for (mod = static_key_mod(key); mod; mod = mod->next) {
  432. struct jump_entry *stop;
  433. struct module *m;
  434. /*
  435. * NULL if the static_key is defined in a module
  436. * that does not use it
  437. */
  438. if (!mod->entries)
  439. continue;
  440. m = mod->mod;
  441. if (!m)
  442. stop = __stop___jump_table;
  443. else
  444. stop = m->jump_entries + m->num_jump_entries;
  445. __jump_label_update(key, mod->entries, stop);
  446. }
  447. }
  448. /***
  449. * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
  450. * @mod: module to patch
  451. *
  452. * Allow for run-time selection of the optimal nops. Before the module
  453. * loads patch these with arch_get_jump_label_nop(), which is specified by
  454. * the arch specific jump label code.
  455. */
  456. void jump_label_apply_nops(struct module *mod)
  457. {
  458. struct jump_entry *iter_start = mod->jump_entries;
  459. struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  460. struct jump_entry *iter;
  461. /* if the module doesn't have jump label entries, just return */
  462. if (iter_start == iter_stop)
  463. return;
  464. for (iter = iter_start; iter < iter_stop; iter++) {
  465. /* Only write NOPs for arch_branch_static(). */
  466. if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
  467. arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
  468. }
  469. }
  470. static int jump_label_add_module(struct module *mod)
  471. {
  472. struct jump_entry *iter_start = mod->jump_entries;
  473. struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  474. struct jump_entry *iter;
  475. struct static_key *key = NULL;
  476. struct static_key_mod *jlm, *jlm2;
  477. /* if the module doesn't have jump label entries, just return */
  478. if (iter_start == iter_stop)
  479. return 0;
  480. jump_label_sort_entries(iter_start, iter_stop);
  481. for (iter = iter_start; iter < iter_stop; iter++) {
  482. struct static_key *iterk;
  483. iterk = jump_entry_key(iter);
  484. if (iterk == key)
  485. continue;
  486. key = iterk;
  487. if (within_module(iter->key, mod)) {
  488. static_key_set_entries(key, iter);
  489. continue;
  490. }
  491. jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
  492. if (!jlm)
  493. return -ENOMEM;
  494. if (!static_key_linked(key)) {
  495. jlm2 = kzalloc(sizeof(struct static_key_mod),
  496. GFP_KERNEL);
  497. if (!jlm2) {
  498. kfree(jlm);
  499. return -ENOMEM;
  500. }
  501. preempt_disable();
  502. jlm2->mod = __module_address((unsigned long)key);
  503. preempt_enable();
  504. jlm2->entries = static_key_entries(key);
  505. jlm2->next = NULL;
  506. static_key_set_mod(key, jlm2);
  507. static_key_set_linked(key);
  508. }
  509. jlm->mod = mod;
  510. jlm->entries = iter;
  511. jlm->next = static_key_mod(key);
  512. static_key_set_mod(key, jlm);
  513. static_key_set_linked(key);
  514. /* Only update if we've changed from our initial state */
  515. if (jump_label_type(iter) != jump_label_init_type(iter))
  516. __jump_label_update(key, iter, iter_stop);
  517. }
  518. return 0;
  519. }
  520. static void jump_label_del_module(struct module *mod)
  521. {
  522. struct jump_entry *iter_start = mod->jump_entries;
  523. struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  524. struct jump_entry *iter;
  525. struct static_key *key = NULL;
  526. struct static_key_mod *jlm, **prev;
  527. for (iter = iter_start; iter < iter_stop; iter++) {
  528. if (jump_entry_key(iter) == key)
  529. continue;
  530. key = jump_entry_key(iter);
  531. if (within_module(iter->key, mod))
  532. continue;
  533. /* No memory during module load */
  534. if (WARN_ON(!static_key_linked(key)))
  535. continue;
  536. prev = &key->next;
  537. jlm = static_key_mod(key);
  538. while (jlm && jlm->mod != mod) {
  539. prev = &jlm->next;
  540. jlm = jlm->next;
  541. }
  542. /* No memory during module load */
  543. if (WARN_ON(!jlm))
  544. continue;
  545. if (prev == &key->next)
  546. static_key_set_mod(key, jlm->next);
  547. else
  548. *prev = jlm->next;
  549. kfree(jlm);
  550. jlm = static_key_mod(key);
  551. /* if only one etry is left, fold it back into the static_key */
  552. if (jlm->next == NULL) {
  553. static_key_set_entries(key, jlm->entries);
  554. static_key_clear_linked(key);
  555. kfree(jlm);
  556. }
  557. }
  558. }
  559. /* Disable any jump label entries in module init code */
  560. static void jump_label_invalidate_module_init(struct module *mod)
  561. {
  562. struct jump_entry *iter_start = mod->jump_entries;
  563. struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  564. struct jump_entry *iter;
  565. for (iter = iter_start; iter < iter_stop; iter++) {
  566. if (within_module_init(iter->code, mod))
  567. iter->code = 0;
  568. }
  569. }
  570. static int
  571. jump_label_module_notify(struct notifier_block *self, unsigned long val,
  572. void *data)
  573. {
  574. struct module *mod = data;
  575. int ret = 0;
  576. cpus_read_lock();
  577. jump_label_lock();
  578. switch (val) {
  579. case MODULE_STATE_COMING:
  580. ret = jump_label_add_module(mod);
  581. if (ret) {
  582. WARN(1, "Failed to allocate memory: jump_label may not work properly.\n");
  583. jump_label_del_module(mod);
  584. }
  585. break;
  586. case MODULE_STATE_GOING:
  587. jump_label_del_module(mod);
  588. break;
  589. case MODULE_STATE_LIVE:
  590. jump_label_invalidate_module_init(mod);
  591. break;
  592. }
  593. jump_label_unlock();
  594. cpus_read_unlock();
  595. return notifier_from_errno(ret);
  596. }
  597. static struct notifier_block jump_label_module_nb = {
  598. .notifier_call = jump_label_module_notify,
  599. .priority = 1, /* higher than tracepoints */
  600. };
  601. static __init int jump_label_init_module(void)
  602. {
  603. return register_module_notifier(&jump_label_module_nb);
  604. }
  605. early_initcall(jump_label_init_module);
  606. #endif /* CONFIG_MODULES */
  607. /***
  608. * jump_label_text_reserved - check if addr range is reserved
  609. * @start: start text addr
  610. * @end: end text addr
  611. *
  612. * checks if the text addr located between @start and @end
  613. * overlaps with any of the jump label patch addresses. Code
  614. * that wants to modify kernel text should first verify that
  615. * it does not overlap with any of the jump label addresses.
  616. * Caller must hold jump_label_mutex.
  617. *
  618. * returns 1 if there is an overlap, 0 otherwise
  619. */
  620. int jump_label_text_reserved(void *start, void *end)
  621. {
  622. int ret = __jump_label_text_reserved(__start___jump_table,
  623. __stop___jump_table, start, end);
  624. if (ret)
  625. return ret;
  626. #ifdef CONFIG_MODULES
  627. ret = __jump_label_mod_text_reserved(start, end);
  628. #endif
  629. return ret;
  630. }
  631. static void jump_label_update(struct static_key *key)
  632. {
  633. struct jump_entry *stop = __stop___jump_table;
  634. struct jump_entry *entry;
  635. #ifdef CONFIG_MODULES
  636. struct module *mod;
  637. if (static_key_linked(key)) {
  638. __jump_label_mod_update(key);
  639. return;
  640. }
  641. preempt_disable();
  642. mod = __module_address((unsigned long)key);
  643. if (mod)
  644. stop = mod->jump_entries + mod->num_jump_entries;
  645. preempt_enable();
  646. #endif
  647. entry = static_key_entries(key);
  648. /* if there are no users, entry can be NULL */
  649. if (entry)
  650. __jump_label_update(key, entry, stop);
  651. }
  652. #ifdef CONFIG_STATIC_KEYS_SELFTEST
  653. static DEFINE_STATIC_KEY_TRUE(sk_true);
  654. static DEFINE_STATIC_KEY_FALSE(sk_false);
  655. static __init int jump_label_test(void)
  656. {
  657. int i;
  658. for (i = 0; i < 2; i++) {
  659. WARN_ON(static_key_enabled(&sk_true.key) != true);
  660. WARN_ON(static_key_enabled(&sk_false.key) != false);
  661. WARN_ON(!static_branch_likely(&sk_true));
  662. WARN_ON(!static_branch_unlikely(&sk_true));
  663. WARN_ON(static_branch_likely(&sk_false));
  664. WARN_ON(static_branch_unlikely(&sk_false));
  665. static_branch_disable(&sk_true);
  666. static_branch_enable(&sk_false);
  667. WARN_ON(static_key_enabled(&sk_true.key) == true);
  668. WARN_ON(static_key_enabled(&sk_false.key) == false);
  669. WARN_ON(static_branch_likely(&sk_true));
  670. WARN_ON(static_branch_unlikely(&sk_true));
  671. WARN_ON(!static_branch_likely(&sk_false));
  672. WARN_ON(!static_branch_unlikely(&sk_false));
  673. static_branch_enable(&sk_true);
  674. static_branch_disable(&sk_false);
  675. }
  676. return 0;
  677. }
  678. early_initcall(jump_label_test);
  679. #endif /* STATIC_KEYS_SELFTEST */