core.c 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929
  1. /*
  2. * Linux Socket Filter - Kernel level socket filtering
  3. *
  4. * Based on the design of the Berkeley Packet Filter. The new
  5. * internal format has been designed by PLUMgrid:
  6. *
  7. * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
  8. *
  9. * Authors:
  10. *
  11. * Jay Schulist <jschlst@samba.org>
  12. * Alexei Starovoitov <ast@plumgrid.com>
  13. * Daniel Borkmann <dborkman@redhat.com>
  14. *
  15. * This program is free software; you can redistribute it and/or
  16. * modify it under the terms of the GNU General Public License
  17. * as published by the Free Software Foundation; either version
  18. * 2 of the License, or (at your option) any later version.
  19. *
  20. * Andi Kleen - Fix a few bad bugs and races.
  21. * Kris Katterjohn - Added many additional checks in bpf_check_classic()
  22. */
  23. #include <linux/filter.h>
  24. #include <linux/skbuff.h>
  25. #include <linux/vmalloc.h>
  26. #include <linux/random.h>
  27. #include <linux/moduleloader.h>
  28. #include <linux/bpf.h>
  29. #include <linux/frame.h>
  30. #include <linux/rbtree_latch.h>
  31. #include <linux/kallsyms.h>
  32. #include <linux/rcupdate.h>
  33. #include <linux/perf_event.h>
  34. #include <asm/unaligned.h>
  35. /* Registers */
  36. #define BPF_R0 regs[BPF_REG_0]
  37. #define BPF_R1 regs[BPF_REG_1]
  38. #define BPF_R2 regs[BPF_REG_2]
  39. #define BPF_R3 regs[BPF_REG_3]
  40. #define BPF_R4 regs[BPF_REG_4]
  41. #define BPF_R5 regs[BPF_REG_5]
  42. #define BPF_R6 regs[BPF_REG_6]
  43. #define BPF_R7 regs[BPF_REG_7]
  44. #define BPF_R8 regs[BPF_REG_8]
  45. #define BPF_R9 regs[BPF_REG_9]
  46. #define BPF_R10 regs[BPF_REG_10]
  47. /* Named registers */
  48. #define DST regs[insn->dst_reg]
  49. #define SRC regs[insn->src_reg]
  50. #define FP regs[BPF_REG_FP]
  51. #define AX regs[BPF_REG_AX]
  52. #define ARG1 regs[BPF_REG_ARG1]
  53. #define CTX regs[BPF_REG_CTX]
  54. #define IMM insn->imm
  55. /* No hurry in this branch
  56. *
  57. * Exported for the bpf jit load helper.
  58. */
  59. void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)
  60. {
  61. u8 *ptr = NULL;
  62. if (k >= SKF_NET_OFF)
  63. ptr = skb_network_header(skb) + k - SKF_NET_OFF;
  64. else if (k >= SKF_LL_OFF)
  65. ptr = skb_mac_header(skb) + k - SKF_LL_OFF;
  66. if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))
  67. return ptr;
  68. return NULL;
  69. }
  70. struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
  71. {
  72. gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
  73. struct bpf_prog_aux *aux;
  74. struct bpf_prog *fp;
  75. size = round_up(size, PAGE_SIZE);
  76. fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
  77. if (fp == NULL)
  78. return NULL;
  79. aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags);
  80. if (aux == NULL) {
  81. vfree(fp);
  82. return NULL;
  83. }
  84. fp->pages = size / PAGE_SIZE;
  85. fp->aux = aux;
  86. fp->aux->prog = fp;
  87. fp->jit_requested = ebpf_jit_enabled();
  88. INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode);
  89. return fp;
  90. }
  91. EXPORT_SYMBOL_GPL(bpf_prog_alloc);
  92. struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
  93. gfp_t gfp_extra_flags)
  94. {
  95. gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
  96. struct bpf_prog *fp;
  97. u32 pages, delta;
  98. int ret;
  99. BUG_ON(fp_old == NULL);
  100. size = round_up(size, PAGE_SIZE);
  101. pages = size / PAGE_SIZE;
  102. if (pages <= fp_old->pages)
  103. return fp_old;
  104. delta = pages - fp_old->pages;
  105. ret = __bpf_prog_charge(fp_old->aux->user, delta);
  106. if (ret)
  107. return NULL;
  108. fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
  109. if (fp == NULL) {
  110. __bpf_prog_uncharge(fp_old->aux->user, delta);
  111. } else {
  112. memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
  113. fp->pages = pages;
  114. fp->aux->prog = fp;
  115. /* We keep fp->aux from fp_old around in the new
  116. * reallocated structure.
  117. */
  118. fp_old->aux = NULL;
  119. __bpf_prog_free(fp_old);
  120. }
  121. return fp;
  122. }
  123. void __bpf_prog_free(struct bpf_prog *fp)
  124. {
  125. kfree(fp->aux);
  126. vfree(fp);
  127. }
  128. int bpf_prog_calc_tag(struct bpf_prog *fp)
  129. {
  130. const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64);
  131. u32 raw_size = bpf_prog_tag_scratch_size(fp);
  132. u32 digest[SHA_DIGEST_WORDS];
  133. u32 ws[SHA_WORKSPACE_WORDS];
  134. u32 i, bsize, psize, blocks;
  135. struct bpf_insn *dst;
  136. bool was_ld_map;
  137. u8 *raw, *todo;
  138. __be32 *result;
  139. __be64 *bits;
  140. raw = vmalloc(raw_size);
  141. if (!raw)
  142. return -ENOMEM;
  143. sha_init(digest);
  144. memset(ws, 0, sizeof(ws));
  145. /* We need to take out the map fd for the digest calculation
  146. * since they are unstable from user space side.
  147. */
  148. dst = (void *)raw;
  149. for (i = 0, was_ld_map = false; i < fp->len; i++) {
  150. dst[i] = fp->insnsi[i];
  151. if (!was_ld_map &&
  152. dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) &&
  153. dst[i].src_reg == BPF_PSEUDO_MAP_FD) {
  154. was_ld_map = true;
  155. dst[i].imm = 0;
  156. } else if (was_ld_map &&
  157. dst[i].code == 0 &&
  158. dst[i].dst_reg == 0 &&
  159. dst[i].src_reg == 0 &&
  160. dst[i].off == 0) {
  161. was_ld_map = false;
  162. dst[i].imm = 0;
  163. } else {
  164. was_ld_map = false;
  165. }
  166. }
  167. psize = bpf_prog_insn_size(fp);
  168. memset(&raw[psize], 0, raw_size - psize);
  169. raw[psize++] = 0x80;
  170. bsize = round_up(psize, SHA_MESSAGE_BYTES);
  171. blocks = bsize / SHA_MESSAGE_BYTES;
  172. todo = raw;
  173. if (bsize - psize >= sizeof(__be64)) {
  174. bits = (__be64 *)(todo + bsize - sizeof(__be64));
  175. } else {
  176. bits = (__be64 *)(todo + bsize + bits_offset);
  177. blocks++;
  178. }
  179. *bits = cpu_to_be64((psize - 1) << 3);
  180. while (blocks--) {
  181. sha_transform(digest, todo, ws);
  182. todo += SHA_MESSAGE_BYTES;
  183. }
  184. result = (__force __be32 *)digest;
  185. for (i = 0; i < SHA_DIGEST_WORDS; i++)
  186. result[i] = cpu_to_be32(digest[i]);
  187. memcpy(fp->tag, result, sizeof(fp->tag));
  188. vfree(raw);
  189. return 0;
  190. }
  191. static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, u32 delta,
  192. u32 curr, const bool probe_pass)
  193. {
  194. const s64 imm_min = S32_MIN, imm_max = S32_MAX;
  195. s64 imm = insn->imm;
  196. if (curr < pos && curr + imm + 1 > pos)
  197. imm += delta;
  198. else if (curr > pos + delta && curr + imm + 1 <= pos + delta)
  199. imm -= delta;
  200. if (imm < imm_min || imm > imm_max)
  201. return -ERANGE;
  202. if (!probe_pass)
  203. insn->imm = imm;
  204. return 0;
  205. }
  206. static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, u32 delta,
  207. u32 curr, const bool probe_pass)
  208. {
  209. const s32 off_min = S16_MIN, off_max = S16_MAX;
  210. s32 off = insn->off;
  211. if (curr < pos && curr + off + 1 > pos)
  212. off += delta;
  213. else if (curr > pos + delta && curr + off + 1 <= pos + delta)
  214. off -= delta;
  215. if (off < off_min || off > off_max)
  216. return -ERANGE;
  217. if (!probe_pass)
  218. insn->off = off;
  219. return 0;
  220. }
  221. static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,
  222. const bool probe_pass)
  223. {
  224. u32 i, insn_cnt = prog->len + (probe_pass ? delta : 0);
  225. struct bpf_insn *insn = prog->insnsi;
  226. int ret = 0;
  227. for (i = 0; i < insn_cnt; i++, insn++) {
  228. u8 code;
  229. /* In the probing pass we still operate on the original,
  230. * unpatched image in order to check overflows before we
  231. * do any other adjustments. Therefore skip the patchlet.
  232. */
  233. if (probe_pass && i == pos) {
  234. i += delta + 1;
  235. insn++;
  236. }
  237. code = insn->code;
  238. if (BPF_CLASS(code) != BPF_JMP ||
  239. BPF_OP(code) == BPF_EXIT)
  240. continue;
  241. /* Adjust offset of jmps if we cross patch boundaries. */
  242. if (BPF_OP(code) == BPF_CALL) {
  243. if (insn->src_reg != BPF_PSEUDO_CALL)
  244. continue;
  245. ret = bpf_adj_delta_to_imm(insn, pos, delta, i,
  246. probe_pass);
  247. } else {
  248. ret = bpf_adj_delta_to_off(insn, pos, delta, i,
  249. probe_pass);
  250. }
  251. if (ret)
  252. break;
  253. }
  254. return ret;
  255. }
  256. struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
  257. const struct bpf_insn *patch, u32 len)
  258. {
  259. u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
  260. const u32 cnt_max = S16_MAX;
  261. struct bpf_prog *prog_adj;
  262. /* Since our patchlet doesn't expand the image, we're done. */
  263. if (insn_delta == 0) {
  264. memcpy(prog->insnsi + off, patch, sizeof(*patch));
  265. return prog;
  266. }
  267. insn_adj_cnt = prog->len + insn_delta;
  268. /* Reject anything that would potentially let the insn->off
  269. * target overflow when we have excessive program expansions.
  270. * We need to probe here before we do any reallocation where
  271. * we afterwards may not fail anymore.
  272. */
  273. if (insn_adj_cnt > cnt_max &&
  274. bpf_adj_branches(prog, off, insn_delta, true))
  275. return NULL;
  276. /* Several new instructions need to be inserted. Make room
  277. * for them. Likely, there's no need for a new allocation as
  278. * last page could have large enough tailroom.
  279. */
  280. prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
  281. GFP_USER);
  282. if (!prog_adj)
  283. return NULL;
  284. prog_adj->len = insn_adj_cnt;
  285. /* Patching happens in 3 steps:
  286. *
  287. * 1) Move over tail of insnsi from next instruction onwards,
  288. * so we can patch the single target insn with one or more
  289. * new ones (patching is always from 1 to n insns, n > 0).
  290. * 2) Inject new instructions at the target location.
  291. * 3) Adjust branch offsets if necessary.
  292. */
  293. insn_rest = insn_adj_cnt - off - len;
  294. memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
  295. sizeof(*patch) * insn_rest);
  296. memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);
  297. /* We are guaranteed to not fail at this point, otherwise
  298. * the ship has sailed to reverse to the original state. An
  299. * overflow cannot happen at this point.
  300. */
  301. BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false));
  302. return prog_adj;
  303. }
  304. void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp)
  305. {
  306. int i;
  307. for (i = 0; i < fp->aux->func_cnt; i++)
  308. bpf_prog_kallsyms_del(fp->aux->func[i]);
  309. }
  310. void bpf_prog_kallsyms_del_all(struct bpf_prog *fp)
  311. {
  312. bpf_prog_kallsyms_del_subprogs(fp);
  313. bpf_prog_kallsyms_del(fp);
  314. }
  315. #ifdef CONFIG_BPF_JIT
  316. /* All BPF JIT sysctl knobs here. */
  317. int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON);
  318. int bpf_jit_harden __read_mostly;
  319. int bpf_jit_kallsyms __read_mostly;
  320. long bpf_jit_limit __read_mostly;
  321. static __always_inline void
  322. bpf_get_prog_addr_region(const struct bpf_prog *prog,
  323. unsigned long *symbol_start,
  324. unsigned long *symbol_end)
  325. {
  326. const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog);
  327. unsigned long addr = (unsigned long)hdr;
  328. WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog));
  329. *symbol_start = addr;
  330. *symbol_end = addr + hdr->pages * PAGE_SIZE;
  331. }
  332. static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
  333. {
  334. const char *end = sym + KSYM_NAME_LEN;
  335. BUILD_BUG_ON(sizeof("bpf_prog_") +
  336. sizeof(prog->tag) * 2 +
  337. /* name has been null terminated.
  338. * We should need +1 for the '_' preceding
  339. * the name. However, the null character
  340. * is double counted between the name and the
  341. * sizeof("bpf_prog_") above, so we omit
  342. * the +1 here.
  343. */
  344. sizeof(prog->aux->name) > KSYM_NAME_LEN);
  345. sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_");
  346. sym = bin2hex(sym, prog->tag, sizeof(prog->tag));
  347. if (prog->aux->name[0])
  348. snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);
  349. else
  350. *sym = 0;
  351. }
  352. static __always_inline unsigned long
  353. bpf_get_prog_addr_start(struct latch_tree_node *n)
  354. {
  355. unsigned long symbol_start, symbol_end;
  356. const struct bpf_prog_aux *aux;
  357. aux = container_of(n, struct bpf_prog_aux, ksym_tnode);
  358. bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);
  359. return symbol_start;
  360. }
  361. static __always_inline bool bpf_tree_less(struct latch_tree_node *a,
  362. struct latch_tree_node *b)
  363. {
  364. return bpf_get_prog_addr_start(a) < bpf_get_prog_addr_start(b);
  365. }
  366. static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n)
  367. {
  368. unsigned long val = (unsigned long)key;
  369. unsigned long symbol_start, symbol_end;
  370. const struct bpf_prog_aux *aux;
  371. aux = container_of(n, struct bpf_prog_aux, ksym_tnode);
  372. bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);
  373. if (val < symbol_start)
  374. return -1;
  375. if (val >= symbol_end)
  376. return 1;
  377. return 0;
  378. }
  379. static const struct latch_tree_ops bpf_tree_ops = {
  380. .less = bpf_tree_less,
  381. .comp = bpf_tree_comp,
  382. };
  383. static DEFINE_SPINLOCK(bpf_lock);
  384. static LIST_HEAD(bpf_kallsyms);
  385. static struct latch_tree_root bpf_tree __cacheline_aligned;
  386. static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux)
  387. {
  388. WARN_ON_ONCE(!list_empty(&aux->ksym_lnode));
  389. list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms);
  390. latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops);
  391. }
  392. static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux)
  393. {
  394. if (list_empty(&aux->ksym_lnode))
  395. return;
  396. latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops);
  397. list_del_rcu(&aux->ksym_lnode);
  398. }
  399. static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp)
  400. {
  401. return fp->jited && !bpf_prog_was_classic(fp);
  402. }
  403. static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
  404. {
  405. return list_empty(&fp->aux->ksym_lnode) ||
  406. fp->aux->ksym_lnode.prev == LIST_POISON2;
  407. }
  408. void bpf_prog_kallsyms_add(struct bpf_prog *fp)
  409. {
  410. if (!bpf_prog_kallsyms_candidate(fp) ||
  411. !capable(CAP_SYS_ADMIN))
  412. return;
  413. spin_lock_bh(&bpf_lock);
  414. bpf_prog_ksym_node_add(fp->aux);
  415. spin_unlock_bh(&bpf_lock);
  416. }
  417. void bpf_prog_kallsyms_del(struct bpf_prog *fp)
  418. {
  419. if (!bpf_prog_kallsyms_candidate(fp))
  420. return;
  421. spin_lock_bh(&bpf_lock);
  422. bpf_prog_ksym_node_del(fp->aux);
  423. spin_unlock_bh(&bpf_lock);
  424. }
  425. static struct bpf_prog *bpf_prog_kallsyms_find(unsigned long addr)
  426. {
  427. struct latch_tree_node *n;
  428. if (!bpf_jit_kallsyms_enabled())
  429. return NULL;
  430. n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops);
  431. return n ?
  432. container_of(n, struct bpf_prog_aux, ksym_tnode)->prog :
  433. NULL;
  434. }
  435. const char *__bpf_address_lookup(unsigned long addr, unsigned long *size,
  436. unsigned long *off, char *sym)
  437. {
  438. unsigned long symbol_start, symbol_end;
  439. struct bpf_prog *prog;
  440. char *ret = NULL;
  441. rcu_read_lock();
  442. prog = bpf_prog_kallsyms_find(addr);
  443. if (prog) {
  444. bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end);
  445. bpf_get_prog_name(prog, sym);
  446. ret = sym;
  447. if (size)
  448. *size = symbol_end - symbol_start;
  449. if (off)
  450. *off = addr - symbol_start;
  451. }
  452. rcu_read_unlock();
  453. return ret;
  454. }
  455. bool is_bpf_text_address(unsigned long addr)
  456. {
  457. bool ret;
  458. rcu_read_lock();
  459. ret = bpf_prog_kallsyms_find(addr) != NULL;
  460. rcu_read_unlock();
  461. return ret;
  462. }
  463. int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
  464. char *sym)
  465. {
  466. unsigned long symbol_start, symbol_end;
  467. struct bpf_prog_aux *aux;
  468. unsigned int it = 0;
  469. int ret = -ERANGE;
  470. if (!bpf_jit_kallsyms_enabled())
  471. return ret;
  472. rcu_read_lock();
  473. list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) {
  474. if (it++ != symnum)
  475. continue;
  476. bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end);
  477. bpf_get_prog_name(aux->prog, sym);
  478. *value = symbol_start;
  479. *type = BPF_SYM_ELF_TYPE;
  480. ret = 0;
  481. break;
  482. }
  483. rcu_read_unlock();
  484. return ret;
  485. }
  486. static atomic_long_t bpf_jit_current;
  487. /* Can be overridden by an arch's JIT compiler if it has a custom,
  488. * dedicated BPF backend memory area, or if neither of the two
  489. * below apply.
  490. */
  491. u64 __weak bpf_jit_alloc_exec_limit(void)
  492. {
  493. #if defined(MODULES_VADDR)
  494. return MODULES_END - MODULES_VADDR;
  495. #else
  496. return VMALLOC_END - VMALLOC_START;
  497. #endif
  498. }
  499. static int __init bpf_jit_charge_init(void)
  500. {
  501. /* Only used as heuristic here to derive limit. */
  502. bpf_jit_limit = min_t(u64, round_up(bpf_jit_alloc_exec_limit() >> 2,
  503. PAGE_SIZE), LONG_MAX);
  504. return 0;
  505. }
  506. pure_initcall(bpf_jit_charge_init);
  507. static int bpf_jit_charge_modmem(u32 pages)
  508. {
  509. if (atomic_long_add_return(pages, &bpf_jit_current) >
  510. (bpf_jit_limit >> PAGE_SHIFT)) {
  511. if (!capable(CAP_SYS_ADMIN)) {
  512. atomic_long_sub(pages, &bpf_jit_current);
  513. return -EPERM;
  514. }
  515. }
  516. return 0;
  517. }
  518. static void bpf_jit_uncharge_modmem(u32 pages)
  519. {
  520. atomic_long_sub(pages, &bpf_jit_current);
  521. }
  522. struct bpf_binary_header *
  523. bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
  524. unsigned int alignment,
  525. bpf_jit_fill_hole_t bpf_fill_ill_insns)
  526. {
  527. struct bpf_binary_header *hdr;
  528. u32 size, hole, start, pages;
  529. /* Most of BPF filters are really small, but if some of them
  530. * fill a page, allow at least 128 extra bytes to insert a
  531. * random section of illegal instructions.
  532. */
  533. size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
  534. pages = size / PAGE_SIZE;
  535. if (bpf_jit_charge_modmem(pages))
  536. return NULL;
  537. hdr = module_alloc(size);
  538. if (!hdr) {
  539. bpf_jit_uncharge_modmem(pages);
  540. return NULL;
  541. }
  542. /* Fill space with illegal/arch-dep instructions. */
  543. bpf_fill_ill_insns(hdr, size);
  544. hdr->pages = pages;
  545. hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
  546. PAGE_SIZE - sizeof(*hdr));
  547. start = (get_random_int() % hole) & ~(alignment - 1);
  548. /* Leave a random number of instructions before BPF code. */
  549. *image_ptr = &hdr->image[start];
  550. return hdr;
  551. }
  552. void bpf_jit_binary_free(struct bpf_binary_header *hdr)
  553. {
  554. u32 pages = hdr->pages;
  555. module_memfree(hdr);
  556. bpf_jit_uncharge_modmem(pages);
  557. }
  558. /* This symbol is only overridden by archs that have different
  559. * requirements than the usual eBPF JITs, f.e. when they only
  560. * implement cBPF JIT, do not set images read-only, etc.
  561. */
  562. void __weak bpf_jit_free(struct bpf_prog *fp)
  563. {
  564. if (fp->jited) {
  565. struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);
  566. bpf_jit_binary_unlock_ro(hdr);
  567. bpf_jit_binary_free(hdr);
  568. WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
  569. }
  570. bpf_prog_unlock_free(fp);
  571. }
  572. static int bpf_jit_blind_insn(const struct bpf_insn *from,
  573. const struct bpf_insn *aux,
  574. struct bpf_insn *to_buff)
  575. {
  576. struct bpf_insn *to = to_buff;
  577. u32 imm_rnd = get_random_int();
  578. s16 off;
  579. BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
  580. BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG);
  581. /* Constraints on AX register:
  582. *
  583. * AX register is inaccessible from user space. It is mapped in
  584. * all JITs, and used here for constant blinding rewrites. It is
  585. * typically "stateless" meaning its contents are only valid within
  586. * the executed instruction, but not across several instructions.
  587. * There are a few exceptions however which are further detailed
  588. * below.
  589. *
  590. * Constant blinding is only used by JITs, not in the interpreter.
  591. * The interpreter uses AX in some occasions as a local temporary
  592. * register e.g. in DIV or MOD instructions.
  593. *
  594. * In restricted circumstances, the verifier can also use the AX
  595. * register for rewrites as long as they do not interfere with
  596. * the above cases!
  597. */
  598. if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX)
  599. goto out;
  600. if (from->imm == 0 &&
  601. (from->code == (BPF_ALU | BPF_MOV | BPF_K) ||
  602. from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) {
  603. *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg);
  604. goto out;
  605. }
  606. switch (from->code) {
  607. case BPF_ALU | BPF_ADD | BPF_K:
  608. case BPF_ALU | BPF_SUB | BPF_K:
  609. case BPF_ALU | BPF_AND | BPF_K:
  610. case BPF_ALU | BPF_OR | BPF_K:
  611. case BPF_ALU | BPF_XOR | BPF_K:
  612. case BPF_ALU | BPF_MUL | BPF_K:
  613. case BPF_ALU | BPF_MOV | BPF_K:
  614. case BPF_ALU | BPF_DIV | BPF_K:
  615. case BPF_ALU | BPF_MOD | BPF_K:
  616. *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
  617. *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  618. *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX);
  619. break;
  620. case BPF_ALU64 | BPF_ADD | BPF_K:
  621. case BPF_ALU64 | BPF_SUB | BPF_K:
  622. case BPF_ALU64 | BPF_AND | BPF_K:
  623. case BPF_ALU64 | BPF_OR | BPF_K:
  624. case BPF_ALU64 | BPF_XOR | BPF_K:
  625. case BPF_ALU64 | BPF_MUL | BPF_K:
  626. case BPF_ALU64 | BPF_MOV | BPF_K:
  627. case BPF_ALU64 | BPF_DIV | BPF_K:
  628. case BPF_ALU64 | BPF_MOD | BPF_K:
  629. *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
  630. *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  631. *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX);
  632. break;
  633. case BPF_JMP | BPF_JEQ | BPF_K:
  634. case BPF_JMP | BPF_JNE | BPF_K:
  635. case BPF_JMP | BPF_JGT | BPF_K:
  636. case BPF_JMP | BPF_JLT | BPF_K:
  637. case BPF_JMP | BPF_JGE | BPF_K:
  638. case BPF_JMP | BPF_JLE | BPF_K:
  639. case BPF_JMP | BPF_JSGT | BPF_K:
  640. case BPF_JMP | BPF_JSLT | BPF_K:
  641. case BPF_JMP | BPF_JSGE | BPF_K:
  642. case BPF_JMP | BPF_JSLE | BPF_K:
  643. case BPF_JMP | BPF_JSET | BPF_K:
  644. /* Accommodate for extra offset in case of a backjump. */
  645. off = from->off;
  646. if (off < 0)
  647. off -= 2;
  648. *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
  649. *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  650. *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
  651. break;
  652. case BPF_LD | BPF_IMM | BPF_DW:
  653. *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);
  654. *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  655. *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
  656. *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);
  657. break;
  658. case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */
  659. *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);
  660. *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  661. *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX);
  662. break;
  663. case BPF_ST | BPF_MEM | BPF_DW:
  664. case BPF_ST | BPF_MEM | BPF_W:
  665. case BPF_ST | BPF_MEM | BPF_H:
  666. case BPF_ST | BPF_MEM | BPF_B:
  667. *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
  668. *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
  669. *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off);
  670. break;
  671. }
  672. out:
  673. return to - to_buff;
  674. }
  675. static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,
  676. gfp_t gfp_extra_flags)
  677. {
  678. gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
  679. struct bpf_prog *fp;
  680. fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL);
  681. if (fp != NULL) {
  682. /* aux->prog still points to the fp_other one, so
  683. * when promoting the clone to the real program,
  684. * this still needs to be adapted.
  685. */
  686. memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE);
  687. }
  688. return fp;
  689. }
  690. static void bpf_prog_clone_free(struct bpf_prog *fp)
  691. {
  692. /* aux was stolen by the other clone, so we cannot free
  693. * it from this path! It will be freed eventually by the
  694. * other program on release.
  695. *
  696. * At this point, we don't need a deferred release since
  697. * clone is guaranteed to not be locked.
  698. */
  699. fp->aux = NULL;
  700. __bpf_prog_free(fp);
  701. }
  702. void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
  703. {
  704. /* We have to repoint aux->prog to self, as we don't
  705. * know whether fp here is the clone or the original.
  706. */
  707. fp->aux->prog = fp;
  708. bpf_prog_clone_free(fp_other);
  709. }
  710. struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
  711. {
  712. struct bpf_insn insn_buff[16], aux[2];
  713. struct bpf_prog *clone, *tmp;
  714. int insn_delta, insn_cnt;
  715. struct bpf_insn *insn;
  716. int i, rewritten;
  717. if (!bpf_jit_blinding_enabled(prog) || prog->blinded)
  718. return prog;
  719. clone = bpf_prog_clone_create(prog, GFP_USER);
  720. if (!clone)
  721. return ERR_PTR(-ENOMEM);
  722. insn_cnt = clone->len;
  723. insn = clone->insnsi;
  724. for (i = 0; i < insn_cnt; i++, insn++) {
  725. /* We temporarily need to hold the original ld64 insn
  726. * so that we can still access the first part in the
  727. * second blinding run.
  728. */
  729. if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) &&
  730. insn[1].code == 0)
  731. memcpy(aux, insn, sizeof(aux));
  732. rewritten = bpf_jit_blind_insn(insn, aux, insn_buff);
  733. if (!rewritten)
  734. continue;
  735. tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
  736. if (!tmp) {
  737. /* Patching may have repointed aux->prog during
  738. * realloc from the original one, so we need to
  739. * fix it up here on error.
  740. */
  741. bpf_jit_prog_release_other(prog, clone);
  742. return ERR_PTR(-ENOMEM);
  743. }
  744. clone = tmp;
  745. insn_delta = rewritten - 1;
  746. /* Walk new program and skip insns we just inserted. */
  747. insn = clone->insnsi + i + insn_delta;
  748. insn_cnt += insn_delta;
  749. i += insn_delta;
  750. }
  751. clone->blinded = 1;
  752. return clone;
  753. }
  754. #endif /* CONFIG_BPF_JIT */
  755. /* Base function for offset calculation. Needs to go into .text section,
  756. * therefore keeping it non-static as well; will also be used by JITs
  757. * anyway later on, so do not let the compiler omit it. This also needs
  758. * to go into kallsyms for correlation from e.g. bpftool, so naming
  759. * must not change.
  760. */
  761. noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
  762. {
  763. return 0;
  764. }
  765. EXPORT_SYMBOL_GPL(__bpf_call_base);
  766. /* All UAPI available opcodes. */
  767. #define BPF_INSN_MAP(INSN_2, INSN_3) \
  768. /* 32 bit ALU operations. */ \
  769. /* Register based. */ \
  770. INSN_3(ALU, ADD, X), \
  771. INSN_3(ALU, SUB, X), \
  772. INSN_3(ALU, AND, X), \
  773. INSN_3(ALU, OR, X), \
  774. INSN_3(ALU, LSH, X), \
  775. INSN_3(ALU, RSH, X), \
  776. INSN_3(ALU, XOR, X), \
  777. INSN_3(ALU, MUL, X), \
  778. INSN_3(ALU, MOV, X), \
  779. INSN_3(ALU, DIV, X), \
  780. INSN_3(ALU, MOD, X), \
  781. INSN_2(ALU, NEG), \
  782. INSN_3(ALU, END, TO_BE), \
  783. INSN_3(ALU, END, TO_LE), \
  784. /* Immediate based. */ \
  785. INSN_3(ALU, ADD, K), \
  786. INSN_3(ALU, SUB, K), \
  787. INSN_3(ALU, AND, K), \
  788. INSN_3(ALU, OR, K), \
  789. INSN_3(ALU, LSH, K), \
  790. INSN_3(ALU, RSH, K), \
  791. INSN_3(ALU, XOR, K), \
  792. INSN_3(ALU, MUL, K), \
  793. INSN_3(ALU, MOV, K), \
  794. INSN_3(ALU, DIV, K), \
  795. INSN_3(ALU, MOD, K), \
  796. /* 64 bit ALU operations. */ \
  797. /* Register based. */ \
  798. INSN_3(ALU64, ADD, X), \
  799. INSN_3(ALU64, SUB, X), \
  800. INSN_3(ALU64, AND, X), \
  801. INSN_3(ALU64, OR, X), \
  802. INSN_3(ALU64, LSH, X), \
  803. INSN_3(ALU64, RSH, X), \
  804. INSN_3(ALU64, XOR, X), \
  805. INSN_3(ALU64, MUL, X), \
  806. INSN_3(ALU64, MOV, X), \
  807. INSN_3(ALU64, ARSH, X), \
  808. INSN_3(ALU64, DIV, X), \
  809. INSN_3(ALU64, MOD, X), \
  810. INSN_2(ALU64, NEG), \
  811. /* Immediate based. */ \
  812. INSN_3(ALU64, ADD, K), \
  813. INSN_3(ALU64, SUB, K), \
  814. INSN_3(ALU64, AND, K), \
  815. INSN_3(ALU64, OR, K), \
  816. INSN_3(ALU64, LSH, K), \
  817. INSN_3(ALU64, RSH, K), \
  818. INSN_3(ALU64, XOR, K), \
  819. INSN_3(ALU64, MUL, K), \
  820. INSN_3(ALU64, MOV, K), \
  821. INSN_3(ALU64, ARSH, K), \
  822. INSN_3(ALU64, DIV, K), \
  823. INSN_3(ALU64, MOD, K), \
  824. /* Call instruction. */ \
  825. INSN_2(JMP, CALL), \
  826. /* Exit instruction. */ \
  827. INSN_2(JMP, EXIT), \
  828. /* Jump instructions. */ \
  829. /* Register based. */ \
  830. INSN_3(JMP, JEQ, X), \
  831. INSN_3(JMP, JNE, X), \
  832. INSN_3(JMP, JGT, X), \
  833. INSN_3(JMP, JLT, X), \
  834. INSN_3(JMP, JGE, X), \
  835. INSN_3(JMP, JLE, X), \
  836. INSN_3(JMP, JSGT, X), \
  837. INSN_3(JMP, JSLT, X), \
  838. INSN_3(JMP, JSGE, X), \
  839. INSN_3(JMP, JSLE, X), \
  840. INSN_3(JMP, JSET, X), \
  841. /* Immediate based. */ \
  842. INSN_3(JMP, JEQ, K), \
  843. INSN_3(JMP, JNE, K), \
  844. INSN_3(JMP, JGT, K), \
  845. INSN_3(JMP, JLT, K), \
  846. INSN_3(JMP, JGE, K), \
  847. INSN_3(JMP, JLE, K), \
  848. INSN_3(JMP, JSGT, K), \
  849. INSN_3(JMP, JSLT, K), \
  850. INSN_3(JMP, JSGE, K), \
  851. INSN_3(JMP, JSLE, K), \
  852. INSN_3(JMP, JSET, K), \
  853. INSN_2(JMP, JA), \
  854. /* Store instructions. */ \
  855. /* Register based. */ \
  856. INSN_3(STX, MEM, B), \
  857. INSN_3(STX, MEM, H), \
  858. INSN_3(STX, MEM, W), \
  859. INSN_3(STX, MEM, DW), \
  860. INSN_3(STX, XADD, W), \
  861. INSN_3(STX, XADD, DW), \
  862. /* Immediate based. */ \
  863. INSN_3(ST, MEM, B), \
  864. INSN_3(ST, MEM, H), \
  865. INSN_3(ST, MEM, W), \
  866. INSN_3(ST, MEM, DW), \
  867. /* Load instructions. */ \
  868. /* Register based. */ \
  869. INSN_3(LDX, MEM, B), \
  870. INSN_3(LDX, MEM, H), \
  871. INSN_3(LDX, MEM, W), \
  872. INSN_3(LDX, MEM, DW), \
  873. /* Immediate based. */ \
  874. INSN_3(LD, IMM, DW)
  875. bool bpf_opcode_in_insntable(u8 code)
  876. {
  877. #define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true
  878. #define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true
  879. static const bool public_insntable[256] = {
  880. [0 ... 255] = false,
  881. /* Now overwrite non-defaults ... */
  882. BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL),
  883. /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */
  884. [BPF_LD | BPF_ABS | BPF_B] = true,
  885. [BPF_LD | BPF_ABS | BPF_H] = true,
  886. [BPF_LD | BPF_ABS | BPF_W] = true,
  887. [BPF_LD | BPF_IND | BPF_B] = true,
  888. [BPF_LD | BPF_IND | BPF_H] = true,
  889. [BPF_LD | BPF_IND | BPF_W] = true,
  890. };
  891. #undef BPF_INSN_3_TBL
  892. #undef BPF_INSN_2_TBL
  893. return public_insntable[code];
  894. }
  895. #ifndef CONFIG_BPF_JIT_ALWAYS_ON
  896. /**
  897. * __bpf_prog_run - run eBPF program on a given context
  898. * @ctx: is the data we are operating on
  899. * @insn: is the array of eBPF instructions
  900. *
  901. * Decode and execute eBPF instructions.
  902. */
  903. static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
  904. {
  905. #define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y
  906. #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z
  907. static const void *jumptable[256] = {
  908. [0 ... 255] = &&default_label,
  909. /* Now overwrite non-defaults ... */
  910. BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL),
  911. /* Non-UAPI available opcodes. */
  912. [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS,
  913. [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL,
  914. };
  915. #undef BPF_INSN_3_LBL
  916. #undef BPF_INSN_2_LBL
  917. u32 tail_call_cnt = 0;
  918. #define CONT ({ insn++; goto select_insn; })
  919. #define CONT_JMP ({ insn++; goto select_insn; })
  920. select_insn:
  921. goto *jumptable[insn->code];
  922. /* ALU */
  923. #define ALU(OPCODE, OP) \
  924. ALU64_##OPCODE##_X: \
  925. DST = DST OP SRC; \
  926. CONT; \
  927. ALU_##OPCODE##_X: \
  928. DST = (u32) DST OP (u32) SRC; \
  929. CONT; \
  930. ALU64_##OPCODE##_K: \
  931. DST = DST OP IMM; \
  932. CONT; \
  933. ALU_##OPCODE##_K: \
  934. DST = (u32) DST OP (u32) IMM; \
  935. CONT;
  936. ALU(ADD, +)
  937. ALU(SUB, -)
  938. ALU(AND, &)
  939. ALU(OR, |)
  940. ALU(LSH, <<)
  941. ALU(RSH, >>)
  942. ALU(XOR, ^)
  943. ALU(MUL, *)
  944. #undef ALU
  945. ALU_NEG:
  946. DST = (u32) -DST;
  947. CONT;
  948. ALU64_NEG:
  949. DST = -DST;
  950. CONT;
  951. ALU_MOV_X:
  952. DST = (u32) SRC;
  953. CONT;
  954. ALU_MOV_K:
  955. DST = (u32) IMM;
  956. CONT;
  957. ALU64_MOV_X:
  958. DST = SRC;
  959. CONT;
  960. ALU64_MOV_K:
  961. DST = IMM;
  962. CONT;
  963. LD_IMM_DW:
  964. DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32;
  965. insn++;
  966. CONT;
  967. ALU64_ARSH_X:
  968. (*(s64 *) &DST) >>= SRC;
  969. CONT;
  970. ALU64_ARSH_K:
  971. (*(s64 *) &DST) >>= IMM;
  972. CONT;
  973. ALU64_MOD_X:
  974. div64_u64_rem(DST, SRC, &AX);
  975. DST = AX;
  976. CONT;
  977. ALU_MOD_X:
  978. AX = (u32) DST;
  979. DST = do_div(AX, (u32) SRC);
  980. CONT;
  981. ALU64_MOD_K:
  982. div64_u64_rem(DST, IMM, &AX);
  983. DST = AX;
  984. CONT;
  985. ALU_MOD_K:
  986. AX = (u32) DST;
  987. DST = do_div(AX, (u32) IMM);
  988. CONT;
  989. ALU64_DIV_X:
  990. DST = div64_u64(DST, SRC);
  991. CONT;
  992. ALU_DIV_X:
  993. AX = (u32) DST;
  994. do_div(AX, (u32) SRC);
  995. DST = (u32) AX;
  996. CONT;
  997. ALU64_DIV_K:
  998. DST = div64_u64(DST, IMM);
  999. CONT;
  1000. ALU_DIV_K:
  1001. AX = (u32) DST;
  1002. do_div(AX, (u32) IMM);
  1003. DST = (u32) AX;
  1004. CONT;
  1005. ALU_END_TO_BE:
  1006. switch (IMM) {
  1007. case 16:
  1008. DST = (__force u16) cpu_to_be16(DST);
  1009. break;
  1010. case 32:
  1011. DST = (__force u32) cpu_to_be32(DST);
  1012. break;
  1013. case 64:
  1014. DST = (__force u64) cpu_to_be64(DST);
  1015. break;
  1016. }
  1017. CONT;
  1018. ALU_END_TO_LE:
  1019. switch (IMM) {
  1020. case 16:
  1021. DST = (__force u16) cpu_to_le16(DST);
  1022. break;
  1023. case 32:
  1024. DST = (__force u32) cpu_to_le32(DST);
  1025. break;
  1026. case 64:
  1027. DST = (__force u64) cpu_to_le64(DST);
  1028. break;
  1029. }
  1030. CONT;
  1031. /* CALL */
  1032. JMP_CALL:
  1033. /* Function call scratches BPF_R1-BPF_R5 registers,
  1034. * preserves BPF_R6-BPF_R9, and stores return value
  1035. * into BPF_R0.
  1036. */
  1037. BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3,
  1038. BPF_R4, BPF_R5);
  1039. CONT;
  1040. JMP_CALL_ARGS:
  1041. BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
  1042. BPF_R3, BPF_R4,
  1043. BPF_R5,
  1044. insn + insn->off + 1);
  1045. CONT;
  1046. JMP_TAIL_CALL: {
  1047. struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2;
  1048. struct bpf_array *array = container_of(map, struct bpf_array, map);
  1049. struct bpf_prog *prog;
  1050. u32 index = BPF_R3;
  1051. if (unlikely(index >= array->map.max_entries))
  1052. goto out;
  1053. if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT))
  1054. goto out;
  1055. tail_call_cnt++;
  1056. prog = READ_ONCE(array->ptrs[index]);
  1057. if (!prog)
  1058. goto out;
  1059. /* ARG1 at this point is guaranteed to point to CTX from
  1060. * the verifier side due to the fact that the tail call is
  1061. * handeled like a helper, that is, bpf_tail_call_proto,
  1062. * where arg1_type is ARG_PTR_TO_CTX.
  1063. */
  1064. insn = prog->insnsi;
  1065. goto select_insn;
  1066. out:
  1067. CONT;
  1068. }
  1069. /* JMP */
  1070. JMP_JA:
  1071. insn += insn->off;
  1072. CONT;
  1073. JMP_JEQ_X:
  1074. if (DST == SRC) {
  1075. insn += insn->off;
  1076. CONT_JMP;
  1077. }
  1078. CONT;
  1079. JMP_JEQ_K:
  1080. if (DST == IMM) {
  1081. insn += insn->off;
  1082. CONT_JMP;
  1083. }
  1084. CONT;
  1085. JMP_JNE_X:
  1086. if (DST != SRC) {
  1087. insn += insn->off;
  1088. CONT_JMP;
  1089. }
  1090. CONT;
  1091. JMP_JNE_K:
  1092. if (DST != IMM) {
  1093. insn += insn->off;
  1094. CONT_JMP;
  1095. }
  1096. CONT;
  1097. JMP_JGT_X:
  1098. if (DST > SRC) {
  1099. insn += insn->off;
  1100. CONT_JMP;
  1101. }
  1102. CONT;
  1103. JMP_JGT_K:
  1104. if (DST > IMM) {
  1105. insn += insn->off;
  1106. CONT_JMP;
  1107. }
  1108. CONT;
  1109. JMP_JLT_X:
  1110. if (DST < SRC) {
  1111. insn += insn->off;
  1112. CONT_JMP;
  1113. }
  1114. CONT;
  1115. JMP_JLT_K:
  1116. if (DST < IMM) {
  1117. insn += insn->off;
  1118. CONT_JMP;
  1119. }
  1120. CONT;
  1121. JMP_JGE_X:
  1122. if (DST >= SRC) {
  1123. insn += insn->off;
  1124. CONT_JMP;
  1125. }
  1126. CONT;
  1127. JMP_JGE_K:
  1128. if (DST >= IMM) {
  1129. insn += insn->off;
  1130. CONT_JMP;
  1131. }
  1132. CONT;
  1133. JMP_JLE_X:
  1134. if (DST <= SRC) {
  1135. insn += insn->off;
  1136. CONT_JMP;
  1137. }
  1138. CONT;
  1139. JMP_JLE_K:
  1140. if (DST <= IMM) {
  1141. insn += insn->off;
  1142. CONT_JMP;
  1143. }
  1144. CONT;
  1145. JMP_JSGT_X:
  1146. if (((s64) DST) > ((s64) SRC)) {
  1147. insn += insn->off;
  1148. CONT_JMP;
  1149. }
  1150. CONT;
  1151. JMP_JSGT_K:
  1152. if (((s64) DST) > ((s64) IMM)) {
  1153. insn += insn->off;
  1154. CONT_JMP;
  1155. }
  1156. CONT;
  1157. JMP_JSLT_X:
  1158. if (((s64) DST) < ((s64) SRC)) {
  1159. insn += insn->off;
  1160. CONT_JMP;
  1161. }
  1162. CONT;
  1163. JMP_JSLT_K:
  1164. if (((s64) DST) < ((s64) IMM)) {
  1165. insn += insn->off;
  1166. CONT_JMP;
  1167. }
  1168. CONT;
  1169. JMP_JSGE_X:
  1170. if (((s64) DST) >= ((s64) SRC)) {
  1171. insn += insn->off;
  1172. CONT_JMP;
  1173. }
  1174. CONT;
  1175. JMP_JSGE_K:
  1176. if (((s64) DST) >= ((s64) IMM)) {
  1177. insn += insn->off;
  1178. CONT_JMP;
  1179. }
  1180. CONT;
  1181. JMP_JSLE_X:
  1182. if (((s64) DST) <= ((s64) SRC)) {
  1183. insn += insn->off;
  1184. CONT_JMP;
  1185. }
  1186. CONT;
  1187. JMP_JSLE_K:
  1188. if (((s64) DST) <= ((s64) IMM)) {
  1189. insn += insn->off;
  1190. CONT_JMP;
  1191. }
  1192. CONT;
  1193. JMP_JSET_X:
  1194. if (DST & SRC) {
  1195. insn += insn->off;
  1196. CONT_JMP;
  1197. }
  1198. CONT;
  1199. JMP_JSET_K:
  1200. if (DST & IMM) {
  1201. insn += insn->off;
  1202. CONT_JMP;
  1203. }
  1204. CONT;
  1205. JMP_EXIT:
  1206. return BPF_R0;
  1207. /* STX and ST and LDX*/
  1208. #define LDST(SIZEOP, SIZE) \
  1209. STX_MEM_##SIZEOP: \
  1210. *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \
  1211. CONT; \
  1212. ST_MEM_##SIZEOP: \
  1213. *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \
  1214. CONT; \
  1215. LDX_MEM_##SIZEOP: \
  1216. DST = *(SIZE *)(unsigned long) (SRC + insn->off); \
  1217. CONT;
  1218. LDST(B, u8)
  1219. LDST(H, u16)
  1220. LDST(W, u32)
  1221. LDST(DW, u64)
  1222. #undef LDST
  1223. STX_XADD_W: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */
  1224. atomic_add((u32) SRC, (atomic_t *)(unsigned long)
  1225. (DST + insn->off));
  1226. CONT;
  1227. STX_XADD_DW: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */
  1228. atomic64_add((u64) SRC, (atomic64_t *)(unsigned long)
  1229. (DST + insn->off));
  1230. CONT;
  1231. default_label:
  1232. /* If we ever reach this, we have a bug somewhere. Die hard here
  1233. * instead of just returning 0; we could be somewhere in a subprog,
  1234. * so execution could continue otherwise which we do /not/ want.
  1235. *
  1236. * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable().
  1237. */
  1238. pr_warn("BPF interpreter: unknown opcode %02x\n", insn->code);
  1239. BUG_ON(1);
  1240. return 0;
  1241. }
  1242. STACK_FRAME_NON_STANDARD(___bpf_prog_run); /* jump table */
  1243. #define PROG_NAME(stack_size) __bpf_prog_run##stack_size
  1244. #define DEFINE_BPF_PROG_RUN(stack_size) \
  1245. static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \
  1246. { \
  1247. u64 stack[stack_size / sizeof(u64)]; \
  1248. u64 regs[MAX_BPF_EXT_REG]; \
  1249. \
  1250. FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
  1251. ARG1 = (u64) (unsigned long) ctx; \
  1252. return ___bpf_prog_run(regs, insn, stack); \
  1253. }
  1254. #define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size
  1255. #define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \
  1256. static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \
  1257. const struct bpf_insn *insn) \
  1258. { \
  1259. u64 stack[stack_size / sizeof(u64)]; \
  1260. u64 regs[MAX_BPF_EXT_REG]; \
  1261. \
  1262. FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
  1263. BPF_R1 = r1; \
  1264. BPF_R2 = r2; \
  1265. BPF_R3 = r3; \
  1266. BPF_R4 = r4; \
  1267. BPF_R5 = r5; \
  1268. return ___bpf_prog_run(regs, insn, stack); \
  1269. }
  1270. #define EVAL1(FN, X) FN(X)
  1271. #define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y)
  1272. #define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y)
  1273. #define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y)
  1274. #define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y)
  1275. #define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y)
  1276. EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192);
  1277. EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384);
  1278. EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512);
  1279. EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192);
  1280. EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384);
  1281. EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512);
  1282. #define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size),
  1283. static unsigned int (*interpreters[])(const void *ctx,
  1284. const struct bpf_insn *insn) = {
  1285. EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192)
  1286. EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384)
  1287. EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
  1288. };
  1289. #undef PROG_NAME_LIST
  1290. #define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size),
  1291. static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
  1292. const struct bpf_insn *insn) = {
  1293. EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192)
  1294. EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384)
  1295. EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
  1296. };
  1297. #undef PROG_NAME_LIST
  1298. void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
  1299. {
  1300. stack_depth = max_t(u32, stack_depth, 1);
  1301. insn->off = (s16) insn->imm;
  1302. insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
  1303. __bpf_call_base_args;
  1304. insn->code = BPF_JMP | BPF_CALL_ARGS;
  1305. }
  1306. #else
  1307. static unsigned int __bpf_prog_ret0_warn(const void *ctx,
  1308. const struct bpf_insn *insn)
  1309. {
  1310. /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
  1311. * is not working properly, so warn about it!
  1312. */
  1313. WARN_ON_ONCE(1);
  1314. return 0;
  1315. }
  1316. #endif
  1317. bool bpf_prog_array_compatible(struct bpf_array *array,
  1318. const struct bpf_prog *fp)
  1319. {
  1320. if (fp->kprobe_override)
  1321. return false;
  1322. if (!array->owner_prog_type) {
  1323. /* There's no owner yet where we could check for
  1324. * compatibility.
  1325. */
  1326. array->owner_prog_type = fp->type;
  1327. array->owner_jited = fp->jited;
  1328. return true;
  1329. }
  1330. return array->owner_prog_type == fp->type &&
  1331. array->owner_jited == fp->jited;
  1332. }
  1333. static int bpf_check_tail_call(const struct bpf_prog *fp)
  1334. {
  1335. struct bpf_prog_aux *aux = fp->aux;
  1336. int i;
  1337. for (i = 0; i < aux->used_map_cnt; i++) {
  1338. struct bpf_map *map = aux->used_maps[i];
  1339. struct bpf_array *array;
  1340. if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
  1341. continue;
  1342. array = container_of(map, struct bpf_array, map);
  1343. if (!bpf_prog_array_compatible(array, fp))
  1344. return -EINVAL;
  1345. }
  1346. return 0;
  1347. }
  1348. static void bpf_prog_select_func(struct bpf_prog *fp)
  1349. {
  1350. #ifndef CONFIG_BPF_JIT_ALWAYS_ON
  1351. u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);
  1352. fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];
  1353. #else
  1354. fp->bpf_func = __bpf_prog_ret0_warn;
  1355. #endif
  1356. }
  1357. /**
  1358. * bpf_prog_select_runtime - select exec runtime for BPF program
  1359. * @fp: bpf_prog populated with internal BPF program
  1360. * @err: pointer to error variable
  1361. *
  1362. * Try to JIT eBPF program, if JIT is not available, use interpreter.
  1363. * The BPF program will be executed via BPF_PROG_RUN() macro.
  1364. */
  1365. struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
  1366. {
  1367. /* In case of BPF to BPF calls, verifier did all the prep
  1368. * work with regards to JITing, etc.
  1369. */
  1370. if (fp->bpf_func)
  1371. goto finalize;
  1372. bpf_prog_select_func(fp);
  1373. /* eBPF JITs can rewrite the program in case constant
  1374. * blinding is active. However, in case of error during
  1375. * blinding, bpf_int_jit_compile() must always return a
  1376. * valid program, which in this case would simply not
  1377. * be JITed, but falls back to the interpreter.
  1378. */
  1379. if (!bpf_prog_is_dev_bound(fp->aux)) {
  1380. fp = bpf_int_jit_compile(fp);
  1381. #ifdef CONFIG_BPF_JIT_ALWAYS_ON
  1382. if (!fp->jited) {
  1383. *err = -ENOTSUPP;
  1384. return fp;
  1385. }
  1386. #endif
  1387. } else {
  1388. *err = bpf_prog_offload_compile(fp);
  1389. if (*err)
  1390. return fp;
  1391. }
  1392. finalize:
  1393. bpf_prog_lock_ro(fp);
  1394. /* The tail call compatibility check can only be done at
  1395. * this late stage as we need to determine, if we deal
  1396. * with JITed or non JITed program concatenations and not
  1397. * all eBPF JITs might immediately support all features.
  1398. */
  1399. *err = bpf_check_tail_call(fp);
  1400. return fp;
  1401. }
  1402. EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
  1403. static unsigned int __bpf_prog_ret1(const void *ctx,
  1404. const struct bpf_insn *insn)
  1405. {
  1406. return 1;
  1407. }
  1408. static struct bpf_prog_dummy {
  1409. struct bpf_prog prog;
  1410. } dummy_bpf_prog = {
  1411. .prog = {
  1412. .bpf_func = __bpf_prog_ret1,
  1413. },
  1414. };
  1415. /* to avoid allocating empty bpf_prog_array for cgroups that
  1416. * don't have bpf program attached use one global 'empty_prog_array'
  1417. * It will not be modified the caller of bpf_prog_array_alloc()
  1418. * (since caller requested prog_cnt == 0)
  1419. * that pointer should be 'freed' by bpf_prog_array_free()
  1420. */
  1421. static struct {
  1422. struct bpf_prog_array hdr;
  1423. struct bpf_prog *null_prog;
  1424. } empty_prog_array = {
  1425. .null_prog = NULL,
  1426. };
  1427. struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
  1428. {
  1429. if (prog_cnt)
  1430. return kzalloc(sizeof(struct bpf_prog_array) +
  1431. sizeof(struct bpf_prog_array_item) *
  1432. (prog_cnt + 1),
  1433. flags);
  1434. return &empty_prog_array.hdr;
  1435. }
  1436. void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
  1437. {
  1438. if (!progs ||
  1439. progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
  1440. return;
  1441. kfree_rcu(progs, rcu);
  1442. }
  1443. int bpf_prog_array_length(struct bpf_prog_array __rcu *array)
  1444. {
  1445. struct bpf_prog_array_item *item;
  1446. u32 cnt = 0;
  1447. rcu_read_lock();
  1448. item = rcu_dereference(array)->items;
  1449. for (; item->prog; item++)
  1450. if (item->prog != &dummy_bpf_prog.prog)
  1451. cnt++;
  1452. rcu_read_unlock();
  1453. return cnt;
  1454. }
  1455. static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
  1456. u32 *prog_ids,
  1457. u32 request_cnt)
  1458. {
  1459. struct bpf_prog_array_item *item;
  1460. int i = 0;
  1461. item = rcu_dereference_check(array, 1)->items;
  1462. for (; item->prog; item++) {
  1463. if (item->prog == &dummy_bpf_prog.prog)
  1464. continue;
  1465. prog_ids[i] = item->prog->aux->id;
  1466. if (++i == request_cnt) {
  1467. item++;
  1468. break;
  1469. }
  1470. }
  1471. return !!(item->prog);
  1472. }
  1473. int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
  1474. __u32 __user *prog_ids, u32 cnt)
  1475. {
  1476. unsigned long err = 0;
  1477. bool nospc;
  1478. u32 *ids;
  1479. /* users of this function are doing:
  1480. * cnt = bpf_prog_array_length();
  1481. * if (cnt > 0)
  1482. * bpf_prog_array_copy_to_user(..., cnt);
  1483. * so below kcalloc doesn't need extra cnt > 0 check, but
  1484. * bpf_prog_array_length() releases rcu lock and
  1485. * prog array could have been swapped with empty or larger array,
  1486. * so always copy 'cnt' prog_ids to the user.
  1487. * In a rare race the user will see zero prog_ids
  1488. */
  1489. ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN);
  1490. if (!ids)
  1491. return -ENOMEM;
  1492. rcu_read_lock();
  1493. nospc = bpf_prog_array_copy_core(array, ids, cnt);
  1494. rcu_read_unlock();
  1495. err = copy_to_user(prog_ids, ids, cnt * sizeof(u32));
  1496. kfree(ids);
  1497. if (err)
  1498. return -EFAULT;
  1499. if (nospc)
  1500. return -ENOSPC;
  1501. return 0;
  1502. }
  1503. void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *array,
  1504. struct bpf_prog *old_prog)
  1505. {
  1506. struct bpf_prog_array_item *item = array->items;
  1507. for (; item->prog; item++)
  1508. if (item->prog == old_prog) {
  1509. WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);
  1510. break;
  1511. }
  1512. }
  1513. int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
  1514. struct bpf_prog *exclude_prog,
  1515. struct bpf_prog *include_prog,
  1516. struct bpf_prog_array **new_array)
  1517. {
  1518. int new_prog_cnt, carry_prog_cnt = 0;
  1519. struct bpf_prog_array_item *existing;
  1520. struct bpf_prog_array *array;
  1521. bool found_exclude = false;
  1522. int new_prog_idx = 0;
  1523. /* Figure out how many existing progs we need to carry over to
  1524. * the new array.
  1525. */
  1526. if (old_array) {
  1527. existing = old_array->items;
  1528. for (; existing->prog; existing++) {
  1529. if (existing->prog == exclude_prog) {
  1530. found_exclude = true;
  1531. continue;
  1532. }
  1533. if (existing->prog != &dummy_bpf_prog.prog)
  1534. carry_prog_cnt++;
  1535. if (existing->prog == include_prog)
  1536. return -EEXIST;
  1537. }
  1538. }
  1539. if (exclude_prog && !found_exclude)
  1540. return -ENOENT;
  1541. /* How many progs (not NULL) will be in the new array? */
  1542. new_prog_cnt = carry_prog_cnt;
  1543. if (include_prog)
  1544. new_prog_cnt += 1;
  1545. /* Do we have any prog (not NULL) in the new array? */
  1546. if (!new_prog_cnt) {
  1547. *new_array = NULL;
  1548. return 0;
  1549. }
  1550. /* +1 as the end of prog_array is marked with NULL */
  1551. array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL);
  1552. if (!array)
  1553. return -ENOMEM;
  1554. /* Fill in the new prog array */
  1555. if (carry_prog_cnt) {
  1556. existing = old_array->items;
  1557. for (; existing->prog; existing++)
  1558. if (existing->prog != exclude_prog &&
  1559. existing->prog != &dummy_bpf_prog.prog) {
  1560. array->items[new_prog_idx++].prog =
  1561. existing->prog;
  1562. }
  1563. }
  1564. if (include_prog)
  1565. array->items[new_prog_idx++].prog = include_prog;
  1566. array->items[new_prog_idx].prog = NULL;
  1567. *new_array = array;
  1568. return 0;
  1569. }
  1570. int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
  1571. u32 *prog_ids, u32 request_cnt,
  1572. u32 *prog_cnt)
  1573. {
  1574. u32 cnt = 0;
  1575. if (array)
  1576. cnt = bpf_prog_array_length(array);
  1577. *prog_cnt = cnt;
  1578. /* return early if user requested only program count or nothing to copy */
  1579. if (!request_cnt || !cnt)
  1580. return 0;
  1581. /* this function is called under trace/bpf_trace.c: bpf_event_mutex */
  1582. return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC
  1583. : 0;
  1584. }
  1585. static void bpf_prog_free_deferred(struct work_struct *work)
  1586. {
  1587. struct bpf_prog_aux *aux;
  1588. int i;
  1589. aux = container_of(work, struct bpf_prog_aux, work);
  1590. if (bpf_prog_is_dev_bound(aux))
  1591. bpf_prog_offload_destroy(aux->prog);
  1592. #ifdef CONFIG_PERF_EVENTS
  1593. if (aux->prog->has_callchain_buf)
  1594. put_callchain_buffers();
  1595. #endif
  1596. for (i = 0; i < aux->func_cnt; i++)
  1597. bpf_jit_free(aux->func[i]);
  1598. if (aux->func_cnt) {
  1599. kfree(aux->func);
  1600. bpf_prog_unlock_free(aux->prog);
  1601. } else {
  1602. bpf_jit_free(aux->prog);
  1603. }
  1604. }
  1605. /* Free internal BPF program */
  1606. void bpf_prog_free(struct bpf_prog *fp)
  1607. {
  1608. struct bpf_prog_aux *aux = fp->aux;
  1609. INIT_WORK(&aux->work, bpf_prog_free_deferred);
  1610. schedule_work(&aux->work);
  1611. }
  1612. EXPORT_SYMBOL_GPL(bpf_prog_free);
  1613. /* RNG for unpriviledged user space with separated state from prandom_u32(). */
  1614. static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state);
  1615. void bpf_user_rnd_init_once(void)
  1616. {
  1617. prandom_init_once(&bpf_user_rnd_state);
  1618. }
  1619. BPF_CALL_0(bpf_user_rnd_u32)
  1620. {
  1621. /* Should someone ever have the rather unwise idea to use some
  1622. * of the registers passed into this function, then note that
  1623. * this function is called from native eBPF and classic-to-eBPF
  1624. * transformations. Register assignments from both sides are
  1625. * different, f.e. classic always sets fn(ctx, A, X) here.
  1626. */
  1627. struct rnd_state *state;
  1628. u32 res;
  1629. state = &get_cpu_var(bpf_user_rnd_state);
  1630. res = prandom_u32_state(state);
  1631. put_cpu_var(bpf_user_rnd_state);
  1632. return res;
  1633. }
  1634. /* Weak definitions of helper functions in case we don't have bpf syscall. */
  1635. const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
  1636. const struct bpf_func_proto bpf_map_update_elem_proto __weak;
  1637. const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
  1638. const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
  1639. const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
  1640. const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;
  1641. const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
  1642. const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
  1643. const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
  1644. const struct bpf_func_proto bpf_get_current_comm_proto __weak;
  1645. const struct bpf_func_proto bpf_sock_map_update_proto __weak;
  1646. const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
  1647. const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;
  1648. const struct bpf_func_proto bpf_get_local_storage_proto __weak;
  1649. const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
  1650. {
  1651. return NULL;
  1652. }
  1653. u64 __weak
  1654. bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
  1655. void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
  1656. {
  1657. return -ENOTSUPP;
  1658. }
  1659. EXPORT_SYMBOL_GPL(bpf_event_output);
  1660. /* Always built-in helper functions. */
  1661. const struct bpf_func_proto bpf_tail_call_proto = {
  1662. .func = NULL,
  1663. .gpl_only = false,
  1664. .ret_type = RET_VOID,
  1665. .arg1_type = ARG_PTR_TO_CTX,
  1666. .arg2_type = ARG_CONST_MAP_PTR,
  1667. .arg3_type = ARG_ANYTHING,
  1668. };
  1669. /* Stub for JITs that only support cBPF. eBPF programs are interpreted.
  1670. * It is encouraged to implement bpf_int_jit_compile() instead, so that
  1671. * eBPF and implicitly also cBPF can get JITed!
  1672. */
  1673. struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
  1674. {
  1675. return prog;
  1676. }
  1677. /* Stub for JITs that support eBPF. All cBPF code gets transformed into
  1678. * eBPF by the kernel and is later compiled by bpf_int_jit_compile().
  1679. */
  1680. void __weak bpf_jit_compile(struct bpf_prog *prog)
  1681. {
  1682. }
  1683. bool __weak bpf_helper_changes_pkt_data(void *func)
  1684. {
  1685. return false;
  1686. }
  1687. /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
  1688. * skb_copy_bits(), so provide a weak definition of it for NET-less config.
  1689. */
  1690. int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to,
  1691. int len)
  1692. {
  1693. return -EFAULT;
  1694. }
  1695. /* All definitions of tracepoints related to BPF. */
  1696. #define CREATE_TRACE_POINTS
  1697. #include <linux/bpf_trace.h>
  1698. EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);