arraymap.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750
  1. /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  2. * Copyright (c) 2016,2017 Facebook
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of version 2 of the GNU General Public
  6. * License as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful, but
  9. * WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. */
  13. #include <linux/bpf.h>
  14. #include <linux/btf.h>
  15. #include <linux/err.h>
  16. #include <linux/slab.h>
  17. #include <linux/mm.h>
  18. #include <linux/filter.h>
  19. #include <linux/perf_event.h>
  20. #include <uapi/linux/btf.h>
  21. #include "map_in_map.h"
  22. #define ARRAY_CREATE_FLAG_MASK \
  23. (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
  24. static void bpf_array_free_percpu(struct bpf_array *array)
  25. {
  26. int i;
  27. for (i = 0; i < array->map.max_entries; i++) {
  28. free_percpu(array->pptrs[i]);
  29. cond_resched();
  30. }
  31. }
  32. static int bpf_array_alloc_percpu(struct bpf_array *array)
  33. {
  34. void __percpu *ptr;
  35. int i;
  36. for (i = 0; i < array->map.max_entries; i++) {
  37. ptr = __alloc_percpu_gfp(array->elem_size, 8,
  38. GFP_USER | __GFP_NOWARN);
  39. if (!ptr) {
  40. bpf_array_free_percpu(array);
  41. return -ENOMEM;
  42. }
  43. array->pptrs[i] = ptr;
  44. cond_resched();
  45. }
  46. return 0;
  47. }
  48. /* Called from syscall */
  49. int array_map_alloc_check(union bpf_attr *attr)
  50. {
  51. bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
  52. int numa_node = bpf_map_attr_numa_node(attr);
  53. /* check sanity of attributes */
  54. if (attr->max_entries == 0 || attr->key_size != 4 ||
  55. attr->value_size == 0 ||
  56. attr->map_flags & ~ARRAY_CREATE_FLAG_MASK ||
  57. (percpu && numa_node != NUMA_NO_NODE))
  58. return -EINVAL;
  59. if (attr->value_size > KMALLOC_MAX_SIZE)
  60. /* if value_size is bigger, the user space won't be able to
  61. * access the elements.
  62. */
  63. return -E2BIG;
  64. return 0;
  65. }
  66. static struct bpf_map *array_map_alloc(union bpf_attr *attr)
  67. {
  68. bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY;
  69. int ret, numa_node = bpf_map_attr_numa_node(attr);
  70. u32 elem_size, index_mask, max_entries;
  71. bool unpriv = !capable(CAP_SYS_ADMIN);
  72. u64 cost, array_size, mask64;
  73. struct bpf_array *array;
  74. elem_size = round_up(attr->value_size, 8);
  75. max_entries = attr->max_entries;
  76. /* On 32 bit archs roundup_pow_of_two() with max_entries that has
  77. * upper most bit set in u32 space is undefined behavior due to
  78. * resulting 1U << 32, so do it manually here in u64 space.
  79. */
  80. mask64 = fls_long(max_entries - 1);
  81. mask64 = 1ULL << mask64;
  82. mask64 -= 1;
  83. index_mask = mask64;
  84. if (unpriv) {
  85. /* round up array size to nearest power of 2,
  86. * since cpu will speculate within index_mask limits
  87. */
  88. max_entries = index_mask + 1;
  89. /* Check for overflows. */
  90. if (max_entries < attr->max_entries)
  91. return ERR_PTR(-E2BIG);
  92. }
  93. array_size = sizeof(*array);
  94. if (percpu)
  95. array_size += (u64) max_entries * sizeof(void *);
  96. else
  97. array_size += (u64) max_entries * elem_size;
  98. /* make sure there is no u32 overflow later in round_up() */
  99. cost = array_size;
  100. if (cost >= U32_MAX - PAGE_SIZE)
  101. return ERR_PTR(-ENOMEM);
  102. if (percpu) {
  103. cost += (u64)attr->max_entries * elem_size * num_possible_cpus();
  104. if (cost >= U32_MAX - PAGE_SIZE)
  105. return ERR_PTR(-ENOMEM);
  106. }
  107. cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
  108. ret = bpf_map_precharge_memlock(cost);
  109. if (ret < 0)
  110. return ERR_PTR(ret);
  111. /* allocate all map elements and zero-initialize them */
  112. array = bpf_map_area_alloc(array_size, numa_node);
  113. if (!array)
  114. return ERR_PTR(-ENOMEM);
  115. array->index_mask = index_mask;
  116. array->map.unpriv_array = unpriv;
  117. /* copy mandatory map attributes */
  118. bpf_map_init_from_attr(&array->map, attr);
  119. array->map.pages = cost;
  120. array->elem_size = elem_size;
  121. if (percpu && bpf_array_alloc_percpu(array)) {
  122. bpf_map_area_free(array);
  123. return ERR_PTR(-ENOMEM);
  124. }
  125. return &array->map;
  126. }
  127. /* Called from syscall or from eBPF program */
  128. static void *array_map_lookup_elem(struct bpf_map *map, void *key)
  129. {
  130. struct bpf_array *array = container_of(map, struct bpf_array, map);
  131. u32 index = *(u32 *)key;
  132. if (unlikely(index >= array->map.max_entries))
  133. return NULL;
  134. return array->value + array->elem_size * (index & array->index_mask);
  135. }
  136. /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
  137. static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
  138. {
  139. struct bpf_array *array = container_of(map, struct bpf_array, map);
  140. struct bpf_insn *insn = insn_buf;
  141. u32 elem_size = round_up(map->value_size, 8);
  142. const int ret = BPF_REG_0;
  143. const int map_ptr = BPF_REG_1;
  144. const int index = BPF_REG_2;
  145. *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
  146. *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
  147. if (map->unpriv_array) {
  148. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4);
  149. *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
  150. } else {
  151. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3);
  152. }
  153. if (is_power_of_2(elem_size)) {
  154. *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
  155. } else {
  156. *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
  157. }
  158. *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
  159. *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
  160. *insn++ = BPF_MOV64_IMM(ret, 0);
  161. return insn - insn_buf;
  162. }
  163. /* Called from eBPF program */
  164. static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key)
  165. {
  166. struct bpf_array *array = container_of(map, struct bpf_array, map);
  167. u32 index = *(u32 *)key;
  168. if (unlikely(index >= array->map.max_entries))
  169. return NULL;
  170. return this_cpu_ptr(array->pptrs[index & array->index_mask]);
  171. }
  172. int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
  173. {
  174. struct bpf_array *array = container_of(map, struct bpf_array, map);
  175. u32 index = *(u32 *)key;
  176. void __percpu *pptr;
  177. int cpu, off = 0;
  178. u32 size;
  179. if (unlikely(index >= array->map.max_entries))
  180. return -ENOENT;
  181. /* per_cpu areas are zero-filled and bpf programs can only
  182. * access 'value_size' of them, so copying rounded areas
  183. * will not leak any kernel data
  184. */
  185. size = round_up(map->value_size, 8);
  186. rcu_read_lock();
  187. pptr = array->pptrs[index & array->index_mask];
  188. for_each_possible_cpu(cpu) {
  189. bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size);
  190. off += size;
  191. }
  192. rcu_read_unlock();
  193. return 0;
  194. }
  195. /* Called from syscall */
  196. static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
  197. {
  198. struct bpf_array *array = container_of(map, struct bpf_array, map);
  199. u32 index = key ? *(u32 *)key : U32_MAX;
  200. u32 *next = (u32 *)next_key;
  201. if (index >= array->map.max_entries) {
  202. *next = 0;
  203. return 0;
  204. }
  205. if (index == array->map.max_entries - 1)
  206. return -ENOENT;
  207. *next = index + 1;
  208. return 0;
  209. }
  210. /* Called from syscall or from eBPF program */
  211. static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
  212. u64 map_flags)
  213. {
  214. struct bpf_array *array = container_of(map, struct bpf_array, map);
  215. u32 index = *(u32 *)key;
  216. if (unlikely(map_flags > BPF_EXIST))
  217. /* unknown flags */
  218. return -EINVAL;
  219. if (unlikely(index >= array->map.max_entries))
  220. /* all elements were pre-allocated, cannot insert a new one */
  221. return -E2BIG;
  222. if (unlikely(map_flags == BPF_NOEXIST))
  223. /* all elements already exist */
  224. return -EEXIST;
  225. if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  226. memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),
  227. value, map->value_size);
  228. else
  229. memcpy(array->value +
  230. array->elem_size * (index & array->index_mask),
  231. value, map->value_size);
  232. return 0;
  233. }
  234. int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
  235. u64 map_flags)
  236. {
  237. struct bpf_array *array = container_of(map, struct bpf_array, map);
  238. u32 index = *(u32 *)key;
  239. void __percpu *pptr;
  240. int cpu, off = 0;
  241. u32 size;
  242. if (unlikely(map_flags > BPF_EXIST))
  243. /* unknown flags */
  244. return -EINVAL;
  245. if (unlikely(index >= array->map.max_entries))
  246. /* all elements were pre-allocated, cannot insert a new one */
  247. return -E2BIG;
  248. if (unlikely(map_flags == BPF_NOEXIST))
  249. /* all elements already exist */
  250. return -EEXIST;
  251. /* the user space will provide round_up(value_size, 8) bytes that
  252. * will be copied into per-cpu area. bpf programs can only access
  253. * value_size of it. During lookup the same extra bytes will be
  254. * returned or zeros which were zero-filled by percpu_alloc,
  255. * so no kernel data leaks possible
  256. */
  257. size = round_up(map->value_size, 8);
  258. rcu_read_lock();
  259. pptr = array->pptrs[index & array->index_mask];
  260. for_each_possible_cpu(cpu) {
  261. bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
  262. off += size;
  263. }
  264. rcu_read_unlock();
  265. return 0;
  266. }
  267. /* Called from syscall or from eBPF program */
  268. static int array_map_delete_elem(struct bpf_map *map, void *key)
  269. {
  270. return -EINVAL;
  271. }
  272. /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
  273. static void array_map_free(struct bpf_map *map)
  274. {
  275. struct bpf_array *array = container_of(map, struct bpf_array, map);
  276. /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
  277. * so the programs (can be more than one that used this map) were
  278. * disconnected from events. Wait for outstanding programs to complete
  279. * and free the array
  280. */
  281. synchronize_rcu();
  282. if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
  283. bpf_array_free_percpu(array);
  284. bpf_map_area_free(array);
  285. }
  286. static void array_map_seq_show_elem(struct bpf_map *map, void *key,
  287. struct seq_file *m)
  288. {
  289. void *value;
  290. rcu_read_lock();
  291. value = array_map_lookup_elem(map, key);
  292. if (!value) {
  293. rcu_read_unlock();
  294. return;
  295. }
  296. seq_printf(m, "%u: ", *(u32 *)key);
  297. btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
  298. seq_puts(m, "\n");
  299. rcu_read_unlock();
  300. }
  301. static int array_map_check_btf(const struct bpf_map *map,
  302. const struct btf_type *key_type,
  303. const struct btf_type *value_type)
  304. {
  305. u32 int_data;
  306. if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
  307. return -EINVAL;
  308. int_data = *(u32 *)(key_type + 1);
  309. /* bpf array can only take a u32 key. This check makes sure
  310. * that the btf matches the attr used during map_create.
  311. */
  312. if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
  313. return -EINVAL;
  314. return 0;
  315. }
  316. const struct bpf_map_ops array_map_ops = {
  317. .map_alloc_check = array_map_alloc_check,
  318. .map_alloc = array_map_alloc,
  319. .map_free = array_map_free,
  320. .map_get_next_key = array_map_get_next_key,
  321. .map_lookup_elem = array_map_lookup_elem,
  322. .map_update_elem = array_map_update_elem,
  323. .map_delete_elem = array_map_delete_elem,
  324. .map_gen_lookup = array_map_gen_lookup,
  325. .map_seq_show_elem = array_map_seq_show_elem,
  326. .map_check_btf = array_map_check_btf,
  327. };
  328. const struct bpf_map_ops percpu_array_map_ops = {
  329. .map_alloc_check = array_map_alloc_check,
  330. .map_alloc = array_map_alloc,
  331. .map_free = array_map_free,
  332. .map_get_next_key = array_map_get_next_key,
  333. .map_lookup_elem = percpu_array_map_lookup_elem,
  334. .map_update_elem = array_map_update_elem,
  335. .map_delete_elem = array_map_delete_elem,
  336. .map_check_btf = array_map_check_btf,
  337. };
  338. static int fd_array_map_alloc_check(union bpf_attr *attr)
  339. {
  340. /* only file descriptors can be stored in this type of map */
  341. if (attr->value_size != sizeof(u32))
  342. return -EINVAL;
  343. return array_map_alloc_check(attr);
  344. }
  345. static void fd_array_map_free(struct bpf_map *map)
  346. {
  347. struct bpf_array *array = container_of(map, struct bpf_array, map);
  348. int i;
  349. synchronize_rcu();
  350. /* make sure it's empty */
  351. for (i = 0; i < array->map.max_entries; i++)
  352. BUG_ON(array->ptrs[i] != NULL);
  353. bpf_map_area_free(array);
  354. }
  355. static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key)
  356. {
  357. return NULL;
  358. }
  359. /* only called from syscall */
  360. int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
  361. {
  362. void **elem, *ptr;
  363. int ret = 0;
  364. if (!map->ops->map_fd_sys_lookup_elem)
  365. return -ENOTSUPP;
  366. rcu_read_lock();
  367. elem = array_map_lookup_elem(map, key);
  368. if (elem && (ptr = READ_ONCE(*elem)))
  369. *value = map->ops->map_fd_sys_lookup_elem(ptr);
  370. else
  371. ret = -ENOENT;
  372. rcu_read_unlock();
  373. return ret;
  374. }
  375. /* only called from syscall */
  376. int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
  377. void *key, void *value, u64 map_flags)
  378. {
  379. struct bpf_array *array = container_of(map, struct bpf_array, map);
  380. void *new_ptr, *old_ptr;
  381. u32 index = *(u32 *)key, ufd;
  382. if (map_flags != BPF_ANY)
  383. return -EINVAL;
  384. if (index >= array->map.max_entries)
  385. return -E2BIG;
  386. ufd = *(u32 *)value;
  387. new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd);
  388. if (IS_ERR(new_ptr))
  389. return PTR_ERR(new_ptr);
  390. old_ptr = xchg(array->ptrs + index, new_ptr);
  391. if (old_ptr)
  392. map->ops->map_fd_put_ptr(old_ptr);
  393. return 0;
  394. }
  395. static int fd_array_map_delete_elem(struct bpf_map *map, void *key)
  396. {
  397. struct bpf_array *array = container_of(map, struct bpf_array, map);
  398. void *old_ptr;
  399. u32 index = *(u32 *)key;
  400. if (index >= array->map.max_entries)
  401. return -E2BIG;
  402. old_ptr = xchg(array->ptrs + index, NULL);
  403. if (old_ptr) {
  404. map->ops->map_fd_put_ptr(old_ptr);
  405. return 0;
  406. } else {
  407. return -ENOENT;
  408. }
  409. }
  410. static void *prog_fd_array_get_ptr(struct bpf_map *map,
  411. struct file *map_file, int fd)
  412. {
  413. struct bpf_array *array = container_of(map, struct bpf_array, map);
  414. struct bpf_prog *prog = bpf_prog_get(fd);
  415. if (IS_ERR(prog))
  416. return prog;
  417. if (!bpf_prog_array_compatible(array, prog)) {
  418. bpf_prog_put(prog);
  419. return ERR_PTR(-EINVAL);
  420. }
  421. return prog;
  422. }
  423. static void prog_fd_array_put_ptr(void *ptr)
  424. {
  425. bpf_prog_put(ptr);
  426. }
  427. static u32 prog_fd_array_sys_lookup_elem(void *ptr)
  428. {
  429. return ((struct bpf_prog *)ptr)->aux->id;
  430. }
  431. /* decrement refcnt of all bpf_progs that are stored in this map */
  432. static void bpf_fd_array_map_clear(struct bpf_map *map)
  433. {
  434. struct bpf_array *array = container_of(map, struct bpf_array, map);
  435. int i;
  436. for (i = 0; i < array->map.max_entries; i++)
  437. fd_array_map_delete_elem(map, &i);
  438. }
  439. const struct bpf_map_ops prog_array_map_ops = {
  440. .map_alloc_check = fd_array_map_alloc_check,
  441. .map_alloc = array_map_alloc,
  442. .map_free = fd_array_map_free,
  443. .map_get_next_key = array_map_get_next_key,
  444. .map_lookup_elem = fd_array_map_lookup_elem,
  445. .map_delete_elem = fd_array_map_delete_elem,
  446. .map_fd_get_ptr = prog_fd_array_get_ptr,
  447. .map_fd_put_ptr = prog_fd_array_put_ptr,
  448. .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem,
  449. .map_release_uref = bpf_fd_array_map_clear,
  450. .map_check_btf = map_check_no_btf,
  451. };
  452. static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,
  453. struct file *map_file)
  454. {
  455. struct bpf_event_entry *ee;
  456. ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
  457. if (ee) {
  458. ee->event = perf_file->private_data;
  459. ee->perf_file = perf_file;
  460. ee->map_file = map_file;
  461. }
  462. return ee;
  463. }
  464. static void __bpf_event_entry_free(struct rcu_head *rcu)
  465. {
  466. struct bpf_event_entry *ee;
  467. ee = container_of(rcu, struct bpf_event_entry, rcu);
  468. fput(ee->perf_file);
  469. kfree(ee);
  470. }
  471. static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee)
  472. {
  473. call_rcu(&ee->rcu, __bpf_event_entry_free);
  474. }
  475. static void *perf_event_fd_array_get_ptr(struct bpf_map *map,
  476. struct file *map_file, int fd)
  477. {
  478. struct bpf_event_entry *ee;
  479. struct perf_event *event;
  480. struct file *perf_file;
  481. u64 value;
  482. perf_file = perf_event_get(fd);
  483. if (IS_ERR(perf_file))
  484. return perf_file;
  485. ee = ERR_PTR(-EOPNOTSUPP);
  486. event = perf_file->private_data;
  487. if (perf_event_read_local(event, &value, NULL, NULL) == -EOPNOTSUPP)
  488. goto err_out;
  489. ee = bpf_event_entry_gen(perf_file, map_file);
  490. if (ee)
  491. return ee;
  492. ee = ERR_PTR(-ENOMEM);
  493. err_out:
  494. fput(perf_file);
  495. return ee;
  496. }
  497. static void perf_event_fd_array_put_ptr(void *ptr)
  498. {
  499. bpf_event_entry_free_rcu(ptr);
  500. }
  501. static void perf_event_fd_array_release(struct bpf_map *map,
  502. struct file *map_file)
  503. {
  504. struct bpf_array *array = container_of(map, struct bpf_array, map);
  505. struct bpf_event_entry *ee;
  506. int i;
  507. rcu_read_lock();
  508. for (i = 0; i < array->map.max_entries; i++) {
  509. ee = READ_ONCE(array->ptrs[i]);
  510. if (ee && ee->map_file == map_file)
  511. fd_array_map_delete_elem(map, &i);
  512. }
  513. rcu_read_unlock();
  514. }
  515. const struct bpf_map_ops perf_event_array_map_ops = {
  516. .map_alloc_check = fd_array_map_alloc_check,
  517. .map_alloc = array_map_alloc,
  518. .map_free = fd_array_map_free,
  519. .map_get_next_key = array_map_get_next_key,
  520. .map_lookup_elem = fd_array_map_lookup_elem,
  521. .map_delete_elem = fd_array_map_delete_elem,
  522. .map_fd_get_ptr = perf_event_fd_array_get_ptr,
  523. .map_fd_put_ptr = perf_event_fd_array_put_ptr,
  524. .map_release = perf_event_fd_array_release,
  525. .map_check_btf = map_check_no_btf,
  526. };
  527. #ifdef CONFIG_CGROUPS
  528. static void *cgroup_fd_array_get_ptr(struct bpf_map *map,
  529. struct file *map_file /* not used */,
  530. int fd)
  531. {
  532. return cgroup_get_from_fd(fd);
  533. }
  534. static void cgroup_fd_array_put_ptr(void *ptr)
  535. {
  536. /* cgroup_put free cgrp after a rcu grace period */
  537. cgroup_put(ptr);
  538. }
  539. static void cgroup_fd_array_free(struct bpf_map *map)
  540. {
  541. bpf_fd_array_map_clear(map);
  542. fd_array_map_free(map);
  543. }
  544. const struct bpf_map_ops cgroup_array_map_ops = {
  545. .map_alloc_check = fd_array_map_alloc_check,
  546. .map_alloc = array_map_alloc,
  547. .map_free = cgroup_fd_array_free,
  548. .map_get_next_key = array_map_get_next_key,
  549. .map_lookup_elem = fd_array_map_lookup_elem,
  550. .map_delete_elem = fd_array_map_delete_elem,
  551. .map_fd_get_ptr = cgroup_fd_array_get_ptr,
  552. .map_fd_put_ptr = cgroup_fd_array_put_ptr,
  553. .map_check_btf = map_check_no_btf,
  554. };
  555. #endif
  556. static struct bpf_map *array_of_map_alloc(union bpf_attr *attr)
  557. {
  558. struct bpf_map *map, *inner_map_meta;
  559. inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
  560. if (IS_ERR(inner_map_meta))
  561. return inner_map_meta;
  562. map = array_map_alloc(attr);
  563. if (IS_ERR(map)) {
  564. bpf_map_meta_free(inner_map_meta);
  565. return map;
  566. }
  567. map->inner_map_meta = inner_map_meta;
  568. return map;
  569. }
  570. static void array_of_map_free(struct bpf_map *map)
  571. {
  572. /* map->inner_map_meta is only accessed by syscall which
  573. * is protected by fdget/fdput.
  574. */
  575. bpf_map_meta_free(map->inner_map_meta);
  576. bpf_fd_array_map_clear(map);
  577. fd_array_map_free(map);
  578. }
  579. static void *array_of_map_lookup_elem(struct bpf_map *map, void *key)
  580. {
  581. struct bpf_map **inner_map = array_map_lookup_elem(map, key);
  582. if (!inner_map)
  583. return NULL;
  584. return READ_ONCE(*inner_map);
  585. }
  586. static u32 array_of_map_gen_lookup(struct bpf_map *map,
  587. struct bpf_insn *insn_buf)
  588. {
  589. struct bpf_array *array = container_of(map, struct bpf_array, map);
  590. u32 elem_size = round_up(map->value_size, 8);
  591. struct bpf_insn *insn = insn_buf;
  592. const int ret = BPF_REG_0;
  593. const int map_ptr = BPF_REG_1;
  594. const int index = BPF_REG_2;
  595. *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
  596. *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
  597. if (map->unpriv_array) {
  598. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6);
  599. *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask);
  600. } else {
  601. *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
  602. }
  603. if (is_power_of_2(elem_size))
  604. *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size));
  605. else
  606. *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size);
  607. *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr);
  608. *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
  609. *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
  610. *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1);
  611. *insn++ = BPF_MOV64_IMM(ret, 0);
  612. return insn - insn_buf;
  613. }
  614. const struct bpf_map_ops array_of_maps_map_ops = {
  615. .map_alloc_check = fd_array_map_alloc_check,
  616. .map_alloc = array_of_map_alloc,
  617. .map_free = array_of_map_free,
  618. .map_get_next_key = array_map_get_next_key,
  619. .map_lookup_elem = array_of_map_lookup_elem,
  620. .map_delete_elem = fd_array_map_delete_elem,
  621. .map_fd_get_ptr = bpf_map_fd_get_ptr,
  622. .map_fd_put_ptr = bpf_map_fd_put_ptr,
  623. .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
  624. .map_gen_lookup = array_of_map_gen_lookup,
  625. .map_check_btf = map_check_no_btf,
  626. };