inode.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689
  1. /*
  2. * Minimal file system backend for holding eBPF maps and programs,
  3. * used by bpf(2) object pinning.
  4. *
  5. * Authors:
  6. *
  7. * Daniel Borkmann <daniel@iogearbox.net>
  8. *
  9. * This program is free software; you can redistribute it and/or
  10. * modify it under the terms of the GNU General Public License
  11. * version 2 as published by the Free Software Foundation.
  12. */
  13. #include <linux/init.h>
  14. #include <linux/magic.h>
  15. #include <linux/major.h>
  16. #include <linux/mount.h>
  17. #include <linux/namei.h>
  18. #include <linux/fs.h>
  19. #include <linux/kdev_t.h>
  20. #include <linux/parser.h>
  21. #include <linux/filter.h>
  22. #include <linux/bpf.h>
  23. #include <linux/bpf_trace.h>
  24. enum bpf_type {
  25. BPF_TYPE_UNSPEC = 0,
  26. BPF_TYPE_PROG,
  27. BPF_TYPE_MAP,
  28. };
  29. static void *bpf_any_get(void *raw, enum bpf_type type)
  30. {
  31. switch (type) {
  32. case BPF_TYPE_PROG:
  33. raw = bpf_prog_inc(raw);
  34. break;
  35. case BPF_TYPE_MAP:
  36. raw = bpf_map_inc(raw, true);
  37. break;
  38. default:
  39. WARN_ON_ONCE(1);
  40. break;
  41. }
  42. return raw;
  43. }
  44. static void bpf_any_put(void *raw, enum bpf_type type)
  45. {
  46. switch (type) {
  47. case BPF_TYPE_PROG:
  48. bpf_prog_put(raw);
  49. break;
  50. case BPF_TYPE_MAP:
  51. bpf_map_put_with_uref(raw);
  52. break;
  53. default:
  54. WARN_ON_ONCE(1);
  55. break;
  56. }
  57. }
  58. static void *bpf_fd_probe_obj(u32 ufd, enum bpf_type *type)
  59. {
  60. void *raw;
  61. *type = BPF_TYPE_MAP;
  62. raw = bpf_map_get_with_uref(ufd);
  63. if (IS_ERR(raw)) {
  64. *type = BPF_TYPE_PROG;
  65. raw = bpf_prog_get(ufd);
  66. }
  67. return raw;
  68. }
  69. static const struct inode_operations bpf_dir_iops;
  70. static const struct inode_operations bpf_prog_iops = { };
  71. static const struct inode_operations bpf_map_iops = { };
  72. static struct inode *bpf_get_inode(struct super_block *sb,
  73. const struct inode *dir,
  74. umode_t mode)
  75. {
  76. struct inode *inode;
  77. switch (mode & S_IFMT) {
  78. case S_IFDIR:
  79. case S_IFREG:
  80. case S_IFLNK:
  81. break;
  82. default:
  83. return ERR_PTR(-EINVAL);
  84. }
  85. inode = new_inode(sb);
  86. if (!inode)
  87. return ERR_PTR(-ENOSPC);
  88. inode->i_ino = get_next_ino();
  89. inode->i_atime = current_time(inode);
  90. inode->i_mtime = inode->i_atime;
  91. inode->i_ctime = inode->i_atime;
  92. inode_init_owner(inode, dir, mode);
  93. return inode;
  94. }
  95. static int bpf_inode_type(const struct inode *inode, enum bpf_type *type)
  96. {
  97. *type = BPF_TYPE_UNSPEC;
  98. if (inode->i_op == &bpf_prog_iops)
  99. *type = BPF_TYPE_PROG;
  100. else if (inode->i_op == &bpf_map_iops)
  101. *type = BPF_TYPE_MAP;
  102. else
  103. return -EACCES;
  104. return 0;
  105. }
  106. static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode,
  107. struct inode *dir)
  108. {
  109. d_instantiate(dentry, inode);
  110. dget(dentry);
  111. dir->i_mtime = current_time(dir);
  112. dir->i_ctime = dir->i_mtime;
  113. }
  114. static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
  115. {
  116. struct inode *inode;
  117. inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR);
  118. if (IS_ERR(inode))
  119. return PTR_ERR(inode);
  120. inode->i_op = &bpf_dir_iops;
  121. inode->i_fop = &simple_dir_operations;
  122. inc_nlink(inode);
  123. inc_nlink(dir);
  124. bpf_dentry_finalize(dentry, inode, dir);
  125. return 0;
  126. }
  127. struct map_iter {
  128. void *key;
  129. bool done;
  130. };
  131. static struct map_iter *map_iter(struct seq_file *m)
  132. {
  133. return m->private;
  134. }
  135. static struct bpf_map *seq_file_to_map(struct seq_file *m)
  136. {
  137. return file_inode(m->file)->i_private;
  138. }
  139. static void map_iter_free(struct map_iter *iter)
  140. {
  141. if (iter) {
  142. kfree(iter->key);
  143. kfree(iter);
  144. }
  145. }
  146. static struct map_iter *map_iter_alloc(struct bpf_map *map)
  147. {
  148. struct map_iter *iter;
  149. iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN);
  150. if (!iter)
  151. goto error;
  152. iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN);
  153. if (!iter->key)
  154. goto error;
  155. return iter;
  156. error:
  157. map_iter_free(iter);
  158. return NULL;
  159. }
  160. static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos)
  161. {
  162. struct bpf_map *map = seq_file_to_map(m);
  163. void *key = map_iter(m)->key;
  164. void *prev_key;
  165. (*pos)++;
  166. if (map_iter(m)->done)
  167. return NULL;
  168. if (unlikely(v == SEQ_START_TOKEN))
  169. prev_key = NULL;
  170. else
  171. prev_key = key;
  172. rcu_read_lock();
  173. if (map->ops->map_get_next_key(map, prev_key, key)) {
  174. map_iter(m)->done = true;
  175. key = NULL;
  176. }
  177. rcu_read_unlock();
  178. return key;
  179. }
  180. static void *map_seq_start(struct seq_file *m, loff_t *pos)
  181. {
  182. if (map_iter(m)->done)
  183. return NULL;
  184. return *pos ? map_iter(m)->key : SEQ_START_TOKEN;
  185. }
  186. static void map_seq_stop(struct seq_file *m, void *v)
  187. {
  188. }
  189. static int map_seq_show(struct seq_file *m, void *v)
  190. {
  191. struct bpf_map *map = seq_file_to_map(m);
  192. void *key = map_iter(m)->key;
  193. if (unlikely(v == SEQ_START_TOKEN)) {
  194. seq_puts(m, "# WARNING!! The output is for debug purpose only\n");
  195. seq_puts(m, "# WARNING!! The output format will change\n");
  196. } else {
  197. map->ops->map_seq_show_elem(map, key, m);
  198. }
  199. return 0;
  200. }
  201. static const struct seq_operations bpffs_map_seq_ops = {
  202. .start = map_seq_start,
  203. .next = map_seq_next,
  204. .show = map_seq_show,
  205. .stop = map_seq_stop,
  206. };
  207. static int bpffs_map_open(struct inode *inode, struct file *file)
  208. {
  209. struct bpf_map *map = inode->i_private;
  210. struct map_iter *iter;
  211. struct seq_file *m;
  212. int err;
  213. iter = map_iter_alloc(map);
  214. if (!iter)
  215. return -ENOMEM;
  216. err = seq_open(file, &bpffs_map_seq_ops);
  217. if (err) {
  218. map_iter_free(iter);
  219. return err;
  220. }
  221. m = file->private_data;
  222. m->private = iter;
  223. return 0;
  224. }
  225. static int bpffs_map_release(struct inode *inode, struct file *file)
  226. {
  227. struct seq_file *m = file->private_data;
  228. map_iter_free(map_iter(m));
  229. return seq_release(inode, file);
  230. }
  231. /* bpffs_map_fops should only implement the basic
  232. * read operation for a BPF map. The purpose is to
  233. * provide a simple user intuitive way to do
  234. * "cat bpffs/pathto/a-pinned-map".
  235. *
  236. * Other operations (e.g. write, lookup...) should be realized by
  237. * the userspace tools (e.g. bpftool) through the
  238. * BPF_OBJ_GET_INFO_BY_FD and the map's lookup/update
  239. * interface.
  240. */
  241. static const struct file_operations bpffs_map_fops = {
  242. .open = bpffs_map_open,
  243. .read = seq_read,
  244. .release = bpffs_map_release,
  245. };
  246. static int bpffs_obj_open(struct inode *inode, struct file *file)
  247. {
  248. return -EIO;
  249. }
  250. static const struct file_operations bpffs_obj_fops = {
  251. .open = bpffs_obj_open,
  252. };
  253. static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
  254. const struct inode_operations *iops,
  255. const struct file_operations *fops)
  256. {
  257. struct inode *dir = dentry->d_parent->d_inode;
  258. struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode);
  259. if (IS_ERR(inode))
  260. return PTR_ERR(inode);
  261. inode->i_op = iops;
  262. inode->i_fop = fops;
  263. inode->i_private = raw;
  264. bpf_dentry_finalize(dentry, inode, dir);
  265. return 0;
  266. }
  267. static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg)
  268. {
  269. return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops,
  270. &bpffs_obj_fops);
  271. }
  272. static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg)
  273. {
  274. struct bpf_map *map = arg;
  275. return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops,
  276. bpf_map_support_seq_show(map) ?
  277. &bpffs_map_fops : &bpffs_obj_fops);
  278. }
  279. static struct dentry *
  280. bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags)
  281. {
  282. /* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future
  283. * extensions.
  284. */
  285. if (strchr(dentry->d_name.name, '.'))
  286. return ERR_PTR(-EPERM);
  287. return simple_lookup(dir, dentry, flags);
  288. }
  289. static int bpf_symlink(struct inode *dir, struct dentry *dentry,
  290. const char *target)
  291. {
  292. char *link = kstrdup(target, GFP_USER | __GFP_NOWARN);
  293. struct inode *inode;
  294. if (!link)
  295. return -ENOMEM;
  296. inode = bpf_get_inode(dir->i_sb, dir, S_IRWXUGO | S_IFLNK);
  297. if (IS_ERR(inode)) {
  298. kfree(link);
  299. return PTR_ERR(inode);
  300. }
  301. inode->i_op = &simple_symlink_inode_operations;
  302. inode->i_link = link;
  303. bpf_dentry_finalize(dentry, inode, dir);
  304. return 0;
  305. }
  306. static const struct inode_operations bpf_dir_iops = {
  307. .lookup = bpf_lookup,
  308. .mkdir = bpf_mkdir,
  309. .symlink = bpf_symlink,
  310. .rmdir = simple_rmdir,
  311. .rename = simple_rename,
  312. .link = simple_link,
  313. .unlink = simple_unlink,
  314. };
  315. static int bpf_obj_do_pin(const struct filename *pathname, void *raw,
  316. enum bpf_type type)
  317. {
  318. struct dentry *dentry;
  319. struct inode *dir;
  320. struct path path;
  321. umode_t mode;
  322. int ret;
  323. dentry = kern_path_create(AT_FDCWD, pathname->name, &path, 0);
  324. if (IS_ERR(dentry))
  325. return PTR_ERR(dentry);
  326. mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask());
  327. ret = security_path_mknod(&path, dentry, mode, 0);
  328. if (ret)
  329. goto out;
  330. dir = d_inode(path.dentry);
  331. if (dir->i_op != &bpf_dir_iops) {
  332. ret = -EPERM;
  333. goto out;
  334. }
  335. switch (type) {
  336. case BPF_TYPE_PROG:
  337. ret = vfs_mkobj(dentry, mode, bpf_mkprog, raw);
  338. break;
  339. case BPF_TYPE_MAP:
  340. ret = vfs_mkobj(dentry, mode, bpf_mkmap, raw);
  341. break;
  342. default:
  343. ret = -EPERM;
  344. }
  345. out:
  346. done_path_create(&path, dentry);
  347. return ret;
  348. }
  349. int bpf_obj_pin_user(u32 ufd, const char __user *pathname)
  350. {
  351. struct filename *pname;
  352. enum bpf_type type;
  353. void *raw;
  354. int ret;
  355. pname = getname(pathname);
  356. if (IS_ERR(pname))
  357. return PTR_ERR(pname);
  358. raw = bpf_fd_probe_obj(ufd, &type);
  359. if (IS_ERR(raw)) {
  360. ret = PTR_ERR(raw);
  361. goto out;
  362. }
  363. ret = bpf_obj_do_pin(pname, raw, type);
  364. if (ret != 0)
  365. bpf_any_put(raw, type);
  366. out:
  367. putname(pname);
  368. return ret;
  369. }
  370. static void *bpf_obj_do_get(const struct filename *pathname,
  371. enum bpf_type *type, int flags)
  372. {
  373. struct inode *inode;
  374. struct path path;
  375. void *raw;
  376. int ret;
  377. ret = kern_path(pathname->name, LOOKUP_FOLLOW, &path);
  378. if (ret)
  379. return ERR_PTR(ret);
  380. inode = d_backing_inode(path.dentry);
  381. ret = inode_permission(inode, ACC_MODE(flags));
  382. if (ret)
  383. goto out;
  384. ret = bpf_inode_type(inode, type);
  385. if (ret)
  386. goto out;
  387. raw = bpf_any_get(inode->i_private, *type);
  388. if (!IS_ERR(raw))
  389. touch_atime(&path);
  390. path_put(&path);
  391. return raw;
  392. out:
  393. path_put(&path);
  394. return ERR_PTR(ret);
  395. }
  396. int bpf_obj_get_user(const char __user *pathname, int flags)
  397. {
  398. enum bpf_type type = BPF_TYPE_UNSPEC;
  399. struct filename *pname;
  400. int ret = -ENOENT;
  401. int f_flags;
  402. void *raw;
  403. f_flags = bpf_get_file_flag(flags);
  404. if (f_flags < 0)
  405. return f_flags;
  406. pname = getname(pathname);
  407. if (IS_ERR(pname))
  408. return PTR_ERR(pname);
  409. raw = bpf_obj_do_get(pname, &type, f_flags);
  410. if (IS_ERR(raw)) {
  411. ret = PTR_ERR(raw);
  412. goto out;
  413. }
  414. if (type == BPF_TYPE_PROG)
  415. ret = bpf_prog_new_fd(raw);
  416. else if (type == BPF_TYPE_MAP)
  417. ret = bpf_map_new_fd(raw, f_flags);
  418. else
  419. goto out;
  420. if (ret < 0)
  421. bpf_any_put(raw, type);
  422. out:
  423. putname(pname);
  424. return ret;
  425. }
  426. static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type)
  427. {
  428. struct bpf_prog *prog;
  429. int ret = inode_permission(inode, MAY_READ);
  430. if (ret)
  431. return ERR_PTR(ret);
  432. if (inode->i_op == &bpf_map_iops)
  433. return ERR_PTR(-EINVAL);
  434. if (inode->i_op != &bpf_prog_iops)
  435. return ERR_PTR(-EACCES);
  436. prog = inode->i_private;
  437. ret = security_bpf_prog(prog);
  438. if (ret < 0)
  439. return ERR_PTR(ret);
  440. if (!bpf_prog_get_ok(prog, &type, false))
  441. return ERR_PTR(-EINVAL);
  442. return bpf_prog_inc(prog);
  443. }
  444. struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type)
  445. {
  446. struct bpf_prog *prog;
  447. struct path path;
  448. int ret = kern_path(name, LOOKUP_FOLLOW, &path);
  449. if (ret)
  450. return ERR_PTR(ret);
  451. prog = __get_prog_inode(d_backing_inode(path.dentry), type);
  452. if (!IS_ERR(prog))
  453. touch_atime(&path);
  454. path_put(&path);
  455. return prog;
  456. }
  457. EXPORT_SYMBOL(bpf_prog_get_type_path);
  458. /*
  459. * Display the mount options in /proc/mounts.
  460. */
  461. static int bpf_show_options(struct seq_file *m, struct dentry *root)
  462. {
  463. umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX;
  464. if (mode != S_IRWXUGO)
  465. seq_printf(m, ",mode=%o", mode);
  466. return 0;
  467. }
  468. static void bpf_destroy_inode_deferred(struct rcu_head *head)
  469. {
  470. struct inode *inode = container_of(head, struct inode, i_rcu);
  471. enum bpf_type type;
  472. if (S_ISLNK(inode->i_mode))
  473. kfree(inode->i_link);
  474. if (!bpf_inode_type(inode, &type))
  475. bpf_any_put(inode->i_private, type);
  476. free_inode_nonrcu(inode);
  477. }
  478. static void bpf_destroy_inode(struct inode *inode)
  479. {
  480. call_rcu(&inode->i_rcu, bpf_destroy_inode_deferred);
  481. }
  482. static const struct super_operations bpf_super_ops = {
  483. .statfs = simple_statfs,
  484. .drop_inode = generic_delete_inode,
  485. .show_options = bpf_show_options,
  486. .destroy_inode = bpf_destroy_inode,
  487. };
  488. enum {
  489. OPT_MODE,
  490. OPT_ERR,
  491. };
  492. static const match_table_t bpf_mount_tokens = {
  493. { OPT_MODE, "mode=%o" },
  494. { OPT_ERR, NULL },
  495. };
  496. struct bpf_mount_opts {
  497. umode_t mode;
  498. };
  499. static int bpf_parse_options(char *data, struct bpf_mount_opts *opts)
  500. {
  501. substring_t args[MAX_OPT_ARGS];
  502. int option, token;
  503. char *ptr;
  504. opts->mode = S_IRWXUGO;
  505. while ((ptr = strsep(&data, ",")) != NULL) {
  506. if (!*ptr)
  507. continue;
  508. token = match_token(ptr, bpf_mount_tokens, args);
  509. switch (token) {
  510. case OPT_MODE:
  511. if (match_octal(&args[0], &option))
  512. return -EINVAL;
  513. opts->mode = option & S_IALLUGO;
  514. break;
  515. /* We might like to report bad mount options here, but
  516. * traditionally we've ignored all mount options, so we'd
  517. * better continue to ignore non-existing options for bpf.
  518. */
  519. }
  520. }
  521. return 0;
  522. }
  523. static int bpf_fill_super(struct super_block *sb, void *data, int silent)
  524. {
  525. static const struct tree_descr bpf_rfiles[] = { { "" } };
  526. struct bpf_mount_opts opts;
  527. struct inode *inode;
  528. int ret;
  529. ret = bpf_parse_options(data, &opts);
  530. if (ret)
  531. return ret;
  532. ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles);
  533. if (ret)
  534. return ret;
  535. sb->s_op = &bpf_super_ops;
  536. inode = sb->s_root->d_inode;
  537. inode->i_op = &bpf_dir_iops;
  538. inode->i_mode &= ~S_IALLUGO;
  539. inode->i_mode |= S_ISVTX | opts.mode;
  540. return 0;
  541. }
  542. static struct dentry *bpf_mount(struct file_system_type *type, int flags,
  543. const char *dev_name, void *data)
  544. {
  545. return mount_nodev(type, flags, data, bpf_fill_super);
  546. }
  547. static struct file_system_type bpf_fs_type = {
  548. .owner = THIS_MODULE,
  549. .name = "bpf",
  550. .mount = bpf_mount,
  551. .kill_sb = kill_litter_super,
  552. };
  553. static int __init bpf_init(void)
  554. {
  555. int ret;
  556. ret = sysfs_create_mount_point(fs_kobj, "bpf");
  557. if (ret)
  558. return ret;
  559. ret = register_filesystem(&bpf_fs_type);
  560. if (ret)
  561. sysfs_remove_mount_point(fs_kobj, "bpf");
  562. return ret;
  563. }
  564. fs_initcall(bpf_init);