libbpf.c 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431
  1. // SPDX-License-Identifier: LGPL-2.1
  2. /*
  3. * Common eBPF ELF object loading operations.
  4. *
  5. * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
  6. * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
  7. * Copyright (C) 2015 Huawei Inc.
  8. * Copyright (C) 2017 Nicira, Inc.
  9. *
  10. * This program is free software; you can redistribute it and/or
  11. * modify it under the terms of the GNU Lesser General Public
  12. * License as published by the Free Software Foundation;
  13. * version 2.1 of the License (not later!)
  14. *
  15. * This program is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. * GNU Lesser General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU Lesser General Public
  21. * License along with this program; if not, see <http://www.gnu.org/licenses>
  22. */
  23. #ifndef _GNU_SOURCE
  24. #define _GNU_SOURCE
  25. #endif
  26. #include <stdlib.h>
  27. #include <stdio.h>
  28. #include <stdarg.h>
  29. #include <libgen.h>
  30. #include <inttypes.h>
  31. #include <string.h>
  32. #include <unistd.h>
  33. #include <fcntl.h>
  34. #include <errno.h>
  35. #include <perf-sys.h>
  36. #include <asm/unistd.h>
  37. #include <linux/err.h>
  38. #include <linux/kernel.h>
  39. #include <linux/bpf.h>
  40. #include <linux/btf.h>
  41. #include <linux/list.h>
  42. #include <linux/limits.h>
  43. #include <sys/stat.h>
  44. #include <sys/types.h>
  45. #include <sys/vfs.h>
  46. #include <tools/libc_compat.h>
  47. #include <libelf.h>
  48. #include <gelf.h>
  49. #include "libbpf.h"
  50. #include "bpf.h"
  51. #include "btf.h"
  52. #include "str_error.h"
  53. #ifndef EM_BPF
  54. #define EM_BPF 247
  55. #endif
  56. #ifndef BPF_FS_MAGIC
  57. #define BPF_FS_MAGIC 0xcafe4a11
  58. #endif
  59. #define __printf(a, b) __attribute__((format(printf, a, b)))
  60. __printf(1, 2)
  61. static int __base_pr(const char *format, ...)
  62. {
  63. va_list args;
  64. int err;
  65. va_start(args, format);
  66. err = vfprintf(stderr, format, args);
  67. va_end(args);
  68. return err;
  69. }
  70. static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  71. static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  72. static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  73. #define __pr(func, fmt, ...) \
  74. do { \
  75. if ((func)) \
  76. (func)("libbpf: " fmt, ##__VA_ARGS__); \
  77. } while (0)
  78. #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
  79. #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
  80. #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
  81. void libbpf_set_print(libbpf_print_fn_t warn,
  82. libbpf_print_fn_t info,
  83. libbpf_print_fn_t debug)
  84. {
  85. __pr_warning = warn;
  86. __pr_info = info;
  87. __pr_debug = debug;
  88. }
  89. #define STRERR_BUFSIZE 128
  90. #define CHECK_ERR(action, err, out) do { \
  91. err = action; \
  92. if (err) \
  93. goto out; \
  94. } while(0)
  95. /* Copied from tools/perf/util/util.h */
  96. #ifndef zfree
  97. # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
  98. #endif
  99. #ifndef zclose
  100. # define zclose(fd) ({ \
  101. int ___err = 0; \
  102. if ((fd) >= 0) \
  103. ___err = close((fd)); \
  104. fd = -1; \
  105. ___err; })
  106. #endif
  107. #ifdef HAVE_LIBELF_MMAP_SUPPORT
  108. # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
  109. #else
  110. # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
  111. #endif
  112. /*
  113. * bpf_prog should be a better name but it has been used in
  114. * linux/filter.h.
  115. */
  116. struct bpf_program {
  117. /* Index in elf obj file, for relocation use. */
  118. int idx;
  119. char *name;
  120. int prog_ifindex;
  121. char *section_name;
  122. struct bpf_insn *insns;
  123. size_t insns_cnt, main_prog_cnt;
  124. enum bpf_prog_type type;
  125. struct reloc_desc {
  126. enum {
  127. RELO_LD64,
  128. RELO_CALL,
  129. } type;
  130. int insn_idx;
  131. union {
  132. int map_idx;
  133. int text_off;
  134. };
  135. } *reloc_desc;
  136. int nr_reloc;
  137. struct {
  138. int nr;
  139. int *fds;
  140. } instances;
  141. bpf_program_prep_t preprocessor;
  142. struct bpf_object *obj;
  143. void *priv;
  144. bpf_program_clear_priv_t clear_priv;
  145. enum bpf_attach_type expected_attach_type;
  146. };
  147. struct bpf_map {
  148. int fd;
  149. char *name;
  150. size_t offset;
  151. int map_ifindex;
  152. struct bpf_map_def def;
  153. __u32 btf_key_type_id;
  154. __u32 btf_value_type_id;
  155. void *priv;
  156. bpf_map_clear_priv_t clear_priv;
  157. };
  158. static LIST_HEAD(bpf_objects_list);
  159. struct bpf_object {
  160. char license[64];
  161. u32 kern_version;
  162. struct bpf_program *programs;
  163. size_t nr_programs;
  164. struct bpf_map *maps;
  165. size_t nr_maps;
  166. bool loaded;
  167. bool has_pseudo_calls;
  168. /*
  169. * Information when doing elf related work. Only valid if fd
  170. * is valid.
  171. */
  172. struct {
  173. int fd;
  174. void *obj_buf;
  175. size_t obj_buf_sz;
  176. Elf *elf;
  177. GElf_Ehdr ehdr;
  178. Elf_Data *symbols;
  179. size_t strtabidx;
  180. struct {
  181. GElf_Shdr shdr;
  182. Elf_Data *data;
  183. } *reloc;
  184. int nr_reloc;
  185. int maps_shndx;
  186. int text_shndx;
  187. } efile;
  188. /*
  189. * All loaded bpf_object is linked in a list, which is
  190. * hidden to caller. bpf_objects__<func> handlers deal with
  191. * all objects.
  192. */
  193. struct list_head list;
  194. struct btf *btf;
  195. void *priv;
  196. bpf_object_clear_priv_t clear_priv;
  197. char path[];
  198. };
  199. #define obj_elf_valid(o) ((o)->efile.elf)
  200. static void bpf_program__unload(struct bpf_program *prog)
  201. {
  202. int i;
  203. if (!prog)
  204. return;
  205. /*
  206. * If the object is opened but the program was never loaded,
  207. * it is possible that prog->instances.nr == -1.
  208. */
  209. if (prog->instances.nr > 0) {
  210. for (i = 0; i < prog->instances.nr; i++)
  211. zclose(prog->instances.fds[i]);
  212. } else if (prog->instances.nr != -1) {
  213. pr_warning("Internal error: instances.nr is %d\n",
  214. prog->instances.nr);
  215. }
  216. prog->instances.nr = -1;
  217. zfree(&prog->instances.fds);
  218. }
  219. static void bpf_program__exit(struct bpf_program *prog)
  220. {
  221. if (!prog)
  222. return;
  223. if (prog->clear_priv)
  224. prog->clear_priv(prog, prog->priv);
  225. prog->priv = NULL;
  226. prog->clear_priv = NULL;
  227. bpf_program__unload(prog);
  228. zfree(&prog->name);
  229. zfree(&prog->section_name);
  230. zfree(&prog->insns);
  231. zfree(&prog->reloc_desc);
  232. prog->nr_reloc = 0;
  233. prog->insns_cnt = 0;
  234. prog->idx = -1;
  235. }
  236. static int
  237. bpf_program__init(void *data, size_t size, char *section_name, int idx,
  238. struct bpf_program *prog)
  239. {
  240. if (size < sizeof(struct bpf_insn)) {
  241. pr_warning("corrupted section '%s'\n", section_name);
  242. return -EINVAL;
  243. }
  244. bzero(prog, sizeof(*prog));
  245. prog->section_name = strdup(section_name);
  246. if (!prog->section_name) {
  247. pr_warning("failed to alloc name for prog under section(%d) %s\n",
  248. idx, section_name);
  249. goto errout;
  250. }
  251. prog->insns = malloc(size);
  252. if (!prog->insns) {
  253. pr_warning("failed to alloc insns for prog under section %s\n",
  254. section_name);
  255. goto errout;
  256. }
  257. prog->insns_cnt = size / sizeof(struct bpf_insn);
  258. memcpy(prog->insns, data,
  259. prog->insns_cnt * sizeof(struct bpf_insn));
  260. prog->idx = idx;
  261. prog->instances.fds = NULL;
  262. prog->instances.nr = -1;
  263. prog->type = BPF_PROG_TYPE_KPROBE;
  264. return 0;
  265. errout:
  266. bpf_program__exit(prog);
  267. return -ENOMEM;
  268. }
  269. static int
  270. bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
  271. char *section_name, int idx)
  272. {
  273. struct bpf_program prog, *progs;
  274. int nr_progs, err;
  275. err = bpf_program__init(data, size, section_name, idx, &prog);
  276. if (err)
  277. return err;
  278. progs = obj->programs;
  279. nr_progs = obj->nr_programs;
  280. progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
  281. if (!progs) {
  282. /*
  283. * In this case the original obj->programs
  284. * is still valid, so don't need special treat for
  285. * bpf_close_object().
  286. */
  287. pr_warning("failed to alloc a new program under section '%s'\n",
  288. section_name);
  289. bpf_program__exit(&prog);
  290. return -ENOMEM;
  291. }
  292. pr_debug("found program %s\n", prog.section_name);
  293. obj->programs = progs;
  294. obj->nr_programs = nr_progs + 1;
  295. prog.obj = obj;
  296. progs[nr_progs] = prog;
  297. return 0;
  298. }
  299. static int
  300. bpf_object__init_prog_names(struct bpf_object *obj)
  301. {
  302. Elf_Data *symbols = obj->efile.symbols;
  303. struct bpf_program *prog;
  304. size_t pi, si;
  305. for (pi = 0; pi < obj->nr_programs; pi++) {
  306. const char *name = NULL;
  307. prog = &obj->programs[pi];
  308. for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
  309. si++) {
  310. GElf_Sym sym;
  311. if (!gelf_getsym(symbols, si, &sym))
  312. continue;
  313. if (sym.st_shndx != prog->idx)
  314. continue;
  315. if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
  316. continue;
  317. name = elf_strptr(obj->efile.elf,
  318. obj->efile.strtabidx,
  319. sym.st_name);
  320. if (!name) {
  321. pr_warning("failed to get sym name string for prog %s\n",
  322. prog->section_name);
  323. return -LIBBPF_ERRNO__LIBELF;
  324. }
  325. }
  326. if (!name && prog->idx == obj->efile.text_shndx)
  327. name = ".text";
  328. if (!name) {
  329. pr_warning("failed to find sym for prog %s\n",
  330. prog->section_name);
  331. return -EINVAL;
  332. }
  333. prog->name = strdup(name);
  334. if (!prog->name) {
  335. pr_warning("failed to allocate memory for prog sym %s\n",
  336. name);
  337. return -ENOMEM;
  338. }
  339. }
  340. return 0;
  341. }
  342. static struct bpf_object *bpf_object__new(const char *path,
  343. void *obj_buf,
  344. size_t obj_buf_sz)
  345. {
  346. struct bpf_object *obj;
  347. obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
  348. if (!obj) {
  349. pr_warning("alloc memory failed for %s\n", path);
  350. return ERR_PTR(-ENOMEM);
  351. }
  352. strcpy(obj->path, path);
  353. obj->efile.fd = -1;
  354. /*
  355. * Caller of this function should also calls
  356. * bpf_object__elf_finish() after data collection to return
  357. * obj_buf to user. If not, we should duplicate the buffer to
  358. * avoid user freeing them before elf finish.
  359. */
  360. obj->efile.obj_buf = obj_buf;
  361. obj->efile.obj_buf_sz = obj_buf_sz;
  362. obj->efile.maps_shndx = -1;
  363. obj->loaded = false;
  364. INIT_LIST_HEAD(&obj->list);
  365. list_add(&obj->list, &bpf_objects_list);
  366. return obj;
  367. }
  368. static void bpf_object__elf_finish(struct bpf_object *obj)
  369. {
  370. if (!obj_elf_valid(obj))
  371. return;
  372. if (obj->efile.elf) {
  373. elf_end(obj->efile.elf);
  374. obj->efile.elf = NULL;
  375. }
  376. obj->efile.symbols = NULL;
  377. zfree(&obj->efile.reloc);
  378. obj->efile.nr_reloc = 0;
  379. zclose(obj->efile.fd);
  380. obj->efile.obj_buf = NULL;
  381. obj->efile.obj_buf_sz = 0;
  382. }
  383. static int bpf_object__elf_init(struct bpf_object *obj)
  384. {
  385. int err = 0;
  386. GElf_Ehdr *ep;
  387. if (obj_elf_valid(obj)) {
  388. pr_warning("elf init: internal error\n");
  389. return -LIBBPF_ERRNO__LIBELF;
  390. }
  391. if (obj->efile.obj_buf_sz > 0) {
  392. /*
  393. * obj_buf should have been validated by
  394. * bpf_object__open_buffer().
  395. */
  396. obj->efile.elf = elf_memory(obj->efile.obj_buf,
  397. obj->efile.obj_buf_sz);
  398. } else {
  399. obj->efile.fd = open(obj->path, O_RDONLY);
  400. if (obj->efile.fd < 0) {
  401. char errmsg[STRERR_BUFSIZE];
  402. char *cp = str_error(errno, errmsg, sizeof(errmsg));
  403. pr_warning("failed to open %s: %s\n", obj->path, cp);
  404. return -errno;
  405. }
  406. obj->efile.elf = elf_begin(obj->efile.fd,
  407. LIBBPF_ELF_C_READ_MMAP,
  408. NULL);
  409. }
  410. if (!obj->efile.elf) {
  411. pr_warning("failed to open %s as ELF file\n",
  412. obj->path);
  413. err = -LIBBPF_ERRNO__LIBELF;
  414. goto errout;
  415. }
  416. if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
  417. pr_warning("failed to get EHDR from %s\n",
  418. obj->path);
  419. err = -LIBBPF_ERRNO__FORMAT;
  420. goto errout;
  421. }
  422. ep = &obj->efile.ehdr;
  423. /* Old LLVM set e_machine to EM_NONE */
  424. if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
  425. pr_warning("%s is not an eBPF object file\n",
  426. obj->path);
  427. err = -LIBBPF_ERRNO__FORMAT;
  428. goto errout;
  429. }
  430. return 0;
  431. errout:
  432. bpf_object__elf_finish(obj);
  433. return err;
  434. }
  435. static int
  436. bpf_object__check_endianness(struct bpf_object *obj)
  437. {
  438. static unsigned int const endian = 1;
  439. switch (obj->efile.ehdr.e_ident[EI_DATA]) {
  440. case ELFDATA2LSB:
  441. /* We are big endian, BPF obj is little endian. */
  442. if (*(unsigned char const *)&endian != 1)
  443. goto mismatch;
  444. break;
  445. case ELFDATA2MSB:
  446. /* We are little endian, BPF obj is big endian. */
  447. if (*(unsigned char const *)&endian != 0)
  448. goto mismatch;
  449. break;
  450. default:
  451. return -LIBBPF_ERRNO__ENDIAN;
  452. }
  453. return 0;
  454. mismatch:
  455. pr_warning("Error: endianness mismatch.\n");
  456. return -LIBBPF_ERRNO__ENDIAN;
  457. }
  458. static int
  459. bpf_object__init_license(struct bpf_object *obj,
  460. void *data, size_t size)
  461. {
  462. memcpy(obj->license, data,
  463. min(size, sizeof(obj->license) - 1));
  464. pr_debug("license of %s is %s\n", obj->path, obj->license);
  465. return 0;
  466. }
  467. static int
  468. bpf_object__init_kversion(struct bpf_object *obj,
  469. void *data, size_t size)
  470. {
  471. u32 kver;
  472. if (size != sizeof(kver)) {
  473. pr_warning("invalid kver section in %s\n", obj->path);
  474. return -LIBBPF_ERRNO__FORMAT;
  475. }
  476. memcpy(&kver, data, sizeof(kver));
  477. obj->kern_version = kver;
  478. pr_debug("kernel version of %s is %x\n", obj->path,
  479. obj->kern_version);
  480. return 0;
  481. }
  482. static int compare_bpf_map(const void *_a, const void *_b)
  483. {
  484. const struct bpf_map *a = _a;
  485. const struct bpf_map *b = _b;
  486. return a->offset - b->offset;
  487. }
  488. static int
  489. bpf_object__init_maps(struct bpf_object *obj)
  490. {
  491. int i, map_idx, map_def_sz, nr_maps = 0;
  492. Elf_Scn *scn;
  493. Elf_Data *data;
  494. Elf_Data *symbols = obj->efile.symbols;
  495. if (obj->efile.maps_shndx < 0)
  496. return -EINVAL;
  497. if (!symbols)
  498. return -EINVAL;
  499. scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
  500. if (scn)
  501. data = elf_getdata(scn, NULL);
  502. if (!scn || !data) {
  503. pr_warning("failed to get Elf_Data from map section %d\n",
  504. obj->efile.maps_shndx);
  505. return -EINVAL;
  506. }
  507. /*
  508. * Count number of maps. Each map has a name.
  509. * Array of maps is not supported: only the first element is
  510. * considered.
  511. *
  512. * TODO: Detect array of map and report error.
  513. */
  514. for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
  515. GElf_Sym sym;
  516. if (!gelf_getsym(symbols, i, &sym))
  517. continue;
  518. if (sym.st_shndx != obj->efile.maps_shndx)
  519. continue;
  520. nr_maps++;
  521. }
  522. /* Alloc obj->maps and fill nr_maps. */
  523. pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
  524. nr_maps, data->d_size);
  525. if (!nr_maps)
  526. return 0;
  527. /* Assume equally sized map definitions */
  528. map_def_sz = data->d_size / nr_maps;
  529. if (!data->d_size || (data->d_size % nr_maps) != 0) {
  530. pr_warning("unable to determine map definition size "
  531. "section %s, %d maps in %zd bytes\n",
  532. obj->path, nr_maps, data->d_size);
  533. return -EINVAL;
  534. }
  535. obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
  536. if (!obj->maps) {
  537. pr_warning("alloc maps for object failed\n");
  538. return -ENOMEM;
  539. }
  540. obj->nr_maps = nr_maps;
  541. /*
  542. * fill all fd with -1 so won't close incorrect
  543. * fd (fd=0 is stdin) when failure (zclose won't close
  544. * negative fd)).
  545. */
  546. for (i = 0; i < nr_maps; i++)
  547. obj->maps[i].fd = -1;
  548. /*
  549. * Fill obj->maps using data in "maps" section.
  550. */
  551. for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
  552. GElf_Sym sym;
  553. const char *map_name;
  554. struct bpf_map_def *def;
  555. if (!gelf_getsym(symbols, i, &sym))
  556. continue;
  557. if (sym.st_shndx != obj->efile.maps_shndx)
  558. continue;
  559. map_name = elf_strptr(obj->efile.elf,
  560. obj->efile.strtabidx,
  561. sym.st_name);
  562. obj->maps[map_idx].offset = sym.st_value;
  563. if (sym.st_value + map_def_sz > data->d_size) {
  564. pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
  565. obj->path, map_name);
  566. return -EINVAL;
  567. }
  568. obj->maps[map_idx].name = strdup(map_name);
  569. if (!obj->maps[map_idx].name) {
  570. pr_warning("failed to alloc map name\n");
  571. return -ENOMEM;
  572. }
  573. pr_debug("map %d is \"%s\"\n", map_idx,
  574. obj->maps[map_idx].name);
  575. def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
  576. /*
  577. * If the definition of the map in the object file fits in
  578. * bpf_map_def, copy it. Any extra fields in our version
  579. * of bpf_map_def will default to zero as a result of the
  580. * calloc above.
  581. */
  582. if (map_def_sz <= sizeof(struct bpf_map_def)) {
  583. memcpy(&obj->maps[map_idx].def, def, map_def_sz);
  584. } else {
  585. /*
  586. * Here the map structure being read is bigger than what
  587. * we expect, truncate if the excess bits are all zero.
  588. * If they are not zero, reject this map as
  589. * incompatible.
  590. */
  591. char *b;
  592. for (b = ((char *)def) + sizeof(struct bpf_map_def);
  593. b < ((char *)def) + map_def_sz; b++) {
  594. if (*b != 0) {
  595. pr_warning("maps section in %s: \"%s\" "
  596. "has unrecognized, non-zero "
  597. "options\n",
  598. obj->path, map_name);
  599. return -EINVAL;
  600. }
  601. }
  602. memcpy(&obj->maps[map_idx].def, def,
  603. sizeof(struct bpf_map_def));
  604. }
  605. map_idx++;
  606. }
  607. qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
  608. return 0;
  609. }
  610. static bool section_have_execinstr(struct bpf_object *obj, int idx)
  611. {
  612. Elf_Scn *scn;
  613. GElf_Shdr sh;
  614. scn = elf_getscn(obj->efile.elf, idx);
  615. if (!scn)
  616. return false;
  617. if (gelf_getshdr(scn, &sh) != &sh)
  618. return false;
  619. if (sh.sh_flags & SHF_EXECINSTR)
  620. return true;
  621. return false;
  622. }
  623. static int bpf_object__elf_collect(struct bpf_object *obj)
  624. {
  625. Elf *elf = obj->efile.elf;
  626. GElf_Ehdr *ep = &obj->efile.ehdr;
  627. Elf_Scn *scn = NULL;
  628. int idx = 0, err = 0;
  629. /* Elf is corrupted/truncated, avoid calling elf_strptr. */
  630. if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
  631. pr_warning("failed to get e_shstrndx from %s\n",
  632. obj->path);
  633. return -LIBBPF_ERRNO__FORMAT;
  634. }
  635. while ((scn = elf_nextscn(elf, scn)) != NULL) {
  636. char *name;
  637. GElf_Shdr sh;
  638. Elf_Data *data;
  639. idx++;
  640. if (gelf_getshdr(scn, &sh) != &sh) {
  641. pr_warning("failed to get section(%d) header from %s\n",
  642. idx, obj->path);
  643. err = -LIBBPF_ERRNO__FORMAT;
  644. goto out;
  645. }
  646. name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
  647. if (!name) {
  648. pr_warning("failed to get section(%d) name from %s\n",
  649. idx, obj->path);
  650. err = -LIBBPF_ERRNO__FORMAT;
  651. goto out;
  652. }
  653. data = elf_getdata(scn, 0);
  654. if (!data) {
  655. pr_warning("failed to get section(%d) data from %s(%s)\n",
  656. idx, name, obj->path);
  657. err = -LIBBPF_ERRNO__FORMAT;
  658. goto out;
  659. }
  660. pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
  661. idx, name, (unsigned long)data->d_size,
  662. (int)sh.sh_link, (unsigned long)sh.sh_flags,
  663. (int)sh.sh_type);
  664. if (strcmp(name, "license") == 0)
  665. err = bpf_object__init_license(obj,
  666. data->d_buf,
  667. data->d_size);
  668. else if (strcmp(name, "version") == 0)
  669. err = bpf_object__init_kversion(obj,
  670. data->d_buf,
  671. data->d_size);
  672. else if (strcmp(name, "maps") == 0)
  673. obj->efile.maps_shndx = idx;
  674. else if (strcmp(name, BTF_ELF_SEC) == 0) {
  675. obj->btf = btf__new(data->d_buf, data->d_size,
  676. __pr_debug);
  677. if (IS_ERR(obj->btf)) {
  678. pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
  679. BTF_ELF_SEC, PTR_ERR(obj->btf));
  680. obj->btf = NULL;
  681. }
  682. } else if (sh.sh_type == SHT_SYMTAB) {
  683. if (obj->efile.symbols) {
  684. pr_warning("bpf: multiple SYMTAB in %s\n",
  685. obj->path);
  686. err = -LIBBPF_ERRNO__FORMAT;
  687. } else {
  688. obj->efile.symbols = data;
  689. obj->efile.strtabidx = sh.sh_link;
  690. }
  691. } else if ((sh.sh_type == SHT_PROGBITS) &&
  692. (sh.sh_flags & SHF_EXECINSTR) &&
  693. (data->d_size > 0)) {
  694. if (strcmp(name, ".text") == 0)
  695. obj->efile.text_shndx = idx;
  696. err = bpf_object__add_program(obj, data->d_buf,
  697. data->d_size, name, idx);
  698. if (err) {
  699. char errmsg[STRERR_BUFSIZE];
  700. char *cp = str_error(-err, errmsg, sizeof(errmsg));
  701. pr_warning("failed to alloc program %s (%s): %s",
  702. name, obj->path, cp);
  703. }
  704. } else if (sh.sh_type == SHT_REL) {
  705. void *reloc = obj->efile.reloc;
  706. int nr_reloc = obj->efile.nr_reloc + 1;
  707. int sec = sh.sh_info; /* points to other section */
  708. /* Only do relo for section with exec instructions */
  709. if (!section_have_execinstr(obj, sec)) {
  710. pr_debug("skip relo %s(%d) for section(%d)\n",
  711. name, idx, sec);
  712. continue;
  713. }
  714. reloc = reallocarray(reloc, nr_reloc,
  715. sizeof(*obj->efile.reloc));
  716. if (!reloc) {
  717. pr_warning("realloc failed\n");
  718. err = -ENOMEM;
  719. } else {
  720. int n = nr_reloc - 1;
  721. obj->efile.reloc = reloc;
  722. obj->efile.nr_reloc = nr_reloc;
  723. obj->efile.reloc[n].shdr = sh;
  724. obj->efile.reloc[n].data = data;
  725. }
  726. } else {
  727. pr_debug("skip section(%d) %s\n", idx, name);
  728. }
  729. if (err)
  730. goto out;
  731. }
  732. if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
  733. pr_warning("Corrupted ELF file: index of strtab invalid\n");
  734. return LIBBPF_ERRNO__FORMAT;
  735. }
  736. if (obj->efile.maps_shndx >= 0) {
  737. err = bpf_object__init_maps(obj);
  738. if (err)
  739. goto out;
  740. }
  741. err = bpf_object__init_prog_names(obj);
  742. out:
  743. return err;
  744. }
  745. static struct bpf_program *
  746. bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
  747. {
  748. struct bpf_program *prog;
  749. size_t i;
  750. for (i = 0; i < obj->nr_programs; i++) {
  751. prog = &obj->programs[i];
  752. if (prog->idx == idx)
  753. return prog;
  754. }
  755. return NULL;
  756. }
  757. struct bpf_program *
  758. bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
  759. {
  760. struct bpf_program *pos;
  761. bpf_object__for_each_program(pos, obj) {
  762. if (pos->section_name && !strcmp(pos->section_name, title))
  763. return pos;
  764. }
  765. return NULL;
  766. }
  767. static int
  768. bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
  769. Elf_Data *data, struct bpf_object *obj)
  770. {
  771. Elf_Data *symbols = obj->efile.symbols;
  772. int text_shndx = obj->efile.text_shndx;
  773. int maps_shndx = obj->efile.maps_shndx;
  774. struct bpf_map *maps = obj->maps;
  775. size_t nr_maps = obj->nr_maps;
  776. int i, nrels;
  777. pr_debug("collecting relocating info for: '%s'\n",
  778. prog->section_name);
  779. nrels = shdr->sh_size / shdr->sh_entsize;
  780. prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
  781. if (!prog->reloc_desc) {
  782. pr_warning("failed to alloc memory in relocation\n");
  783. return -ENOMEM;
  784. }
  785. prog->nr_reloc = nrels;
  786. for (i = 0; i < nrels; i++) {
  787. GElf_Sym sym;
  788. GElf_Rel rel;
  789. unsigned int insn_idx;
  790. struct bpf_insn *insns = prog->insns;
  791. size_t map_idx;
  792. if (!gelf_getrel(data, i, &rel)) {
  793. pr_warning("relocation: failed to get %d reloc\n", i);
  794. return -LIBBPF_ERRNO__FORMAT;
  795. }
  796. if (!gelf_getsym(symbols,
  797. GELF_R_SYM(rel.r_info),
  798. &sym)) {
  799. pr_warning("relocation: symbol %"PRIx64" not found\n",
  800. GELF_R_SYM(rel.r_info));
  801. return -LIBBPF_ERRNO__FORMAT;
  802. }
  803. pr_debug("relo for %lld value %lld name %d\n",
  804. (long long) (rel.r_info >> 32),
  805. (long long) sym.st_value, sym.st_name);
  806. if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
  807. pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
  808. prog->section_name, sym.st_shndx);
  809. return -LIBBPF_ERRNO__RELOC;
  810. }
  811. insn_idx = rel.r_offset / sizeof(struct bpf_insn);
  812. pr_debug("relocation: insn_idx=%u\n", insn_idx);
  813. if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
  814. if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
  815. pr_warning("incorrect bpf_call opcode\n");
  816. return -LIBBPF_ERRNO__RELOC;
  817. }
  818. prog->reloc_desc[i].type = RELO_CALL;
  819. prog->reloc_desc[i].insn_idx = insn_idx;
  820. prog->reloc_desc[i].text_off = sym.st_value;
  821. obj->has_pseudo_calls = true;
  822. continue;
  823. }
  824. if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
  825. pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
  826. insn_idx, insns[insn_idx].code);
  827. return -LIBBPF_ERRNO__RELOC;
  828. }
  829. /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
  830. for (map_idx = 0; map_idx < nr_maps; map_idx++) {
  831. if (maps[map_idx].offset == sym.st_value) {
  832. pr_debug("relocation: find map %zd (%s) for insn %u\n",
  833. map_idx, maps[map_idx].name, insn_idx);
  834. break;
  835. }
  836. }
  837. if (map_idx >= nr_maps) {
  838. pr_warning("bpf relocation: map_idx %d large than %d\n",
  839. (int)map_idx, (int)nr_maps - 1);
  840. return -LIBBPF_ERRNO__RELOC;
  841. }
  842. prog->reloc_desc[i].type = RELO_LD64;
  843. prog->reloc_desc[i].insn_idx = insn_idx;
  844. prog->reloc_desc[i].map_idx = map_idx;
  845. }
  846. return 0;
  847. }
  848. static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
  849. {
  850. const struct btf_type *container_type;
  851. const struct btf_member *key, *value;
  852. struct bpf_map_def *def = &map->def;
  853. const size_t max_name = 256;
  854. char container_name[max_name];
  855. __s64 key_size, value_size;
  856. __s32 container_id;
  857. if (snprintf(container_name, max_name, "____btf_map_%s", map->name) ==
  858. max_name) {
  859. pr_warning("map:%s length of '____btf_map_%s' is too long\n",
  860. map->name, map->name);
  861. return -EINVAL;
  862. }
  863. container_id = btf__find_by_name(btf, container_name);
  864. if (container_id < 0) {
  865. pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
  866. map->name, container_name);
  867. return container_id;
  868. }
  869. container_type = btf__type_by_id(btf, container_id);
  870. if (!container_type) {
  871. pr_warning("map:%s cannot find BTF type for container_id:%u\n",
  872. map->name, container_id);
  873. return -EINVAL;
  874. }
  875. if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT ||
  876. BTF_INFO_VLEN(container_type->info) < 2) {
  877. pr_warning("map:%s container_name:%s is an invalid container struct\n",
  878. map->name, container_name);
  879. return -EINVAL;
  880. }
  881. key = (struct btf_member *)(container_type + 1);
  882. value = key + 1;
  883. key_size = btf__resolve_size(btf, key->type);
  884. if (key_size < 0) {
  885. pr_warning("map:%s invalid BTF key_type_size\n",
  886. map->name);
  887. return key_size;
  888. }
  889. if (def->key_size != key_size) {
  890. pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
  891. map->name, (__u32)key_size, def->key_size);
  892. return -EINVAL;
  893. }
  894. value_size = btf__resolve_size(btf, value->type);
  895. if (value_size < 0) {
  896. pr_warning("map:%s invalid BTF value_type_size\n", map->name);
  897. return value_size;
  898. }
  899. if (def->value_size != value_size) {
  900. pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
  901. map->name, (__u32)value_size, def->value_size);
  902. return -EINVAL;
  903. }
  904. map->btf_key_type_id = key->type;
  905. map->btf_value_type_id = value->type;
  906. return 0;
  907. }
  908. int bpf_map__reuse_fd(struct bpf_map *map, int fd)
  909. {
  910. struct bpf_map_info info = {};
  911. __u32 len = sizeof(info);
  912. int new_fd, err;
  913. char *new_name;
  914. err = bpf_obj_get_info_by_fd(fd, &info, &len);
  915. if (err)
  916. return err;
  917. new_name = strdup(info.name);
  918. if (!new_name)
  919. return -errno;
  920. new_fd = open("/", O_RDONLY | O_CLOEXEC);
  921. if (new_fd < 0) {
  922. err = -errno;
  923. goto err_free_new_name;
  924. }
  925. new_fd = dup3(fd, new_fd, O_CLOEXEC);
  926. if (new_fd < 0) {
  927. err = -errno;
  928. goto err_close_new_fd;
  929. }
  930. err = zclose(map->fd);
  931. if (err) {
  932. err = -errno;
  933. goto err_close_new_fd;
  934. }
  935. free(map->name);
  936. map->fd = new_fd;
  937. map->name = new_name;
  938. map->def.type = info.type;
  939. map->def.key_size = info.key_size;
  940. map->def.value_size = info.value_size;
  941. map->def.max_entries = info.max_entries;
  942. map->def.map_flags = info.map_flags;
  943. map->btf_key_type_id = info.btf_key_type_id;
  944. map->btf_value_type_id = info.btf_value_type_id;
  945. return 0;
  946. err_close_new_fd:
  947. close(new_fd);
  948. err_free_new_name:
  949. free(new_name);
  950. return err;
  951. }
  952. static int
  953. bpf_object__create_maps(struct bpf_object *obj)
  954. {
  955. struct bpf_create_map_attr create_attr = {};
  956. unsigned int i;
  957. int err;
  958. for (i = 0; i < obj->nr_maps; i++) {
  959. struct bpf_map *map = &obj->maps[i];
  960. struct bpf_map_def *def = &map->def;
  961. char *cp, errmsg[STRERR_BUFSIZE];
  962. int *pfd = &map->fd;
  963. if (map->fd >= 0) {
  964. pr_debug("skip map create (preset) %s: fd=%d\n",
  965. map->name, map->fd);
  966. continue;
  967. }
  968. create_attr.name = map->name;
  969. create_attr.map_ifindex = map->map_ifindex;
  970. create_attr.map_type = def->type;
  971. create_attr.map_flags = def->map_flags;
  972. create_attr.key_size = def->key_size;
  973. create_attr.value_size = def->value_size;
  974. create_attr.max_entries = def->max_entries;
  975. create_attr.btf_fd = 0;
  976. create_attr.btf_key_type_id = 0;
  977. create_attr.btf_value_type_id = 0;
  978. if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
  979. create_attr.btf_fd = btf__fd(obj->btf);
  980. create_attr.btf_key_type_id = map->btf_key_type_id;
  981. create_attr.btf_value_type_id = map->btf_value_type_id;
  982. }
  983. *pfd = bpf_create_map_xattr(&create_attr);
  984. if (*pfd < 0 && create_attr.btf_key_type_id) {
  985. cp = str_error(errno, errmsg, sizeof(errmsg));
  986. pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
  987. map->name, cp, errno);
  988. create_attr.btf_fd = 0;
  989. create_attr.btf_key_type_id = 0;
  990. create_attr.btf_value_type_id = 0;
  991. map->btf_key_type_id = 0;
  992. map->btf_value_type_id = 0;
  993. *pfd = bpf_create_map_xattr(&create_attr);
  994. }
  995. if (*pfd < 0) {
  996. size_t j;
  997. err = *pfd;
  998. cp = str_error(errno, errmsg, sizeof(errmsg));
  999. pr_warning("failed to create map (name: '%s'): %s\n",
  1000. map->name, cp);
  1001. for (j = 0; j < i; j++)
  1002. zclose(obj->maps[j].fd);
  1003. return err;
  1004. }
  1005. pr_debug("create map %s: fd=%d\n", map->name, *pfd);
  1006. }
  1007. return 0;
  1008. }
  1009. static int
  1010. bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
  1011. struct reloc_desc *relo)
  1012. {
  1013. struct bpf_insn *insn, *new_insn;
  1014. struct bpf_program *text;
  1015. size_t new_cnt;
  1016. if (relo->type != RELO_CALL)
  1017. return -LIBBPF_ERRNO__RELOC;
  1018. if (prog->idx == obj->efile.text_shndx) {
  1019. pr_warning("relo in .text insn %d into off %d\n",
  1020. relo->insn_idx, relo->text_off);
  1021. return -LIBBPF_ERRNO__RELOC;
  1022. }
  1023. if (prog->main_prog_cnt == 0) {
  1024. text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
  1025. if (!text) {
  1026. pr_warning("no .text section found yet relo into text exist\n");
  1027. return -LIBBPF_ERRNO__RELOC;
  1028. }
  1029. new_cnt = prog->insns_cnt + text->insns_cnt;
  1030. new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
  1031. if (!new_insn) {
  1032. pr_warning("oom in prog realloc\n");
  1033. return -ENOMEM;
  1034. }
  1035. memcpy(new_insn + prog->insns_cnt, text->insns,
  1036. text->insns_cnt * sizeof(*insn));
  1037. prog->insns = new_insn;
  1038. prog->main_prog_cnt = prog->insns_cnt;
  1039. prog->insns_cnt = new_cnt;
  1040. pr_debug("added %zd insn from %s to prog %s\n",
  1041. text->insns_cnt, text->section_name,
  1042. prog->section_name);
  1043. }
  1044. insn = &prog->insns[relo->insn_idx];
  1045. insn->imm += prog->main_prog_cnt - relo->insn_idx;
  1046. return 0;
  1047. }
  1048. static int
  1049. bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
  1050. {
  1051. int i, err;
  1052. if (!prog || !prog->reloc_desc)
  1053. return 0;
  1054. for (i = 0; i < prog->nr_reloc; i++) {
  1055. if (prog->reloc_desc[i].type == RELO_LD64) {
  1056. struct bpf_insn *insns = prog->insns;
  1057. int insn_idx, map_idx;
  1058. insn_idx = prog->reloc_desc[i].insn_idx;
  1059. map_idx = prog->reloc_desc[i].map_idx;
  1060. if (insn_idx >= (int)prog->insns_cnt) {
  1061. pr_warning("relocation out of range: '%s'\n",
  1062. prog->section_name);
  1063. return -LIBBPF_ERRNO__RELOC;
  1064. }
  1065. insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
  1066. insns[insn_idx].imm = obj->maps[map_idx].fd;
  1067. } else {
  1068. err = bpf_program__reloc_text(prog, obj,
  1069. &prog->reloc_desc[i]);
  1070. if (err)
  1071. return err;
  1072. }
  1073. }
  1074. zfree(&prog->reloc_desc);
  1075. prog->nr_reloc = 0;
  1076. return 0;
  1077. }
  1078. static int
  1079. bpf_object__relocate(struct bpf_object *obj)
  1080. {
  1081. struct bpf_program *prog;
  1082. size_t i;
  1083. int err;
  1084. for (i = 0; i < obj->nr_programs; i++) {
  1085. prog = &obj->programs[i];
  1086. err = bpf_program__relocate(prog, obj);
  1087. if (err) {
  1088. pr_warning("failed to relocate '%s'\n",
  1089. prog->section_name);
  1090. return err;
  1091. }
  1092. }
  1093. return 0;
  1094. }
  1095. static int bpf_object__collect_reloc(struct bpf_object *obj)
  1096. {
  1097. int i, err;
  1098. if (!obj_elf_valid(obj)) {
  1099. pr_warning("Internal error: elf object is closed\n");
  1100. return -LIBBPF_ERRNO__INTERNAL;
  1101. }
  1102. for (i = 0; i < obj->efile.nr_reloc; i++) {
  1103. GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
  1104. Elf_Data *data = obj->efile.reloc[i].data;
  1105. int idx = shdr->sh_info;
  1106. struct bpf_program *prog;
  1107. if (shdr->sh_type != SHT_REL) {
  1108. pr_warning("internal error at %d\n", __LINE__);
  1109. return -LIBBPF_ERRNO__INTERNAL;
  1110. }
  1111. prog = bpf_object__find_prog_by_idx(obj, idx);
  1112. if (!prog) {
  1113. pr_warning("relocation failed: no section(%d)\n", idx);
  1114. return -LIBBPF_ERRNO__RELOC;
  1115. }
  1116. err = bpf_program__collect_reloc(prog,
  1117. shdr, data,
  1118. obj);
  1119. if (err)
  1120. return err;
  1121. }
  1122. return 0;
  1123. }
  1124. static int
  1125. load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
  1126. const char *name, struct bpf_insn *insns, int insns_cnt,
  1127. char *license, u32 kern_version, int *pfd, int prog_ifindex)
  1128. {
  1129. struct bpf_load_program_attr load_attr;
  1130. char *cp, errmsg[STRERR_BUFSIZE];
  1131. char *log_buf;
  1132. int ret;
  1133. memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
  1134. load_attr.prog_type = type;
  1135. load_attr.expected_attach_type = expected_attach_type;
  1136. load_attr.name = name;
  1137. load_attr.insns = insns;
  1138. load_attr.insns_cnt = insns_cnt;
  1139. load_attr.license = license;
  1140. load_attr.kern_version = kern_version;
  1141. load_attr.prog_ifindex = prog_ifindex;
  1142. if (!load_attr.insns || !load_attr.insns_cnt)
  1143. return -EINVAL;
  1144. log_buf = malloc(BPF_LOG_BUF_SIZE);
  1145. if (!log_buf)
  1146. pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
  1147. ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
  1148. if (ret >= 0) {
  1149. *pfd = ret;
  1150. ret = 0;
  1151. goto out;
  1152. }
  1153. ret = -LIBBPF_ERRNO__LOAD;
  1154. cp = str_error(errno, errmsg, sizeof(errmsg));
  1155. pr_warning("load bpf program failed: %s\n", cp);
  1156. if (log_buf && log_buf[0] != '\0') {
  1157. ret = -LIBBPF_ERRNO__VERIFY;
  1158. pr_warning("-- BEGIN DUMP LOG ---\n");
  1159. pr_warning("\n%s\n", log_buf);
  1160. pr_warning("-- END LOG --\n");
  1161. } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
  1162. pr_warning("Program too large (%zu insns), at most %d insns\n",
  1163. load_attr.insns_cnt, BPF_MAXINSNS);
  1164. ret = -LIBBPF_ERRNO__PROG2BIG;
  1165. } else {
  1166. /* Wrong program type? */
  1167. if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
  1168. int fd;
  1169. load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
  1170. load_attr.expected_attach_type = 0;
  1171. fd = bpf_load_program_xattr(&load_attr, NULL, 0);
  1172. if (fd >= 0) {
  1173. close(fd);
  1174. ret = -LIBBPF_ERRNO__PROGTYPE;
  1175. goto out;
  1176. }
  1177. }
  1178. if (log_buf)
  1179. ret = -LIBBPF_ERRNO__KVER;
  1180. }
  1181. out:
  1182. free(log_buf);
  1183. return ret;
  1184. }
  1185. static int
  1186. bpf_program__load(struct bpf_program *prog,
  1187. char *license, u32 kern_version)
  1188. {
  1189. int err = 0, fd, i;
  1190. if (prog->instances.nr < 0 || !prog->instances.fds) {
  1191. if (prog->preprocessor) {
  1192. pr_warning("Internal error: can't load program '%s'\n",
  1193. prog->section_name);
  1194. return -LIBBPF_ERRNO__INTERNAL;
  1195. }
  1196. prog->instances.fds = malloc(sizeof(int));
  1197. if (!prog->instances.fds) {
  1198. pr_warning("Not enough memory for BPF fds\n");
  1199. return -ENOMEM;
  1200. }
  1201. prog->instances.nr = 1;
  1202. prog->instances.fds[0] = -1;
  1203. }
  1204. if (!prog->preprocessor) {
  1205. if (prog->instances.nr != 1) {
  1206. pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
  1207. prog->section_name, prog->instances.nr);
  1208. }
  1209. err = load_program(prog->type, prog->expected_attach_type,
  1210. prog->name, prog->insns, prog->insns_cnt,
  1211. license, kern_version, &fd,
  1212. prog->prog_ifindex);
  1213. if (!err)
  1214. prog->instances.fds[0] = fd;
  1215. goto out;
  1216. }
  1217. for (i = 0; i < prog->instances.nr; i++) {
  1218. struct bpf_prog_prep_result result;
  1219. bpf_program_prep_t preprocessor = prog->preprocessor;
  1220. bzero(&result, sizeof(result));
  1221. err = preprocessor(prog, i, prog->insns,
  1222. prog->insns_cnt, &result);
  1223. if (err) {
  1224. pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
  1225. i, prog->section_name);
  1226. goto out;
  1227. }
  1228. if (!result.new_insn_ptr || !result.new_insn_cnt) {
  1229. pr_debug("Skip loading the %dth instance of program '%s'\n",
  1230. i, prog->section_name);
  1231. prog->instances.fds[i] = -1;
  1232. if (result.pfd)
  1233. *result.pfd = -1;
  1234. continue;
  1235. }
  1236. err = load_program(prog->type, prog->expected_attach_type,
  1237. prog->name, result.new_insn_ptr,
  1238. result.new_insn_cnt,
  1239. license, kern_version, &fd,
  1240. prog->prog_ifindex);
  1241. if (err) {
  1242. pr_warning("Loading the %dth instance of program '%s' failed\n",
  1243. i, prog->section_name);
  1244. goto out;
  1245. }
  1246. if (result.pfd)
  1247. *result.pfd = fd;
  1248. prog->instances.fds[i] = fd;
  1249. }
  1250. out:
  1251. if (err)
  1252. pr_warning("failed to load program '%s'\n",
  1253. prog->section_name);
  1254. zfree(&prog->insns);
  1255. prog->insns_cnt = 0;
  1256. return err;
  1257. }
  1258. static bool bpf_program__is_function_storage(struct bpf_program *prog,
  1259. struct bpf_object *obj)
  1260. {
  1261. return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
  1262. }
  1263. static int
  1264. bpf_object__load_progs(struct bpf_object *obj)
  1265. {
  1266. size_t i;
  1267. int err;
  1268. for (i = 0; i < obj->nr_programs; i++) {
  1269. if (bpf_program__is_function_storage(&obj->programs[i], obj))
  1270. continue;
  1271. err = bpf_program__load(&obj->programs[i],
  1272. obj->license,
  1273. obj->kern_version);
  1274. if (err)
  1275. return err;
  1276. }
  1277. return 0;
  1278. }
  1279. static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
  1280. {
  1281. switch (type) {
  1282. case BPF_PROG_TYPE_SOCKET_FILTER:
  1283. case BPF_PROG_TYPE_SCHED_CLS:
  1284. case BPF_PROG_TYPE_SCHED_ACT:
  1285. case BPF_PROG_TYPE_XDP:
  1286. case BPF_PROG_TYPE_CGROUP_SKB:
  1287. case BPF_PROG_TYPE_CGROUP_SOCK:
  1288. case BPF_PROG_TYPE_LWT_IN:
  1289. case BPF_PROG_TYPE_LWT_OUT:
  1290. case BPF_PROG_TYPE_LWT_XMIT:
  1291. case BPF_PROG_TYPE_LWT_SEG6LOCAL:
  1292. case BPF_PROG_TYPE_SOCK_OPS:
  1293. case BPF_PROG_TYPE_SK_SKB:
  1294. case BPF_PROG_TYPE_CGROUP_DEVICE:
  1295. case BPF_PROG_TYPE_SK_MSG:
  1296. case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
  1297. case BPF_PROG_TYPE_LIRC_MODE2:
  1298. case BPF_PROG_TYPE_SK_REUSEPORT:
  1299. return false;
  1300. case BPF_PROG_TYPE_UNSPEC:
  1301. case BPF_PROG_TYPE_KPROBE:
  1302. case BPF_PROG_TYPE_TRACEPOINT:
  1303. case BPF_PROG_TYPE_PERF_EVENT:
  1304. case BPF_PROG_TYPE_RAW_TRACEPOINT:
  1305. default:
  1306. return true;
  1307. }
  1308. }
  1309. static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
  1310. {
  1311. if (needs_kver && obj->kern_version == 0) {
  1312. pr_warning("%s doesn't provide kernel version\n",
  1313. obj->path);
  1314. return -LIBBPF_ERRNO__KVERSION;
  1315. }
  1316. return 0;
  1317. }
  1318. static struct bpf_object *
  1319. __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
  1320. bool needs_kver)
  1321. {
  1322. struct bpf_object *obj;
  1323. int err;
  1324. if (elf_version(EV_CURRENT) == EV_NONE) {
  1325. pr_warning("failed to init libelf for %s\n", path);
  1326. return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
  1327. }
  1328. obj = bpf_object__new(path, obj_buf, obj_buf_sz);
  1329. if (IS_ERR(obj))
  1330. return obj;
  1331. CHECK_ERR(bpf_object__elf_init(obj), err, out);
  1332. CHECK_ERR(bpf_object__check_endianness(obj), err, out);
  1333. CHECK_ERR(bpf_object__elf_collect(obj), err, out);
  1334. CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
  1335. CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
  1336. bpf_object__elf_finish(obj);
  1337. return obj;
  1338. out:
  1339. bpf_object__close(obj);
  1340. return ERR_PTR(err);
  1341. }
  1342. struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
  1343. {
  1344. /* param validation */
  1345. if (!attr->file)
  1346. return NULL;
  1347. pr_debug("loading %s\n", attr->file);
  1348. return __bpf_object__open(attr->file, NULL, 0,
  1349. bpf_prog_type__needs_kver(attr->prog_type));
  1350. }
  1351. struct bpf_object *bpf_object__open(const char *path)
  1352. {
  1353. struct bpf_object_open_attr attr = {
  1354. .file = path,
  1355. .prog_type = BPF_PROG_TYPE_UNSPEC,
  1356. };
  1357. return bpf_object__open_xattr(&attr);
  1358. }
  1359. struct bpf_object *bpf_object__open_buffer(void *obj_buf,
  1360. size_t obj_buf_sz,
  1361. const char *name)
  1362. {
  1363. char tmp_name[64];
  1364. /* param validation */
  1365. if (!obj_buf || obj_buf_sz <= 0)
  1366. return NULL;
  1367. if (!name) {
  1368. snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
  1369. (unsigned long)obj_buf,
  1370. (unsigned long)obj_buf_sz);
  1371. tmp_name[sizeof(tmp_name) - 1] = '\0';
  1372. name = tmp_name;
  1373. }
  1374. pr_debug("loading object '%s' from buffer\n",
  1375. name);
  1376. return __bpf_object__open(name, obj_buf, obj_buf_sz, true);
  1377. }
  1378. int bpf_object__unload(struct bpf_object *obj)
  1379. {
  1380. size_t i;
  1381. if (!obj)
  1382. return -EINVAL;
  1383. for (i = 0; i < obj->nr_maps; i++)
  1384. zclose(obj->maps[i].fd);
  1385. for (i = 0; i < obj->nr_programs; i++)
  1386. bpf_program__unload(&obj->programs[i]);
  1387. return 0;
  1388. }
  1389. int bpf_object__load(struct bpf_object *obj)
  1390. {
  1391. int err;
  1392. if (!obj)
  1393. return -EINVAL;
  1394. if (obj->loaded) {
  1395. pr_warning("object should not be loaded twice\n");
  1396. return -EINVAL;
  1397. }
  1398. obj->loaded = true;
  1399. CHECK_ERR(bpf_object__create_maps(obj), err, out);
  1400. CHECK_ERR(bpf_object__relocate(obj), err, out);
  1401. CHECK_ERR(bpf_object__load_progs(obj), err, out);
  1402. return 0;
  1403. out:
  1404. bpf_object__unload(obj);
  1405. pr_warning("failed to load object '%s'\n", obj->path);
  1406. return err;
  1407. }
  1408. static int check_path(const char *path)
  1409. {
  1410. char *cp, errmsg[STRERR_BUFSIZE];
  1411. struct statfs st_fs;
  1412. char *dname, *dir;
  1413. int err = 0;
  1414. if (path == NULL)
  1415. return -EINVAL;
  1416. dname = strdup(path);
  1417. if (dname == NULL)
  1418. return -ENOMEM;
  1419. dir = dirname(dname);
  1420. if (statfs(dir, &st_fs)) {
  1421. cp = str_error(errno, errmsg, sizeof(errmsg));
  1422. pr_warning("failed to statfs %s: %s\n", dir, cp);
  1423. err = -errno;
  1424. }
  1425. free(dname);
  1426. if (!err && st_fs.f_type != BPF_FS_MAGIC) {
  1427. pr_warning("specified path %s is not on BPF FS\n", path);
  1428. err = -EINVAL;
  1429. }
  1430. return err;
  1431. }
  1432. int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
  1433. int instance)
  1434. {
  1435. char *cp, errmsg[STRERR_BUFSIZE];
  1436. int err;
  1437. err = check_path(path);
  1438. if (err)
  1439. return err;
  1440. if (prog == NULL) {
  1441. pr_warning("invalid program pointer\n");
  1442. return -EINVAL;
  1443. }
  1444. if (instance < 0 || instance >= prog->instances.nr) {
  1445. pr_warning("invalid prog instance %d of prog %s (max %d)\n",
  1446. instance, prog->section_name, prog->instances.nr);
  1447. return -EINVAL;
  1448. }
  1449. if (bpf_obj_pin(prog->instances.fds[instance], path)) {
  1450. cp = str_error(errno, errmsg, sizeof(errmsg));
  1451. pr_warning("failed to pin program: %s\n", cp);
  1452. return -errno;
  1453. }
  1454. pr_debug("pinned program '%s'\n", path);
  1455. return 0;
  1456. }
  1457. static int make_dir(const char *path)
  1458. {
  1459. char *cp, errmsg[STRERR_BUFSIZE];
  1460. int err = 0;
  1461. if (mkdir(path, 0700) && errno != EEXIST)
  1462. err = -errno;
  1463. if (err) {
  1464. cp = str_error(-err, errmsg, sizeof(errmsg));
  1465. pr_warning("failed to mkdir %s: %s\n", path, cp);
  1466. }
  1467. return err;
  1468. }
  1469. int bpf_program__pin(struct bpf_program *prog, const char *path)
  1470. {
  1471. int i, err;
  1472. err = check_path(path);
  1473. if (err)
  1474. return err;
  1475. if (prog == NULL) {
  1476. pr_warning("invalid program pointer\n");
  1477. return -EINVAL;
  1478. }
  1479. if (prog->instances.nr <= 0) {
  1480. pr_warning("no instances of prog %s to pin\n",
  1481. prog->section_name);
  1482. return -EINVAL;
  1483. }
  1484. err = make_dir(path);
  1485. if (err)
  1486. return err;
  1487. for (i = 0; i < prog->instances.nr; i++) {
  1488. char buf[PATH_MAX];
  1489. int len;
  1490. len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
  1491. if (len < 0)
  1492. return -EINVAL;
  1493. else if (len >= PATH_MAX)
  1494. return -ENAMETOOLONG;
  1495. err = bpf_program__pin_instance(prog, buf, i);
  1496. if (err)
  1497. return err;
  1498. }
  1499. return 0;
  1500. }
  1501. int bpf_map__pin(struct bpf_map *map, const char *path)
  1502. {
  1503. char *cp, errmsg[STRERR_BUFSIZE];
  1504. int err;
  1505. err = check_path(path);
  1506. if (err)
  1507. return err;
  1508. if (map == NULL) {
  1509. pr_warning("invalid map pointer\n");
  1510. return -EINVAL;
  1511. }
  1512. if (bpf_obj_pin(map->fd, path)) {
  1513. cp = str_error(errno, errmsg, sizeof(errmsg));
  1514. pr_warning("failed to pin map: %s\n", cp);
  1515. return -errno;
  1516. }
  1517. pr_debug("pinned map '%s'\n", path);
  1518. return 0;
  1519. }
  1520. int bpf_object__pin(struct bpf_object *obj, const char *path)
  1521. {
  1522. struct bpf_program *prog;
  1523. struct bpf_map *map;
  1524. int err;
  1525. if (!obj)
  1526. return -ENOENT;
  1527. if (!obj->loaded) {
  1528. pr_warning("object not yet loaded; load it first\n");
  1529. return -ENOENT;
  1530. }
  1531. err = make_dir(path);
  1532. if (err)
  1533. return err;
  1534. bpf_map__for_each(map, obj) {
  1535. char buf[PATH_MAX];
  1536. int len;
  1537. len = snprintf(buf, PATH_MAX, "%s/%s", path,
  1538. bpf_map__name(map));
  1539. if (len < 0)
  1540. return -EINVAL;
  1541. else if (len >= PATH_MAX)
  1542. return -ENAMETOOLONG;
  1543. err = bpf_map__pin(map, buf);
  1544. if (err)
  1545. return err;
  1546. }
  1547. bpf_object__for_each_program(prog, obj) {
  1548. char buf[PATH_MAX];
  1549. int len;
  1550. len = snprintf(buf, PATH_MAX, "%s/%s", path,
  1551. prog->section_name);
  1552. if (len < 0)
  1553. return -EINVAL;
  1554. else if (len >= PATH_MAX)
  1555. return -ENAMETOOLONG;
  1556. err = bpf_program__pin(prog, buf);
  1557. if (err)
  1558. return err;
  1559. }
  1560. return 0;
  1561. }
  1562. void bpf_object__close(struct bpf_object *obj)
  1563. {
  1564. size_t i;
  1565. if (!obj)
  1566. return;
  1567. if (obj->clear_priv)
  1568. obj->clear_priv(obj, obj->priv);
  1569. bpf_object__elf_finish(obj);
  1570. bpf_object__unload(obj);
  1571. btf__free(obj->btf);
  1572. for (i = 0; i < obj->nr_maps; i++) {
  1573. zfree(&obj->maps[i].name);
  1574. if (obj->maps[i].clear_priv)
  1575. obj->maps[i].clear_priv(&obj->maps[i],
  1576. obj->maps[i].priv);
  1577. obj->maps[i].priv = NULL;
  1578. obj->maps[i].clear_priv = NULL;
  1579. }
  1580. zfree(&obj->maps);
  1581. obj->nr_maps = 0;
  1582. if (obj->programs && obj->nr_programs) {
  1583. for (i = 0; i < obj->nr_programs; i++)
  1584. bpf_program__exit(&obj->programs[i]);
  1585. }
  1586. zfree(&obj->programs);
  1587. list_del(&obj->list);
  1588. free(obj);
  1589. }
  1590. struct bpf_object *
  1591. bpf_object__next(struct bpf_object *prev)
  1592. {
  1593. struct bpf_object *next;
  1594. if (!prev)
  1595. next = list_first_entry(&bpf_objects_list,
  1596. struct bpf_object,
  1597. list);
  1598. else
  1599. next = list_next_entry(prev, list);
  1600. /* Empty list is noticed here so don't need checking on entry. */
  1601. if (&next->list == &bpf_objects_list)
  1602. return NULL;
  1603. return next;
  1604. }
  1605. const char *bpf_object__name(struct bpf_object *obj)
  1606. {
  1607. return obj ? obj->path : ERR_PTR(-EINVAL);
  1608. }
  1609. unsigned int bpf_object__kversion(struct bpf_object *obj)
  1610. {
  1611. return obj ? obj->kern_version : 0;
  1612. }
  1613. int bpf_object__btf_fd(const struct bpf_object *obj)
  1614. {
  1615. return obj->btf ? btf__fd(obj->btf) : -1;
  1616. }
  1617. int bpf_object__set_priv(struct bpf_object *obj, void *priv,
  1618. bpf_object_clear_priv_t clear_priv)
  1619. {
  1620. if (obj->priv && obj->clear_priv)
  1621. obj->clear_priv(obj, obj->priv);
  1622. obj->priv = priv;
  1623. obj->clear_priv = clear_priv;
  1624. return 0;
  1625. }
  1626. void *bpf_object__priv(struct bpf_object *obj)
  1627. {
  1628. return obj ? obj->priv : ERR_PTR(-EINVAL);
  1629. }
  1630. static struct bpf_program *
  1631. __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
  1632. {
  1633. size_t idx;
  1634. if (!obj->programs)
  1635. return NULL;
  1636. /* First handler */
  1637. if (prev == NULL)
  1638. return &obj->programs[0];
  1639. if (prev->obj != obj) {
  1640. pr_warning("error: program handler doesn't match object\n");
  1641. return NULL;
  1642. }
  1643. idx = (prev - obj->programs) + 1;
  1644. if (idx >= obj->nr_programs)
  1645. return NULL;
  1646. return &obj->programs[idx];
  1647. }
  1648. struct bpf_program *
  1649. bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
  1650. {
  1651. struct bpf_program *prog = prev;
  1652. do {
  1653. prog = __bpf_program__next(prog, obj);
  1654. } while (prog && bpf_program__is_function_storage(prog, obj));
  1655. return prog;
  1656. }
  1657. int bpf_program__set_priv(struct bpf_program *prog, void *priv,
  1658. bpf_program_clear_priv_t clear_priv)
  1659. {
  1660. if (prog->priv && prog->clear_priv)
  1661. prog->clear_priv(prog, prog->priv);
  1662. prog->priv = priv;
  1663. prog->clear_priv = clear_priv;
  1664. return 0;
  1665. }
  1666. void *bpf_program__priv(struct bpf_program *prog)
  1667. {
  1668. return prog ? prog->priv : ERR_PTR(-EINVAL);
  1669. }
  1670. void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
  1671. {
  1672. prog->prog_ifindex = ifindex;
  1673. }
  1674. const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
  1675. {
  1676. const char *title;
  1677. title = prog->section_name;
  1678. if (needs_copy) {
  1679. title = strdup(title);
  1680. if (!title) {
  1681. pr_warning("failed to strdup program title\n");
  1682. return ERR_PTR(-ENOMEM);
  1683. }
  1684. }
  1685. return title;
  1686. }
  1687. int bpf_program__fd(struct bpf_program *prog)
  1688. {
  1689. return bpf_program__nth_fd(prog, 0);
  1690. }
  1691. int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
  1692. bpf_program_prep_t prep)
  1693. {
  1694. int *instances_fds;
  1695. if (nr_instances <= 0 || !prep)
  1696. return -EINVAL;
  1697. if (prog->instances.nr > 0 || prog->instances.fds) {
  1698. pr_warning("Can't set pre-processor after loading\n");
  1699. return -EINVAL;
  1700. }
  1701. instances_fds = malloc(sizeof(int) * nr_instances);
  1702. if (!instances_fds) {
  1703. pr_warning("alloc memory failed for fds\n");
  1704. return -ENOMEM;
  1705. }
  1706. /* fill all fd with -1 */
  1707. memset(instances_fds, -1, sizeof(int) * nr_instances);
  1708. prog->instances.nr = nr_instances;
  1709. prog->instances.fds = instances_fds;
  1710. prog->preprocessor = prep;
  1711. return 0;
  1712. }
  1713. int bpf_program__nth_fd(struct bpf_program *prog, int n)
  1714. {
  1715. int fd;
  1716. if (!prog)
  1717. return -EINVAL;
  1718. if (n >= prog->instances.nr || n < 0) {
  1719. pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
  1720. n, prog->section_name, prog->instances.nr);
  1721. return -EINVAL;
  1722. }
  1723. fd = prog->instances.fds[n];
  1724. if (fd < 0) {
  1725. pr_warning("%dth instance of program '%s' is invalid\n",
  1726. n, prog->section_name);
  1727. return -ENOENT;
  1728. }
  1729. return fd;
  1730. }
  1731. void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
  1732. {
  1733. prog->type = type;
  1734. }
  1735. static bool bpf_program__is_type(struct bpf_program *prog,
  1736. enum bpf_prog_type type)
  1737. {
  1738. return prog ? (prog->type == type) : false;
  1739. }
  1740. #define BPF_PROG_TYPE_FNS(NAME, TYPE) \
  1741. int bpf_program__set_##NAME(struct bpf_program *prog) \
  1742. { \
  1743. if (!prog) \
  1744. return -EINVAL; \
  1745. bpf_program__set_type(prog, TYPE); \
  1746. return 0; \
  1747. } \
  1748. \
  1749. bool bpf_program__is_##NAME(struct bpf_program *prog) \
  1750. { \
  1751. return bpf_program__is_type(prog, TYPE); \
  1752. } \
  1753. BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
  1754. BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
  1755. BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
  1756. BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
  1757. BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
  1758. BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
  1759. BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
  1760. BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
  1761. void bpf_program__set_expected_attach_type(struct bpf_program *prog,
  1762. enum bpf_attach_type type)
  1763. {
  1764. prog->expected_attach_type = type;
  1765. }
  1766. #define BPF_PROG_SEC_FULL(string, ptype, atype) \
  1767. { string, sizeof(string) - 1, ptype, atype }
  1768. #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0)
  1769. #define BPF_S_PROG_SEC(string, ptype) \
  1770. BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK, ptype)
  1771. #define BPF_SA_PROG_SEC(string, ptype) \
  1772. BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype)
  1773. static const struct {
  1774. const char *sec;
  1775. size_t len;
  1776. enum bpf_prog_type prog_type;
  1777. enum bpf_attach_type expected_attach_type;
  1778. } section_names[] = {
  1779. BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
  1780. BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),
  1781. BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),
  1782. BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
  1783. BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
  1784. BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),
  1785. BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),
  1786. BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
  1787. BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
  1788. BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
  1789. BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK),
  1790. BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE),
  1791. BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
  1792. BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
  1793. BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
  1794. BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
  1795. BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS),
  1796. BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB),
  1797. BPF_PROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG),
  1798. BPF_PROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2),
  1799. BPF_SA_PROG_SEC("cgroup/bind4", BPF_CGROUP_INET4_BIND),
  1800. BPF_SA_PROG_SEC("cgroup/bind6", BPF_CGROUP_INET6_BIND),
  1801. BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT),
  1802. BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT),
  1803. BPF_SA_PROG_SEC("cgroup/sendmsg4", BPF_CGROUP_UDP4_SENDMSG),
  1804. BPF_SA_PROG_SEC("cgroup/sendmsg6", BPF_CGROUP_UDP6_SENDMSG),
  1805. BPF_S_PROG_SEC("cgroup/post_bind4", BPF_CGROUP_INET4_POST_BIND),
  1806. BPF_S_PROG_SEC("cgroup/post_bind6", BPF_CGROUP_INET6_POST_BIND),
  1807. };
  1808. #undef BPF_PROG_SEC
  1809. #undef BPF_PROG_SEC_FULL
  1810. #undef BPF_S_PROG_SEC
  1811. #undef BPF_SA_PROG_SEC
  1812. int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
  1813. enum bpf_attach_type *expected_attach_type)
  1814. {
  1815. int i;
  1816. if (!name)
  1817. return -EINVAL;
  1818. for (i = 0; i < ARRAY_SIZE(section_names); i++) {
  1819. if (strncmp(name, section_names[i].sec, section_names[i].len))
  1820. continue;
  1821. *prog_type = section_names[i].prog_type;
  1822. *expected_attach_type = section_names[i].expected_attach_type;
  1823. return 0;
  1824. }
  1825. return -EINVAL;
  1826. }
  1827. static int
  1828. bpf_program__identify_section(struct bpf_program *prog,
  1829. enum bpf_prog_type *prog_type,
  1830. enum bpf_attach_type *expected_attach_type)
  1831. {
  1832. return libbpf_prog_type_by_name(prog->section_name, prog_type,
  1833. expected_attach_type);
  1834. }
  1835. int bpf_map__fd(struct bpf_map *map)
  1836. {
  1837. return map ? map->fd : -EINVAL;
  1838. }
  1839. const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
  1840. {
  1841. return map ? &map->def : ERR_PTR(-EINVAL);
  1842. }
  1843. const char *bpf_map__name(struct bpf_map *map)
  1844. {
  1845. return map ? map->name : NULL;
  1846. }
  1847. __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
  1848. {
  1849. return map ? map->btf_key_type_id : 0;
  1850. }
  1851. __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
  1852. {
  1853. return map ? map->btf_value_type_id : 0;
  1854. }
  1855. int bpf_map__set_priv(struct bpf_map *map, void *priv,
  1856. bpf_map_clear_priv_t clear_priv)
  1857. {
  1858. if (!map)
  1859. return -EINVAL;
  1860. if (map->priv) {
  1861. if (map->clear_priv)
  1862. map->clear_priv(map, map->priv);
  1863. }
  1864. map->priv = priv;
  1865. map->clear_priv = clear_priv;
  1866. return 0;
  1867. }
  1868. void *bpf_map__priv(struct bpf_map *map)
  1869. {
  1870. return map ? map->priv : ERR_PTR(-EINVAL);
  1871. }
  1872. bool bpf_map__is_offload_neutral(struct bpf_map *map)
  1873. {
  1874. return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
  1875. }
  1876. void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
  1877. {
  1878. map->map_ifindex = ifindex;
  1879. }
  1880. struct bpf_map *
  1881. bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
  1882. {
  1883. size_t idx;
  1884. struct bpf_map *s, *e;
  1885. if (!obj || !obj->maps)
  1886. return NULL;
  1887. s = obj->maps;
  1888. e = obj->maps + obj->nr_maps;
  1889. if (prev == NULL)
  1890. return s;
  1891. if ((prev < s) || (prev >= e)) {
  1892. pr_warning("error in %s: map handler doesn't belong to object\n",
  1893. __func__);
  1894. return NULL;
  1895. }
  1896. idx = (prev - obj->maps) + 1;
  1897. if (idx >= obj->nr_maps)
  1898. return NULL;
  1899. return &obj->maps[idx];
  1900. }
  1901. struct bpf_map *
  1902. bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
  1903. {
  1904. struct bpf_map *pos;
  1905. bpf_map__for_each(pos, obj) {
  1906. if (pos->name && !strcmp(pos->name, name))
  1907. return pos;
  1908. }
  1909. return NULL;
  1910. }
  1911. struct bpf_map *
  1912. bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
  1913. {
  1914. int i;
  1915. for (i = 0; i < obj->nr_maps; i++) {
  1916. if (obj->maps[i].offset == offset)
  1917. return &obj->maps[i];
  1918. }
  1919. return ERR_PTR(-ENOENT);
  1920. }
  1921. long libbpf_get_error(const void *ptr)
  1922. {
  1923. if (IS_ERR(ptr))
  1924. return PTR_ERR(ptr);
  1925. return 0;
  1926. }
  1927. int bpf_prog_load(const char *file, enum bpf_prog_type type,
  1928. struct bpf_object **pobj, int *prog_fd)
  1929. {
  1930. struct bpf_prog_load_attr attr;
  1931. memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
  1932. attr.file = file;
  1933. attr.prog_type = type;
  1934. attr.expected_attach_type = 0;
  1935. return bpf_prog_load_xattr(&attr, pobj, prog_fd);
  1936. }
  1937. int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
  1938. struct bpf_object **pobj, int *prog_fd)
  1939. {
  1940. struct bpf_object_open_attr open_attr = {};
  1941. struct bpf_program *prog, *first_prog = NULL;
  1942. enum bpf_attach_type expected_attach_type;
  1943. enum bpf_prog_type prog_type;
  1944. struct bpf_object *obj;
  1945. struct bpf_map *map;
  1946. int err;
  1947. if (!attr)
  1948. return -EINVAL;
  1949. if (!attr->file)
  1950. return -EINVAL;
  1951. open_attr.file = attr->file;
  1952. open_attr.prog_type = attr->prog_type;
  1953. obj = bpf_object__open_xattr(&open_attr);
  1954. if (IS_ERR_OR_NULL(obj))
  1955. return -ENOENT;
  1956. bpf_object__for_each_program(prog, obj) {
  1957. /*
  1958. * If type is not specified, try to guess it based on
  1959. * section name.
  1960. */
  1961. prog_type = attr->prog_type;
  1962. prog->prog_ifindex = attr->ifindex;
  1963. expected_attach_type = attr->expected_attach_type;
  1964. if (prog_type == BPF_PROG_TYPE_UNSPEC) {
  1965. err = bpf_program__identify_section(prog, &prog_type,
  1966. &expected_attach_type);
  1967. if (err < 0) {
  1968. pr_warning("failed to guess program type based on section name %s\n",
  1969. prog->section_name);
  1970. bpf_object__close(obj);
  1971. return -EINVAL;
  1972. }
  1973. }
  1974. bpf_program__set_type(prog, prog_type);
  1975. bpf_program__set_expected_attach_type(prog,
  1976. expected_attach_type);
  1977. if (!bpf_program__is_function_storage(prog, obj) && !first_prog)
  1978. first_prog = prog;
  1979. }
  1980. bpf_map__for_each(map, obj) {
  1981. if (!bpf_map__is_offload_neutral(map))
  1982. map->map_ifindex = attr->ifindex;
  1983. }
  1984. if (!first_prog) {
  1985. pr_warning("object file doesn't contain bpf program\n");
  1986. bpf_object__close(obj);
  1987. return -ENOENT;
  1988. }
  1989. err = bpf_object__load(obj);
  1990. if (err) {
  1991. bpf_object__close(obj);
  1992. return -EINVAL;
  1993. }
  1994. *pobj = obj;
  1995. *prog_fd = bpf_program__fd(first_prog);
  1996. return 0;
  1997. }
  1998. enum bpf_perf_event_ret
  1999. bpf_perf_event_read_simple(void *mem, unsigned long size,
  2000. unsigned long page_size, void **buf, size_t *buf_len,
  2001. bpf_perf_event_print_t fn, void *priv)
  2002. {
  2003. volatile struct perf_event_mmap_page *header = mem;
  2004. __u64 data_tail = header->data_tail;
  2005. __u64 data_head = header->data_head;
  2006. int ret = LIBBPF_PERF_EVENT_ERROR;
  2007. void *base, *begin, *end;
  2008. asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
  2009. if (data_head == data_tail)
  2010. return LIBBPF_PERF_EVENT_CONT;
  2011. base = ((char *)header) + page_size;
  2012. begin = base + data_tail % size;
  2013. end = base + data_head % size;
  2014. while (begin != end) {
  2015. struct perf_event_header *ehdr;
  2016. ehdr = begin;
  2017. if (begin + ehdr->size > base + size) {
  2018. long len = base + size - begin;
  2019. if (*buf_len < ehdr->size) {
  2020. free(*buf);
  2021. *buf = malloc(ehdr->size);
  2022. if (!*buf) {
  2023. ret = LIBBPF_PERF_EVENT_ERROR;
  2024. break;
  2025. }
  2026. *buf_len = ehdr->size;
  2027. }
  2028. memcpy(*buf, begin, len);
  2029. memcpy(*buf + len, base, ehdr->size - len);
  2030. ehdr = (void *)*buf;
  2031. begin = base + ehdr->size - len;
  2032. } else if (begin + ehdr->size == base + size) {
  2033. begin = base;
  2034. } else {
  2035. begin += ehdr->size;
  2036. }
  2037. ret = fn(ehdr, priv);
  2038. if (ret != LIBBPF_PERF_EVENT_CONT)
  2039. break;
  2040. data_tail += ehdr->size;
  2041. }
  2042. __sync_synchronize(); /* smp_mb() */
  2043. header->data_tail = data_tail;
  2044. return ret;
  2045. }