123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691 |
- // SPDX-License-Identifier: LGPL-2.1
- /*
- * common eBPF ELF operations.
- *
- * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
- * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
- * Copyright (C) 2015 Huawei Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation;
- * version 2.1 of the License (not later!)
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this program; if not, see <http://www.gnu.org/licenses>
- */
- #include <stdlib.h>
- #include <memory.h>
- #include <unistd.h>
- #include <asm/unistd.h>
- #include <linux/bpf.h>
- #include "bpf.h"
- #include "libbpf.h"
- #include "nlattr.h"
- #include <linux/rtnetlink.h>
- #include <linux/if_link.h>
- #include <sys/socket.h>
- #include <errno.h>
- #ifndef SOL_NETLINK
- #define SOL_NETLINK 270
- #endif
- /*
- * When building perf, unistd.h is overridden. __NR_bpf is
- * required to be defined explicitly.
- */
- #ifndef __NR_bpf
- # if defined(__i386__)
- # define __NR_bpf 357
- # elif defined(__x86_64__)
- # define __NR_bpf 321
- # elif defined(__aarch64__)
- # define __NR_bpf 280
- # elif defined(__sparc__)
- # define __NR_bpf 349
- # elif defined(__s390__)
- # define __NR_bpf 351
- # elif defined(__arc__)
- # define __NR_bpf 280
- # else
- # error __NR_bpf not defined. libbpf does not support your arch.
- # endif
- #endif
- #ifndef min
- #define min(x, y) ((x) < (y) ? (x) : (y))
- #endif
- static inline __u64 ptr_to_u64(const void *ptr)
- {
- return (__u64) (unsigned long) ptr;
- }
- static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
- unsigned int size)
- {
- return syscall(__NR_bpf, cmd, attr, size);
- }
- static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size)
- {
- int fd;
- do {
- fd = sys_bpf(BPF_PROG_LOAD, attr, size);
- } while (fd < 0 && errno == EAGAIN);
- return fd;
- }
- int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr)
- {
- __u32 name_len = create_attr->name ? strlen(create_attr->name) : 0;
- union bpf_attr attr;
- int ret;
- memset(&attr, '\0', sizeof(attr));
- attr.map_type = create_attr->map_type;
- attr.key_size = create_attr->key_size;
- attr.value_size = create_attr->value_size;
- attr.max_entries = create_attr->max_entries;
- attr.map_flags = create_attr->map_flags;
- memcpy(attr.map_name, create_attr->name,
- min(name_len, BPF_OBJ_NAME_LEN - 1));
- attr.numa_node = create_attr->numa_node;
- attr.btf_fd = create_attr->btf_fd;
- attr.btf_key_type_id = create_attr->btf_key_type_id;
- attr.btf_value_type_id = create_attr->btf_value_type_id;
- attr.map_ifindex = create_attr->map_ifindex;
- attr.inner_map_fd = create_attr->inner_map_fd;
- ret = sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
- if (ret < 0 && errno == EINVAL && create_attr->name) {
- /* Retry the same syscall, but without the name.
- * Pre v4.14 kernels don't support map names.
- */
- memset(attr.map_name, 0, sizeof(attr.map_name));
- return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
- }
- return ret;
- }
- int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
- int key_size, int value_size, int max_entries,
- __u32 map_flags, int node)
- {
- struct bpf_create_map_attr map_attr = {};
- map_attr.name = name;
- map_attr.map_type = map_type;
- map_attr.map_flags = map_flags;
- map_attr.key_size = key_size;
- map_attr.value_size = value_size;
- map_attr.max_entries = max_entries;
- if (node >= 0) {
- map_attr.numa_node = node;
- map_attr.map_flags |= BPF_F_NUMA_NODE;
- }
- return bpf_create_map_xattr(&map_attr);
- }
- int bpf_create_map(enum bpf_map_type map_type, int key_size,
- int value_size, int max_entries, __u32 map_flags)
- {
- struct bpf_create_map_attr map_attr = {};
- map_attr.map_type = map_type;
- map_attr.map_flags = map_flags;
- map_attr.key_size = key_size;
- map_attr.value_size = value_size;
- map_attr.max_entries = max_entries;
- return bpf_create_map_xattr(&map_attr);
- }
- int bpf_create_map_name(enum bpf_map_type map_type, const char *name,
- int key_size, int value_size, int max_entries,
- __u32 map_flags)
- {
- struct bpf_create_map_attr map_attr = {};
- map_attr.name = name;
- map_attr.map_type = map_type;
- map_attr.map_flags = map_flags;
- map_attr.key_size = key_size;
- map_attr.value_size = value_size;
- map_attr.max_entries = max_entries;
- return bpf_create_map_xattr(&map_attr);
- }
- int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name,
- int key_size, int inner_map_fd, int max_entries,
- __u32 map_flags, int node)
- {
- __u32 name_len = name ? strlen(name) : 0;
- union bpf_attr attr;
- memset(&attr, '\0', sizeof(attr));
- attr.map_type = map_type;
- attr.key_size = key_size;
- attr.value_size = 4;
- attr.inner_map_fd = inner_map_fd;
- attr.max_entries = max_entries;
- attr.map_flags = map_flags;
- memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1));
- if (node >= 0) {
- attr.map_flags |= BPF_F_NUMA_NODE;
- attr.numa_node = node;
- }
- return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
- }
- int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
- int key_size, int inner_map_fd, int max_entries,
- __u32 map_flags)
- {
- return bpf_create_map_in_map_node(map_type, name, key_size,
- inner_map_fd, max_entries, map_flags,
- -1);
- }
- int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
- char *log_buf, size_t log_buf_sz)
- {
- union bpf_attr attr;
- __u32 name_len;
- int fd;
- if (!load_attr)
- return -EINVAL;
- name_len = load_attr->name ? strlen(load_attr->name) : 0;
- bzero(&attr, sizeof(attr));
- attr.prog_type = load_attr->prog_type;
- attr.expected_attach_type = load_attr->expected_attach_type;
- attr.insn_cnt = (__u32)load_attr->insns_cnt;
- attr.insns = ptr_to_u64(load_attr->insns);
- attr.license = ptr_to_u64(load_attr->license);
- attr.log_buf = ptr_to_u64(NULL);
- attr.log_size = 0;
- attr.log_level = 0;
- attr.kern_version = load_attr->kern_version;
- attr.prog_ifindex = load_attr->prog_ifindex;
- memcpy(attr.prog_name, load_attr->name,
- min(name_len, BPF_OBJ_NAME_LEN - 1));
- fd = sys_bpf_prog_load(&attr, sizeof(attr));
- if (fd >= 0 || !log_buf || !log_buf_sz)
- return fd;
- /* Try again with log */
- attr.log_buf = ptr_to_u64(log_buf);
- attr.log_size = log_buf_sz;
- attr.log_level = 1;
- log_buf[0] = 0;
- return sys_bpf_prog_load(&attr, sizeof(attr));
- }
- int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
- size_t insns_cnt, const char *license,
- __u32 kern_version, char *log_buf,
- size_t log_buf_sz)
- {
- struct bpf_load_program_attr load_attr;
- memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
- load_attr.prog_type = type;
- load_attr.expected_attach_type = 0;
- load_attr.name = NULL;
- load_attr.insns = insns;
- load_attr.insns_cnt = insns_cnt;
- load_attr.license = license;
- load_attr.kern_version = kern_version;
- return bpf_load_program_xattr(&load_attr, log_buf, log_buf_sz);
- }
- int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
- size_t insns_cnt, int strict_alignment,
- const char *license, __u32 kern_version,
- char *log_buf, size_t log_buf_sz, int log_level)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.prog_type = type;
- attr.insn_cnt = (__u32)insns_cnt;
- attr.insns = ptr_to_u64(insns);
- attr.license = ptr_to_u64(license);
- attr.log_buf = ptr_to_u64(log_buf);
- attr.log_size = log_buf_sz;
- attr.log_level = log_level;
- log_buf[0] = 0;
- attr.kern_version = kern_version;
- attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0;
- return sys_bpf_prog_load(&attr, sizeof(attr));
- }
- int bpf_map_update_elem(int fd, const void *key, const void *value,
- __u64 flags)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.map_fd = fd;
- attr.key = ptr_to_u64(key);
- attr.value = ptr_to_u64(value);
- attr.flags = flags;
- return sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
- }
- int bpf_map_lookup_elem(int fd, const void *key, void *value)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.map_fd = fd;
- attr.key = ptr_to_u64(key);
- attr.value = ptr_to_u64(value);
- return sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
- }
- int bpf_map_delete_elem(int fd, const void *key)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.map_fd = fd;
- attr.key = ptr_to_u64(key);
- return sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
- }
- int bpf_map_get_next_key(int fd, const void *key, void *next_key)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.map_fd = fd;
- attr.key = ptr_to_u64(key);
- attr.next_key = ptr_to_u64(next_key);
- return sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
- }
- int bpf_obj_pin(int fd, const char *pathname)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.pathname = ptr_to_u64((void *)pathname);
- attr.bpf_fd = fd;
- return sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr));
- }
- int bpf_obj_get(const char *pathname)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.pathname = ptr_to_u64((void *)pathname);
- return sys_bpf(BPF_OBJ_GET, &attr, sizeof(attr));
- }
- int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
- unsigned int flags)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.target_fd = target_fd;
- attr.attach_bpf_fd = prog_fd;
- attr.attach_type = type;
- attr.attach_flags = flags;
- return sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
- }
- int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.target_fd = target_fd;
- attr.attach_type = type;
- return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
- }
- int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.target_fd = target_fd;
- attr.attach_bpf_fd = prog_fd;
- attr.attach_type = type;
- return sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
- }
- int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
- __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
- {
- union bpf_attr attr;
- int ret;
- bzero(&attr, sizeof(attr));
- attr.query.target_fd = target_fd;
- attr.query.attach_type = type;
- attr.query.query_flags = query_flags;
- attr.query.prog_cnt = *prog_cnt;
- attr.query.prog_ids = ptr_to_u64(prog_ids);
- ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
- if (attach_flags)
- *attach_flags = attr.query.attach_flags;
- *prog_cnt = attr.query.prog_cnt;
- return ret;
- }
- int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
- void *data_out, __u32 *size_out, __u32 *retval,
- __u32 *duration)
- {
- union bpf_attr attr;
- int ret;
- bzero(&attr, sizeof(attr));
- attr.test.prog_fd = prog_fd;
- attr.test.data_in = ptr_to_u64(data);
- attr.test.data_out = ptr_to_u64(data_out);
- attr.test.data_size_in = size;
- attr.test.repeat = repeat;
- ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
- if (size_out)
- *size_out = attr.test.data_size_out;
- if (retval)
- *retval = attr.test.retval;
- if (duration)
- *duration = attr.test.duration;
- return ret;
- }
- int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id)
- {
- union bpf_attr attr;
- int err;
- bzero(&attr, sizeof(attr));
- attr.start_id = start_id;
- err = sys_bpf(BPF_PROG_GET_NEXT_ID, &attr, sizeof(attr));
- if (!err)
- *next_id = attr.next_id;
- return err;
- }
- int bpf_map_get_next_id(__u32 start_id, __u32 *next_id)
- {
- union bpf_attr attr;
- int err;
- bzero(&attr, sizeof(attr));
- attr.start_id = start_id;
- err = sys_bpf(BPF_MAP_GET_NEXT_ID, &attr, sizeof(attr));
- if (!err)
- *next_id = attr.next_id;
- return err;
- }
- int bpf_prog_get_fd_by_id(__u32 id)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.prog_id = id;
- return sys_bpf(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr));
- }
- int bpf_map_get_fd_by_id(__u32 id)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.map_id = id;
- return sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
- }
- int bpf_btf_get_fd_by_id(__u32 id)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.btf_id = id;
- return sys_bpf(BPF_BTF_GET_FD_BY_ID, &attr, sizeof(attr));
- }
- int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len)
- {
- union bpf_attr attr;
- int err;
- bzero(&attr, sizeof(attr));
- attr.info.bpf_fd = prog_fd;
- attr.info.info_len = *info_len;
- attr.info.info = ptr_to_u64(info);
- err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
- if (!err)
- *info_len = attr.info.info_len;
- return err;
- }
- int bpf_raw_tracepoint_open(const char *name, int prog_fd)
- {
- union bpf_attr attr;
- bzero(&attr, sizeof(attr));
- attr.raw_tracepoint.name = ptr_to_u64(name);
- attr.raw_tracepoint.prog_fd = prog_fd;
- return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr));
- }
- int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags)
- {
- struct sockaddr_nl sa;
- int sock, seq = 0, len, ret = -1;
- char buf[4096];
- struct nlattr *nla, *nla_xdp;
- struct {
- struct nlmsghdr nh;
- struct ifinfomsg ifinfo;
- char attrbuf[64];
- } req;
- struct nlmsghdr *nh;
- struct nlmsgerr *err;
- socklen_t addrlen;
- int one = 1;
- memset(&sa, 0, sizeof(sa));
- sa.nl_family = AF_NETLINK;
- sock = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
- if (sock < 0) {
- return -errno;
- }
- if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK,
- &one, sizeof(one)) < 0) {
- fprintf(stderr, "Netlink error reporting not supported\n");
- }
- if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
- ret = -errno;
- goto cleanup;
- }
- addrlen = sizeof(sa);
- if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) {
- ret = -errno;
- goto cleanup;
- }
- if (addrlen != sizeof(sa)) {
- ret = -LIBBPF_ERRNO__INTERNAL;
- goto cleanup;
- }
- memset(&req, 0, sizeof(req));
- req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
- req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
- req.nh.nlmsg_type = RTM_SETLINK;
- req.nh.nlmsg_pid = 0;
- req.nh.nlmsg_seq = ++seq;
- req.ifinfo.ifi_family = AF_UNSPEC;
- req.ifinfo.ifi_index = ifindex;
- /* started nested attribute for XDP */
- nla = (struct nlattr *)(((char *)&req)
- + NLMSG_ALIGN(req.nh.nlmsg_len));
- nla->nla_type = NLA_F_NESTED | IFLA_XDP;
- nla->nla_len = NLA_HDRLEN;
- /* add XDP fd */
- nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len);
- nla_xdp->nla_type = IFLA_XDP_FD;
- nla_xdp->nla_len = NLA_HDRLEN + sizeof(int);
- memcpy((char *)nla_xdp + NLA_HDRLEN, &fd, sizeof(fd));
- nla->nla_len += nla_xdp->nla_len;
- /* if user passed in any flags, add those too */
- if (flags) {
- nla_xdp = (struct nlattr *)((char *)nla + nla->nla_len);
- nla_xdp->nla_type = IFLA_XDP_FLAGS;
- nla_xdp->nla_len = NLA_HDRLEN + sizeof(flags);
- memcpy((char *)nla_xdp + NLA_HDRLEN, &flags, sizeof(flags));
- nla->nla_len += nla_xdp->nla_len;
- }
- req.nh.nlmsg_len += NLA_ALIGN(nla->nla_len);
- if (send(sock, &req, req.nh.nlmsg_len, 0) < 0) {
- ret = -errno;
- goto cleanup;
- }
- len = recv(sock, buf, sizeof(buf), 0);
- if (len < 0) {
- ret = -errno;
- goto cleanup;
- }
- for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len);
- nh = NLMSG_NEXT(nh, len)) {
- if (nh->nlmsg_pid != sa.nl_pid) {
- ret = -LIBBPF_ERRNO__WRNGPID;
- goto cleanup;
- }
- if (nh->nlmsg_seq != seq) {
- ret = -LIBBPF_ERRNO__INVSEQ;
- goto cleanup;
- }
- switch (nh->nlmsg_type) {
- case NLMSG_ERROR:
- err = (struct nlmsgerr *)NLMSG_DATA(nh);
- if (!err->error)
- continue;
- ret = err->error;
- nla_dump_errormsg(nh);
- goto cleanup;
- case NLMSG_DONE:
- break;
- default:
- break;
- }
- }
- ret = 0;
- cleanup:
- close(sock);
- return ret;
- }
- int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size,
- bool do_log)
- {
- union bpf_attr attr = {};
- int fd;
- attr.btf = ptr_to_u64(btf);
- attr.btf_size = btf_size;
- retry:
- if (do_log && log_buf && log_buf_size) {
- attr.btf_log_level = 1;
- attr.btf_log_size = log_buf_size;
- attr.btf_log_buf = ptr_to_u64(log_buf);
- }
- fd = sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr));
- if (fd == -1 && !do_log && log_buf && log_buf_size) {
- do_log = true;
- goto retry;
- }
- return fd;
- }
- int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
- __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset,
- __u64 *probe_addr)
- {
- union bpf_attr attr = {};
- int err;
- attr.task_fd_query.pid = pid;
- attr.task_fd_query.fd = fd;
- attr.task_fd_query.flags = flags;
- attr.task_fd_query.buf = ptr_to_u64(buf);
- attr.task_fd_query.buf_len = *buf_len;
- err = sys_bpf(BPF_TASK_FD_QUERY, &attr, sizeof(attr));
- *buf_len = attr.task_fd_query.buf_len;
- *prog_id = attr.task_fd_query.prog_id;
- *fd_type = attr.task_fd_query.fd_type;
- *probe_offset = attr.task_fd_query.probe_offset;
- *probe_addr = attr.task_fd_query.probe_addr;
- return err;
- }
|