summaryrefslogtreecommitdiff
path: root/tools/lib/bpf/libbpf.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib/bpf/libbpf.c')
-rw-r--r--tools/lib/bpf/libbpf.c328
1 files changed, 292 insertions, 36 deletions
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 214f828ece6b..17883f5a44b9 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -118,6 +118,8 @@ static const char * const attach_type_name[] = {
[BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
[BPF_STRUCT_OPS] = "struct_ops",
[BPF_NETFILTER] = "netfilter",
+ [BPF_TCX_INGRESS] = "tcx_ingress",
+ [BPF_TCX_EGRESS] = "tcx_egress",
};
static const char * const link_type_name[] = {
@@ -132,6 +134,7 @@ static const char * const link_type_name[] = {
[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
[BPF_LINK_TYPE_NETFILTER] = "netfilter",
+ [BPF_LINK_TYPE_TCX] = "tcx",
};
static const char * const map_type_name[] = {
@@ -5471,6 +5474,10 @@ static int load_module_btfs(struct bpf_object *obj)
err = bpf_btf_get_next_id(id, &id);
if (err && errno == ENOENT)
return 0;
+ if (err && errno == EPERM) {
+ pr_debug("skipping module BTFs loading, missing privileges\n");
+ return 0;
+ }
if (err) {
err = -errno;
pr_warn("failed to iterate BTF objects: %d\n", err);
@@ -6157,7 +6164,11 @@ static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_progra
if (main_prog == subprog)
return 0;
relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
- if (!relos)
+ /* if new count is zero, reallocarray can return a valid NULL result;
+ * in this case the previous pointer will be freed, so we *have to*
+ * reassign old pointer to the new value (even if it's NULL)
+ */
+ if (!relos && new_cnt)
return -ENOMEM;
if (subprog->nr_reloc)
memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
@@ -8528,7 +8539,8 @@ int bpf_program__set_insns(struct bpf_program *prog,
return -EBUSY;
insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
- if (!insns) {
+ /* NULL is a valid return from reallocarray if the new count is zero */
+ if (!insns && new_insn_cnt) {
pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
return -ENOMEM;
}
@@ -8558,13 +8570,31 @@ enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
return prog->type;
}
+static size_t custom_sec_def_cnt;
+static struct bpf_sec_def *custom_sec_defs;
+static struct bpf_sec_def custom_fallback_def;
+static bool has_custom_fallback_def;
+static int last_custom_sec_def_handler_id;
+
int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
{
if (prog->obj->loaded)
return libbpf_err(-EBUSY);
+ /* if type is not changed, do nothing */
+ if (prog->type == type)
+ return 0;
+
prog->type = type;
- prog->sec_def = NULL;
+
+ /* If a program type was changed, we need to reset associated SEC()
+ * handler, as it will be invalid now. The only exception is a generic
+ * fallback handler, which by definition is program type-agnostic and
+ * is a catch-all custom handler, optionally set by the application,
+ * so should be able to handle any type of BPF program.
+ */
+ if (prog->sec_def != &custom_fallback_def)
+ prog->sec_def = NULL;
return 0;
}
@@ -8669,9 +8699,13 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
- SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
- SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
- SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
+ SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */
+ SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */
+ SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE),
+ SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE),
+ SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
+ SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */
+ SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */
SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
@@ -8740,13 +8774,6 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE),
};
-static size_t custom_sec_def_cnt;
-static struct bpf_sec_def *custom_sec_defs;
-static struct bpf_sec_def custom_fallback_def;
-static bool has_custom_fallback_def;
-
-static int last_custom_sec_def_handler_id;
-
int libbpf_register_prog_handler(const char *sec,
enum bpf_prog_type prog_type,
enum bpf_attach_type exp_attach_type,
@@ -8826,7 +8853,11 @@ int libbpf_unregister_prog_handler(int handler_id)
/* try to shrink the array, but it's ok if we couldn't */
sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs));
- if (sec_defs)
+ /* if new count is zero, reallocarray can return a valid NULL result;
+ * in this case the previous pointer will be freed, so we *have to*
+ * reassign old pointer to the new value (even if it's NULL)
+ */
+ if (sec_defs || custom_sec_def_cnt == 0)
custom_sec_defs = sec_defs;
return 0;
@@ -10224,6 +10255,18 @@ static const char *tracefs_uprobe_events(void)
return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
}
+static const char *tracefs_available_filter_functions(void)
+{
+ return use_debugfs() ? DEBUGFS"/available_filter_functions"
+ : TRACEFS"/available_filter_functions";
+}
+
+static const char *tracefs_available_filter_functions_addrs(void)
+{
+ return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs"
+ : TRACEFS"/available_filter_functions_addrs";
+}
+
static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
const char *kfunc_name, size_t offset)
{
@@ -10539,25 +10582,158 @@ struct kprobe_multi_resolve {
size_t cnt;
};
-static int
-resolve_kprobe_multi_cb(unsigned long long sym_addr, char sym_type,
- const char *sym_name, void *ctx)
+struct avail_kallsyms_data {
+ char **syms;
+ size_t cnt;
+ struct kprobe_multi_resolve *res;
+};
+
+static int avail_func_cmp(const void *a, const void *b)
{
- struct kprobe_multi_resolve *res = ctx;
+ return strcmp(*(const char **)a, *(const char **)b);
+}
+
+static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type,
+ const char *sym_name, void *ctx)
+{
+ struct avail_kallsyms_data *data = ctx;
+ struct kprobe_multi_resolve *res = data->res;
int err;
- if (!glob_match(sym_name, res->pattern))
+ if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))
return 0;
- err = libbpf_ensure_mem((void **) &res->addrs, &res->cap, sizeof(unsigned long),
- res->cnt + 1);
+ err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);
if (err)
return err;
- res->addrs[res->cnt++] = (unsigned long) sym_addr;
+ res->addrs[res->cnt++] = (unsigned long)sym_addr;
return 0;
}
+static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res)
+{
+ const char *available_functions_file = tracefs_available_filter_functions();
+ struct avail_kallsyms_data data;
+ char sym_name[500];
+ FILE *f;
+ int err = 0, ret, i;
+ char **syms = NULL;
+ size_t cap = 0, cnt = 0;
+
+ f = fopen(available_functions_file, "re");
+ if (!f) {
+ err = -errno;
+ pr_warn("failed to open %s: %d\n", available_functions_file, err);
+ return err;
+ }
+
+ while (true) {
+ char *name;
+
+ ret = fscanf(f, "%499s%*[^\n]\n", sym_name);
+ if (ret == EOF && feof(f))
+ break;
+
+ if (ret != 1) {
+ pr_warn("failed to parse available_filter_functions entry: %d\n", ret);
+ err = -EINVAL;
+ goto cleanup;
+ }
+
+ if (!glob_match(sym_name, res->pattern))
+ continue;
+
+ err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1);
+ if (err)
+ goto cleanup;
+
+ name = strdup(sym_name);
+ if (!name) {
+ err = -errno;
+ goto cleanup;
+ }
+
+ syms[cnt++] = name;
+ }
+
+ /* no entries found, bail out */
+ if (cnt == 0) {
+ err = -ENOENT;
+ goto cleanup;
+ }
+
+ /* sort available functions */
+ qsort(syms, cnt, sizeof(*syms), avail_func_cmp);
+
+ data.syms = syms;
+ data.res = res;
+ data.cnt = cnt;
+ libbpf_kallsyms_parse(avail_kallsyms_cb, &data);
+
+ if (res->cnt == 0)
+ err = -ENOENT;
+
+cleanup:
+ for (i = 0; i < cnt; i++)
+ free((char *)syms[i]);
+ free(syms);
+
+ fclose(f);
+ return err;
+}
+
+static bool has_available_filter_functions_addrs(void)
+{
+ return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;
+}
+
+static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res)
+{
+ const char *available_path = tracefs_available_filter_functions_addrs();
+ char sym_name[500];
+ FILE *f;
+ int ret, err = 0;
+ unsigned long long sym_addr;
+
+ f = fopen(available_path, "re");
+ if (!f) {
+ err = -errno;
+ pr_warn("failed to open %s: %d\n", available_path, err);
+ return err;
+ }
+
+ while (true) {
+ ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name);
+ if (ret == EOF && feof(f))
+ break;
+
+ if (ret != 2) {
+ pr_warn("failed to parse available_filter_functions_addrs entry: %d\n",
+ ret);
+ err = -EINVAL;
+ goto cleanup;
+ }
+
+ if (!glob_match(sym_name, res->pattern))
+ continue;
+
+ err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,
+ sizeof(*res->addrs), res->cnt + 1);
+ if (err)
+ goto cleanup;
+
+ res->addrs[res->cnt++] = (unsigned long)sym_addr;
+ }
+
+ if (res->cnt == 0)
+ err = -ENOENT;
+
+cleanup:
+ fclose(f);
+ return err;
+}
+
struct bpf_link *
bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
const char *pattern,
@@ -10594,13 +10770,12 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
return libbpf_err_ptr(-EINVAL);
if (pattern) {
- err = libbpf_kallsyms_parse(resolve_kprobe_multi_cb, &res);
+ if (has_available_filter_functions_addrs())
+ err = libbpf_available_kprobes_parse(&res);
+ else
+ err = libbpf_available_kallsyms_parse(&res);
if (err)
goto error;
- if (!res.cnt) {
- err = -ENOENT;
- goto error;
- }
addrs = res.addrs;
cnt = res.cnt;
}
@@ -11680,11 +11855,10 @@ static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_li
}
static struct bpf_link *
-bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
- const char *target_name)
+bpf_program_attach_fd(const struct bpf_program *prog,
+ int target_fd, const char *target_name,
+ const struct bpf_link_create_opts *opts)
{
- DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
- .target_btf_id = btf_id);
enum bpf_attach_type attach_type;
char errmsg[STRERR_BUFSIZE];
struct bpf_link *link;
@@ -11702,7 +11876,7 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id
link->detach = &bpf_link__detach_fd;
attach_type = bpf_program__expected_attach_type(prog);
- link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
+ link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts);
if (link_fd < 0) {
link_fd = -errno;
free(link);
@@ -11718,19 +11892,54 @@ bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id
struct bpf_link *
bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
{
- return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
+ return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
}
struct bpf_link *
bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
{
- return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
+ return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
}
struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
{
/* target_fd/target_ifindex use the same field in LINK_CREATE */
- return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
+ return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
+}
+
+struct bpf_link *
+bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
+ const struct bpf_tcx_opts *opts)
+{
+ LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
+ __u32 relative_id;
+ int relative_fd;
+
+ if (!OPTS_VALID(opts, bpf_tcx_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ relative_id = OPTS_GET(opts, relative_id, 0);
+ relative_fd = OPTS_GET(opts, relative_fd, 0);
+
+ /* validate we don't have unexpected combinations of non-zero fields */
+ if (!ifindex) {
+ pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+ if (relative_fd && relative_id) {
+ pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
+ prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
+ link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0);
+ link_create_opts.tcx.relative_fd = relative_fd;
+ link_create_opts.tcx.relative_id = relative_id;
+ link_create_opts.flags = OPTS_GET(opts, flags, 0);
+
+ /* target_fd/target_ifindex use the same field in LINK_CREATE */
+ return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
}
struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
@@ -11752,11 +11961,16 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
}
if (target_fd) {
+ LIBBPF_OPTS(bpf_link_create_opts, target_opts);
+
btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
if (btf_id < 0)
return libbpf_err_ptr(btf_id);
- return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
+ target_opts.target_btf_id = btf_id;
+
+ return bpf_program_attach_fd(prog, target_fd, "freplace",
+ &target_opts);
} else {
/* no target, so use raw_tracepoint_open for compatibility
* with old kernels
@@ -11811,6 +12025,48 @@ static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_l
return libbpf_get_error(*link);
}
+struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
+ const struct bpf_netfilter_opts *opts)
+{
+ LIBBPF_OPTS(bpf_link_create_opts, lopts);
+ struct bpf_link *link;
+ int prog_fd, link_fd;
+
+ if (!OPTS_VALID(opts, bpf_netfilter_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ prog_fd = bpf_program__fd(prog);
+ if (prog_fd < 0) {
+ pr_warn("prog '%s': can't attach before loaded\n", prog->name);
+ return libbpf_err_ptr(-EINVAL);
+ }
+
+ link = calloc(1, sizeof(*link));
+ if (!link)
+ return libbpf_err_ptr(-ENOMEM);
+
+ link->detach = &bpf_link__detach_fd;
+
+ lopts.netfilter.pf = OPTS_GET(opts, pf, 0);
+ lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0);
+ lopts.netfilter.priority = OPTS_GET(opts, priority, 0);
+ lopts.netfilter.flags = OPTS_GET(opts, flags, 0);
+
+ link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
+ if (link_fd < 0) {
+ char errmsg[STRERR_BUFSIZE];
+
+ link_fd = -errno;
+ free(link);
+ pr_warn("prog '%s': failed to attach to netfilter: %s\n",
+ prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
+ return libbpf_err_ptr(link_fd);
+ }
+ link->fd = link_fd;
+
+ return link;
+}
+
struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
{
struct bpf_link *link = NULL;
generated by cgit v1.2.3 (git 2.39.0) at 2024-07-23 03:15:52 +0300