summaryrefslogtreecommitdiff
path: root/tools/lib
diff options
context:
space:
mode:
authorJakub Kicinski <kuba@kernel.org>2020-11-14 20:13:40 +0300
committerJakub Kicinski <kuba@kernel.org>2020-11-14 20:13:41 +0300
commit07cbce2e466cabb46b7c2317bd456584aa4ceacc (patch)
tree17c5bd5574a13aa7db481f650b622839597a8d29 /tools/lib
parent774626fa440e4c01bcbe5213cd5220dea545c9f7 (diff)
parentc14d61fca0d10498bf267c0ab1f381dd0b35d96b (diff)
downloadlinux-07cbce2e466cabb46b7c2317bd456584aa4ceacc.tar.xz
Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next
Daniel Borkmann says: ==================== pull-request: bpf-next 2020-11-14 1) Add BTF generation for kernel modules and extend BTF infra in kernel e.g. support for split BTF loading and validation, from Andrii Nakryiko. 2) Support for pointers beyond pkt_end to recognize LLVM generated patterns on inlined branch conditions, from Alexei Starovoitov. 3) Implements bpf_local_storage for task_struct for BPF LSM, from KP Singh. 4) Enable FENTRY/FEXIT/RAW_TP tracing program to use the bpf_sk_storage infra, from Martin KaFai Lau. 5) Add XDP bulk APIs that introduce a defer/flush mechanism to optimize the XDP_REDIRECT path, from Lorenzo Bianconi. 6) Fix a potential (although rather theoretical) deadlock of hashtab in NMI context, from Song Liu. 7) Fixes for cross and out-of-tree build of bpftool and runqslower allowing build for different target archs on same source tree, from Jean-Philippe Brucker. 8) Fix error path in htab_map_alloc() triggered from syzbot, from Eric Dumazet. 9) Move functionality from test_tcpbpf_user into the test_progs framework so it can run in BPF CI, from Alexander Duyck. 10) Lift hashtab key_size limit to be larger than MAX_BPF_STACK, from Florian Lehner. Note that for the fix from Song we have seen a sparse report on context imbalance which requires changes in sparse itself for proper annotation detection where this is currently being discussed on linux-sparse among developers [0]. Once we have more clarification/guidance after their fix, Song will follow-up. [0] https://lore.kernel.org/linux-sparse/CAHk-=wh4bx8A8dHnX612MsDO13st6uzAz1mJ1PaHHVevJx_ZCw@mail.gmail.com/T/ https://lore.kernel.org/linux-sparse/20201109221345.uklbp3lzgq6g42zb@ltop.local/T/ * git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (66 commits) net: mlx5: Add xdp tx return bulking support net: mvpp2: Add xdp tx return bulking support net: mvneta: Add xdp tx return bulking support net: page_pool: Add bulk support for ptr_ring net: xdp: Introduce bulking for xdp tx return path bpf: Expose bpf_d_path helper to sleepable LSM hooks bpf: Augment the set of sleepable LSM hooks bpf: selftest: Use bpf_sk_storage in FENTRY/FEXIT/RAW_TP bpf: Allow using bpf_sk_storage in FENTRY/FEXIT/RAW_TP bpf: Rename some functions in bpf_sk_storage bpf: Folding omem_charge() into sk_storage_charge() selftests/bpf: Add asm tests for pkt vs pkt_end comparison. selftests/bpf: Add skb_pkt_end test bpf: Support for pointers beyond pkt_end. tools/bpf: Always run the *-clean recipes tools/bpf: Add bootstrap/ to .gitignore bpf: Fix NULL dereference in bpf_task_storage tools/bpftool: Fix build slowdown tools/runqslower: Build bpftool using HOSTCC tools/runqslower: Enable out-of-tree build ... ==================== Link: https://lore.kernel.org/r/20201114020819.29584-1-daniel@iogearbox.net Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Diffstat (limited to 'tools/lib')
-rw-r--r--tools/lib/bpf/btf.c807
-rw-r--r--tools/lib/bpf/btf.h8
-rw-r--r--tools/lib/bpf/libbpf.map9
-rw-r--r--tools/lib/bpf/libbpf_probes.c1
4 files changed, 490 insertions, 335 deletions
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 231b07203e3d..2d0d064c6d31 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -78,10 +78,32 @@ struct btf {
void *types_data;
size_t types_data_cap; /* used size stored in hdr->type_len */
- /* type ID to `struct btf_type *` lookup index */
+ /* type ID to `struct btf_type *` lookup index
+ * type_offs[0] corresponds to the first non-VOID type:
+ * - for base BTF it's type [1];
+ * - for split BTF it's the first non-base BTF type.
+ */
__u32 *type_offs;
size_t type_offs_cap;
+ /* number of types in this BTF instance:
+ * - doesn't include special [0] void type;
+ * - for split BTF counts number of types added on top of base BTF.
+ */
__u32 nr_types;
+ /* if not NULL, points to the base BTF on top of which the current
+ * split BTF is based
+ */
+ struct btf *base_btf;
+ /* BTF type ID of the first type in this BTF instance:
+ * - for base BTF it's equal to 1;
+ * - for split BTF it's equal to biggest type ID of base BTF plus 1.
+ */
+ int start_id;
+ /* logical string offset of this BTF instance:
+ * - for base BTF it's equal to 0;
+ * - for split BTF it's equal to total size of base BTF's string section size.
+ */
+ int start_str_off;
void *strs_data;
size_t strs_data_cap; /* used size stored in hdr->str_len */
@@ -90,6 +112,14 @@ struct btf {
struct hashmap *strs_hash;
/* whether strings are already deduplicated */
bool strs_deduped;
+ /* extra indirection layer to make strings hashmap work with stable
+ * string offsets and ability to transparently choose between
+ * btf->strs_data or btf_dedup->strs_data as a source of strings.
+ * This is used for BTF strings dedup to transfer deduplicated strings
+ * data back to struct btf without re-building strings index.
+ */
+ void **strs_data_ptr;
+
/* BTF object FD, if loaded into kernel */
int fd;
@@ -168,7 +198,7 @@ static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
__u32 *p;
p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
- btf->nr_types + 1, BTF_MAX_NR_TYPES, 1);
+ btf->nr_types, BTF_MAX_NR_TYPES, 1);
if (!p)
return -ENOMEM;
@@ -215,22 +245,18 @@ static int btf_parse_hdr(struct btf *btf)
return -EINVAL;
}
- if (meta_left < hdr->type_off) {
- pr_debug("Invalid BTF type section offset:%u\n", hdr->type_off);
- return -EINVAL;
- }
-
- if (meta_left < hdr->str_off) {
- pr_debug("Invalid BTF string section offset:%u\n", hdr->str_off);
+ if (meta_left < hdr->str_off + hdr->str_len) {
+ pr_debug("Invalid BTF total size:%u\n", btf->raw_size);
return -EINVAL;
}
- if (hdr->type_off >= hdr->str_off) {
- pr_debug("BTF type section offset >= string section offset. No type?\n");
+ if (hdr->type_off + hdr->type_len > hdr->str_off) {
+ pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
+ hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
return -EINVAL;
}
- if (hdr->type_off & 0x02) {
+ if (hdr->type_off % 4) {
pr_debug("BTF type section is not aligned to 4 bytes\n");
return -EINVAL;
}
@@ -244,12 +270,16 @@ static int btf_parse_str_sec(struct btf *btf)
const char *start = btf->strs_data;
const char *end = start + btf->hdr->str_len;
- if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET ||
- start[0] || end[-1]) {
+ if (btf->base_btf && hdr->str_len == 0)
+ return 0;
+ if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
+ pr_debug("Invalid BTF string section\n");
+ return -EINVAL;
+ }
+ if (!btf->base_btf && start[0]) {
pr_debug("Invalid BTF string section\n");
return -EINVAL;
}
-
return 0;
}
@@ -364,19 +394,9 @@ static int btf_parse_type_sec(struct btf *btf)
struct btf_header *hdr = btf->hdr;
void *next_type = btf->types_data;
void *end_type = next_type + hdr->type_len;
- int err, i = 0, type_size;
-
- /* VOID (type_id == 0) is specially handled by btf__get_type_by_id(),
- * so ensure we can never properly use its offset from index by
- * setting it to a large value
- */
- err = btf_add_type_idx_entry(btf, UINT_MAX);
- if (err)
- return err;
+ int err, type_size;
while (next_type + sizeof(struct btf_type) <= end_type) {
- i++;
-
if (btf->swapped_endian)
btf_bswap_type_base(next_type);
@@ -384,7 +404,7 @@ static int btf_parse_type_sec(struct btf *btf)
if (type_size < 0)
return type_size;
if (next_type + type_size > end_type) {
- pr_warn("BTF type [%d] is malformed\n", i);
+ pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
return -EINVAL;
}
@@ -409,7 +429,7 @@ static int btf_parse_type_sec(struct btf *btf)
__u32 btf__get_nr_types(const struct btf *btf)
{
- return btf->nr_types;
+ return btf->start_id + btf->nr_types - 1;
}
/* internal helper returning non-const pointer to a type */
@@ -417,13 +437,14 @@ static struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id)
{
if (type_id == 0)
return &btf_void;
-
- return btf->types_data + btf->type_offs[type_id];
+ if (type_id < btf->start_id)
+ return btf_type_by_id(btf->base_btf, type_id);
+ return btf->types_data + btf->type_offs[type_id - btf->start_id];
}
const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
{
- if (type_id > btf->nr_types)
+ if (type_id >= btf->start_id + btf->nr_types)
return NULL;
return btf_type_by_id((struct btf *)btf, type_id);
}
@@ -432,9 +453,13 @@ static int determine_ptr_size(const struct btf *btf)
{
const struct btf_type *t;
const char *name;
- int i;
+ int i, n;
- for (i = 1; i <= btf->nr_types; i++) {
+ if (btf->base_btf && btf->base_btf->ptr_sz > 0)
+ return btf->base_btf->ptr_sz;
+
+ n = btf__get_nr_types(btf);
+ for (i = 1; i <= n; i++) {
t = btf__type_by_id(btf, i);
if (!btf_is_int(t))
continue;
@@ -717,7 +742,7 @@ void btf__free(struct btf *btf)
free(btf);
}
-struct btf *btf__new_empty(void)
+static struct btf *btf_new_empty(struct btf *base_btf)
{
struct btf *btf;
@@ -725,12 +750,21 @@ struct btf *btf__new_empty(void)
if (!btf)
return ERR_PTR(-ENOMEM);
+ btf->nr_types = 0;
+ btf->start_id = 1;
+ btf->start_str_off = 0;
btf->fd = -1;
btf->ptr_sz = sizeof(void *);
btf->swapped_endian = false;
+ if (base_btf) {
+ btf->base_btf = base_btf;
+ btf->start_id = btf__get_nr_types(base_btf) + 1;
+ btf->start_str_off = base_btf->hdr->str_len;
+ }
+
/* +1 for empty string at offset 0 */
- btf->raw_size = sizeof(struct btf_header) + 1;
+ btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
btf->raw_data = calloc(1, btf->raw_size);
if (!btf->raw_data) {
free(btf);
@@ -744,12 +778,22 @@ struct btf *btf__new_empty(void)
btf->types_data = btf->raw_data + btf->hdr->hdr_len;
btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
- btf->hdr->str_len = 1; /* empty string at offset 0 */
+ btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
return btf;
}
-struct btf *btf__new(const void *data, __u32 size)
+struct btf *btf__new_empty(void)
+{
+ return btf_new_empty(NULL);
+}
+
+struct btf *btf__new_empty_split(struct btf *base_btf)
+{
+ return btf_new_empty(base_btf);
+}
+
+static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
{
struct btf *btf;
int err;
@@ -758,6 +802,16 @@ struct btf *btf__new(const void *data, __u32 size)
if (!btf)
return ERR_PTR(-ENOMEM);
+ btf->nr_types = 0;
+ btf->start_id = 1;
+ btf->start_str_off = 0;
+
+ if (base_btf) {
+ btf->base_btf = base_btf;
+ btf->start_id = btf__get_nr_types(base_btf) + 1;
+ btf->start_str_off = base_btf->hdr->str_len;
+ }
+
btf->raw_data = malloc(size);
if (!btf->raw_data) {
err = -ENOMEM;
@@ -790,7 +844,13 @@ done:
return btf;
}
-struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
+struct btf *btf__new(const void *data, __u32 size)
+{
+ return btf_new(data, size, NULL);
+}
+
+static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
+ struct btf_ext **btf_ext)
{
Elf_Data *btf_data = NULL, *btf_ext_data = NULL;
int err = 0, fd = -1, idx = 0;
@@ -868,7 +928,7 @@ struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
err = -ENOENT;
goto done;
}
- btf = btf__new(btf_data->d_buf, btf_data->d_size);
+ btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf);
if (IS_ERR(btf))
goto done;
@@ -913,7 +973,17 @@ done:
return btf;
}
-struct btf *btf__parse_raw(const char *path)
+struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
+{
+ return btf_parse_elf(path, NULL, btf_ext);
+}
+
+struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse_elf(path, base_btf, NULL);
+}
+
+static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
{
struct btf *btf = NULL;
void *data = NULL;
@@ -967,7 +1037,7 @@ struct btf *btf__parse_raw(const char *path)
}
/* finally parse BTF data */
- btf = btf__new(data, sz);
+ btf = btf_new(data, sz, base_btf);
err_out:
free(data);
@@ -976,18 +1046,38 @@ err_out:
return err ? ERR_PTR(err) : btf;
}
-struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
+struct btf *btf__parse_raw(const char *path)
+{
+ return btf_parse_raw(path, NULL);
+}
+
+struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse_raw(path, base_btf);
+}
+
+static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
{
struct btf *btf;
if (btf_ext)
*btf_ext = NULL;
- btf = btf__parse_raw(path);
+ btf = btf_parse_raw(path, base_btf);
if (!IS_ERR(btf) || PTR_ERR(btf) != -EPROTO)
return btf;
- return btf__parse_elf(path, btf_ext);
+ return btf_parse_elf(path, base_btf, btf_ext);
+}
+
+struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
+{
+ return btf_parse(path, NULL, btf_ext);
+}
+
+struct btf *btf__parse_split(const char *path, struct btf *base_btf)
+{
+ return btf_parse(path, base_btf, NULL);
}
static int compare_vsi_off(const void *_a, const void *_b)
@@ -1171,8 +1261,8 @@ static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endi
memcpy(p, btf->types_data, hdr->type_len);
if (swap_endian) {
- for (i = 1; i <= btf->nr_types; i++) {
- t = p + btf->type_offs[i];
+ for (i = 0; i < btf->nr_types; i++) {
+ t = p + btf->type_offs[i];
/* btf_bswap_type_rest() relies on native t->info, so
* we swap base type info after we swapped all the
* additional information
@@ -1215,8 +1305,10 @@ const void *btf__get_raw_data(const struct btf *btf_ro, __u32 *size)
const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
{
- if (offset < btf->hdr->str_len)
- return btf->strs_data + offset;
+ if (offset < btf->start_str_off)
+ return btf__str_by_offset(btf->base_btf, offset);
+ else if (offset - btf->start_str_off < btf->hdr->str_len)
+ return btf->strs_data + (offset - btf->start_str_off);
else
return NULL;
}
@@ -1363,17 +1455,19 @@ int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
static size_t strs_hash_fn(const void *key, void *ctx)
{
- struct btf *btf = ctx;
- const char *str = btf->strs_data + (long)key;
+ const struct btf *btf = ctx;
+ const char *strs = *btf->strs_data_ptr;
+ const char *str = strs + (long)key;
return str_hash(str);
}
static bool strs_hash_equal_fn(const void *key1, const void *key2, void *ctx)
{
- struct btf *btf = ctx;
- const char *str1 = btf->strs_data + (long)key1;
- const char *str2 = btf->strs_data + (long)key2;
+ const struct btf *btf = ctx;
+ const char *strs = *btf->strs_data_ptr;
+ const char *str1 = strs + (long)key1;
+ const char *str2 = strs + (long)key2;
return strcmp(str1, str2) == 0;
}
@@ -1418,6 +1512,9 @@ static int btf_ensure_modifiable(struct btf *btf)
memcpy(types, btf->types_data, btf->hdr->type_len);
memcpy(strs, btf->strs_data, btf->hdr->str_len);
+ /* make hashmap below use btf->strs_data as a source of strings */
+ btf->strs_data_ptr = &btf->strs_data;
+
/* build lookup index for all strings */
hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, btf);
if (IS_ERR(hash)) {
@@ -1448,7 +1545,10 @@ static int btf_ensure_modifiable(struct btf *btf)
/* if BTF was created from scratch, all strings are guaranteed to be
* unique and deduplicated
*/
- btf->strs_deduped = btf->hdr->str_len <= 1;
+ if (btf->hdr->str_len == 0)
+ btf->strs_deduped = true;
+ if (!btf->base_btf && btf->hdr->str_len == 1)
+ btf->strs_deduped = true;
/* invalidate raw_data representation */
btf_invalidate_raw_data(btf);
@@ -1480,6 +1580,14 @@ int btf__find_str(struct btf *btf, const char *s)
long old_off, new_off, len;
void *p;
+ if (btf->base_btf) {
+ int ret;
+
+ ret = btf__find_str(btf->base_btf, s);
+ if (ret != -ENOENT)
+ return ret;
+ }
+
/* BTF needs to be in a modifiable state to build string lookup index */
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1494,7 +1602,7 @@ int btf__find_str(struct btf *btf, const char *s)
memcpy(p, s, len);
if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off))
- return old_off;
+ return btf->start_str_off + old_off;
return -ENOENT;
}
@@ -1510,6 +1618,14 @@ int btf__add_str(struct btf *btf, const char *s)
void *p;
int err;
+ if (btf->base_btf) {
+ int ret;
+
+ ret = btf__find_str(btf->base_btf, s);
+ if (ret != -ENOENT)
+ return ret;
+ }
+
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1536,12 +1652,12 @@ int btf__add_str(struct btf *btf, const char *s)
err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off,
HASHMAP_ADD, (const void **)&old_off, NULL);
if (err == -EEXIST)
- return old_off; /* duplicated string, return existing offset */
+ return btf->start_str_off + old_off; /* duplicated string, return existing offset */
if (err)
return err;
btf->hdr->str_len += len; /* new unique string, adjust data length */
- return new_off;
+ return btf->start_str_off + new_off;
}
static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
@@ -1560,6 +1676,20 @@ static void btf_type_inc_vlen(struct btf_type *t)
t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
}
+static int btf_commit_type(struct btf *btf, int data_sz)
+{
+ int err;
+
+ err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
+ if (err)
+ return err;
+
+ btf->hdr->type_len += data_sz;
+ btf->hdr->str_off += data_sz;
+ btf->nr_types++;
+ return btf->start_id + btf->nr_types - 1;
+}
+
/*
* Append new BTF_KIND_INT type with:
* - *name* - non-empty, non-NULL type name;
@@ -1572,7 +1702,7 @@ static void btf_type_inc_vlen(struct btf_type *t)
int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
{
struct btf_type *t;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -1606,14 +1736,7 @@ int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding
/* set INT info, we don't allow setting legacy bit offset/size */
*(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/* it's completely legal to append BTF types with type IDs pointing forward to
@@ -1631,7 +1754,7 @@ static int validate_type_id(int id)
static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
{
struct btf_type *t;
- int sz, name_off = 0, err;
+ int sz, name_off = 0;
if (validate_type_id(ref_type_id))
return -EINVAL;
@@ -1654,14 +1777,7 @@ static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref
t->info = btf_type_info(kind, 0, 0);
t->type = ref_type_id;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1689,7 +1805,7 @@ int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 n
{
struct btf_type *t;
struct btf_array *a;
- int sz, err;
+ int sz;
if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
return -EINVAL;
@@ -1711,21 +1827,14 @@ int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 n
a->index_type = index_type_id;
a->nelems = nr_elems;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/* generic STRUCT/UNION append function */
static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
{
struct btf_type *t;
- int sz, err, name_off = 0;
+ int sz, name_off = 0;
if (btf_ensure_modifiable(btf))
return -ENOMEM;
@@ -1748,14 +1857,7 @@ static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32
t->info = btf_type_info(kind, 0, 0);
t->size = bytes_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1793,6 +1895,11 @@ int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
}
+static struct btf_type *btf_last_type(struct btf *btf)
+{
+ return btf_type_by_id(btf, btf__get_nr_types(btf));
+}
+
/*
* Append new field for the current STRUCT/UNION type with:
* - *name* - name of the field, can be NULL or empty for anonymous field;
@@ -1814,7 +1921,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
/* last type should be union/struct */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_composite(t))
return -EINVAL;
@@ -1849,7 +1956,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
m->offset = bit_offset | (bit_size << 24);
/* btf_add_type_mem can invalidate t pointer */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
/* update parent type's vlen and kflag */
t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
@@ -1874,7 +1981,7 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
{
struct btf_type *t;
- int sz, err, name_off = 0;
+ int sz, name_off = 0;
/* byte_sz must be power of 2 */
if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
@@ -1899,14 +2006,7 @@ int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
t->size = byte_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -1926,7 +2026,7 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
/* last type should be BTF_KIND_ENUM */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_enum(t))
return -EINVAL;
@@ -1953,7 +2053,7 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
v->val = value;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2093,7 +2193,7 @@ int btf__add_func(struct btf *btf, const char *name,
int btf__add_func_proto(struct btf *btf, int ret_type_id)
{
struct btf_type *t;
- int sz, err;
+ int sz;
if (validate_type_id(ret_type_id))
return -EINVAL;
@@ -2113,14 +2213,7 @@ int btf__add_func_proto(struct btf *btf, int ret_type_id)
t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
t->type = ret_type_id;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2143,7 +2236,7 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id)
/* last type should be BTF_KIND_FUNC_PROTO */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_func_proto(t))
return -EINVAL;
@@ -2166,7 +2259,7 @@ int btf__add_func_param(struct btf *btf, const char *name, int type_id)
p->type = type_id;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2188,7 +2281,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
{
struct btf_type *t;
struct btf_var *v;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -2219,14 +2312,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
v = btf_var(t);
v->linkage = linkage;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2244,7 +2330,7 @@ int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
{
struct btf_type *t;
- int sz, err, name_off;
+ int sz, name_off;
/* non-empty name */
if (!name || !name[0])
@@ -2267,14 +2353,7 @@ int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
t->size = byte_sz;
- err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
- if (err)
- return err;
-
- btf->hdr->type_len += sz;
- btf->hdr->str_off += sz;
- btf->nr_types++;
- return btf->nr_types;
+ return btf_commit_type(btf, sz);
}
/*
@@ -2296,7 +2375,7 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __
/* last type should be BTF_KIND_DATASEC */
if (btf->nr_types == 0)
return -EINVAL;
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
if (!btf_is_datasec(t))
return -EINVAL;
@@ -2317,7 +2396,7 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __
v->size = byte_sz;
/* update parent type's vlen */
- t = btf_type_by_id(btf, btf->nr_types);
+ t = btf_last_type(btf);
btf_type_inc_vlen(t);
btf->hdr->type_len += sz;
@@ -2639,6 +2718,7 @@ struct btf_dedup;
static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
const struct btf_dedup_opts *opts);
static void btf_dedup_free(struct btf_dedup *d);
+static int btf_dedup_prep(struct btf_dedup *d);
static int btf_dedup_strings(struct btf_dedup *d);
static int btf_dedup_prim_types(struct btf_dedup *d);
static int btf_dedup_struct_types(struct btf_dedup *d);
@@ -2797,6 +2877,11 @@ int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
if (btf_ensure_modifiable(btf))
return -ENOMEM;
+ err = btf_dedup_prep(d);
+ if (err) {
+ pr_debug("btf_dedup_prep failed:%d\n", err);
+ goto done;
+ }
err = btf_dedup_strings(d);
if (err < 0) {
pr_debug("btf_dedup_strings failed:%d\n", err);
@@ -2859,21 +2944,20 @@ struct btf_dedup {
__u32 *hypot_list;
size_t hypot_cnt;
size_t hypot_cap;
+ /* Whether hypothetical mapping, if successful, would need to adjust
+ * already canonicalized types (due to a new forward declaration to
+ * concrete type resolution). In such case, during split BTF dedup
+ * candidate type would still be considered as different, because base
+ * BTF is considered to be immutable.
+ */
+ bool hypot_adjust_canon;
/* Various option modifying behavior of algorithm */
struct btf_dedup_opts opts;
-};
-
-struct btf_str_ptr {
- const char *str;
- __u32 new_off;
- bool used;
-};
-
-struct btf_str_ptrs {
- struct btf_str_ptr *ptrs;
- const char *data;
- __u32 cnt;
- __u32 cap;
+ /* temporary strings deduplication state */
+ void *strs_data;
+ size_t strs_cap;
+ size_t strs_len;
+ struct hashmap* strs_hash;
};
static long hash_combine(long h, long value)
@@ -2914,6 +2998,7 @@ static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
for (i = 0; i < d->hypot_cnt; i++)
d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
d->hypot_cnt = 0;
+ d->hypot_adjust_canon = false;
}
static void btf_dedup_free(struct btf_dedup *d)
@@ -2953,7 +3038,7 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
{
struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
- int i, err = 0;
+ int i, err = 0, type_cnt;
if (!d)
return ERR_PTR(-ENOMEM);
@@ -2973,14 +3058,15 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
goto done;
}
- d->map = malloc(sizeof(__u32) * (1 + btf->nr_types));
+ type_cnt = btf__get_nr_types(btf) + 1;
+ d->map = malloc(sizeof(__u32) * type_cnt);
if (!d->map) {
err = -ENOMEM;
goto done;
}
/* special BTF "void" type is made canonical immediately */
d->map[0] = 0;
- for (i = 1; i <= btf->nr_types; i++) {
+ for (i = 1; i < type_cnt; i++) {
struct btf_type *t = btf_type_by_id(d->btf, i);
/* VAR and DATASEC are never deduped and are self-canonical */
@@ -2990,12 +3076,12 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
d->map[i] = BTF_UNPROCESSED_ID;
}
- d->hypot_map = malloc(sizeof(__u32) * (1 + btf->nr_types));
+ d->hypot_map = malloc(sizeof(__u32) * type_cnt);
if (!d->hypot_map) {
err = -ENOMEM;
goto done;
}
- for (i = 0; i <= btf->nr_types; i++)
+ for (i = 0; i < type_cnt; i++)
d->hypot_map[i] = BTF_UNPROCESSED_ID;
done:
@@ -3019,8 +3105,8 @@ static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx)
int i, j, r, rec_size;
struct btf_type *t;
- for (i = 1; i <= d->btf->nr_types; i++) {
- t = btf_type_by_id(d->btf, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ t = btf_type_by_id(d->btf, d->btf->start_id + i);
r = fn(&t->name_off, ctx);
if (r)
return r;
@@ -3100,64 +3186,53 @@ static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx)
return 0;
}
-static int str_sort_by_content(const void *a1, const void *a2)
+static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
{
- const struct btf_str_ptr *p1 = a1;
- const struct btf_str_ptr *p2 = a2;
-
- return strcmp(p1->str, p2->str);
-}
-
-static int str_sort_by_offset(const void *a1, const void *a2)
-{
- const struct btf_str_ptr *p1 = a1;
- const struct btf_str_ptr *p2 = a2;
-
- if (p1->str != p2->str)
- return p1->str < p2->str ? -1 : 1;
- return 0;
-}
-
-static int btf_dedup_str_ptr_cmp(const void *str_ptr, const void *pelem)
-{
- const struct btf_str_ptr *p = pelem;
-
- if (str_ptr != p->str)
- return (const char *)str_ptr < p->str ? -1 : 1;
- return 0;
-}
-
-static int btf_str_mark_as_used(__u32 *str_off_ptr, void *ctx)
-{
- struct btf_str_ptrs *strs;
- struct btf_str_ptr *s;
+ struct btf_dedup *d = ctx;
+ __u32 str_off = *str_off_ptr;
+ long old_off, new_off, len;
+ const char *s;
+ void *p;
+ int err;
- if (*str_off_ptr == 0)
+ /* don't touch empty string or string in main BTF */
+ if (str_off == 0 || str_off < d->btf->start_str_off)
return 0;
- strs = ctx;
- s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
- sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
- if (!s)
- return -EINVAL;
- s->used = true;
- return 0;
-}
+ s = btf__str_by_offset(d->btf, str_off);
+ if (d->btf->base_btf) {
+ err = btf__find_str(d->btf->base_btf, s);
+ if (err >= 0) {
+ *str_off_ptr = err;
+ return 0;
+ }
+ if (err != -ENOENT)
+ return err;
+ }
-static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx)
-{
- struct btf_str_ptrs *strs;
- struct btf_str_ptr *s;
+ len = strlen(s) + 1;
- if (*str_off_ptr == 0)
- return 0;
+ new_off = d->strs_len;
+ p = btf_add_mem(&d->strs_data, &d->strs_cap, 1, new_off, BTF_MAX_STR_OFFSET, len);
+ if (!p)
+ return -ENOMEM;
- strs = ctx;
- s = bsearch(strs->data + *str_off_ptr, strs->ptrs, strs->cnt,
- sizeof(struct btf_str_ptr), btf_dedup_str_ptr_cmp);
- if (!s)
- return -EINVAL;
- *str_off_ptr = s->new_off;
+ memcpy(p, s, len);
+
+ /* Now attempt to add the string, but only if the string with the same
+ * contents doesn't exist already (HASHMAP_ADD strategy). If such
+ * string exists, we'll get its offset in old_off (that's old_key).
+ */
+ err = hashmap__insert(d->strs_hash, (void *)new_off, (void *)new_off,
+ HASHMAP_ADD, (const void **)&old_off, NULL);
+ if (err == -EEXIST) {
+ *str_off_ptr = d->btf->start_str_off + old_off;
+ } else if (err) {
+ return err;
+ } else {
+ *str_off_ptr = d->btf->start_str_off + new_off;
+ d->strs_len += len;
+ }
return 0;
}
@@ -3174,118 +3249,71 @@ static int btf_str_remap_offset(__u32 *str_off_ptr, void *ctx)
*/
static int btf_dedup_strings(struct btf_dedup *d)
{
- char *start = d->btf->strs_data;
- char *end = start + d->btf->hdr->str_len;
- char *p = start, *tmp_strs = NULL;
- struct btf_str_ptrs strs = {
- .cnt = 0,
- .cap = 0,
- .ptrs = NULL,
- .data = start,
- };
- int i, j, err = 0, grp_idx;
- bool grp_used;
+ char *s;
+ int err;
if (d->btf->strs_deduped)
return 0;
- /* build index of all strings */
- while (p < end) {
- if (strs.cnt + 1 > strs.cap) {
- struct btf_str_ptr *new_ptrs;
-
- strs.cap += max(strs.cnt / 2, 16U);
- new_ptrs = libbpf_reallocarray(strs.ptrs, strs.cap, sizeof(strs.ptrs[0]));
- if (!new_ptrs) {
- err = -ENOMEM;
- goto done;
- }
- strs.ptrs = new_ptrs;
- }
-
- strs.ptrs[strs.cnt].str = p;
- strs.ptrs[strs.cnt].used = false;
-
- p += strlen(p) + 1;
- strs.cnt++;
- }
+ /* temporarily switch to use btf_dedup's strs_data for strings for hash
+ * functions; later we'll just transfer hashmap to struct btf as is,
+ * along the strs_data
+ */
+ d->btf->strs_data_ptr = &d->strs_data;
- /* temporary storage for deduplicated strings */
- tmp_strs = malloc(d->btf->hdr->str_len);
- if (!tmp_strs) {
- err = -ENOMEM;
- goto done;
+ d->strs_hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, d->btf);
+ if (IS_ERR(d->strs_hash)) {
+ err = PTR_ERR(d->strs_hash);
+ d->strs_hash = NULL;
+ goto err_out;
}
- /* mark all used strings */
- strs.ptrs[0].used = true;
- err = btf_for_each_str_off(d, btf_str_mark_as_used, &strs);
- if (err)
- goto done;
-
- /* sort strings by context, so that we can identify duplicates */
- qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_content);
-
- /*
- * iterate groups of equal strings and if any instance in a group was
- * referenced, emit single instance and remember new offset
- */
- p = tmp_strs;
- grp_idx = 0;
- grp_used = strs.ptrs[0].used;
- /* iterate past end to avoid code duplication after loop */
- for (i = 1; i <= strs.cnt; i++) {
- /*
- * when i == strs.cnt, we want to skip string comparison and go
- * straight to handling last group of strings (otherwise we'd
- * need to handle last group after the loop w/ duplicated code)
- */
- if (i < strs.cnt &&
- !strcmp(strs.ptrs[i].str, strs.ptrs[grp_idx].str)) {
- grp_used = grp_used || strs.ptrs[i].used;
- continue;
- }
+ if (!d->btf->base_btf) {
+ s = btf_add_mem(&d->strs_data, &d->strs_cap, 1, d->strs_len, BTF_MAX_STR_OFFSET, 1);
+ if (!s)
+ return -ENOMEM;
+ /* initial empty string */
+ s[0] = 0;
+ d->strs_len = 1;
- /*
- * this check would have been required after the loop to handle
- * last group of strings, but due to <= condition in a loop
- * we avoid that duplication
+ /* insert empty string; we won't be looking it up during strings
+ * dedup, but it's good to have it for generic BTF string lookups
*/
- if (grp_used) {
- int new_off = p - tmp_strs;
- __u32 len = strlen(strs.ptrs[grp_idx].str);
-
- memmove(p, strs.ptrs[grp_idx].str, len + 1);
- for (j = grp_idx; j < i; j++)
- strs.ptrs[j].new_off = new_off;
- p += len + 1;
- }
-
- if (i < strs.cnt) {
- grp_idx = i;
- grp_used = strs.ptrs[i].used;
- }
+ err = hashmap__insert(d->strs_hash, (void *)0, (void *)0,
+ HASHMAP_ADD, NULL, NULL);
+ if (err)
+ goto err_out;
}
- /* replace original strings with deduped ones */
- d->btf->hdr->str_len = p - tmp_strs;
- memmove(start, tmp_strs, d->btf->hdr->str_len);
- end = start + d->btf->hdr->str_len;
-
- /* restore original order for further binary search lookups */
- qsort(strs.ptrs, strs.cnt, sizeof(strs.ptrs[0]), str_sort_by_offset);
-
/* remap string offsets */
- err = btf_for_each_str_off(d, btf_str_remap_offset, &strs);
+ err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
if (err)
- goto done;
+ goto err_out;
- d->btf->hdr->str_len = end - start;
+ /* replace BTF string data and hash with deduped ones */
+ free(d->btf->strs_data);
+ hashmap__free(d->btf->strs_hash);
+ d->btf->strs_data = d->strs_data;
+ d->btf->strs_data_cap = d->strs_cap;
+ d->btf->hdr->str_len = d->strs_len;
+ d->btf->strs_hash = d->strs_hash;
+ /* now point strs_data_ptr back to btf->strs_data */
+ d->btf->strs_data_ptr = &d->btf->strs_data;
+
+ d->strs_data = d->strs_hash = NULL;
+ d->strs_len = d->strs_cap = 0;
d->btf->strs_deduped = true;
+ return 0;
+
+err_out:
+ free(d->strs_data);
+ hashmap__free(d->strs_hash);
+ d->strs_data = d->strs_hash = NULL;
+ d->strs_len = d->strs_cap = 0;
+
+ /* restore strings pointer for existing d->btf->strs_hash back */
+ d->btf->strs_data_ptr = &d->strs_data;
-done:
- free(tmp_strs);
- free(strs.ptrs);
return err;
}
@@ -3550,6 +3578,66 @@ static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
return true;
}
+/* Prepare split BTF for deduplication by calculating hashes of base BTF's
+ * types and initializing the rest of the state (canonical type mapping) for
+ * the fixed base BTF part.
+ */
+static int btf_dedup_prep(struct btf_dedup *d)
+{
+ struct btf_type *t;
+ int type_id;
+ long h;
+
+ if (!d->btf->base_btf)
+ return 0;
+
+ for (type_id = 1; type_id < d->btf->start_id; type_id++) {
+ t = btf_type_by_id(d->btf, type_id);
+
+ /* all base BTF types are self-canonical by definition */
+ d->map[type_id] = type_id;
+
+ switch (btf_kind(t)) {
+ case BTF_KIND_VAR:
+ case BTF_KIND_DATASEC:
+ /* VAR and DATASEC are never hash/deduplicated */
+ continue;
+ case BTF_KIND_CONST:
+ case BTF_KIND_VOLATILE:
+ case BTF_KIND_RESTRICT:
+ case BTF_KIND_PTR:
+ case BTF_KIND_FWD:
+ case BTF_KIND_TYPEDEF:
+ case BTF_KIND_FUNC:
+ h = btf_hash_common(t);
+ break;
+ case BTF_KIND_INT:
+ h = btf_hash_int(t);
+ break;
+ case BTF_KIND_ENUM:
+ h = btf_hash_enum(t);
+ break;
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION:
+ h = btf_hash_struct(t);
+ break;
+ case BTF_KIND_ARRAY:
+ h = btf_hash_array(t);
+ break;
+ case BTF_KIND_FUNC_PROTO:
+ h = btf_hash_fnproto(t);
+ break;
+ default:
+ pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
+ return -EINVAL;
+ }
+ if (btf_dedup_table_add(d, h, type_id))
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
/*
* Deduplicate primitive types, that can't reference other types, by calculating
* their type signature hash and comparing them with any possible canonical
@@ -3643,8 +3731,8 @@ static int btf_dedup_prim_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_prim_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_prim_type(d, d->btf->start_id + i);
if (err)
return err;
}
@@ -3697,6 +3785,19 @@ static inline __u16 btf_fwd_kind(struct btf_type *t)
return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
}
+/* Check if given two types are identical ARRAY definitions */
+static int btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
+{
+ struct btf_type *t1, *t2;
+
+ t1 = btf_type_by_id(d->btf, id1);
+ t2 = btf_type_by_id(d->btf, id2);
+ if (!btf_is_array(t1) || !btf_is_array(t2))
+ return 0;
+
+ return btf_equal_array(t1, t2);
+}
+
/*
* Check equivalence of BTF type graph formed by candidate struct/union (we'll
* call it "candidate graph" in this description for brevity) to a type graph
@@ -3807,8 +3908,18 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
canon_id = resolve_fwd_id(d, canon_id);
hypot_type_id = d->hypot_map[canon_id];
- if (hypot_type_id <= BTF_MAX_NR_TYPES)
- return hypot_type_id == cand_id;
+ if (hypot_type_id <= BTF_MAX_NR_TYPES) {
+ /* In some cases compiler will generate different DWARF types
+ * for *identical* array type definitions and use them for
+ * different fields within the *same* struct. This breaks type
+ * equivalence check, which makes an assumption that candidate
+ * types sub-graph has a consistent and deduped-by-compiler
+ * types within a single CU. So work around that by explicitly
+ * allowing identical array types here.
+ */
+ return hypot_type_id == cand_id ||
+ btf_dedup_identical_arrays(d, hypot_type_id, cand_id);
+ }
if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
return -ENOMEM;
@@ -3834,6 +3945,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
} else {
real_kind = cand_kind;
fwd_kind = btf_fwd_kind(canon_type);
+ /* we'd need to resolve base FWD to STRUCT/UNION */
+ if (fwd_kind == real_kind && canon_id < d->btf->start_id)
+ d->hypot_adjust_canon = true;
}
return fwd_kind == real_kind;
}
@@ -3871,8 +3985,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
return 0;
cand_arr = btf_array(cand_type);
canon_arr = btf_array(canon_type);
- eq = btf_dedup_is_equiv(d,
- cand_arr->index_type, canon_arr->index_type);
+ eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
if (eq <= 0)
return eq;
return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
@@ -3955,16 +4068,16 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
*/
static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
{
- __u32 cand_type_id, targ_type_id;
+ __u32 canon_type_id, targ_type_id;
__u16 t_kind, c_kind;
__u32 t_id, c_id;
int i;
for (i = 0; i < d->hypot_cnt; i++) {
- cand_type_id = d->hypot_list[i];
- targ_type_id = d->hypot_map[cand_type_id];
+ canon_type_id = d->hypot_list[i];
+ targ_type_id = d->hypot_map[canon_type_id];
t_id = resolve_type_id(d, targ_type_id);
- c_id = resolve_type_id(d, cand_type_id);
+ c_id = resolve_type_id(d, canon_type_id);
t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
/*
@@ -3979,9 +4092,26 @@ static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
* stability is not a requirement for STRUCT/UNION equivalence
* checks, though.
*/
+
+ /* if it's the split BTF case, we still need to point base FWD
+ * to STRUCT/UNION in a split BTF, because FWDs from split BTF
+ * will be resolved against base FWD. If we don't point base
+ * canonical FWD to the resolved STRUCT/UNION, then all the
+ * FWDs in split BTF won't be correctly resolved to a proper
+ * STRUCT/UNION.
+ */
if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
d->map[c_id] = t_id;
- else if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
+
+ /* if graph equivalence determined that we'd need to adjust
+ * base canonical types, then we need to only point base FWDs
+ * to STRUCTs/UNIONs and do no more modifications. For all
+ * other purposes the type graphs were not equivalent.
+ */
+ if (d->hypot_adjust_canon)
+ continue;
+
+ if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
d->map[t_id] = c_id;
if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
@@ -4065,8 +4195,10 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
return eq;
if (!eq)
continue;
- new_id = cand_id;
btf_dedup_merge_hypot_map(d);
+ if (d->hypot_adjust_canon) /* not really equivalent */
+ continue;
+ new_id = cand_id;
break;
}
@@ -4081,8 +4213,8 @@ static int btf_dedup_struct_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_struct_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_struct_type(d, d->btf->start_id + i);
if (err)
return err;
}
@@ -4225,8 +4357,8 @@ static int btf_dedup_ref_types(struct btf_dedup *d)
{
int i, err;
- for (i = 1; i <= d->btf->nr_types; i++) {
- err = btf_dedup_ref_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ err = btf_dedup_ref_type(d, d->btf->start_id + i);
if (err < 0)
return err;
}
@@ -4250,39 +4382,44 @@ static int btf_dedup_ref_types(struct btf_dedup *d)
static int btf_dedup_compact_types(struct btf_dedup *d)
{
__u32 *new_offs;
- __u32 next_type_id = 1;
+ __u32 next_type_id = d->btf->start_id;
+ const struct btf_type *t;
void *p;
- int i, len;
+ int i, id, len;
/* we are going to reuse hypot_map to store compaction remapping */
d->hypot_map[0] = 0;
- for (i = 1; i <= d->btf->nr_types; i++)
- d->hypot_map[i] = BTF_UNPROCESSED_ID;
+ /* base BTF types are not renumbered */
+ for (id = 1; id < d->btf->start_id; id++)
+ d->hypot_map[id] = id;
+ for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
+ d->hypot_map[id] = BTF_UNPROCESSED_ID;
p = d->btf->types_data;
- for (i = 1; i <= d->btf->nr_types; i++) {
- if (d->map[i] != i)
+ for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
+ if (d->map[id] != id)
continue;
- len = btf_type_size(btf__type_by_id(d->btf, i));
+ t = btf__type_by_id(d->btf, id);
+ len = btf_type_size(t);
if (len < 0)
return len;
- memmove(p, btf__type_by_id(d->btf, i), len);
- d->hypot_map[i] = next_type_id;
- d->btf->type_offs[next_type_id] = p - d->btf->types_data;
+ memmove(p, t, len);
+ d->hypot_map[id] = next_type_id;
+ d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
p += len;
next_type_id++;
}
/* shrink struct btf's internal types index and update btf_header */
- d->btf->nr_types = next_type_id - 1;
- d->btf->type_offs_cap = d->btf->nr_types + 1;
+ d->btf->nr_types = next_type_id - d->btf->start_id;
+ d->btf->type_offs_cap = d->btf->nr_types;
d->btf->hdr->type_len = p - d->btf->types_data;
new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
sizeof(*new_offs));
- if (!new_offs)
+ if (d->btf->type_offs_cap && !new_offs)
return -ENOMEM;
d->btf->type_offs = new_offs;
d->btf->hdr->str_off = d->btf->hdr->type_len;
@@ -4414,8 +4551,8 @@ static int btf_dedup_remap_types(struct btf_dedup *d)
{
int i, r;
- for (i = 1; i <= d->btf->nr_types; i++) {
- r = btf_dedup_remap_type(d, i);
+ for (i = 0; i < d->btf->nr_types; i++) {
+ r = btf_dedup_remap_type(d, d->btf->start_id + i);
if (r < 0)
return r;
}
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 57247240a20a..1093f6fe6800 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -31,11 +31,19 @@ enum btf_endianness {
};
LIBBPF_API void btf__free(struct btf *btf);
+
LIBBPF_API struct btf *btf__new(const void *data, __u32 size);
+LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf);
LIBBPF_API struct btf *btf__new_empty(void);
+LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);
+
LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext);
+LIBBPF_API struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_raw(const char *path);
+LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf);
+
LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
LIBBPF_API int btf__load(struct btf *btf);
LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 4ebfadf45b47..29ff4807b909 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -337,3 +337,12 @@ LIBBPF_0.2.0 {
perf_buffer__consume_buffer;
xsk_socket__create_shared;
} LIBBPF_0.1.0;
+
+LIBBPF_0.3.0 {
+ global:
+ btf__parse_elf_split;
+ btf__parse_raw_split;
+ btf__parse_split;
+ btf__new_empty_split;
+ btf__new_split;
+} LIBBPF_0.2.0;
diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c
index 5482a9b7ae2d..ecaae2927ab8 100644
--- a/tools/lib/bpf/libbpf_probes.c
+++ b/tools/lib/bpf/libbpf_probes.c
@@ -230,6 +230,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
break;
case BPF_MAP_TYPE_SK_STORAGE:
case BPF_MAP_TYPE_INODE_STORAGE:
+ case BPF_MAP_TYPE_TASK_STORAGE:
btf_key_type_id = 1;
btf_value_type_id = 3;
value_size = 8;