From fbb0de795078190a9834b3409e4b009cfb18a6d4 Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Mon, 27 Aug 2018 11:34:44 +0200 Subject: Add udmabuf misc device A driver to let userspace turn memfd regions into dma-bufs. Use case: Allows qemu create dmabufs for the vga framebuffer or virtio-gpu ressources. Then they can be passed around to display those guest things on the host. To spice client for classic full framebuffer display, and hopefully some day to wayland server for seamless guest window display. qemu test branch: https://git.kraxel.org/cgit/qemu/log/?h=sirius/udmabuf Cc: David Airlie Cc: Tomeu Vizoso Cc: Laurent Pinchart Cc: Daniel Vetter Signed-off-by: Gerd Hoffmann Acked-by: Daniel Vetter Link: http://patchwork.freedesktop.org/patch/msgid/20180827093444.23623-1-kraxel@redhat.com --- tools/testing/selftests/drivers/dma-buf/Makefile | 5 ++ tools/testing/selftests/drivers/dma-buf/udmabuf.c | 96 +++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tools/testing/selftests/drivers/dma-buf/Makefile create mode 100644 tools/testing/selftests/drivers/dma-buf/udmabuf.c (limited to 'tools') diff --git a/tools/testing/selftests/drivers/dma-buf/Makefile b/tools/testing/selftests/drivers/dma-buf/Makefile new file mode 100644 index 000000000000..4154c3d7aa58 --- /dev/null +++ b/tools/testing/selftests/drivers/dma-buf/Makefile @@ -0,0 +1,5 @@ +CFLAGS += -I../../../../../usr/include/ + +TEST_GEN_PROGS := udmabuf + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/dma-buf/udmabuf.c b/tools/testing/selftests/drivers/dma-buf/udmabuf.c new file mode 100644 index 000000000000..376b1d6730bd --- /dev/null +++ b/tools/testing/selftests/drivers/dma-buf/udmabuf.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define TEST_PREFIX "drivers/dma-buf/udmabuf" +#define NUM_PAGES 4 + +static int memfd_create(const char *name, unsigned int flags) +{ + return syscall(__NR_memfd_create, name, flags); +} + +int main(int argc, char *argv[]) +{ + struct udmabuf_create create; + int devfd, memfd, buf, ret; + off_t size; + void *mem; + + devfd = open("/dev/udmabuf", O_RDWR); + if (devfd < 0) { + printf("%s: [skip,no-udmabuf]\n", TEST_PREFIX); + exit(77); + } + + memfd = memfd_create("udmabuf-test", MFD_CLOEXEC); + if (memfd < 0) { + printf("%s: [skip,no-memfd]\n", TEST_PREFIX); + exit(77); + } + + size = getpagesize() * NUM_PAGES; + ret = ftruncate(memfd, size); + if (ret == -1) { + printf("%s: [FAIL,memfd-truncate]\n", TEST_PREFIX); + exit(1); + } + + memset(&create, 0, sizeof(create)); + + /* should fail (offset not page aligned) */ + create.memfd = memfd; + create.offset = getpagesize()/2; + create.size = getpagesize(); + buf = ioctl(devfd, UDMABUF_CREATE, &create); + if (buf >= 0) { + printf("%s: [FAIL,test-1]\n", TEST_PREFIX); + exit(1); + } + + /* should fail (size not multiple of page) */ + create.memfd = memfd; + create.offset = 0; + create.size = getpagesize()/2; + buf = ioctl(devfd, UDMABUF_CREATE, &create); + if (buf >= 0) { + printf("%s: [FAIL,test-2]\n", TEST_PREFIX); + exit(1); + } + + /* should fail (not memfd) */ + create.memfd = 0; /* stdin */ + create.offset = 0; + create.size = size; + buf = ioctl(devfd, UDMABUF_CREATE, &create); + if (buf >= 0) { + printf("%s: [FAIL,test-3]\n", TEST_PREFIX); + exit(1); + } + + /* should work */ + create.memfd = memfd; + create.offset = 0; + create.size = size; + buf = ioctl(devfd, UDMABUF_CREATE, &create); + if (buf < 0) { + printf("%s: [FAIL,test-4]\n", TEST_PREFIX); + exit(1); + } + + fprintf(stderr, "%s: ok\n", TEST_PREFIX); + close(buf); + close(memfd); + close(devfd); + return 0; +} -- cgit v1.2.3 From 5a5e3d3ceae2121590d67258a4e658d938b4a46d Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Mon, 20 Aug 2018 10:12:50 +0530 Subject: perf probe: Support SDT markers having reference counter (semaphore) With this, perf buildid-cache will save SDT markers with reference counter in probe cache. Perf probe will be able to probe markers having reference counter. Ex, # readelf -n /tmp/tick | grep -A1 loop2 Name: loop2 ... Semaphore: 0x0000000010020036 # ./perf buildid-cache --add /tmp/tick # ./perf probe sdt_tick:loop2 # ./perf stat -e sdt_tick:loop2 /tmp/tick hi: 0 hi: 1 hi: 2 ^C Performance counter stats for '/tmp/tick': 3 sdt_tick:loop2 2.561851452 seconds time elapsed Link: http://lkml.kernel.org/r/20180820044250.11659-5-ravi.bangoria@linux.ibm.com Acked-by: Arnaldo Carvalho de Melo Acked-by: Masami Hiramatsu Acked-by: Srikar Dronamraju Reviewed-by: Song Liu Tested-by: Song Liu Signed-off-by: Ravi Bangoria Signed-off-by: Steven Rostedt (VMware) --- tools/perf/util/probe-event.c | 39 ++++++++++++++++++++++++++++++++---- tools/perf/util/probe-event.h | 1 + tools/perf/util/probe-file.c | 34 ++++++++++++++++++++++++++------ tools/perf/util/probe-file.h | 1 + tools/perf/util/symbol-elf.c | 46 ++++++++++++++++++++++++++++++++----------- tools/perf/util/symbol.h | 7 +++++++ 6 files changed, 106 insertions(+), 22 deletions(-) (limited to 'tools') diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index f119eb628dbb..e86f8be89157 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -1819,6 +1819,12 @@ int parse_probe_trace_command(const char *cmd, struct probe_trace_event *tev) tp->offset = strtoul(fmt2_str, NULL, 10); } + if (tev->uprobes) { + fmt2_str = strchr(p, '('); + if (fmt2_str) + tp->ref_ctr_offset = strtoul(fmt2_str + 1, NULL, 0); + } + tev->nargs = argc - 2; tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs); if (tev->args == NULL) { @@ -2012,6 +2018,22 @@ static int synthesize_probe_trace_arg(struct probe_trace_arg *arg, return err; } +static int +synthesize_uprobe_trace_def(struct probe_trace_event *tev, struct strbuf *buf) +{ + struct probe_trace_point *tp = &tev->point; + int err; + + err = strbuf_addf(buf, "%s:0x%lx", tp->module, tp->address); + + if (err >= 0 && tp->ref_ctr_offset) { + if (!uprobe_ref_ctr_is_supported()) + return -1; + err = strbuf_addf(buf, "(0x%lx)", tp->ref_ctr_offset); + } + return err >= 0 ? 0 : -1; +} + char *synthesize_probe_trace_command(struct probe_trace_event *tev) { struct probe_trace_point *tp = &tev->point; @@ -2041,15 +2063,17 @@ char *synthesize_probe_trace_command(struct probe_trace_event *tev) } /* Use the tp->address for uprobes */ - if (tev->uprobes) - err = strbuf_addf(&buf, "%s:0x%lx", tp->module, tp->address); - else if (!strncmp(tp->symbol, "0x", 2)) + if (tev->uprobes) { + err = synthesize_uprobe_trace_def(tev, &buf); + } else if (!strncmp(tp->symbol, "0x", 2)) { /* Absolute address. See try_to_find_absolute_address() */ err = strbuf_addf(&buf, "%s%s0x%lx", tp->module ?: "", tp->module ? ":" : "", tp->address); - else + } else { err = strbuf_addf(&buf, "%s%s%s+%lu", tp->module ?: "", tp->module ? ":" : "", tp->symbol, tp->offset); + } + if (err) goto error; @@ -2633,6 +2657,13 @@ static void warn_uprobe_event_compat(struct probe_trace_event *tev) { int i; char *buf = synthesize_probe_trace_command(tev); + struct probe_trace_point *tp = &tev->point; + + if (tp->ref_ctr_offset && !uprobe_ref_ctr_is_supported()) { + pr_warning("A semaphore is associated with %s:%s and " + "seems your kernel doesn't support it.\n", + tev->group, tev->event); + } /* Old uprobe event doesn't support memory dereference */ if (!tev->uprobes || tev->nargs == 0 || !buf) diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h index 45b14f020558..15a98c3a2a2f 100644 --- a/tools/perf/util/probe-event.h +++ b/tools/perf/util/probe-event.h @@ -27,6 +27,7 @@ struct probe_trace_point { char *symbol; /* Base symbol */ char *module; /* Module name */ unsigned long offset; /* Offset from symbol */ + unsigned long ref_ctr_offset; /* SDT reference counter offset */ unsigned long address; /* Actual address of the trace point */ bool retprobe; /* Return probe flag */ }; diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c index b76088fadf3d..aac7817d9e14 100644 --- a/tools/perf/util/probe-file.c +++ b/tools/perf/util/probe-file.c @@ -696,8 +696,16 @@ out_err: #ifdef HAVE_GELF_GETNOTE_SUPPORT static unsigned long long sdt_note__get_addr(struct sdt_note *note) { - return note->bit32 ? (unsigned long long)note->addr.a32[0] - : (unsigned long long)note->addr.a64[0]; + return note->bit32 ? + (unsigned long long)note->addr.a32[SDT_NOTE_IDX_LOC] : + (unsigned long long)note->addr.a64[SDT_NOTE_IDX_LOC]; +} + +static unsigned long long sdt_note__get_ref_ctr_offset(struct sdt_note *note) +{ + return note->bit32 ? + (unsigned long long)note->addr.a32[SDT_NOTE_IDX_REFCTR] : + (unsigned long long)note->addr.a64[SDT_NOTE_IDX_REFCTR]; } static const char * const type_to_suffix[] = { @@ -775,14 +783,21 @@ static char *synthesize_sdt_probe_command(struct sdt_note *note, { struct strbuf buf; char *ret = NULL, **args; - int i, args_count; + int i, args_count, err; + unsigned long long ref_ctr_offset; if (strbuf_init(&buf, 32) < 0) return NULL; - if (strbuf_addf(&buf, "p:%s/%s %s:0x%llx", - sdtgrp, note->name, pathname, - sdt_note__get_addr(note)) < 0) + err = strbuf_addf(&buf, "p:%s/%s %s:0x%llx", + sdtgrp, note->name, pathname, + sdt_note__get_addr(note)); + + ref_ctr_offset = sdt_note__get_ref_ctr_offset(note); + if (ref_ctr_offset && err >= 0) + err = strbuf_addf(&buf, "(0x%llx)", ref_ctr_offset); + + if (err < 0) goto error; if (!note->args) @@ -998,6 +1013,7 @@ int probe_cache__show_all_caches(struct strfilter *filter) enum ftrace_readme { FTRACE_README_PROBE_TYPE_X = 0, FTRACE_README_KRETPROBE_OFFSET, + FTRACE_README_UPROBE_REF_CTR, FTRACE_README_END, }; @@ -1009,6 +1025,7 @@ static struct { [idx] = {.pattern = pat, .avail = false} DEFINE_TYPE(FTRACE_README_PROBE_TYPE_X, "*type: * x8/16/32/64,*"), DEFINE_TYPE(FTRACE_README_KRETPROBE_OFFSET, "*place (kretprobe): *"), + DEFINE_TYPE(FTRACE_README_UPROBE_REF_CTR, "*ref_ctr_offset*"), }; static bool scan_ftrace_readme(enum ftrace_readme type) @@ -1064,3 +1081,8 @@ bool kretprobe_offset_is_supported(void) { return scan_ftrace_readme(FTRACE_README_KRETPROBE_OFFSET); } + +bool uprobe_ref_ctr_is_supported(void) +{ + return scan_ftrace_readme(FTRACE_README_UPROBE_REF_CTR); +} diff --git a/tools/perf/util/probe-file.h b/tools/perf/util/probe-file.h index 63f29b1d22c1..2a249182f2a6 100644 --- a/tools/perf/util/probe-file.h +++ b/tools/perf/util/probe-file.h @@ -69,6 +69,7 @@ struct probe_cache_entry *probe_cache__find_by_name(struct probe_cache *pcache, int probe_cache__show_all_caches(struct strfilter *filter); bool probe_type_is_available(enum probe_type type); bool kretprobe_offset_is_supported(void); +bool uprobe_ref_ctr_is_supported(void); #else /* ! HAVE_LIBELF_SUPPORT */ static inline struct probe_cache *probe_cache__new(const char *tgt __maybe_unused, struct nsinfo *nsi __maybe_unused) { diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 29770ea61768..0281d5e2cd67 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1947,6 +1947,34 @@ void kcore_extract__delete(struct kcore_extract *kce) } #ifdef HAVE_GELF_GETNOTE_SUPPORT + +static void sdt_adjust_loc(struct sdt_note *tmp, GElf_Addr base_off) +{ + if (!base_off) + return; + + if (tmp->bit32) + tmp->addr.a32[SDT_NOTE_IDX_LOC] = + tmp->addr.a32[SDT_NOTE_IDX_LOC] + base_off - + tmp->addr.a32[SDT_NOTE_IDX_BASE]; + else + tmp->addr.a64[SDT_NOTE_IDX_LOC] = + tmp->addr.a64[SDT_NOTE_IDX_LOC] + base_off - + tmp->addr.a64[SDT_NOTE_IDX_BASE]; +} + +static void sdt_adjust_refctr(struct sdt_note *tmp, GElf_Addr base_addr, + GElf_Addr base_off) +{ + if (!base_off) + return; + + if (tmp->bit32 && tmp->addr.a32[SDT_NOTE_IDX_REFCTR]) + tmp->addr.a32[SDT_NOTE_IDX_REFCTR] -= (base_addr - base_off); + else if (tmp->addr.a64[SDT_NOTE_IDX_REFCTR]) + tmp->addr.a64[SDT_NOTE_IDX_REFCTR] -= (base_addr - base_off); +} + /** * populate_sdt_note : Parse raw data and identify SDT note * @elf: elf of the opened file @@ -1964,7 +1992,6 @@ static int populate_sdt_note(Elf **elf, const char *data, size_t len, const char *provider, *name, *args; struct sdt_note *tmp = NULL; GElf_Ehdr ehdr; - GElf_Addr base_off = 0; GElf_Shdr shdr; int ret = -EINVAL; @@ -2060,17 +2087,12 @@ static int populate_sdt_note(Elf **elf, const char *data, size_t len, * base address in the description of the SDT note. If its different, * then accordingly, adjust the note location. */ - if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_BASE_SCN, NULL)) { - base_off = shdr.sh_offset; - if (base_off) { - if (tmp->bit32) - tmp->addr.a32[0] = tmp->addr.a32[0] + base_off - - tmp->addr.a32[1]; - else - tmp->addr.a64[0] = tmp->addr.a64[0] + base_off - - tmp->addr.a64[1]; - } - } + if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_BASE_SCN, NULL)) + sdt_adjust_loc(tmp, shdr.sh_offset); + + /* Adjust reference counter offset */ + if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_PROBES_SCN, NULL)) + sdt_adjust_refctr(tmp, shdr.sh_addr, shdr.sh_offset); list_add_tail(&tmp->note_list, sdt_notes); return 0; diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h index f25fae4b5743..20f49779116b 100644 --- a/tools/perf/util/symbol.h +++ b/tools/perf/util/symbol.h @@ -379,12 +379,19 @@ int get_sdt_note_list(struct list_head *head, const char *target); int cleanup_sdt_note_list(struct list_head *sdt_notes); int sdt_notes__get_count(struct list_head *start); +#define SDT_PROBES_SCN ".probes" #define SDT_BASE_SCN ".stapsdt.base" #define SDT_NOTE_SCN ".note.stapsdt" #define SDT_NOTE_TYPE 3 #define SDT_NOTE_NAME "stapsdt" #define NR_ADDR 3 +enum { + SDT_NOTE_IDX_LOC = 0, + SDT_NOTE_IDX_BASE, + SDT_NOTE_IDX_REFCTR, +}; + struct mem_info *mem_info__new(void); struct mem_info *mem_info__get(struct mem_info *mi); void mem_info__put(struct mem_info *mi); -- cgit v1.2.3 From 66ee620f06f99d72475db6eb638559ba608c7dee Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 25 Jun 2018 06:56:50 -0400 Subject: idr: Permit any valid kernel pointer to be stored An upcoming change to the encoding of internal entries will set the bottom two bits to 0b10. Unfortunately, m68k only aligns some data structures to 2 bytes, so the IDR will interpret them as internal entries and things will go badly wrong. Change the radix tree so that it stops either when the node indicates that it's the bottom of the tree (shift == 0) or when the entry is not an internal entry. This means we cannot insert an arbitrary kernel pointer as a multiorder entry, but the IDR does not permit multiorder entries. Annoyingly, this means the IDR can no longer take advantage of the radix tree's ability to store a single entry at offset 0 without allocating memory. A pointer which is 2-byte aligned cannot be stored directly in the root as it would be indistinguishable from a node, so we must allocate a node in order to store a 2-byte pointer at index 0. The idr_replace() function does not take a GFP flags argument, so cannot allocate memory. If a user inserts a 4-byte aligned pointer at index 0 and then replaces it with a 2-byte aligned pointer, we must be able to store it. Arbitrary pointer values are still not permitted; pointers of the form 2 + (i * 4) for values of i between 0 and 1023 are reserved for the implementation. These are not valid kernel pointers as they would point into the zero page. This change does cause a runtime memory consumption regression for the IDA. I will recover that later. Signed-off-by: Matthew Wilcox Tested-by: Guenter Roeck --- lib/idr.c | 4 --- lib/radix-tree.c | 21 +++++++++---- tools/testing/radix-tree/idr-test.c | 63 +++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 10 deletions(-) (limited to 'tools') diff --git a/lib/idr.c b/lib/idr.c index fab2fd5bc326..729e381e23b4 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -39,8 +39,6 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid, unsigned int base = idr->idr_base; unsigned int id = *nextid; - if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr))) - return -EINVAL; if (WARN_ON_ONCE(!(idr->idr_rt.gfp_mask & ROOT_IS_IDR))) idr->idr_rt.gfp_mask |= IDR_RT_MARKER; @@ -295,8 +293,6 @@ void *idr_replace(struct idr *idr, void *ptr, unsigned long id) void __rcu **slot = NULL; void *entry; - if (WARN_ON_ONCE(radix_tree_is_internal_node(ptr))) - return ERR_PTR(-EINVAL); id -= idr->idr_base; entry = __radix_tree_lookup(&idr->idr_rt, id, &node, &slot); diff --git a/lib/radix-tree.c b/lib/radix-tree.c index bc03ecc4dfd2..a904a8ddd174 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -703,6 +703,14 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, if (!radix_tree_is_internal_node(child) && node->shift) break; + /* + * For an IDR, we must not shrink entry 0 into the root in + * case somebody calls idr_replace() with a pointer that + * appears to be an internal entry + */ + if (!node->shift && is_idr(root)) + break; + if (radix_tree_is_internal_node(child)) entry_to_node(child)->parent = NULL; @@ -875,8 +883,8 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) for (;;) { void *entry = rcu_dereference_raw(child->slots[offset]); - if (radix_tree_is_internal_node(entry) && - !is_sibling_entry(child, entry)) { + if (radix_tree_is_internal_node(entry) && child->shift && + !is_sibling_entry(child, entry)) { child = entry_to_node(entry); offset = 0; continue; @@ -1049,6 +1057,8 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, parent = entry_to_node(node); offset = radix_tree_descend(parent, &node, index); slot = parent->slots + offset; + if (parent->shift == 0) + break; } if (nodep) @@ -1123,9 +1133,6 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, static void replace_slot(void __rcu **slot, void *item, struct radix_tree_node *node, int count, int exceptional) { - if (WARN_ON_ONCE(radix_tree_is_internal_node(item))) - return; - if (node && (count || exceptional)) { node->count += count; node->exceptional += exceptional; @@ -1784,7 +1791,7 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, goto restart; if (child == RADIX_TREE_RETRY) break; - } while (radix_tree_is_internal_node(child)); + } while (node->shift && radix_tree_is_internal_node(child)); /* Update the iterator state */ iter->index = (index &~ node_maxindex(node)) | (offset << node->shift); @@ -2150,6 +2157,8 @@ void __rcu **idr_get_free(struct radix_tree_root *root, shift = error; child = rcu_dereference_raw(root->rnode); } + if (start == 0 && shift == 0) + shift = RADIX_TREE_MAP_SHIFT; while (shift) { shift -= RADIX_TREE_MAP_SHIFT; diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index 321ba92c70d2..f620c831a4b5 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -227,6 +227,66 @@ void idr_u32_test(int base) idr_u32_test1(&idr, 0xffffffff); } +static void idr_align_test(struct idr *idr) +{ + char name[] = "Motorola 68000"; + int i, id; + void *entry; + + for (i = 0; i < 9; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 1; i < 10; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 1); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 2; i < 11; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 2); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 3; i < 12; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 3); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0); + BUG_ON(idr_alloc(idr, &name[i + 1], 0, 0, GFP_KERNEL) != 1); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 1); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 0); + BUG_ON(!idr_is_empty(idr)); + } + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 0); + idr_for_each_entry(idr, entry, id); + idr_replace(idr, &name[i], 0); + idr_for_each_entry(idr, entry, id); + BUG_ON(idr_find(idr, 0) != &name[i]); + idr_remove(idr, 0); + } + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0); + BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 1); + idr_remove(idr, 1); + idr_for_each_entry(idr, entry, id); + idr_replace(idr, &name[i + 1], 0); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 0); + } +} + void idr_checks(void) { unsigned long i; @@ -307,6 +367,7 @@ void idr_checks(void) idr_u32_test(4); idr_u32_test(1); idr_u32_test(0); + idr_align_test(&idr); } #define module_init(x) @@ -341,6 +402,7 @@ void ida_check_nomem(void) */ void ida_check_conv_user(void) { +#if 0 DEFINE_IDA(ida); unsigned long i; @@ -358,6 +420,7 @@ void ida_check_conv_user(void) IDA_BUG_ON(&ida, id != i); } ida_destroy(&ida); +#endif } void ida_check_random(void) -- cgit v1.2.3 From 3159f943aafdbacb2f94c38fdaadabf2bbde2a14 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Fri, 3 Nov 2017 13:30:42 -0400 Subject: xarray: Replace exceptional entries Introduce xarray value entries and tagged pointers to replace radix tree exceptional entries. This is a slight change in encoding to allow the use of an extra bit (we can now store BITS_PER_LONG - 1 bits in a value entry). It is also a change in emphasis; exceptional entries are intimidating and different. As the comment explains, you can choose to store values or pointers in the xarray and they are both first-class citizens. Signed-off-by: Matthew Wilcox Reviewed-by: Josef Bacik --- arch/powerpc/include/asm/book3s/64/pgtable.h | 4 +- arch/powerpc/include/asm/nohash/64/pgtable.h | 4 +- drivers/gpu/drm/i915/i915_gem.c | 17 ++-- drivers/staging/erofs/utils.c | 18 ++--- fs/btrfs/compression.c | 2 +- fs/dax.c | 112 +++++++++++++-------------- fs/proc/task_mmu.c | 2 +- include/linux/radix-tree.h | 36 ++------- include/linux/swapops.h | 19 ++--- include/linux/xarray.h | 102 ++++++++++++++++++++++++ lib/idr.c | 60 ++++++-------- lib/radix-tree.c | 21 +++-- mm/filemap.c | 10 +-- mm/khugepaged.c | 2 +- mm/madvise.c | 2 +- mm/memcontrol.c | 2 +- mm/mincore.c | 2 +- mm/readahead.c | 2 +- mm/shmem.c | 10 +-- mm/swap.c | 2 +- mm/truncate.c | 12 +-- mm/workingset.c | 13 ++-- tools/testing/radix-tree/idr-test.c | 6 +- tools/testing/radix-tree/linux/radix-tree.h | 1 - tools/testing/radix-tree/multiorder.c | 47 ++++++----- tools/testing/radix-tree/test.c | 2 +- 26 files changed, 278 insertions(+), 232 deletions(-) (limited to 'tools') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 2fdc865ca374..62039e557ac0 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -723,9 +723,7 @@ static inline bool pte_user(pte_t pte) BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \ BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \ } while (0) -/* - * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT; - */ + #define SWP_TYPE_BITS 5 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \ & ((1UL << SWP_TYPE_BITS) - 1)) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 7cd6809f4d33..05765c2d2c1f 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -313,9 +313,7 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, #define MAX_SWAPFILES_CHECK() do { \ BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \ } while (0) -/* - * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT; - */ + #define SWP_TYPE_BITS 5 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \ & ((1UL << SWP_TYPE_BITS) - 1)) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index fcc73a6ab503..316730b45f84 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -5996,7 +5996,8 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, count = __sg_page_count(sg); while (idx + count <= n) { - unsigned long exception, i; + void *entry; + unsigned long i; int ret; /* If we cannot allocate and insert this entry, or the @@ -6011,12 +6012,9 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj, if (ret && ret != -EEXIST) goto scan; - exception = - RADIX_TREE_EXCEPTIONAL_ENTRY | - idx << RADIX_TREE_EXCEPTIONAL_SHIFT; + entry = xa_mk_value(idx); for (i = 1; i < count; i++) { - ret = radix_tree_insert(&iter->radix, idx + i, - (void *)exception); + ret = radix_tree_insert(&iter->radix, idx + i, entry); if (ret && ret != -EEXIST) goto scan; } @@ -6054,15 +6052,14 @@ lookup: GEM_BUG_ON(!sg); /* If this index is in the middle of multi-page sg entry, - * the radixtree will contain an exceptional entry that points + * the radix tree will contain a value entry that points * to the start of that range. We will return the pointer to * the base page and the offset of this page within the * sg entry's range. */ *offset = 0; - if (unlikely(radix_tree_exception(sg))) { - unsigned long base = - (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT; + if (unlikely(xa_is_value(sg))) { + unsigned long base = xa_to_value(sg); sg = radix_tree_lookup(&iter->radix, base); GEM_BUG_ON(!sg); diff --git a/drivers/staging/erofs/utils.c b/drivers/staging/erofs/utils.c index 595cf90af9bb..bdee9bd09f11 100644 --- a/drivers/staging/erofs/utils.c +++ b/drivers/staging/erofs/utils.c @@ -35,7 +35,6 @@ static atomic_long_t erofs_global_shrink_cnt; #ifdef CONFIG_EROFS_FS_ZIP -/* radix_tree and the future XArray both don't use tagptr_t yet */ struct erofs_workgroup *erofs_find_workgroup( struct super_block *sb, pgoff_t index, bool *tag) { @@ -47,9 +46,8 @@ repeat: rcu_read_lock(); grp = radix_tree_lookup(&sbi->workstn_tree, index); if (grp != NULL) { - *tag = radix_tree_exceptional_entry(grp); - grp = (void *)((unsigned long)grp & - ~RADIX_TREE_EXCEPTIONAL_ENTRY); + *tag = xa_pointer_tag(grp); + grp = xa_untag_pointer(grp); if (erofs_workgroup_get(grp, &oldcount)) { /* prefer to relax rcu read side */ @@ -83,9 +81,7 @@ int erofs_register_workgroup(struct super_block *sb, sbi = EROFS_SB(sb); erofs_workstn_lock(sbi); - if (tag) - grp = (void *)((unsigned long)grp | - 1UL << RADIX_TREE_EXCEPTIONAL_SHIFT); + grp = xa_tag_pointer(grp, tag); err = radix_tree_insert(&sbi->workstn_tree, grp->index, grp); @@ -131,9 +127,7 @@ repeat: for (i = 0; i < found; ++i) { int cnt; - struct erofs_workgroup *grp = (void *) - ((unsigned long)batch[i] & - ~RADIX_TREE_EXCEPTIONAL_ENTRY); + struct erofs_workgroup *grp = xa_untag_pointer(batch[i]); first_index = grp->index + 1; @@ -150,8 +144,8 @@ repeat: #endif continue; - if (radix_tree_delete(&sbi->workstn_tree, - grp->index) != grp) { + if (xa_untag_pointer(radix_tree_delete(&sbi->workstn_tree, + grp->index)) != grp) { #ifdef EROFS_FS_HAS_MANAGED_CACHE skip: erofs_workgroup_unfreeze(grp, 1); diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 9bfa66592aa7..fd25e125303c 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -440,7 +440,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, rcu_read_lock(); page = radix_tree_lookup(&mapping->i_pages, pg_index); rcu_read_unlock(); - if (page && !radix_tree_exceptional_entry(page)) { + if (page && !xa_is_value(page)) { misses++; if (misses > 4) break; diff --git a/fs/dax.c b/fs/dax.c index b68ce484e1be..ebcec36335eb 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -59,56 +59,57 @@ static int __init init_dax_wait_table(void) fs_initcall(init_dax_wait_table); /* - * We use lowest available bit in exceptional entry for locking, one bit for - * the entry size (PMD) and two more to tell us if the entry is a zero page or - * an empty entry that is just used for locking. In total four special bits. + * DAX pagecache entries use XArray value entries so they can't be mistaken + * for pages. We use one bit for locking, one bit for the entry size (PMD) + * and two more to tell us if the entry is a zero page or an empty entry that + * is just used for locking. In total four special bits. * * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem * block allocation. */ -#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4) -#define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) -#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) -#define RADIX_DAX_ZERO_PAGE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) -#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)) +#define DAX_SHIFT (4) +#define DAX_LOCKED (1UL << 0) +#define DAX_PMD (1UL << 1) +#define DAX_ZERO_PAGE (1UL << 2) +#define DAX_EMPTY (1UL << 3) static unsigned long dax_radix_pfn(void *entry) { - return (unsigned long)entry >> RADIX_DAX_SHIFT; + return xa_to_value(entry) >> DAX_SHIFT; } static void *dax_radix_locked_entry(unsigned long pfn, unsigned long flags) { - return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags | - (pfn << RADIX_DAX_SHIFT) | RADIX_DAX_ENTRY_LOCK); + return xa_mk_value(flags | ((unsigned long)pfn << DAX_SHIFT) | + DAX_LOCKED); } static unsigned int dax_radix_order(void *entry) { - if ((unsigned long)entry & RADIX_DAX_PMD) + if (xa_to_value(entry) & DAX_PMD) return PMD_SHIFT - PAGE_SHIFT; return 0; } static int dax_is_pmd_entry(void *entry) { - return (unsigned long)entry & RADIX_DAX_PMD; + return xa_to_value(entry) & DAX_PMD; } static int dax_is_pte_entry(void *entry) { - return !((unsigned long)entry & RADIX_DAX_PMD); + return !(xa_to_value(entry) & DAX_PMD); } static int dax_is_zero_entry(void *entry) { - return (unsigned long)entry & RADIX_DAX_ZERO_PAGE; + return xa_to_value(entry) & DAX_ZERO_PAGE; } static int dax_is_empty_entry(void *entry) { - return (unsigned long)entry & RADIX_DAX_EMPTY; + return xa_to_value(entry) & DAX_EMPTY; } /* @@ -186,9 +187,9 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, */ static inline int slot_locked(struct address_space *mapping, void **slot) { - unsigned long entry = (unsigned long) - radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - return entry & RADIX_DAX_ENTRY_LOCK; + unsigned long entry = xa_to_value( + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock)); + return entry & DAX_LOCKED; } /* @@ -196,12 +197,11 @@ static inline int slot_locked(struct address_space *mapping, void **slot) */ static inline void *lock_slot(struct address_space *mapping, void **slot) { - unsigned long entry = (unsigned long) - radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - - entry |= RADIX_DAX_ENTRY_LOCK; - radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); - return (void *)entry; + unsigned long v = xa_to_value( + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock)); + void *entry = xa_mk_value(v | DAX_LOCKED); + radix_tree_replace_slot(&mapping->i_pages, slot, entry); + return entry; } /* @@ -209,17 +209,16 @@ static inline void *lock_slot(struct address_space *mapping, void **slot) */ static inline void *unlock_slot(struct address_space *mapping, void **slot) { - unsigned long entry = (unsigned long) - radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - - entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; - radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); - return (void *)entry; + unsigned long v = xa_to_value( + radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock)); + void *entry = xa_mk_value(v & ~DAX_LOCKED); + radix_tree_replace_slot(&mapping->i_pages, slot, entry); + return entry; } /* * Lookup entry in radix tree, wait for it to become unlocked if it is - * exceptional entry and return it. The caller must call + * a DAX entry and return it. The caller must call * put_unlocked_mapping_entry() when he decided not to lock the entry or * put_locked_mapping_entry() when he locked the entry and now wants to * unlock it. @@ -242,7 +241,7 @@ static void *__get_unlocked_mapping_entry(struct address_space *mapping, entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); if (!entry || - WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || + WARN_ON_ONCE(!xa_is_value(entry)) || !slot_locked(mapping, slot)) { if (slotp) *slotp = slot; @@ -283,7 +282,7 @@ static void unlock_mapping_entry(struct address_space *mapping, pgoff_t index) xa_lock_irq(&mapping->i_pages); entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); - if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || + if (WARN_ON_ONCE(!entry || !xa_is_value(entry) || !slot_locked(mapping, slot))) { xa_unlock_irq(&mapping->i_pages); return; @@ -472,12 +471,11 @@ void dax_unlock_mapping_entry(struct page *page) } /* - * Find radix tree entry at given index. If it points to an exceptional entry, - * return it with the radix tree entry locked. If the radix tree doesn't - * contain given index, create an empty exceptional entry for the index and - * return with it locked. + * Find radix tree entry at given index. If it is a DAX entry, return it + * with the radix tree entry locked. If the radix tree doesn't contain the + * given index, create an empty entry for the index and return with it locked. * - * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will + * When requesting an entry with size DAX_PMD, grab_mapping_entry() will * either return that locked entry or will return an error. This error will * happen if there are any 4k entries within the 2MiB range that we are * requesting. @@ -507,13 +505,13 @@ restart: xa_lock_irq(&mapping->i_pages); entry = get_unlocked_mapping_entry(mapping, index, &slot); - if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { + if (WARN_ON_ONCE(entry && !xa_is_value(entry))) { entry = ERR_PTR(-EIO); goto out_unlock; } if (entry) { - if (size_flag & RADIX_DAX_PMD) { + if (size_flag & DAX_PMD) { if (dax_is_pte_entry(entry)) { put_unlocked_mapping_entry(mapping, index, entry); @@ -584,7 +582,7 @@ restart: true); } - entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); + entry = dax_radix_locked_entry(0, size_flag | DAX_EMPTY); err = __radix_tree_insert(&mapping->i_pages, index, dax_radix_order(entry), entry); @@ -673,8 +671,7 @@ struct page *dax_layout_busy_page(struct address_space *mapping) if (index >= end) break; - if (WARN_ON_ONCE( - !radix_tree_exceptional_entry(pvec_ent))) + if (WARN_ON_ONCE(!xa_is_value(pvec_ent))) continue; xa_lock_irq(&mapping->i_pages); @@ -713,7 +710,7 @@ static int __dax_invalidate_mapping_entry(struct address_space *mapping, xa_lock_irq(pages); entry = get_unlocked_mapping_entry(mapping, index, NULL); - if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) + if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) goto out; if (!trunc && (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) || @@ -729,8 +726,8 @@ out: return ret; } /* - * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree - * entry to get unlocked before deleting it. + * Delete DAX entry at @index from @mapping. Wait for it + * to be unlocked before deleting it. */ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) { @@ -740,7 +737,7 @@ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) * This gets called from truncate / punch_hole path. As such, the caller * must hold locks protecting against concurrent modifications of the * radix tree (usually fs-private i_mmap_sem for writing). Since the - * caller has seen exceptional entry for this index, we better find it + * caller has seen a DAX entry for this index, we better find it * at that index as well... */ WARN_ON_ONCE(!ret); @@ -748,7 +745,7 @@ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) } /* - * Invalidate exceptional DAX entry if it is clean. + * Invalidate DAX entry if it is clean. */ int dax_invalidate_mapping_entry_sync(struct address_space *mapping, pgoff_t index) @@ -802,7 +799,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, if (dirty) __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); - if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) { + if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) { /* we are replacing a zero page with block mapping */ if (dax_is_pmd_entry(entry)) unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR, @@ -940,13 +937,13 @@ static int dax_writeback_one(struct dax_device *dax_dev, * A page got tagged dirty in DAX mapping? Something is seriously * wrong. */ - if (WARN_ON(!radix_tree_exceptional_entry(entry))) + if (WARN_ON(!xa_is_value(entry))) return -EIO; xa_lock_irq(pages); entry2 = get_unlocked_mapping_entry(mapping, index, &slot); /* Entry got punched out / reallocated? */ - if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) + if (!entry2 || WARN_ON_ONCE(!xa_is_value(entry2))) goto put_unlocked; /* * Entry got reallocated elsewhere? No need to writeback. We have to @@ -1123,8 +1120,9 @@ static vm_fault_t dax_load_hole(struct address_space *mapping, void *entry, pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr)); vm_fault_t ret; - dax_insert_mapping_entry(mapping, vmf, entry, pfn, RADIX_DAX_ZERO_PAGE, - false); + dax_insert_mapping_entry(mapping, vmf, entry, pfn, + DAX_ZERO_PAGE, false); + ret = vmf_insert_mixed(vmf->vma, vaddr, pfn); trace_dax_load_hole(inode, vmf, ret); return ret; @@ -1514,7 +1512,7 @@ static vm_fault_t dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap, pfn = page_to_pfn_t(zero_page); ret = dax_insert_mapping_entry(mapping, vmf, entry, pfn, - RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false); + DAX_PMD | DAX_ZERO_PAGE, false); ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); if (!pmd_none(*(vmf->pmd))) { @@ -1597,7 +1595,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, * is already in the tree, for instance), it will return -EEXIST and * we just fall back to 4k entries. */ - entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); + entry = grab_mapping_entry(mapping, pgoff, DAX_PMD); if (IS_ERR(entry)) goto fallback; @@ -1635,7 +1633,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp, goto finish_iomap; entry = dax_insert_mapping_entry(mapping, vmf, entry, pfn, - RADIX_DAX_PMD, write && !sync); + DAX_PMD, write && !sync); /* * If we are doing synchronous page fault and inode needs fsync, diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5ea1d64cb0b4..669abb617321 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -521,7 +521,7 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) mss->swap += PAGE_SIZE; else put_page(page); diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 34149e8b5f73..e9e76ab4fbbf 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -28,34 +28,26 @@ #include #include #include +#include /* * The bottom two bits of the slot determine how the remaining bits in the * slot are interpreted: * * 00 - data pointer - * 01 - internal entry - * 10 - exceptional entry - * 11 - this bit combination is currently unused/reserved + * 10 - internal entry + * x1 - value entry * * The internal entry may be a pointer to the next level in the tree, a * sibling entry, or an indicator that the entry in this slot has been moved * to another location in the tree and the lookup should be restarted. While * NULL fits the 'data pointer' pattern, it means that there is no entry in * the tree for this index (no matter what level of the tree it is found at). - * This means that you cannot store NULL in the tree as a value for the index. + * This means that storing a NULL entry in the tree is the same as deleting + * the entry from the tree. */ #define RADIX_TREE_ENTRY_MASK 3UL -#define RADIX_TREE_INTERNAL_NODE 1UL - -/* - * Most users of the radix tree store pointers but shmem/tmpfs stores swap - * entries in the same tree. They are marked as exceptional entries to - * distinguish them from pointers to struct page. - * EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it. - */ -#define RADIX_TREE_EXCEPTIONAL_ENTRY 2 -#define RADIX_TREE_EXCEPTIONAL_SHIFT 2 +#define RADIX_TREE_INTERNAL_NODE 2UL static inline bool radix_tree_is_internal_node(void *ptr) { @@ -83,11 +75,10 @@ static inline bool radix_tree_is_internal_node(void *ptr) /* * @count is the count of every non-NULL element in the ->slots array - * whether that is an exceptional entry, a retry entry, a user pointer, + * whether that is a value entry, a retry entry, a user pointer, * a sibling entry or a pointer to the next level of the tree. * @exceptional is the count of every element in ->slots which is - * either radix_tree_exceptional_entry() or is a sibling entry for an - * exceptional entry. + * either a value entry or a sibling of a value entry. */ struct radix_tree_node { unsigned char shift; /* Bits remaining in each slot */ @@ -268,17 +259,6 @@ static inline int radix_tree_deref_retry(void *arg) return unlikely(radix_tree_is_internal_node(arg)); } -/** - * radix_tree_exceptional_entry - radix_tree_deref_slot gave exceptional entry? - * @arg: value returned by radix_tree_deref_slot - * Returns: 0 if well-aligned pointer, non-0 if exceptional entry. - */ -static inline int radix_tree_exceptional_entry(void *arg) -{ - /* Not unlikely because radix_tree_exception often tested first */ - return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY; -} - /** * radix_tree_exception - radix_tree_deref_slot returned either exception? * @arg: value returned by radix_tree_deref_slot diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 22af9d8a84ae..4d961668e5fc 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -18,9 +18,8 @@ * * swp_entry_t's are *never* stored anywhere in their arch-dependent format. */ -#define SWP_TYPE_SHIFT(e) ((sizeof(e.val) * 8) - \ - (MAX_SWAPFILES_SHIFT + RADIX_TREE_EXCEPTIONAL_SHIFT)) -#define SWP_OFFSET_MASK(e) ((1UL << SWP_TYPE_SHIFT(e)) - 1) +#define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) +#define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) /* * Store a type+offset into a swp_entry_t in an arch-independent format @@ -29,8 +28,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) { swp_entry_t ret; - ret.val = (type << SWP_TYPE_SHIFT(ret)) | - (offset & SWP_OFFSET_MASK(ret)); + ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); return ret; } @@ -40,7 +38,7 @@ static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) */ static inline unsigned swp_type(swp_entry_t entry) { - return (entry.val >> SWP_TYPE_SHIFT(entry)); + return (entry.val >> SWP_TYPE_SHIFT); } /* @@ -49,7 +47,7 @@ static inline unsigned swp_type(swp_entry_t entry) */ static inline pgoff_t swp_offset(swp_entry_t entry) { - return entry.val & SWP_OFFSET_MASK(entry); + return entry.val & SWP_OFFSET_MASK; } #ifdef CONFIG_MMU @@ -90,16 +88,13 @@ static inline swp_entry_t radix_to_swp_entry(void *arg) { swp_entry_t entry; - entry.val = (unsigned long)arg >> RADIX_TREE_EXCEPTIONAL_SHIFT; + entry.val = xa_to_value(arg); return entry; } static inline void *swp_to_radix_entry(swp_entry_t entry) { - unsigned long value; - - value = entry.val << RADIX_TREE_EXCEPTIONAL_SHIFT; - return (void *)(value | RADIX_TREE_EXCEPTIONAL_ENTRY); + return xa_mk_value(entry.val); } #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 9e4c86853fa4..5c8acfc4ff55 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -5,9 +5,111 @@ * eXtensible Arrays * Copyright (c) 2017 Microsoft Corporation * Author: Matthew Wilcox + * + * See Documentation/core-api/xarray.rst for how to use the XArray. */ +#include #include +#include + +/* + * The bottom two bits of the entry determine how the XArray interprets + * the contents: + * + * 00: Pointer entry + * 10: Internal entry + * x1: Value entry or tagged pointer + * + * Attempting to store internal entries in the XArray is a bug. + */ + +#define BITS_PER_XA_VALUE (BITS_PER_LONG - 1) + +/** + * xa_mk_value() - Create an XArray entry from an integer. + * @v: Value to store in XArray. + * + * Context: Any context. + * Return: An entry suitable for storing in the XArray. + */ +static inline void *xa_mk_value(unsigned long v) +{ + WARN_ON((long)v < 0); + return (void *)((v << 1) | 1); +} + +/** + * xa_to_value() - Get value stored in an XArray entry. + * @entry: XArray entry. + * + * Context: Any context. + * Return: The value stored in the XArray entry. + */ +static inline unsigned long xa_to_value(const void *entry) +{ + return (unsigned long)entry >> 1; +} + +/** + * xa_is_value() - Determine if an entry is a value. + * @entry: XArray entry. + * + * Context: Any context. + * Return: True if the entry is a value, false if it is a pointer. + */ +static inline bool xa_is_value(const void *entry) +{ + return (unsigned long)entry & 1; +} + +/** + * xa_tag_pointer() - Create an XArray entry for a tagged pointer. + * @p: Plain pointer. + * @tag: Tag value (0, 1 or 3). + * + * If the user of the XArray prefers, they can tag their pointers instead + * of storing value entries. Three tags are available (0, 1 and 3). + * These are distinct from the xa_mark_t as they are not replicated up + * through the array and cannot be searched for. + * + * Context: Any context. + * Return: An XArray entry. + */ +static inline void *xa_tag_pointer(void *p, unsigned long tag) +{ + return (void *)((unsigned long)p | tag); +} + +/** + * xa_untag_pointer() - Turn an XArray entry into a plain pointer. + * @entry: XArray entry. + * + * If you have stored a tagged pointer in the XArray, call this function + * to get the untagged version of the pointer. + * + * Context: Any context. + * Return: A pointer. + */ +static inline void *xa_untag_pointer(void *entry) +{ + return (void *)((unsigned long)entry & ~3UL); +} + +/** + * xa_pointer_tag() - Get the tag stored in an XArray entry. + * @entry: XArray entry. + * + * If you have stored a tagged pointer in the XArray, call this function + * to get the tag of that pointer. + * + * Context: Any context. + * Return: A tag. + */ +static inline unsigned int xa_pointer_tag(void *entry) +{ + return (unsigned long)entry & 3UL; +} #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) diff --git a/lib/idr.c b/lib/idr.c index 729e381e23b4..88419fbc5737 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -338,11 +338,8 @@ EXPORT_SYMBOL(idr_replace); * by the number of bits in the leaf bitmap before doing a radix tree lookup. * * As an optimisation, if there are only a few low bits set in any given - * leaf, instead of allocating a 128-byte bitmap, we use the 'exceptional - * entry' functionality of the radix tree to store BITS_PER_LONG - 2 bits - * directly in the entry. By being really tricksy, we could store - * BITS_PER_LONG - 1 bits, but there're diminishing returns after optimising - * for 0-3 allocated IDs. + * leaf, instead of allocating a 128-byte bitmap, we store the bits + * directly in the entry. * * We allow the radix tree 'exceptional' count to get out of date. Nothing * in the IDA nor the radix tree code checks it. If it becomes important @@ -366,12 +363,11 @@ static int ida_get_new_above(struct ida *ida, int start) struct radix_tree_iter iter; struct ida_bitmap *bitmap; unsigned long index; - unsigned bit, ebit; + unsigned bit; int new; index = start / IDA_BITMAP_BITS; bit = start % IDA_BITMAP_BITS; - ebit = bit + RADIX_TREE_EXCEPTIONAL_SHIFT; slot = radix_tree_iter_init(&iter, index); for (;;) { @@ -386,25 +382,23 @@ static int ida_get_new_above(struct ida *ida, int start) return PTR_ERR(slot); } } - if (iter.index > index) { + if (iter.index > index) bit = 0; - ebit = RADIX_TREE_EXCEPTIONAL_SHIFT; - } new = iter.index * IDA_BITMAP_BITS; bitmap = rcu_dereference_raw(*slot); - if (radix_tree_exception(bitmap)) { - unsigned long tmp = (unsigned long)bitmap; - ebit = find_next_zero_bit(&tmp, BITS_PER_LONG, ebit); - if (ebit < BITS_PER_LONG) { - tmp |= 1UL << ebit; - rcu_assign_pointer(*slot, (void *)tmp); - return new + ebit - - RADIX_TREE_EXCEPTIONAL_SHIFT; + if (xa_is_value(bitmap)) { + unsigned long tmp = xa_to_value(bitmap); + int vbit = find_next_zero_bit(&tmp, BITS_PER_XA_VALUE, + bit); + if (vbit < BITS_PER_XA_VALUE) { + tmp |= 1UL << vbit; + rcu_assign_pointer(*slot, xa_mk_value(tmp)); + return new + vbit; } bitmap = this_cpu_xchg(ida_bitmap, NULL); if (!bitmap) return -EAGAIN; - bitmap->bitmap[0] = tmp >> RADIX_TREE_EXCEPTIONAL_SHIFT; + bitmap->bitmap[0] = tmp; rcu_assign_pointer(*slot, bitmap); } @@ -425,17 +419,14 @@ static int ida_get_new_above(struct ida *ida, int start) new += bit; if (new < 0) return -ENOSPC; - if (ebit < BITS_PER_LONG) { - bitmap = (void *)((1UL << ebit) | - RADIX_TREE_EXCEPTIONAL_ENTRY); - radix_tree_iter_replace(root, &iter, slot, - bitmap); - return new; + if (bit < BITS_PER_XA_VALUE) { + bitmap = xa_mk_value(1UL << bit); + } else { + bitmap = this_cpu_xchg(ida_bitmap, NULL); + if (!bitmap) + return -EAGAIN; + __set_bit(bit, bitmap->bitmap); } - bitmap = this_cpu_xchg(ida_bitmap, NULL); - if (!bitmap) - return -EAGAIN; - __set_bit(bit, bitmap->bitmap); radix_tree_iter_replace(root, &iter, slot, bitmap); } @@ -457,9 +448,9 @@ static void ida_remove(struct ida *ida, int id) goto err; bitmap = rcu_dereference_raw(*slot); - if (radix_tree_exception(bitmap)) { + if (xa_is_value(bitmap)) { btmp = (unsigned long *)slot; - offset += RADIX_TREE_EXCEPTIONAL_SHIFT; + offset += 1; /* Intimate knowledge of the value encoding */ if (offset >= BITS_PER_LONG) goto err; } else { @@ -470,9 +461,8 @@ static void ida_remove(struct ida *ida, int id) __clear_bit(offset, btmp); radix_tree_iter_tag_set(&ida->ida_rt, &iter, IDR_FREE); - if (radix_tree_exception(bitmap)) { - if (rcu_dereference_raw(*slot) == - (void *)RADIX_TREE_EXCEPTIONAL_ENTRY) + if (xa_is_value(bitmap)) { + if (xa_to_value(rcu_dereference_raw(*slot)) == 0) radix_tree_iter_delete(&ida->ida_rt, &iter, slot); } else if (bitmap_empty(btmp, IDA_BITMAP_BITS)) { kfree(bitmap); @@ -503,7 +493,7 @@ void ida_destroy(struct ida *ida) xa_lock_irqsave(&ida->ida_rt, flags); radix_tree_for_each_slot(slot, &ida->ida_rt, &iter, 0) { struct ida_bitmap *bitmap = rcu_dereference_raw(*slot); - if (!radix_tree_exception(bitmap)) + if (!xa_is_value(bitmap)) kfree(bitmap); radix_tree_iter_delete(&ida->ida_rt, &iter, slot); } diff --git a/lib/radix-tree.c b/lib/radix-tree.c index a904a8ddd174..b6c0e7f3a894 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -340,14 +340,12 @@ static void dump_ida_node(void *entry, unsigned long index) for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) dump_ida_node(node->slots[i], index | (i << node->shift)); - } else if (radix_tree_exceptional_entry(entry)) { + } else if (xa_is_value(entry)) { pr_debug("ida excp: %p offset %d indices %lu-%lu data %lx\n", entry, (int)(index & RADIX_TREE_MAP_MASK), index * IDA_BITMAP_BITS, - index * IDA_BITMAP_BITS + BITS_PER_LONG - - RADIX_TREE_EXCEPTIONAL_SHIFT, - (unsigned long)entry >> - RADIX_TREE_EXCEPTIONAL_SHIFT); + index * IDA_BITMAP_BITS + BITS_PER_XA_VALUE, + xa_to_value(entry)); } else { struct ida_bitmap *bitmap = entry; @@ -656,7 +654,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, BUG_ON(shift > BITS_PER_LONG); if (radix_tree_is_internal_node(entry)) { entry_to_node(entry)->parent = node; - } else if (radix_tree_exceptional_entry(entry)) { + } else if (xa_is_value(entry)) { /* Moving an exceptional root->rnode to a node */ node->exceptional = 1; } @@ -955,12 +953,12 @@ static inline int insert_entries(struct radix_tree_node *node, !is_sibling_entry(node, old) && (old != RADIX_TREE_RETRY)) radix_tree_free_nodes(old); - if (radix_tree_exceptional_entry(old)) + if (xa_is_value(old)) node->exceptional--; } if (node) { node->count += n; - if (radix_tree_exceptional_entry(item)) + if (xa_is_value(item)) node->exceptional += n; } return n; @@ -974,7 +972,7 @@ static inline int insert_entries(struct radix_tree_node *node, rcu_assign_pointer(*slot, item); if (node) { node->count++; - if (radix_tree_exceptional_entry(item)) + if (xa_is_value(item)) node->exceptional++; } return 1; @@ -1190,8 +1188,7 @@ void __radix_tree_replace(struct radix_tree_root *root, radix_tree_update_node_t update_node) { void *old = rcu_dereference_raw(*slot); - int exceptional = !!radix_tree_exceptional_entry(item) - - !!radix_tree_exceptional_entry(old); + int exceptional = !!xa_is_value(item) - !!xa_is_value(old); int count = calculate_count(root, node, slot, item, old); /* @@ -1992,7 +1989,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, struct radix_tree_node *node, void __rcu **slot) { void *old = rcu_dereference_raw(*slot); - int exceptional = radix_tree_exceptional_entry(old) ? -1 : 0; + int exceptional = xa_is_value(old) ? -1 : 0; unsigned offset = get_slot_offset(node, slot); int tag; diff --git a/mm/filemap.c b/mm/filemap.c index 52517f28e6f4..4de14e75c4ec 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -127,7 +127,7 @@ static int page_cache_tree_insert(struct address_space *mapping, p = radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - if (!radix_tree_exceptional_entry(p)) + if (!xa_is_value(p)) return -EEXIST; mapping->nrexceptional--; @@ -336,7 +336,7 @@ page_cache_tree_delete_batch(struct address_space *mapping, break; page = radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) continue; if (!tail_pages) { /* @@ -1355,7 +1355,7 @@ pgoff_t page_cache_next_hole(struct address_space *mapping, struct page *page; page = radix_tree_lookup(&mapping->i_pages, index); - if (!page || radix_tree_exceptional_entry(page)) + if (!page || xa_is_value(page)) break; index++; if (index == 0) @@ -1396,7 +1396,7 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping, struct page *page; page = radix_tree_lookup(&mapping->i_pages, index); - if (!page || radix_tree_exceptional_entry(page)) + if (!page || xa_is_value(page)) break; index--; if (index == ULONG_MAX) @@ -1539,7 +1539,7 @@ struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset, repeat: page = find_get_entry(mapping, offset); - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) page = NULL; if (!page) goto no_page; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index a31d740e6cd1..4820e4adf853 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1369,7 +1369,7 @@ static void collapse_shmem(struct mm_struct *mm, page = radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); - if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { + if (xa_is_value(page) || !PageUptodate(page)) { xa_unlock_irq(&mapping->i_pages); /* swap in or instantiate fallocated page */ if (shmem_getpage(mapping->host, index, &page, diff --git a/mm/madvise.c b/mm/madvise.c index 972a9eaa898b..9d802566c494 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -251,7 +251,7 @@ static void force_shm_swapin_readahead(struct vm_area_struct *vma, index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; page = find_get_entry(mapping, index); - if (!radix_tree_exceptional_entry(page)) { + if (!xa_is_value(page)) { if (page) put_page(page); continue; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e79cb59552d9..29d9d1a69b36 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -4750,7 +4750,7 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma, /* shmem/tmpfs may report page out on swap: account for that too. */ if (shmem_mapping(mapping)) { page = find_get_entry(mapping, pgoff); - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { swp_entry_t swp = radix_to_swp_entry(page); if (do_memsw_account()) *entry = swp; diff --git a/mm/mincore.c b/mm/mincore.c index fc37afe226e6..4985965aa20a 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -66,7 +66,7 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff) * shmem/tmpfs may return swap: account for swapcache * page too. */ - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { swp_entry_t swp = radix_to_swp_entry(page); page = find_get_page(swap_address_space(swp), swp_offset(swp)); diff --git a/mm/readahead.c b/mm/readahead.c index 4e630143a0ba..de657077d41d 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -179,7 +179,7 @@ unsigned int __do_page_cache_readahead(struct address_space *mapping, rcu_read_lock(); page = radix_tree_lookup(&mapping->i_pages, page_offset); rcu_read_unlock(); - if (page && !radix_tree_exceptional_entry(page)) { + if (page && !xa_is_value(page)) { /* * Page already present? Kick off the current batch of * contiguous pages before continuing with the next diff --git a/mm/shmem.c b/mm/shmem.c index 446942677cd4..c1062760fe41 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -709,7 +709,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, continue; } - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) swapped++; if (need_resched()) { @@ -824,7 +824,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (index >= end) break; - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { if (unfalloc) continue; nr_swaps_freed += !shmem_free_swap(mapping, @@ -921,7 +921,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, if (index >= end) break; - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { if (unfalloc) continue; if (shmem_free_swap(mapping, index, page)) { @@ -1643,7 +1643,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, repeat: swap.val = 0; page = find_lock_entry(mapping, index); - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { swap = radix_to_swp_entry(page); page = NULL; } @@ -2578,7 +2578,7 @@ static pgoff_t shmem_seek_hole_data(struct address_space *mapping, index = indices[i]; } page = pvec.pages[i]; - if (page && !radix_tree_exceptional_entry(page)) { + if (page && !xa_is_value(page)) { if (!PageUptodate(page)) page = NULL; } diff --git a/mm/swap.c b/mm/swap.c index 26fc9b5f1b6c..4c5c7fcc6e46 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -965,7 +965,7 @@ void pagevec_remove_exceptionals(struct pagevec *pvec) for (i = 0, j = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; - if (!radix_tree_exceptional_entry(page)) + if (!xa_is_value(page)) pvec->pages[j++] = page; } pvec->nr = j; diff --git a/mm/truncate.c b/mm/truncate.c index 1d2fb2dca96f..ed778555c9f3 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -70,7 +70,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, return; for (j = 0; j < pagevec_count(pvec); j++) - if (radix_tree_exceptional_entry(pvec->pages[j])) + if (xa_is_value(pvec->pages[j])) break; if (j == pagevec_count(pvec)) @@ -85,7 +85,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, struct page *page = pvec->pages[i]; pgoff_t index = indices[i]; - if (!radix_tree_exceptional_entry(page)) { + if (!xa_is_value(page)) { pvec->pages[j++] = page; continue; } @@ -347,7 +347,7 @@ void truncate_inode_pages_range(struct address_space *mapping, if (index >= end) break; - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) continue; if (!trylock_page(page)) @@ -442,7 +442,7 @@ void truncate_inode_pages_range(struct address_space *mapping, break; } - if (radix_tree_exceptional_entry(page)) + if (xa_is_value(page)) continue; lock_page(page); @@ -561,7 +561,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, if (index > end) break; - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { invalidate_exceptional_entry(mapping, index, page); continue; @@ -692,7 +692,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping, if (index > end) break; - if (radix_tree_exceptional_entry(page)) { + if (xa_is_value(page)) { if (!invalidate_exceptional_entry2(mapping, index, page)) ret = -EBUSY; diff --git a/mm/workingset.c b/mm/workingset.c index 4516dd790129..bb109b3afac2 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -155,8 +155,8 @@ * refault distance will immediately activate the refaulting page. */ -#define EVICTION_SHIFT (RADIX_TREE_EXCEPTIONAL_ENTRY + \ - NODES_SHIFT + \ +#define EVICTION_SHIFT ((BITS_PER_LONG - BITS_PER_XA_VALUE) + \ + NODES_SHIFT + \ MEM_CGROUP_ID_SHIFT) #define EVICTION_MASK (~0UL >> EVICTION_SHIFT) @@ -173,20 +173,19 @@ static unsigned int bucket_order __read_mostly; static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction) { eviction >>= bucket_order; + eviction &= EVICTION_MASK; eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; eviction = (eviction << NODES_SHIFT) | pgdat->node_id; - eviction = (eviction << RADIX_TREE_EXCEPTIONAL_SHIFT); - return (void *)(eviction | RADIX_TREE_EXCEPTIONAL_ENTRY); + return xa_mk_value(eviction); } static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, unsigned long *evictionp) { - unsigned long entry = (unsigned long)shadow; + unsigned long entry = xa_to_value(shadow); int memcgid, nid; - entry >>= RADIX_TREE_EXCEPTIONAL_SHIFT; nid = entry & ((1UL << NODES_SHIFT) - 1); entry >>= NODES_SHIFT; memcgid = entry & ((1UL << MEM_CGROUP_ID_SHIFT) - 1); @@ -453,7 +452,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, goto out_invalid; for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) { if (node->slots[i]) { - if (WARN_ON_ONCE(!radix_tree_exceptional_entry(node->slots[i]))) + if (WARN_ON_ONCE(!xa_is_value(node->slots[i]))) goto out_invalid; if (WARN_ON_ONCE(!node->exceptional)) goto out_invalid; diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index f620c831a4b5..a5a4494922a6 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -19,7 +19,7 @@ #include "test.h" -#define DUMMY_PTR ((void *)0x12) +#define DUMMY_PTR ((void *)0x10) int item_idr_free(int id, void *p, void *data) { @@ -411,11 +411,11 @@ void ida_check_conv_user(void) int id = ida_alloc(&ida, GFP_NOWAIT); if (id == -ENOMEM) { IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) != - BITS_PER_LONG - 2); + BITS_PER_XA_VALUE); id = ida_alloc(&ida, GFP_KERNEL); } else { IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) == - BITS_PER_LONG - 2); + BITS_PER_XA_VALUE); } IDA_BUG_ON(&ida, id != i); } diff --git a/tools/testing/radix-tree/linux/radix-tree.h b/tools/testing/radix-tree/linux/radix-tree.h index 24f13d27a8da..d1635a5bef02 100644 --- a/tools/testing/radix-tree/linux/radix-tree.h +++ b/tools/testing/radix-tree/linux/radix-tree.h @@ -2,7 +2,6 @@ #ifndef _TEST_RADIX_TREE_H #define _TEST_RADIX_TREE_H -#include "generated/map-shift.h" #include "../../../../include/linux/radix-tree.h" extern int kmalloc_verbose; diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 7bf405638b0b..2b4f4dba1882 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -39,12 +39,11 @@ static void __multiorder_tag_test(int index, int order) /* * Verify we get collisions for covered indices. We try and fail to - * insert an exceptional entry so we don't leak memory via + * insert a value entry so we don't leak memory via * item_insert_order(). */ for_each_index(i, base, order) { - err = __radix_tree_insert(&tree, i, order, - (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY)); + err = __radix_tree_insert(&tree, i, order, xa_mk_value(0xA0)); assert(err == -EEXIST); } @@ -380,8 +379,8 @@ static void multiorder_join1(unsigned long index, } /* - * Check that the accounting of exceptional entries is handled correctly - * by joining an exceptional entry to a normal pointer. + * Check that the accounting of value entries is handled correctly + * by joining a value entry to a normal pointer. */ static void multiorder_join2(unsigned order1, unsigned order2) { @@ -391,9 +390,9 @@ static void multiorder_join2(unsigned order1, unsigned order2) void *item2; item_insert_order(&tree, 0, order2); - radix_tree_insert(&tree, 1 << order2, (void *)0x12UL); + radix_tree_insert(&tree, 1 << order2, xa_mk_value(5)); item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); - assert(item2 == (void *)0x12UL); + assert(item2 == xa_mk_value(5)); assert(node->exceptional == 1); item2 = radix_tree_lookup(&tree, 0); @@ -407,7 +406,7 @@ static void multiorder_join2(unsigned order1, unsigned order2) } /* - * This test revealed an accounting bug for exceptional entries at one point. + * This test revealed an accounting bug for value entries at one point. * Nodes were being freed back into the pool with an elevated exception count * by radix_tree_join() and then radix_tree_split() was failing to zero the * count of exceptional entries. @@ -421,16 +420,16 @@ static void multiorder_join3(unsigned int order) unsigned long i; for (i = 0; i < (1 << order); i++) { - radix_tree_insert(&tree, i, (void *)0x12UL); + radix_tree_insert(&tree, i, xa_mk_value(5)); } - radix_tree_join(&tree, 0, order, (void *)0x16UL); + radix_tree_join(&tree, 0, order, xa_mk_value(7)); rcu_barrier(); radix_tree_split(&tree, 0, 0); radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, (void *)0x12UL); + radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(5)); } __radix_tree_lookup(&tree, 0, &node, NULL); @@ -517,10 +516,10 @@ static void __multiorder_split2(int old_order, int new_order) struct radix_tree_node *node; void *item; - __radix_tree_insert(&tree, 0, old_order, (void *)0x12); + __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == (void *)0x12); + assert(item == xa_mk_value(5)); assert(node->exceptional > 0); radix_tree_split(&tree, 0, new_order); @@ -530,7 +529,7 @@ static void __multiorder_split2(int old_order, int new_order) } item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item != (void *)0x12); + assert(item != xa_mk_value(5)); assert(node->exceptional == 0); item_kill_tree(&tree); @@ -544,40 +543,40 @@ static void __multiorder_split3(int old_order, int new_order) struct radix_tree_node *node; void *item; - __radix_tree_insert(&tree, 0, old_order, (void *)0x12); + __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == (void *)0x12); + assert(item == xa_mk_value(5)); assert(node->exceptional > 0); radix_tree_split(&tree, 0, new_order); radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16); + radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(7)); } item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == (void *)0x16); + assert(item == xa_mk_value(7)); assert(node->exceptional > 0); item_kill_tree(&tree); - __radix_tree_insert(&tree, 0, old_order, (void *)0x12); + __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == (void *)0x12); + assert(item == xa_mk_value(5)); assert(node->exceptional > 0); radix_tree_split(&tree, 0, new_order); radix_tree_for_each_slot(slot, &tree, &iter, 0) { if (iter.index == (1 << new_order)) radix_tree_iter_replace(&tree, &iter, slot, - (void *)0x16); + xa_mk_value(7)); else radix_tree_iter_replace(&tree, &iter, slot, NULL); } item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); - assert(item == (void *)0x16); + assert(item == xa_mk_value(7)); assert(node->count == node->exceptional); do { node = node->parent; @@ -610,13 +609,13 @@ static void multiorder_account(void) item_insert_order(&tree, 0, 5); - __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); + __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); __radix_tree_lookup(&tree, 0, &node, NULL); assert(node->count == node->exceptional * 2); radix_tree_delete(&tree, 1 << 5); assert(node->exceptional == 0); - __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); + __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); __radix_tree_lookup(&tree, 1 << 5, &node, &slot); assert(node->count == node->exceptional * 2); __radix_tree_replace(&tree, node, slot, NULL, NULL); diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index def6015570b2..62de66c314b7 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -295,7 +295,7 @@ void item_kill_tree(struct radix_tree_root *root) int nfound; radix_tree_for_each_slot(slot, root, &iter, 0) { - if (radix_tree_exceptional_entry(*slot)) + if (xa_is_value(*slot)) radix_tree_delete(root, iter.index); } -- cgit v1.2.3 From 02c02bf12c5d838603eed44195d3e91f094e2ab2 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Fri, 3 Nov 2017 23:09:45 -0400 Subject: xarray: Change definition of sibling entries Instead of storing a pointer to the slot containing the canonical entry, store the offset of the slot. Produces slightly more efficient code (~300 bytes) and simplifies the implementation. Signed-off-by: Matthew Wilcox Reviewed-by: Josef Bacik --- include/linux/radix-tree.h | 5 +- include/linux/xarray.h | 92 +++++++++++++++++++++++++++ lib/Kconfig | 7 ++ lib/radix-tree.c | 64 ++++++------------- tools/testing/radix-tree/Makefile | 2 +- tools/testing/radix-tree/generated/autoconf.h | 1 + 6 files changed, 121 insertions(+), 50 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index e9e76ab4fbbf..60f3d8eb2cb7 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -59,10 +59,7 @@ static inline bool radix_tree_is_internal_node(void *ptr) #define RADIX_TREE_MAX_TAGS 3 -#ifndef RADIX_TREE_MAP_SHIFT -#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6) -#endif - +#define RADIX_TREE_MAP_SHIFT XA_CHUNK_SHIFT #define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT) #define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 5c8acfc4ff55..4d1cd7a083e8 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -22,6 +22,12 @@ * x1: Value entry or tagged pointer * * Attempting to store internal entries in the XArray is a bug. + * + * Most internal entries are pointers to the next node in the tree. + * The following internal entries have a special meaning: + * + * 0-62: Sibling entries + * 256: Retry entry */ #define BITS_PER_XA_VALUE (BITS_PER_LONG - 1) @@ -111,6 +117,42 @@ static inline unsigned int xa_pointer_tag(void *entry) return (unsigned long)entry & 3UL; } +/* + * xa_mk_internal() - Create an internal entry. + * @v: Value to turn into an internal entry. + * + * Context: Any context. + * Return: An XArray internal entry corresponding to this value. + */ +static inline void *xa_mk_internal(unsigned long v) +{ + return (void *)((v << 2) | 2); +} + +/* + * xa_to_internal() - Extract the value from an internal entry. + * @entry: XArray entry. + * + * Context: Any context. + * Return: The value which was stored in the internal entry. + */ +static inline unsigned long xa_to_internal(const void *entry) +{ + return (unsigned long)entry >> 2; +} + +/* + * xa_is_internal() - Is the entry an internal entry? + * @entry: XArray entry. + * + * Context: Any context. + * Return: %true if the entry is an internal entry. + */ +static inline bool xa_is_internal(const void *entry) +{ + return ((unsigned long)entry & 3) == 2; +} + #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) #define xa_unlock(xa) spin_unlock(&(xa)->xa_lock) @@ -123,4 +165,54 @@ static inline unsigned int xa_pointer_tag(void *entry) #define xa_unlock_irqrestore(xa, flags) \ spin_unlock_irqrestore(&(xa)->xa_lock, flags) +/* Everything below here is the Advanced API. Proceed with caution. */ + +/* + * The xarray is constructed out of a set of 'chunks' of pointers. Choosing + * the best chunk size requires some tradeoffs. A power of two recommends + * itself so that we can walk the tree based purely on shifts and masks. + * Generally, the larger the better; as the number of slots per level of the + * tree increases, the less tall the tree needs to be. But that needs to be + * balanced against the memory consumption of each node. On a 64-bit system, + * xa_node is currently 576 bytes, and we get 7 of them per 4kB page. If we + * doubled the number of slots per node, we'd get only 3 nodes per 4kB page. + */ +#ifndef XA_CHUNK_SHIFT +#define XA_CHUNK_SHIFT (CONFIG_BASE_SMALL ? 4 : 6) +#endif +#define XA_CHUNK_SIZE (1UL << XA_CHUNK_SHIFT) +#define XA_CHUNK_MASK (XA_CHUNK_SIZE - 1) + +/* Private */ +static inline bool xa_is_node(const void *entry) +{ + return xa_is_internal(entry) && (unsigned long)entry > 4096; +} + +/* Private */ +static inline void *xa_mk_sibling(unsigned int offset) +{ + return xa_mk_internal(offset); +} + +/* Private */ +static inline unsigned long xa_to_sibling(const void *entry) +{ + return xa_to_internal(entry); +} + +/** + * xa_is_sibling() - Is the entry a sibling entry? + * @entry: Entry retrieved from the XArray + * + * Return: %true if the entry is a sibling entry. + */ +static inline bool xa_is_sibling(const void *entry) +{ + return IS_ENABLED(CONFIG_XARRAY_MULTI) && xa_is_internal(entry) && + (entry < xa_mk_sibling(XA_CHUNK_SIZE - 1)); +} + +#define XA_RETRY_ENTRY xa_mk_internal(256) + #endif /* _LINUX_XARRAY_H */ diff --git a/lib/Kconfig b/lib/Kconfig index a3928d4438b5..40bfa6ccd294 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -399,8 +399,15 @@ config INTERVAL_TREE for more information. +config XARRAY_MULTI + bool + help + Support entries which occupy multiple consecutive indices in the + XArray. + config RADIX_TREE_MULTIORDER bool + select XARRAY_MULTI config ASSOCIATIVE_ARRAY bool diff --git a/lib/radix-tree.c b/lib/radix-tree.c index b6c0e7f3a894..59b28111eabc 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -38,6 +38,7 @@ #include #include #include +#include /* Number of nodes in fully populated tree of given height */ @@ -98,24 +99,7 @@ static inline void *node_to_entry(void *ptr) return (void *)((unsigned long)ptr | RADIX_TREE_INTERNAL_NODE); } -#define RADIX_TREE_RETRY node_to_entry(NULL) - -#ifdef CONFIG_RADIX_TREE_MULTIORDER -/* Sibling slots point directly to another slot in the same node */ -static inline -bool is_sibling_entry(const struct radix_tree_node *parent, void *node) -{ - void __rcu **ptr = node; - return (parent->slots <= ptr) && - (ptr < parent->slots + RADIX_TREE_MAP_SIZE); -} -#else -static inline -bool is_sibling_entry(const struct radix_tree_node *parent, void *node) -{ - return false; -} -#endif +#define RADIX_TREE_RETRY XA_RETRY_ENTRY static inline unsigned long get_slot_offset(const struct radix_tree_node *parent, void __rcu **slot) @@ -129,16 +113,10 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK; void __rcu **entry = rcu_dereference_raw(parent->slots[offset]); -#ifdef CONFIG_RADIX_TREE_MULTIORDER - if (radix_tree_is_internal_node(entry)) { - if (is_sibling_entry(parent, entry)) { - void __rcu **sibentry; - sibentry = (void __rcu **) entry_to_node(entry); - offset = get_slot_offset(parent, sibentry); - entry = rcu_dereference_raw(*sibentry); - } + if (xa_is_sibling(entry)) { + offset = xa_to_sibling(entry); + entry = rcu_dereference_raw(parent->slots[offset]); } -#endif *nodep = (void *)entry; return offset; @@ -300,10 +278,10 @@ static void dump_node(struct radix_tree_node *node, unsigned long index) } else if (!radix_tree_is_internal_node(entry)) { pr_debug("radix entry %p offset %ld indices %lu-%lu parent %p\n", entry, i, first, last, node); - } else if (is_sibling_entry(node, entry)) { + } else if (xa_is_sibling(entry)) { pr_debug("radix sblng %p offset %ld indices %lu-%lu parent %p val %p\n", entry, i, first, last, node, - *(void **)entry_to_node(entry)); + node->slots[xa_to_sibling(entry)]); } else { dump_node(entry_to_node(entry), first); } @@ -881,8 +859,7 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) for (;;) { void *entry = rcu_dereference_raw(child->slots[offset]); - if (radix_tree_is_internal_node(entry) && child->shift && - !is_sibling_entry(child, entry)) { + if (xa_is_node(entry) && child->shift) { child = entry_to_node(entry); offset = 0; continue; @@ -904,7 +881,7 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) static inline int insert_entries(struct radix_tree_node *node, void __rcu **slot, void *item, unsigned order, bool replace) { - struct radix_tree_node *child; + void *sibling; unsigned i, n, tag, offset, tags = 0; if (node) { @@ -922,7 +899,7 @@ static inline int insert_entries(struct radix_tree_node *node, offset = offset & ~(n - 1); slot = &node->slots[offset]; } - child = node_to_entry(slot); + sibling = xa_mk_sibling(offset); for (i = 0; i < n; i++) { if (slot[i]) { @@ -939,7 +916,7 @@ static inline int insert_entries(struct radix_tree_node *node, for (i = 0; i < n; i++) { struct radix_tree_node *old = rcu_dereference_raw(slot[i]); if (i) { - rcu_assign_pointer(slot[i], child); + rcu_assign_pointer(slot[i], sibling); for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) if (tags & (1 << tag)) tag_clear(node, tag, offset + i); @@ -949,9 +926,7 @@ static inline int insert_entries(struct radix_tree_node *node, if (tags & (1 << tag)) tag_set(node, tag, offset); } - if (radix_tree_is_internal_node(old) && - !is_sibling_entry(node, old) && - (old != RADIX_TREE_RETRY)) + if (xa_is_node(old)) radix_tree_free_nodes(old); if (xa_is_value(old)) node->exceptional--; @@ -1112,10 +1087,10 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, void __rcu **slot, int count, int exceptional) { #ifdef CONFIG_RADIX_TREE_MULTIORDER - void *ptr = node_to_entry(slot); - unsigned offset = get_slot_offset(node, slot) + 1; + unsigned offset = get_slot_offset(node, slot); + void *ptr = xa_mk_sibling(offset); - while (offset < RADIX_TREE_MAP_SIZE) { + while (++offset < RADIX_TREE_MAP_SIZE) { if (rcu_dereference_raw(node->slots[offset]) != ptr) break; if (count < 0) { @@ -1123,7 +1098,6 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, node->count--; } node->exceptional += exceptional; - offset++; } #endif } @@ -1319,8 +1293,7 @@ int radix_tree_split(struct radix_tree_root *root, unsigned long index, tags |= 1 << tag; for (end = offset + 1; end < RADIX_TREE_MAP_SIZE; end++) { - if (!is_sibling_entry(parent, - rcu_dereference_raw(parent->slots[end]))) + if (!xa_is_sibling(rcu_dereference_raw(parent->slots[end]))) break; for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) if (tags & (1 << tag)) @@ -1618,7 +1591,7 @@ static void __rcu **skip_siblings(struct radix_tree_node **nodep, { while (iter->index < iter->next_index) { *nodep = rcu_dereference_raw(*slot); - if (*nodep && !is_sibling_entry(iter->node, *nodep)) + if (*nodep && !xa_is_sibling(*nodep)) return slot; slot++; iter->index = __radix_tree_iter_add(iter, 1); @@ -1769,7 +1742,7 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, while (++offset < RADIX_TREE_MAP_SIZE) { void *slot = rcu_dereference_raw( node->slots[offset]); - if (is_sibling_entry(node, slot)) + if (xa_is_sibling(slot)) continue; if (slot) break; @@ -2283,6 +2256,7 @@ void __init radix_tree_init(void) BUILD_BUG_ON(RADIX_TREE_MAX_TAGS + __GFP_BITS_SHIFT > 32); BUILD_BUG_ON(ROOT_IS_IDR & ~GFP_ZONEMASK); + BUILD_BUG_ON(XA_CHUNK_SIZE > 255); radix_tree_node_cachep = kmem_cache_create("radix_tree_node", sizeof(struct radix_tree_node), 0, SLAB_PANIC | SLAB_RECLAIM_ACCOUNT, diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index 37baecc3766f..12adcf9ffe86 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -46,6 +46,6 @@ idr.c: ../../../lib/idr.c generated/map-shift.h: @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ - echo "#define RADIX_TREE_MAP_SHIFT $(SHIFT)" > \ + echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ generated/map-shift.h; \ fi diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h index cf88dc5b8832..ca8e03ad19ac 100644 --- a/tools/testing/radix-tree/generated/autoconf.h +++ b/tools/testing/radix-tree/generated/autoconf.h @@ -1 +1,2 @@ #define CONFIG_RADIX_TREE_MULTIORDER 1 +#define CONFIG_XARRAY_MULTI 1 -- cgit v1.2.3 From f8d5d0cc145cc21bfc56ef807dc28102aebbf228 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Tue, 7 Nov 2017 16:30:10 -0500 Subject: xarray: Add definition of struct xarray This is a direct replacement for struct radix_tree_root. Some of the struct members have changed name; convert those, and use a #define so that radix_tree users continue to work without change. Signed-off-by: Matthew Wilcox Reviewed-by: Josef Bacik --- include/linux/radix-tree.h | 28 ++++-------- include/linux/xarray.h | 70 +++++++++++++++++++++++++++++ lib/Makefile | 2 +- lib/idr.c | 4 +- lib/radix-tree.c | 75 ++++++++++++++++---------------- lib/xarray.c | 44 +++++++++++++++++++ tools/testing/radix-tree/Makefile | 5 ++- tools/testing/radix-tree/linux/bug.h | 1 + tools/testing/radix-tree/linux/kconfig.h | 1 + tools/testing/radix-tree/multiorder.c | 6 +-- tools/testing/radix-tree/test.c | 6 +-- tools/testing/radix-tree/xarray.c | 7 +++ 12 files changed, 181 insertions(+), 68 deletions(-) create mode 100644 lib/xarray.c create mode 100644 tools/testing/radix-tree/linux/kconfig.h create mode 100644 tools/testing/radix-tree/xarray.c (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 60f3d8eb2cb7..0b080bd88ab7 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -30,6 +30,9 @@ #include #include +/* Keep unconverted code working */ +#define radix_tree_root xarray + /* * The bottom two bits of the slot determine how the remaining bits in the * slot are interpreted: @@ -92,36 +95,21 @@ struct radix_tree_node { unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS]; }; -/* The IDR tag is stored in the low bits of the GFP flags */ +/* The IDR tag is stored in the low bits of xa_flags */ #define ROOT_IS_IDR ((__force gfp_t)4) -/* The top bits of gfp_mask are used to store the root tags */ +/* The top bits of xa_flags are used to store the root tags */ #define ROOT_TAG_SHIFT (__GFP_BITS_SHIFT) -struct radix_tree_root { - spinlock_t xa_lock; - gfp_t gfp_mask; - struct radix_tree_node __rcu *rnode; -}; - -#define RADIX_TREE_INIT(name, mask) { \ - .xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \ - .gfp_mask = (mask), \ - .rnode = NULL, \ -} +#define RADIX_TREE_INIT(name, mask) XARRAY_INIT(name, mask) #define RADIX_TREE(name, mask) \ struct radix_tree_root name = RADIX_TREE_INIT(name, mask) -#define INIT_RADIX_TREE(root, mask) \ -do { \ - spin_lock_init(&(root)->xa_lock); \ - (root)->gfp_mask = (mask); \ - (root)->rnode = NULL; \ -} while (0) +#define INIT_RADIX_TREE(root, mask) xa_init_flags(root, mask) static inline bool radix_tree_empty(const struct radix_tree_root *root) { - return root->rnode == NULL; + return root->xa_head == NULL; } /** diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 4d1cd7a083e8..9122cf8bf52a 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -10,6 +10,8 @@ */ #include +#include +#include #include #include @@ -153,6 +155,74 @@ static inline bool xa_is_internal(const void *entry) return ((unsigned long)entry & 3) == 2; } +/** + * struct xarray - The anchor of the XArray. + * @xa_lock: Lock that protects the contents of the XArray. + * + * To use the xarray, define it statically or embed it in your data structure. + * It is a very small data structure, so it does not usually make sense to + * allocate it separately and keep a pointer to it in your data structure. + * + * You may use the xa_lock to protect your own data structures as well. + */ +/* + * If all of the entries in the array are NULL, @xa_head is a NULL pointer. + * If the only non-NULL entry in the array is at index 0, @xa_head is that + * entry. If any other entry in the array is non-NULL, @xa_head points + * to an @xa_node. + */ +struct xarray { + spinlock_t xa_lock; +/* private: The rest of the data structure is not to be used directly. */ + gfp_t xa_flags; + void __rcu * xa_head; +}; + +#define XARRAY_INIT(name, flags) { \ + .xa_lock = __SPIN_LOCK_UNLOCKED(name.xa_lock), \ + .xa_flags = flags, \ + .xa_head = NULL, \ +} + +/** + * DEFINE_XARRAY_FLAGS() - Define an XArray with custom flags. + * @name: A string that names your XArray. + * @flags: XA_FLAG values. + * + * This is intended for file scope definitions of XArrays. It declares + * and initialises an empty XArray with the chosen name and flags. It is + * equivalent to calling xa_init_flags() on the array, but it does the + * initialisation at compiletime instead of runtime. + */ +#define DEFINE_XARRAY_FLAGS(name, flags) \ + struct xarray name = XARRAY_INIT(name, flags) + +/** + * DEFINE_XARRAY() - Define an XArray. + * @name: A string that names your XArray. + * + * This is intended for file scope definitions of XArrays. It declares + * and initialises an empty XArray with the chosen name. It is equivalent + * to calling xa_init() on the array, but it does the initialisation at + * compiletime instead of runtime. + */ +#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) + +void xa_init_flags(struct xarray *, gfp_t flags); + +/** + * xa_init() - Initialise an empty XArray. + * @xa: XArray. + * + * An empty XArray is full of NULL entries. + * + * Context: Any context. + */ +static inline void xa_init(struct xarray *xa) +{ + xa_init_flags(xa, 0); +} + #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) #define xa_unlock(xa) spin_unlock(&(xa)->xa_lock) diff --git a/lib/Makefile b/lib/Makefile index ca3f7ebb900d..057385f1f145 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -18,7 +18,7 @@ KCOV_INSTRUMENT_debugobjects.o := n KCOV_INSTRUMENT_dynamic_debug.o := n lib-y := ctype.o string.o vsprintf.o cmdline.o \ - rbtree.o radix-tree.o timerqueue.o\ + rbtree.o radix-tree.o timerqueue.o xarray.o \ idr.o int_sqrt.o extable.o \ sha1.o chacha20.o irq_regs.o argv_split.o \ flex_proportions.o ratelimit.o show_mem.o \ diff --git a/lib/idr.c b/lib/idr.c index 88419fbc5737..9a366b5be2c2 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -39,8 +39,8 @@ int idr_alloc_u32(struct idr *idr, void *ptr, u32 *nextid, unsigned int base = idr->idr_base; unsigned int id = *nextid; - if (WARN_ON_ONCE(!(idr->idr_rt.gfp_mask & ROOT_IS_IDR))) - idr->idr_rt.gfp_mask |= IDR_RT_MARKER; + if (WARN_ON_ONCE(!(idr->idr_rt.xa_flags & ROOT_IS_IDR))) + idr->idr_rt.xa_flags |= IDR_RT_MARKER; id = (id < base) ? 0 : id - base; radix_tree_iter_init(&iter, id); diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 59b28111eabc..299d4bdba109 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -124,7 +124,7 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, static inline gfp_t root_gfp_mask(const struct radix_tree_root *root) { - return root->gfp_mask & (__GFP_BITS_MASK & ~GFP_ZONEMASK); + return root->xa_flags & (__GFP_BITS_MASK & ~GFP_ZONEMASK); } static inline void tag_set(struct radix_tree_node *node, unsigned int tag, @@ -147,32 +147,32 @@ static inline int tag_get(const struct radix_tree_node *node, unsigned int tag, static inline void root_tag_set(struct radix_tree_root *root, unsigned tag) { - root->gfp_mask |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT)); + root->xa_flags |= (__force gfp_t)(1 << (tag + ROOT_TAG_SHIFT)); } static inline void root_tag_clear(struct radix_tree_root *root, unsigned tag) { - root->gfp_mask &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT)); + root->xa_flags &= (__force gfp_t)~(1 << (tag + ROOT_TAG_SHIFT)); } static inline void root_tag_clear_all(struct radix_tree_root *root) { - root->gfp_mask &= (1 << ROOT_TAG_SHIFT) - 1; + root->xa_flags &= (__force gfp_t)((1 << ROOT_TAG_SHIFT) - 1); } static inline int root_tag_get(const struct radix_tree_root *root, unsigned tag) { - return (__force int)root->gfp_mask & (1 << (tag + ROOT_TAG_SHIFT)); + return (__force int)root->xa_flags & (1 << (tag + ROOT_TAG_SHIFT)); } static inline unsigned root_tags_get(const struct radix_tree_root *root) { - return (__force unsigned)root->gfp_mask >> ROOT_TAG_SHIFT; + return (__force unsigned)root->xa_flags >> ROOT_TAG_SHIFT; } static inline bool is_idr(const struct radix_tree_root *root) { - return !!(root->gfp_mask & ROOT_IS_IDR); + return !!(root->xa_flags & ROOT_IS_IDR); } /* @@ -291,12 +291,12 @@ static void dump_node(struct radix_tree_node *node, unsigned long index) /* For debug */ static void radix_tree_dump(struct radix_tree_root *root) { - pr_debug("radix root: %p rnode %p tags %x\n", - root, root->rnode, - root->gfp_mask >> ROOT_TAG_SHIFT); - if (!radix_tree_is_internal_node(root->rnode)) + pr_debug("radix root: %p xa_head %p tags %x\n", + root, root->xa_head, + root->xa_flags >> ROOT_TAG_SHIFT); + if (!radix_tree_is_internal_node(root->xa_head)) return; - dump_node(entry_to_node(root->rnode), 0); + dump_node(entry_to_node(root->xa_head), 0); } static void dump_ida_node(void *entry, unsigned long index) @@ -340,9 +340,9 @@ static void dump_ida_node(void *entry, unsigned long index) static void ida_dump(struct ida *ida) { struct radix_tree_root *root = &ida->ida_rt; - pr_debug("ida: %p node %p free %d\n", ida, root->rnode, - root->gfp_mask >> ROOT_TAG_SHIFT); - dump_ida_node(root->rnode, 0); + pr_debug("ida: %p node %p free %d\n", ida, root->xa_head, + root->xa_flags >> ROOT_TAG_SHIFT); + dump_ida_node(root->xa_head, 0); } #endif @@ -576,7 +576,7 @@ int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order) static unsigned radix_tree_load_root(const struct radix_tree_root *root, struct radix_tree_node **nodep, unsigned long *maxindex) { - struct radix_tree_node *node = rcu_dereference_raw(root->rnode); + struct radix_tree_node *node = rcu_dereference_raw(root->xa_head); *nodep = node; @@ -605,7 +605,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, while (index > shift_maxindex(maxshift)) maxshift += RADIX_TREE_MAP_SHIFT; - entry = rcu_dereference_raw(root->rnode); + entry = rcu_dereference_raw(root->xa_head); if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE))) goto out; @@ -633,7 +633,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, if (radix_tree_is_internal_node(entry)) { entry_to_node(entry)->parent = node; } else if (xa_is_value(entry)) { - /* Moving an exceptional root->rnode to a node */ + /* Moving an exceptional root->xa_head to a node */ node->exceptional = 1; } /* @@ -642,7 +642,7 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, */ node->slots[0] = (void __rcu *)entry; entry = node_to_entry(node); - rcu_assign_pointer(root->rnode, entry); + rcu_assign_pointer(root->xa_head, entry); shift += RADIX_TREE_MAP_SHIFT; } while (shift <= maxshift); out: @@ -659,7 +659,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, bool shrunk = false; for (;;) { - struct radix_tree_node *node = rcu_dereference_raw(root->rnode); + struct radix_tree_node *node = rcu_dereference_raw(root->xa_head); struct radix_tree_node *child; if (!radix_tree_is_internal_node(node)) @@ -695,9 +695,9 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, * moving the node from one part of the tree to another: if it * was safe to dereference the old pointer to it * (node->slots[0]), it will be safe to dereference the new - * one (root->rnode) as far as dependent read barriers go. + * one (root->xa_head) as far as dependent read barriers go. */ - root->rnode = (void __rcu *)child; + root->xa_head = (void __rcu *)child; if (is_idr(root) && !tag_get(node, IDR_FREE, 0)) root_tag_clear(root, IDR_FREE); @@ -745,9 +745,8 @@ static bool delete_node(struct radix_tree_root *root, if (node->count) { if (node_to_entry(node) == - rcu_dereference_raw(root->rnode)) - deleted |= radix_tree_shrink(root, - update_node); + rcu_dereference_raw(root->xa_head)) + deleted |= radix_tree_shrink(root, update_node); return deleted; } @@ -762,7 +761,7 @@ static bool delete_node(struct radix_tree_root *root, */ if (!is_idr(root)) root_tag_clear_all(root); - root->rnode = NULL; + root->xa_head = NULL; } WARN_ON_ONCE(!list_empty(&node->private_list)); @@ -787,7 +786,7 @@ static bool delete_node(struct radix_tree_root *root, * at position @index in the radix tree @root. * * Until there is more than one item in the tree, no nodes are - * allocated and @root->rnode is used as a direct slot instead of + * allocated and @root->xa_head is used as a direct slot instead of * pointing to a node, in which case *@nodep will be NULL. * * Returns -ENOMEM, or 0 for success. @@ -797,7 +796,7 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, void __rcu ***slotp) { struct radix_tree_node *node = NULL, *child; - void __rcu **slot = (void __rcu **)&root->rnode; + void __rcu **slot = (void __rcu **)&root->xa_head; unsigned long maxindex; unsigned int shift, offset = 0; unsigned long max = index | ((1UL << order) - 1); @@ -813,7 +812,7 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, if (error < 0) return error; shift = error; - child = rcu_dereference_raw(root->rnode); + child = rcu_dereference_raw(root->xa_head); } while (shift > order) { @@ -1004,7 +1003,7 @@ EXPORT_SYMBOL(__radix_tree_insert); * tree @root. * * Until there is more than one item in the tree, no nodes are - * allocated and @root->rnode is used as a direct slot instead of + * allocated and @root->xa_head is used as a direct slot instead of * pointing to a node, in which case *@nodep will be NULL. */ void *__radix_tree_lookup(const struct radix_tree_root *root, @@ -1017,7 +1016,7 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, restart: parent = NULL; - slot = (void __rcu **)&root->rnode; + slot = (void __rcu **)&root->xa_head; radix_tree_load_root(root, &node, &maxindex); if (index > maxindex) return NULL; @@ -1168,9 +1167,9 @@ void __radix_tree_replace(struct radix_tree_root *root, /* * This function supports replacing exceptional entries and * deleting entries, but that needs accounting against the - * node unless the slot is root->rnode. + * node unless the slot is root->xa_head. */ - WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->rnode) && + WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->xa_head) && (count || exceptional)); replace_slot(slot, item, node, count, exceptional); @@ -1722,7 +1721,7 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, iter->tags = 1; iter->node = NULL; __set_iter_shift(iter, 0); - return (void __rcu **)&root->rnode; + return (void __rcu **)&root->xa_head; } do { @@ -2109,7 +2108,7 @@ void __rcu **idr_get_free(struct radix_tree_root *root, unsigned long max) { struct radix_tree_node *node = NULL, *child; - void __rcu **slot = (void __rcu **)&root->rnode; + void __rcu **slot = (void __rcu **)&root->xa_head; unsigned long maxindex, start = iter->next_index; unsigned int shift, offset = 0; @@ -2125,7 +2124,7 @@ void __rcu **idr_get_free(struct radix_tree_root *root, if (error < 0) return ERR_PTR(error); shift = error; - child = rcu_dereference_raw(root->rnode); + child = rcu_dereference_raw(root->xa_head); } if (start == 0 && shift == 0) shift = RADIX_TREE_MAP_SHIFT; @@ -2190,10 +2189,10 @@ void __rcu **idr_get_free(struct radix_tree_root *root, */ void idr_destroy(struct idr *idr) { - struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.rnode); + struct radix_tree_node *node = rcu_dereference_raw(idr->idr_rt.xa_head); if (radix_tree_is_internal_node(node)) radix_tree_free_nodes(node); - idr->idr_rt.rnode = NULL; + idr->idr_rt.xa_head = NULL; root_tag_set(&idr->idr_rt, IDR_FREE); } EXPORT_SYMBOL(idr_destroy); diff --git a/lib/xarray.c b/lib/xarray.c new file mode 100644 index 000000000000..862f4c64c754 --- /dev/null +++ b/lib/xarray.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * XArray implementation + * Copyright (c) 2017 Microsoft Corporation + * Author: Matthew Wilcox + */ + +#include +#include + +/* + * Coding conventions in this file: + * + * @xa is used to refer to the entire xarray. + * @xas is the 'xarray operation state'. It may be either a pointer to + * an xa_state, or an xa_state stored on the stack. This is an unfortunate + * ambiguity. + * @index is the index of the entry being operated on + * @mark is an xa_mark_t; a small number indicating one of the mark bits. + * @node refers to an xa_node; usually the primary one being operated on by + * this function. + * @offset is the index into the slots array inside an xa_node. + * @parent refers to the @xa_node closer to the head than @node. + * @entry refers to something stored in a slot in the xarray + */ + +/** + * xa_init_flags() - Initialise an empty XArray with flags. + * @xa: XArray. + * @flags: XA_FLAG values. + * + * If you need to initialise an XArray with special flags (eg you need + * to take the lock from interrupt context), use this function instead + * of xa_init(). + * + * Context: Any context. + */ +void xa_init_flags(struct xarray *xa, gfp_t flags) +{ + spin_lock_init(&xa->xa_lock); + xa->xa_flags = flags; + xa->xa_head = NULL; +} +EXPORT_SYMBOL(xa_init_flags); diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index 12adcf9ffe86..c0cf1c79efd5 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -5,7 +5,7 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ LDFLAGS += -fsanitize=address -fsanitize=undefined LDLIBS+= -lpthread -lurcu TARGETS = main idr-test multiorder -CORE_OFILES := radix-tree.o idr.o linux.o test.o find_bit.o +CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ tag_check.o multiorder.o idr-test.o iteration_check.o benchmark.o @@ -35,6 +35,7 @@ vpath %.c ../../lib $(OFILES): Makefile *.h */*.h generated/map-shift.h \ ../../include/linux/*.h \ ../../include/asm/*.h \ + ../../../include/linux/xarray.h \ ../../../include/linux/radix-tree.h \ ../../../include/linux/idr.h @@ -44,6 +45,8 @@ radix-tree.c: ../../../lib/radix-tree.c idr.c: ../../../lib/idr.c sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@ +xarray.o: ../../../lib/xarray.c + generated/map-shift.h: @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ diff --git a/tools/testing/radix-tree/linux/bug.h b/tools/testing/radix-tree/linux/bug.h index 23b8ed52f8c8..03dc8a57eb99 100644 --- a/tools/testing/radix-tree/linux/bug.h +++ b/tools/testing/radix-tree/linux/bug.h @@ -1 +1,2 @@ +#include #include "asm/bug.h" diff --git a/tools/testing/radix-tree/linux/kconfig.h b/tools/testing/radix-tree/linux/kconfig.h new file mode 100644 index 000000000000..6c8675859913 --- /dev/null +++ b/tools/testing/radix-tree/linux/kconfig.h @@ -0,0 +1 @@ +#include "../../../../include/linux/kconfig.h" diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 2b4f4dba1882..080aea450430 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -192,13 +192,13 @@ static void multiorder_shrink(unsigned long index, int order) assert(item_insert_order(&tree, 0, order) == 0); - node = tree.rnode; + node = tree.xa_head; assert(item_insert(&tree, index) == 0); - assert(node != tree.rnode); + assert(node != tree.xa_head); assert(item_delete(&tree, index) != 0); - assert(node == tree.rnode); + assert(node == tree.xa_head); for (i = 0; i < max; i++) { struct item *item = item_lookup(&tree, i); diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 62de66c314b7..70ddf964d51c 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -281,7 +281,7 @@ static int verify_node(struct radix_tree_node *slot, unsigned int tag, void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag) { - struct radix_tree_node *node = root->rnode; + struct radix_tree_node *node = root->xa_head; if (!radix_tree_is_internal_node(node)) return; verify_node(node, tag, !!root_tag_get(root, tag)); @@ -311,13 +311,13 @@ void item_kill_tree(struct radix_tree_root *root) } } assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0); - assert(root->rnode == NULL); + assert(root->xa_head == NULL); } void tree_verify_min_height(struct radix_tree_root *root, int maxindex) { unsigned shift; - struct radix_tree_node *node = root->rnode; + struct radix_tree_node *node = root->xa_head; if (!radix_tree_is_internal_node(node)) { assert(maxindex == 0); return; diff --git a/tools/testing/radix-tree/xarray.c b/tools/testing/radix-tree/xarray.c new file mode 100644 index 000000000000..9bbd667172a7 --- /dev/null +++ b/tools/testing/radix-tree/xarray.c @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * xarray.c: Userspace shim for XArray test-suite + * Copyright (c) 2018 Matthew Wilcox + */ + +#include "../../../lib/xarray.c" -- cgit v1.2.3 From 01959dfe771c6893365482ec78dc1d9cbbbe6de8 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 9 Nov 2017 09:23:56 -0500 Subject: xarray: Define struct xa_node This is a direct replacement for struct radix_tree_node. A couple of struct members have changed name, so convert those. Use a #define so that radix tree users continue to work without change. Signed-off-by: Matthew Wilcox Reviewed-by: Josef Bacik --- include/linux/radix-tree.h | 29 +++------------------ include/linux/xarray.h | 27 ++++++++++++++++++++ lib/radix-tree.c | 48 +++++++++++++++++------------------ mm/workingset.c | 16 ++++++------ tools/testing/radix-tree/multiorder.c | 30 +++++++++++----------- 5 files changed, 77 insertions(+), 73 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 0b080bd88ab7..15388b7e38b9 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -32,6 +32,7 @@ /* Keep unconverted code working */ #define radix_tree_root xarray +#define radix_tree_node xa_node /* * The bottom two bits of the slot determine how the remaining bits in the @@ -60,41 +61,17 @@ static inline bool radix_tree_is_internal_node(void *ptr) /*** radix-tree API starts here ***/ -#define RADIX_TREE_MAX_TAGS 3 - #define RADIX_TREE_MAP_SHIFT XA_CHUNK_SHIFT #define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT) #define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1) -#define RADIX_TREE_TAG_LONGS \ - ((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG) +#define RADIX_TREE_MAX_TAGS XA_MAX_MARKS +#define RADIX_TREE_TAG_LONGS XA_MARK_LONGS #define RADIX_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long)) #define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \ RADIX_TREE_MAP_SHIFT)) -/* - * @count is the count of every non-NULL element in the ->slots array - * whether that is a value entry, a retry entry, a user pointer, - * a sibling entry or a pointer to the next level of the tree. - * @exceptional is the count of every element in ->slots which is - * either a value entry or a sibling of a value entry. - */ -struct radix_tree_node { - unsigned char shift; /* Bits remaining in each slot */ - unsigned char offset; /* Slot offset in parent */ - unsigned char count; /* Total entry count */ - unsigned char exceptional; /* Exceptional entry count */ - struct radix_tree_node *parent; /* Used when ascending tree */ - struct radix_tree_root *root; /* The tree we belong to */ - union { - struct list_head private_list; /* For tree user */ - struct rcu_head rcu_head; /* Used when freeing node */ - }; - void __rcu *slots[RADIX_TREE_MAP_SIZE]; - unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS]; -}; - /* The IDR tag is stored in the low bits of xa_flags */ #define ROOT_IS_IDR ((__force gfp_t)4) /* The top bits of xa_flags are used to store the root tags */ diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 9122cf8bf52a..52141dfc5a90 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -252,6 +252,33 @@ static inline void xa_init(struct xarray *xa) #endif #define XA_CHUNK_SIZE (1UL << XA_CHUNK_SHIFT) #define XA_CHUNK_MASK (XA_CHUNK_SIZE - 1) +#define XA_MAX_MARKS 3 +#define XA_MARK_LONGS DIV_ROUND_UP(XA_CHUNK_SIZE, BITS_PER_LONG) + +/* + * @count is the count of every non-NULL element in the ->slots array + * whether that is a value entry, a retry entry, a user pointer, + * a sibling entry or a pointer to the next level of the tree. + * @nr_values is the count of every element in ->slots which is + * either a value entry or a sibling of a value entry. + */ +struct xa_node { + unsigned char shift; /* Bits remaining in each slot */ + unsigned char offset; /* Slot offset in parent */ + unsigned char count; /* Total entry count */ + unsigned char nr_values; /* Value entry count */ + struct xa_node __rcu *parent; /* NULL at top of tree */ + struct xarray *array; /* The array we belong to */ + union { + struct list_head private_list; /* For tree user */ + struct rcu_head rcu_head; /* Used when freeing node */ + }; + void __rcu *slots[XA_CHUNK_SIZE]; + union { + unsigned long tags[XA_MAX_MARKS][XA_MARK_LONGS]; + unsigned long marks[XA_MAX_MARKS][XA_MARK_LONGS]; + }; +}; /* Private */ static inline bool xa_is_node(const void *entry) diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 299d4bdba109..8a568cea1237 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -260,11 +260,11 @@ static void dump_node(struct radix_tree_node *node, unsigned long index) { unsigned long i; - pr_debug("radix node: %p offset %d indices %lu-%lu parent %p tags %lx %lx %lx shift %d count %d exceptional %d\n", + pr_debug("radix node: %p offset %d indices %lu-%lu parent %p tags %lx %lx %lx shift %d count %d nr_values %d\n", node, node->offset, index, index | node_maxindex(node), node->parent, node->tags[0][0], node->tags[1][0], node->tags[2][0], - node->shift, node->count, node->exceptional); + node->shift, node->count, node->nr_values); for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) { unsigned long first = index | (i << node->shift); @@ -354,7 +354,7 @@ static struct radix_tree_node * radix_tree_node_alloc(gfp_t gfp_mask, struct radix_tree_node *parent, struct radix_tree_root *root, unsigned int shift, unsigned int offset, - unsigned int count, unsigned int exceptional) + unsigned int count, unsigned int nr_values) { struct radix_tree_node *ret = NULL; @@ -401,9 +401,9 @@ out: ret->shift = shift; ret->offset = offset; ret->count = count; - ret->exceptional = exceptional; + ret->nr_values = nr_values; ret->parent = parent; - ret->root = root; + ret->array = root; } return ret; } @@ -633,8 +633,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, if (radix_tree_is_internal_node(entry)) { entry_to_node(entry)->parent = node; } else if (xa_is_value(entry)) { - /* Moving an exceptional root->xa_head to a node */ - node->exceptional = 1; + /* Moving a value entry root->xa_head to a node */ + node->nr_values = 1; } /* * entry was already in the radix tree, so we do not need @@ -928,12 +928,12 @@ static inline int insert_entries(struct radix_tree_node *node, if (xa_is_node(old)) radix_tree_free_nodes(old); if (xa_is_value(old)) - node->exceptional--; + node->nr_values--; } if (node) { node->count += n; if (xa_is_value(item)) - node->exceptional += n; + node->nr_values += n; } return n; } @@ -947,7 +947,7 @@ static inline int insert_entries(struct radix_tree_node *node, if (node) { node->count++; if (xa_is_value(item)) - node->exceptional++; + node->nr_values++; } return 1; } @@ -1083,7 +1083,7 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) EXPORT_SYMBOL(radix_tree_lookup); static inline void replace_sibling_entries(struct radix_tree_node *node, - void __rcu **slot, int count, int exceptional) + void __rcu **slot, int count, int values) { #ifdef CONFIG_RADIX_TREE_MULTIORDER unsigned offset = get_slot_offset(node, slot); @@ -1096,18 +1096,18 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, node->slots[offset] = NULL; node->count--; } - node->exceptional += exceptional; + node->nr_values += values; } #endif } static void replace_slot(void __rcu **slot, void *item, - struct radix_tree_node *node, int count, int exceptional) + struct radix_tree_node *node, int count, int values) { - if (node && (count || exceptional)) { + if (node && (count || values)) { node->count += count; - node->exceptional += exceptional; - replace_sibling_entries(node, slot, count, exceptional); + node->nr_values += values; + replace_sibling_entries(node, slot, count, values); } rcu_assign_pointer(*slot, item); @@ -1161,17 +1161,17 @@ void __radix_tree_replace(struct radix_tree_root *root, radix_tree_update_node_t update_node) { void *old = rcu_dereference_raw(*slot); - int exceptional = !!xa_is_value(item) - !!xa_is_value(old); + int values = !!xa_is_value(item) - !!xa_is_value(old); int count = calculate_count(root, node, slot, item, old); /* - * This function supports replacing exceptional entries and + * This function supports replacing value entries and * deleting entries, but that needs accounting against the * node unless the slot is root->xa_head. */ WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->xa_head) && - (count || exceptional)); - replace_slot(slot, item, node, count, exceptional); + (count || values)); + replace_slot(slot, item, node, count, values); if (!node) return; @@ -1193,7 +1193,7 @@ void __radix_tree_replace(struct radix_tree_root *root, * across slot lookup and replacement. * * NOTE: This cannot be used to switch between non-entries (empty slots), - * regular entries, and exceptional entries, as that requires accounting + * regular entries, and value entries, as that requires accounting * inside the radix tree node. When switching from one type of entry or * deleting, use __radix_tree_lookup() and __radix_tree_replace() or * radix_tree_iter_replace(). @@ -1301,7 +1301,7 @@ int radix_tree_split(struct radix_tree_root *root, unsigned long index, rcu_assign_pointer(parent->slots[end], RADIX_TREE_RETRY); } rcu_assign_pointer(parent->slots[offset], RADIX_TREE_RETRY); - parent->exceptional -= (end - offset); + parent->nr_values -= (end - offset); if (order == parent->shift) return 0; @@ -1961,7 +1961,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, struct radix_tree_node *node, void __rcu **slot) { void *old = rcu_dereference_raw(*slot); - int exceptional = xa_is_value(old) ? -1 : 0; + int values = xa_is_value(old) ? -1 : 0; unsigned offset = get_slot_offset(node, slot); int tag; @@ -1971,7 +1971,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) node_tag_clear(root, node, tag, offset); - replace_slot(slot, NULL, node, -1, exceptional); + replace_slot(slot, NULL, node, -1, values); return node && delete_node(root, node, NULL); } diff --git a/mm/workingset.c b/mm/workingset.c index bb109b3afac2..c201cbb8c00f 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -349,7 +349,7 @@ void workingset_update_node(struct radix_tree_node *node) * already where they should be. The list_empty() test is safe * as node->private_list is protected by the i_pages lock. */ - if (node->count && node->count == node->exceptional) { + if (node->count && node->count == node->nr_values) { if (list_empty(&node->private_list)) list_lru_add(&shadow_nodes, &node->private_list); } else { @@ -428,8 +428,8 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, * to reclaim, take the node off-LRU, and drop the lru_lock. */ - node = container_of(item, struct radix_tree_node, private_list); - mapping = container_of(node->root, struct address_space, i_pages); + node = container_of(item, struct xa_node, private_list); + mapping = container_of(node->array, struct address_space, i_pages); /* Coming from the list, invert the lock order */ if (!xa_trylock(&mapping->i_pages)) { @@ -446,25 +446,25 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, * no pages, so we expect to be able to remove them all and * delete and free the empty node afterwards. */ - if (WARN_ON_ONCE(!node->exceptional)) + if (WARN_ON_ONCE(!node->nr_values)) goto out_invalid; - if (WARN_ON_ONCE(node->count != node->exceptional)) + if (WARN_ON_ONCE(node->count != node->nr_values)) goto out_invalid; for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) { if (node->slots[i]) { if (WARN_ON_ONCE(!xa_is_value(node->slots[i]))) goto out_invalid; - if (WARN_ON_ONCE(!node->exceptional)) + if (WARN_ON_ONCE(!node->nr_values)) goto out_invalid; if (WARN_ON_ONCE(!mapping->nrexceptional)) goto out_invalid; node->slots[i] = NULL; - node->exceptional--; + node->nr_values--; node->count--; mapping->nrexceptional--; } } - if (WARN_ON_ONCE(node->exceptional)) + if (WARN_ON_ONCE(node->nr_values)) goto out_invalid; inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); __radix_tree_delete_node(&mapping->i_pages, node, diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 080aea450430..60786fa55302 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -393,7 +393,7 @@ static void multiorder_join2(unsigned order1, unsigned order2) radix_tree_insert(&tree, 1 << order2, xa_mk_value(5)); item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); assert(item2 == xa_mk_value(5)); - assert(node->exceptional == 1); + assert(node->nr_values == 1); item2 = radix_tree_lookup(&tree, 0); free(item2); @@ -401,7 +401,7 @@ static void multiorder_join2(unsigned order1, unsigned order2) radix_tree_join(&tree, 0, order1, item1); item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); assert(item2 == item1); - assert(node->exceptional == 0); + assert(node->nr_values == 0); item_kill_tree(&tree); } @@ -409,7 +409,7 @@ static void multiorder_join2(unsigned order1, unsigned order2) * This test revealed an accounting bug for value entries at one point. * Nodes were being freed back into the pool with an elevated exception count * by radix_tree_join() and then radix_tree_split() was failing to zero the - * count of exceptional entries. + * count of value entries. */ static void multiorder_join3(unsigned int order) { @@ -433,7 +433,7 @@ static void multiorder_join3(unsigned int order) } __radix_tree_lookup(&tree, 0, &node, NULL); - assert(node->exceptional == node->count); + assert(node->nr_values == node->count); item_kill_tree(&tree); } @@ -520,7 +520,7 @@ static void __multiorder_split2(int old_order, int new_order) item = __radix_tree_lookup(&tree, 0, &node, NULL); assert(item == xa_mk_value(5)); - assert(node->exceptional > 0); + assert(node->nr_values > 0); radix_tree_split(&tree, 0, new_order); radix_tree_for_each_slot(slot, &tree, &iter, 0) { @@ -530,7 +530,7 @@ static void __multiorder_split2(int old_order, int new_order) item = __radix_tree_lookup(&tree, 0, &node, NULL); assert(item != xa_mk_value(5)); - assert(node->exceptional == 0); + assert(node->nr_values == 0); item_kill_tree(&tree); } @@ -547,7 +547,7 @@ static void __multiorder_split3(int old_order, int new_order) item = __radix_tree_lookup(&tree, 0, &node, NULL); assert(item == xa_mk_value(5)); - assert(node->exceptional > 0); + assert(node->nr_values > 0); radix_tree_split(&tree, 0, new_order); radix_tree_for_each_slot(slot, &tree, &iter, 0) { @@ -556,7 +556,7 @@ static void __multiorder_split3(int old_order, int new_order) item = __radix_tree_lookup(&tree, 0, &node, NULL); assert(item == xa_mk_value(7)); - assert(node->exceptional > 0); + assert(node->nr_values > 0); item_kill_tree(&tree); @@ -564,7 +564,7 @@ static void __multiorder_split3(int old_order, int new_order) item = __radix_tree_lookup(&tree, 0, &node, NULL); assert(item == xa_mk_value(5)); - assert(node->exceptional > 0); + assert(node->nr_values > 0); radix_tree_split(&tree, 0, new_order); radix_tree_for_each_slot(slot, &tree, &iter, 0) { @@ -577,13 +577,13 @@ static void __multiorder_split3(int old_order, int new_order) item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); assert(item == xa_mk_value(7)); - assert(node->count == node->exceptional); + assert(node->count == node->nr_values); do { node = node->parent; if (!node) break; assert(node->count == 1); - assert(node->exceptional == 0); + assert(node->nr_values == 0); } while (1); item_kill_tree(&tree); @@ -611,15 +611,15 @@ static void multiorder_account(void) __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); __radix_tree_lookup(&tree, 0, &node, NULL); - assert(node->count == node->exceptional * 2); + assert(node->count == node->nr_values * 2); radix_tree_delete(&tree, 1 << 5); - assert(node->exceptional == 0); + assert(node->nr_values == 0); __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); __radix_tree_lookup(&tree, 1 << 5, &node, &slot); - assert(node->count == node->exceptional * 2); + assert(node->count == node->nr_values * 2); __radix_tree_replace(&tree, node, slot, NULL, NULL); - assert(node->exceptional == 0); + assert(node->nr_values == 0); item_kill_tree(&tree); } -- cgit v1.2.3 From ad3d6c7263e368abdc151e1cc13dc78aa39cc7a7 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Tue, 7 Nov 2017 14:57:46 -0500 Subject: xarray: Add XArray load operation The xa_load function brings with it a lot of infrastructure; xa_empty(), xa_is_err(), and large chunks of the XArray advanced API that are used to implement xa_load. As the test-suite demonstrates, it is possible to use the XArray functions on a radix tree. The radix tree functions depend on the GFP flags being stored in the root of the tree, so it's not possible to use the radix tree functions on an XArray. Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 336 ++++++++++++++++++++++++++++++ lib/Kconfig.debug | 3 + lib/Makefile | 1 + lib/radix-tree.c | 43 ---- lib/test_xarray.c | 87 ++++++++ lib/xarray.c | 195 +++++++++++++++++ tools/include/linux/kernel.h | 1 + tools/testing/radix-tree/.gitignore | 1 + tools/testing/radix-tree/Makefile | 6 +- tools/testing/radix-tree/linux/kernel.h | 1 + tools/testing/radix-tree/linux/rcupdate.h | 2 + tools/testing/radix-tree/main.c | 1 + tools/testing/radix-tree/test.h | 1 + tools/testing/radix-tree/xarray.c | 28 +++ 14 files changed, 661 insertions(+), 45 deletions(-) create mode 100644 lib/test_xarray.c (limited to 'tools') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 52141dfc5a90..a0df8217068c 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -30,6 +32,10 @@ * * 0-62: Sibling entries * 256: Retry entry + * + * Errors are also represented as internal entries, but use the negative + * space (-4094 to -2). They're never stored in the slots array; only + * returned by the normal API. */ #define BITS_PER_XA_VALUE (BITS_PER_LONG - 1) @@ -155,6 +161,42 @@ static inline bool xa_is_internal(const void *entry) return ((unsigned long)entry & 3) == 2; } +/** + * xa_is_err() - Report whether an XArray operation returned an error + * @entry: Result from calling an XArray function + * + * If an XArray operation cannot complete an operation, it will return + * a special value indicating an error. This function tells you + * whether an error occurred; xa_err() tells you which error occurred. + * + * Context: Any context. + * Return: %true if the entry indicates an error. + */ +static inline bool xa_is_err(const void *entry) +{ + return unlikely(xa_is_internal(entry)); +} + +/** + * xa_err() - Turn an XArray result into an errno. + * @entry: Result from calling an XArray function. + * + * If an XArray operation cannot complete an operation, it will return + * a special pointer value which encodes an errno. This function extracts + * the errno from the pointer value, or returns 0 if the pointer does not + * represent an errno. + * + * Context: Any context. + * Return: A negative errno or 0. + */ +static inline int xa_err(void *entry) +{ + /* xa_to_internal() would not do sign extension. */ + if (xa_is_err(entry)) + return (long)entry >> 2; + return 0; +} + /** * struct xarray - The anchor of the XArray. * @xa_lock: Lock that protects the contents of the XArray. @@ -209,6 +251,7 @@ struct xarray { #define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0) void xa_init_flags(struct xarray *, gfp_t flags); +void *xa_load(struct xarray *, unsigned long index); /** * xa_init() - Initialise an empty XArray. @@ -223,6 +266,18 @@ static inline void xa_init(struct xarray *xa) xa_init_flags(xa, 0); } +/** + * xa_empty() - Determine if an array has any present entries. + * @xa: XArray. + * + * Context: Any context. + * Return: %true if the array contains only NULL pointers. + */ +static inline bool xa_empty(const struct xarray *xa) +{ + return xa->xa_head == NULL; +} + #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) #define xa_unlock(xa) spin_unlock(&(xa)->xa_lock) @@ -280,6 +335,65 @@ struct xa_node { }; }; +void xa_dump(const struct xarray *); +void xa_dump_node(const struct xa_node *); + +#ifdef XA_DEBUG +#define XA_BUG_ON(xa, x) do { \ + if (x) { \ + xa_dump(xa); \ + BUG(); \ + } \ + } while (0) +#define XA_NODE_BUG_ON(node, x) do { \ + if (x) { \ + if (node) xa_dump_node(node); \ + BUG(); \ + } \ + } while (0) +#else +#define XA_BUG_ON(xa, x) do { } while (0) +#define XA_NODE_BUG_ON(node, x) do { } while (0) +#endif + +/* Private */ +static inline void *xa_head(const struct xarray *xa) +{ + return rcu_dereference_check(xa->xa_head, + lockdep_is_held(&xa->xa_lock)); +} + +/* Private */ +static inline void *xa_head_locked(const struct xarray *xa) +{ + return rcu_dereference_protected(xa->xa_head, + lockdep_is_held(&xa->xa_lock)); +} + +/* Private */ +static inline void *xa_entry(const struct xarray *xa, + const struct xa_node *node, unsigned int offset) +{ + XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE); + return rcu_dereference_check(node->slots[offset], + lockdep_is_held(&xa->xa_lock)); +} + +/* Private */ +static inline void *xa_entry_locked(const struct xarray *xa, + const struct xa_node *node, unsigned int offset) +{ + XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE); + return rcu_dereference_protected(node->slots[offset], + lockdep_is_held(&xa->xa_lock)); +} + +/* Private */ +static inline struct xa_node *xa_to_node(const void *entry) +{ + return (struct xa_node *)((unsigned long)entry - 2); +} + /* Private */ static inline bool xa_is_node(const void *entry) { @@ -312,4 +426,226 @@ static inline bool xa_is_sibling(const void *entry) #define XA_RETRY_ENTRY xa_mk_internal(256) +/** + * xa_is_retry() - Is the entry a retry entry? + * @entry: Entry retrieved from the XArray + * + * Return: %true if the entry is a retry entry. + */ +static inline bool xa_is_retry(const void *entry) +{ + return unlikely(entry == XA_RETRY_ENTRY); +} + +/** + * typedef xa_update_node_t - A callback function from the XArray. + * @node: The node which is being processed + * + * This function is called every time the XArray updates the count of + * present and value entries in a node. It allows advanced users to + * maintain the private_list in the node. + * + * Context: The xa_lock is held and interrupts may be disabled. + * Implementations should not drop the xa_lock, nor re-enable + * interrupts. + */ +typedef void (*xa_update_node_t)(struct xa_node *node); + +/* + * The xa_state is opaque to its users. It contains various different pieces + * of state involved in the current operation on the XArray. It should be + * declared on the stack and passed between the various internal routines. + * The various elements in it should not be accessed directly, but only + * through the provided accessor functions. The below documentation is for + * the benefit of those working on the code, not for users of the XArray. + * + * @xa_node usually points to the xa_node containing the slot we're operating + * on (and @xa_offset is the offset in the slots array). If there is a + * single entry in the array at index 0, there are no allocated xa_nodes to + * point to, and so we store %NULL in @xa_node. @xa_node is set to + * the value %XAS_RESTART if the xa_state is not walked to the correct + * position in the tree of nodes for this operation. If an error occurs + * during an operation, it is set to an %XAS_ERROR value. If we run off the + * end of the allocated nodes, it is set to %XAS_BOUNDS. + */ +struct xa_state { + struct xarray *xa; + unsigned long xa_index; + unsigned char xa_shift; + unsigned char xa_sibs; + unsigned char xa_offset; + unsigned char xa_pad; /* Helps gcc generate better code */ + struct xa_node *xa_node; + struct xa_node *xa_alloc; + xa_update_node_t xa_update; +}; + +/* + * We encode errnos in the xas->xa_node. If an error has happened, we need to + * drop the lock to fix it, and once we've done so the xa_state is invalid. + */ +#define XA_ERROR(errno) ((struct xa_node *)(((unsigned long)errno << 2) | 2UL)) +#define XAS_BOUNDS ((struct xa_node *)1UL) +#define XAS_RESTART ((struct xa_node *)3UL) + +#define __XA_STATE(array, index, shift, sibs) { \ + .xa = array, \ + .xa_index = index, \ + .xa_shift = shift, \ + .xa_sibs = sibs, \ + .xa_offset = 0, \ + .xa_pad = 0, \ + .xa_node = XAS_RESTART, \ + .xa_alloc = NULL, \ + .xa_update = NULL \ +} + +/** + * XA_STATE() - Declare an XArray operation state. + * @name: Name of this operation state (usually xas). + * @array: Array to operate on. + * @index: Initial index of interest. + * + * Declare and initialise an xa_state on the stack. + */ +#define XA_STATE(name, array, index) \ + struct xa_state name = __XA_STATE(array, index, 0, 0) + +/** + * XA_STATE_ORDER() - Declare an XArray operation state. + * @name: Name of this operation state (usually xas). + * @array: Array to operate on. + * @index: Initial index of interest. + * @order: Order of entry. + * + * Declare and initialise an xa_state on the stack. This variant of + * XA_STATE() allows you to specify the 'order' of the element you + * want to operate on.` + */ +#define XA_STATE_ORDER(name, array, index, order) \ + struct xa_state name = __XA_STATE(array, \ + (index >> order) << order, \ + order - (order % XA_CHUNK_SHIFT), \ + (1U << (order % XA_CHUNK_SHIFT)) - 1) + +#define xas_marked(xas, mark) xa_marked((xas)->xa, (mark)) +#define xas_trylock(xas) xa_trylock((xas)->xa) +#define xas_lock(xas) xa_lock((xas)->xa) +#define xas_unlock(xas) xa_unlock((xas)->xa) +#define xas_lock_bh(xas) xa_lock_bh((xas)->xa) +#define xas_unlock_bh(xas) xa_unlock_bh((xas)->xa) +#define xas_lock_irq(xas) xa_lock_irq((xas)->xa) +#define xas_unlock_irq(xas) xa_unlock_irq((xas)->xa) +#define xas_lock_irqsave(xas, flags) \ + xa_lock_irqsave((xas)->xa, flags) +#define xas_unlock_irqrestore(xas, flags) \ + xa_unlock_irqrestore((xas)->xa, flags) + +/** + * xas_error() - Return an errno stored in the xa_state. + * @xas: XArray operation state. + * + * Return: 0 if no error has been noted. A negative errno if one has. + */ +static inline int xas_error(const struct xa_state *xas) +{ + return xa_err(xas->xa_node); +} + +/** + * xas_set_err() - Note an error in the xa_state. + * @xas: XArray operation state. + * @err: Negative error number. + * + * Only call this function with a negative @err; zero or positive errors + * will probably not behave the way you think they should. If you want + * to clear the error from an xa_state, use xas_reset(). + */ +static inline void xas_set_err(struct xa_state *xas, long err) +{ + xas->xa_node = XA_ERROR(err); +} + +/** + * xas_invalid() - Is the xas in a retry or error state? + * @xas: XArray operation state. + * + * Return: %true if the xas cannot be used for operations. + */ +static inline bool xas_invalid(const struct xa_state *xas) +{ + return (unsigned long)xas->xa_node & 3; +} + +/** + * xas_valid() - Is the xas a valid cursor into the array? + * @xas: XArray operation state. + * + * Return: %true if the xas can be used for operations. + */ +static inline bool xas_valid(const struct xa_state *xas) +{ + return !xas_invalid(xas); +} + +/** + * xas_reset() - Reset an XArray operation state. + * @xas: XArray operation state. + * + * Resets the error or walk state of the @xas so future walks of the + * array will start from the root. Use this if you have dropped the + * xarray lock and want to reuse the xa_state. + * + * Context: Any context. + */ +static inline void xas_reset(struct xa_state *xas) +{ + xas->xa_node = XAS_RESTART; +} + +/** + * xas_retry() - Retry the operation if appropriate. + * @xas: XArray operation state. + * @entry: Entry from xarray. + * + * The advanced functions may sometimes return an internal entry, such as + * a retry entry or a zero entry. This function sets up the @xas to restart + * the walk from the head of the array if needed. + * + * Context: Any context. + * Return: true if the operation needs to be retried. + */ +static inline bool xas_retry(struct xa_state *xas, const void *entry) +{ + if (!xa_is_retry(entry)) + return false; + xas_reset(xas); + return true; +} + +void *xas_load(struct xa_state *); + +/** + * xas_reload() - Refetch an entry from the xarray. + * @xas: XArray operation state. + * + * Use this function to check that a previously loaded entry still has + * the same value. This is useful for the lockless pagecache lookup where + * we walk the array with only the RCU lock to protect us, lock the page, + * then check that the page hasn't moved since we looked it up. + * + * The caller guarantees that @xas is still valid. If it may be in an + * error or restart state, call xas_load() instead. + * + * Return: The entry at this location in the xarray. + */ +static inline void *xas_reload(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_node; + + if (node) + return xa_entry(xas->xa, node, xas->xa_offset); + return xa_head(xas->xa); +} + #endif /* _LINUX_XARRAY_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 4966c4fbe7f7..091155e12422 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1813,6 +1813,9 @@ config TEST_BITFIELD config TEST_UUID tristate "Test functions located in the uuid module at runtime" +config TEST_XARRAY + tristate "Test the XArray code at runtime" + config TEST_OVERFLOW tristate "Test check_*_overflow() functions at runtime" diff --git a/lib/Makefile b/lib/Makefile index 057385f1f145..e6f809556af5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -68,6 +68,7 @@ obj-$(CONFIG_TEST_PRINTF) += test_printf.o obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o obj-$(CONFIG_TEST_BITFIELD) += test_bitfield.o obj-$(CONFIG_TEST_UUID) += test_uuid.o +obj-$(CONFIG_TEST_XARRAY) += test_xarray.o obj-$(CONFIG_TEST_PARMAN) += test_parman.o obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 8a568cea1237..b8e961428484 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -256,49 +256,6 @@ static unsigned long next_index(unsigned long index, } #ifndef __KERNEL__ -static void dump_node(struct radix_tree_node *node, unsigned long index) -{ - unsigned long i; - - pr_debug("radix node: %p offset %d indices %lu-%lu parent %p tags %lx %lx %lx shift %d count %d nr_values %d\n", - node, node->offset, index, index | node_maxindex(node), - node->parent, - node->tags[0][0], node->tags[1][0], node->tags[2][0], - node->shift, node->count, node->nr_values); - - for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) { - unsigned long first = index | (i << node->shift); - unsigned long last = first | ((1UL << node->shift) - 1); - void *entry = node->slots[i]; - if (!entry) - continue; - if (entry == RADIX_TREE_RETRY) { - pr_debug("radix retry offset %ld indices %lu-%lu parent %p\n", - i, first, last, node); - } else if (!radix_tree_is_internal_node(entry)) { - pr_debug("radix entry %p offset %ld indices %lu-%lu parent %p\n", - entry, i, first, last, node); - } else if (xa_is_sibling(entry)) { - pr_debug("radix sblng %p offset %ld indices %lu-%lu parent %p val %p\n", - entry, i, first, last, node, - node->slots[xa_to_sibling(entry)]); - } else { - dump_node(entry_to_node(entry), first); - } - } -} - -/* For debug */ -static void radix_tree_dump(struct radix_tree_root *root) -{ - pr_debug("radix root: %p xa_head %p tags %x\n", - root, root->xa_head, - root->xa_flags >> ROOT_TAG_SHIFT); - if (!radix_tree_is_internal_node(root->xa_head)) - return; - dump_node(entry_to_node(root->xa_head), 0); -} - static void dump_ida_node(void *entry, unsigned long index) { unsigned long i; diff --git a/lib/test_xarray.c b/lib/test_xarray.c new file mode 100644 index 000000000000..a7248b87617f --- /dev/null +++ b/lib/test_xarray.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * test_xarray.c: Test the XArray API + * Copyright (c) 2017-2018 Microsoft Corporation + * Author: Matthew Wilcox + */ + +#include +#include + +static unsigned int tests_run; +static unsigned int tests_passed; + +#ifndef XA_DEBUG +# ifdef __KERNEL__ +void xa_dump(const struct xarray *xa) { } +# endif +#undef XA_BUG_ON +#define XA_BUG_ON(xa, x) do { \ + tests_run++; \ + if (x) { \ + printk("BUG at %s:%d\n", __func__, __LINE__); \ + xa_dump(xa); \ + dump_stack(); \ + } else { \ + tests_passed++; \ + } \ +} while (0) +#endif + +static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) +{ + radix_tree_insert(xa, index, xa_mk_value(index)); + return NULL; +} + +static void xa_erase_index(struct xarray *xa, unsigned long index) +{ + radix_tree_delete(xa, index); +} + +static noinline void check_xa_load(struct xarray *xa) +{ + unsigned long i, j; + + for (i = 0; i < 1024; i++) { + for (j = 0; j < 1024; j++) { + void *entry = xa_load(xa, j); + if (j < i) + XA_BUG_ON(xa, xa_to_value(entry) != j); + else + XA_BUG_ON(xa, entry); + } + XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL); + } + + for (i = 0; i < 1024; i++) { + for (j = 0; j < 1024; j++) { + void *entry = xa_load(xa, j); + if (j >= i) + XA_BUG_ON(xa, xa_to_value(entry) != j); + else + XA_BUG_ON(xa, entry); + } + xa_erase_index(xa, i); + } + XA_BUG_ON(xa, !xa_empty(xa)); +} + +static RADIX_TREE(array, GFP_KERNEL); + +static int xarray_checks(void) +{ + check_xa_load(&array); + + printk("XArray: %u of %u tests passed\n", tests_passed, tests_run); + return (tests_run == tests_passed) ? 0 : -EINVAL; +} + +static void xarray_exit(void) +{ +} + +module_init(xarray_checks); +module_exit(xarray_exit); +MODULE_AUTHOR("Matthew Wilcox "); +MODULE_LICENSE("GPL"); diff --git a/lib/xarray.c b/lib/xarray.c index 862f4c64c754..19cfcbc69a68 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -24,6 +24,100 @@ * @entry refers to something stored in a slot in the xarray */ +/* extracts the offset within this node from the index */ +static unsigned int get_offset(unsigned long index, struct xa_node *node) +{ + return (index >> node->shift) & XA_CHUNK_MASK; +} + +/* move the index either forwards (find) or backwards (sibling slot) */ +static void xas_move_index(struct xa_state *xas, unsigned long offset) +{ + unsigned int shift = xas->xa_node->shift; + xas->xa_index &= ~XA_CHUNK_MASK << shift; + xas->xa_index += offset << shift; +} + +static void *set_bounds(struct xa_state *xas) +{ + xas->xa_node = XAS_BOUNDS; + return NULL; +} + +/* + * Starts a walk. If the @xas is already valid, we assume that it's on + * the right path and just return where we've got to. If we're in an + * error state, return NULL. If the index is outside the current scope + * of the xarray, return NULL without changing @xas->xa_node. Otherwise + * set @xas->xa_node to NULL and return the current head of the array. + */ +static void *xas_start(struct xa_state *xas) +{ + void *entry; + + if (xas_valid(xas)) + return xas_reload(xas); + if (xas_error(xas)) + return NULL; + + entry = xa_head(xas->xa); + if (!xa_is_node(entry)) { + if (xas->xa_index) + return set_bounds(xas); + } else { + if ((xas->xa_index >> xa_to_node(entry)->shift) > XA_CHUNK_MASK) + return set_bounds(xas); + } + + xas->xa_node = NULL; + return entry; +} + +static void *xas_descend(struct xa_state *xas, struct xa_node *node) +{ + unsigned int offset = get_offset(xas->xa_index, node); + void *entry = xa_entry(xas->xa, node, offset); + + xas->xa_node = node; + if (xa_is_sibling(entry)) { + offset = xa_to_sibling(entry); + entry = xa_entry(xas->xa, node, offset); + } + + xas->xa_offset = offset; + return entry; +} + +/** + * xas_load() - Load an entry from the XArray (advanced). + * @xas: XArray operation state. + * + * Usually walks the @xas to the appropriate state to load the entry + * stored at xa_index. However, it will do nothing and return %NULL if + * @xas is in an error state. xas_load() will never expand the tree. + * + * If the xa_state is set up to operate on a multi-index entry, xas_load() + * may return %NULL or an internal entry, even if there are entries + * present within the range specified by @xas. + * + * Context: Any context. The caller should hold the xa_lock or the RCU lock. + * Return: Usually an entry in the XArray, but see description for exceptions. + */ +void *xas_load(struct xa_state *xas) +{ + void *entry = xas_start(xas); + + while (xa_is_node(entry)) { + struct xa_node *node = xa_to_node(entry); + + if (xas->xa_shift > node->shift) + break; + entry = xas_descend(xas, node); + } + return entry; +} +EXPORT_SYMBOL_GPL(xas_load); + /** * xa_init_flags() - Initialise an empty XArray with flags. * @xa: XArray. @@ -42,3 +136,104 @@ void xa_init_flags(struct xarray *xa, gfp_t flags) xa->xa_head = NULL; } EXPORT_SYMBOL(xa_init_flags); + +/** + * xa_load() - Load an entry from an XArray. + * @xa: XArray. + * @index: index into array. + * + * Context: Any context. Takes and releases the RCU lock. + * Return: The entry at @index in @xa. + */ +void *xa_load(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + void *entry; + + rcu_read_lock(); + do { + entry = xas_load(&xas); + } while (xas_retry(&xas, entry)); + rcu_read_unlock(); + + return entry; +} +EXPORT_SYMBOL(xa_load); + +#ifdef XA_DEBUG +void xa_dump_node(const struct xa_node *node) +{ + unsigned i, j; + + if (!node) + return; + if ((unsigned long)node & 3) { + pr_cont("node %px\n", node); + return; + } + + pr_cont("node %px %s %d parent %px shift %d count %d values %d " + "array %px list %px %px marks", + node, node->parent ? "offset" : "max", node->offset, + node->parent, node->shift, node->count, node->nr_values, + node->array, node->private_list.prev, node->private_list.next); + for (i = 0; i < XA_MAX_MARKS; i++) + for (j = 0; j < XA_MARK_LONGS; j++) + pr_cont(" %lx", node->marks[i][j]); + pr_cont("\n"); +} + +void xa_dump_index(unsigned long index, unsigned int shift) +{ + if (!shift) + pr_info("%lu: ", index); + else if (shift >= BITS_PER_LONG) + pr_info("0-%lu: ", ~0UL); + else + pr_info("%lu-%lu: ", index, index | ((1UL << shift) - 1)); +} + +void xa_dump_entry(const void *entry, unsigned long index, unsigned long shift) +{ + if (!entry) + return; + + xa_dump_index(index, shift); + + if (xa_is_node(entry)) { + if (shift == 0) { + pr_cont("%px\n", entry); + } else { + unsigned long i; + struct xa_node *node = xa_to_node(entry); + xa_dump_node(node); + for (i = 0; i < XA_CHUNK_SIZE; i++) + xa_dump_entry(node->slots[i], + index + (i << node->shift), node->shift); + } + } else if (xa_is_value(entry)) + pr_cont("value %ld (0x%lx) [%px]\n", xa_to_value(entry), + xa_to_value(entry), entry); + else if (!xa_is_internal(entry)) + pr_cont("%px\n", entry); + else if (xa_is_retry(entry)) + pr_cont("retry (%ld)\n", xa_to_internal(entry)); + else if (xa_is_sibling(entry)) + pr_cont("sibling (slot %ld)\n", xa_to_sibling(entry)); + else + pr_cont("UNKNOWN ENTRY (%px)\n", entry); +} + +void xa_dump(const struct xarray *xa) +{ + void *entry = xa->xa_head; + unsigned int shift = 0; + + pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry, + xa->xa_flags, radix_tree_tagged(xa, 0), + radix_tree_tagged(xa, 1), radix_tree_tagged(xa, 2)); + if (xa_is_node(entry)) + shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT; + xa_dump_entry(entry, 0, shift); +} +#endif diff --git a/tools/include/linux/kernel.h b/tools/include/linux/kernel.h index 0ad884452c5c..6935ef94e77a 100644 --- a/tools/include/linux/kernel.h +++ b/tools/include/linux/kernel.h @@ -70,6 +70,7 @@ #define BUG_ON(cond) assert(!(cond)) #endif #endif +#define BUG() BUG_ON(1) #if __BYTE_ORDER == __BIG_ENDIAN #define cpu_to_le16 bswap_16 diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore index d4706c0ffceb..3834899b6693 100644 --- a/tools/testing/radix-tree/.gitignore +++ b/tools/testing/radix-tree/.gitignore @@ -4,3 +4,4 @@ idr-test main multiorder radix-tree.c +xarray diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index c0cf1c79efd5..1379f1d78d0b 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -4,7 +4,7 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ -fsanitize=undefined LDFLAGS += -fsanitize=address -fsanitize=undefined LDLIBS+= -lpthread -lurcu -TARGETS = main idr-test multiorder +TARGETS = main idr-test multiorder xarray CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ tag_check.o multiorder.o idr-test.o iteration_check.o benchmark.o @@ -25,6 +25,8 @@ main: $(OFILES) idr-test.o: ../../../lib/test_ida.c idr-test: idr-test.o $(CORE_OFILES) +xarray: $(CORE_OFILES) + multiorder: multiorder.o $(CORE_OFILES) clean: @@ -45,7 +47,7 @@ radix-tree.c: ../../../lib/radix-tree.c idr.c: ../../../lib/idr.c sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@ -xarray.o: ../../../lib/xarray.c +xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c generated/map-shift.h: @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h index 426f32f28547..5d06ac75a14d 100644 --- a/tools/testing/radix-tree/linux/kernel.h +++ b/tools/testing/radix-tree/linux/kernel.h @@ -14,6 +14,7 @@ #include "../../../include/linux/kconfig.h" #define printk printf +#define pr_info printk #define pr_debug printk #define pr_cont printk diff --git a/tools/testing/radix-tree/linux/rcupdate.h b/tools/testing/radix-tree/linux/rcupdate.h index 73ed33658203..fd280b070fdb 100644 --- a/tools/testing/radix-tree/linux/rcupdate.h +++ b/tools/testing/radix-tree/linux/rcupdate.h @@ -6,5 +6,7 @@ #define rcu_dereference_raw(p) rcu_dereference(p) #define rcu_dereference_protected(p, cond) rcu_dereference(p) +#define rcu_dereference_check(p, cond) rcu_dereference(p) +#define RCU_INIT_POINTER(p, v) (p) = (v) #endif diff --git a/tools/testing/radix-tree/main.c b/tools/testing/radix-tree/main.c index b741686e53d6..09deaf4f0959 100644 --- a/tools/testing/radix-tree/main.c +++ b/tools/testing/radix-tree/main.c @@ -365,6 +365,7 @@ int main(int argc, char **argv) rcu_register_thread(); radix_tree_init(); + xarray_tests(); regression1_test(); regression2_test(); regression3_test(); diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index 92d901eacf49..e3dc7a16f09b 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -34,6 +34,7 @@ int tag_tagged_items(struct radix_tree_root *, pthread_mutex_t *, unsigned iftag, unsigned thentag); unsigned long find_item(struct radix_tree_root *, void *item); +void xarray_tests(void); void tag_check(void); void multiorder_checks(void); void iteration_test(unsigned order, unsigned duration); diff --git a/tools/testing/radix-tree/xarray.c b/tools/testing/radix-tree/xarray.c index 9bbd667172a7..e61e43efe463 100644 --- a/tools/testing/radix-tree/xarray.c +++ b/tools/testing/radix-tree/xarray.c @@ -4,4 +4,32 @@ * Copyright (c) 2018 Matthew Wilcox */ +#define XA_DEBUG +#include "test.h" + +#define module_init(x) +#define module_exit(x) +#define MODULE_AUTHOR(x) +#define MODULE_LICENSE(x) +#define dump_stack() assert(0) + #include "../../../lib/xarray.c" +#undef XA_DEBUG +#include "../../../lib/test_xarray.c" + +void xarray_tests(void) +{ + xarray_checks(); + xarray_exit(); +} + +int __weak main(void) +{ + radix_tree_init(); + xarray_tests(); + radix_tree_cpu_dead(1); + rcu_barrier(); + if (nr_allocated) + printf("nr_allocated = %d\n", nr_allocated); + return 0; +} -- cgit v1.2.3 From 9b89a0355144685a787b0dc5bcf7bdd6f2d02968 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Fri, 10 Nov 2017 09:34:31 -0500 Subject: xarray: Add XArray marks XArray marks are like the radix tree tags, only slightly more strongly typed. They are renamed in order to distinguish them from tagged pointers. This commit adds the basic get/set/clear operations. Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 63 +++++++ lib/test_xarray.c | 34 ++++ lib/xarray.c | 232 +++++++++++++++++++++++++- tools/include/asm-generic/bitops.h | 1 + tools/include/asm-generic/bitops/atomic.h | 9 - tools/include/asm-generic/bitops/non-atomic.h | 109 ++++++++++++ tools/include/linux/spinlock.h | 10 +- 7 files changed, 445 insertions(+), 13 deletions(-) create mode 100644 tools/include/asm-generic/bitops/non-atomic.h (limited to 'tools') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index a0df8217068c..2de504ae9ba4 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -197,6 +198,20 @@ static inline int xa_err(void *entry) return 0; } +typedef unsigned __bitwise xa_mark_t; +#define XA_MARK_0 ((__force xa_mark_t)0U) +#define XA_MARK_1 ((__force xa_mark_t)1U) +#define XA_MARK_2 ((__force xa_mark_t)2U) +#define XA_PRESENT ((__force xa_mark_t)8U) +#define XA_MARK_MAX XA_MARK_2 + +/* + * Values for xa_flags. The radix tree stores its GFP flags in the xa_flags, + * and we remain compatible with that. + */ +#define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ + (__force unsigned)(mark))) + /** * struct xarray - The anchor of the XArray. * @xa_lock: Lock that protects the contents of the XArray. @@ -252,6 +267,9 @@ struct xarray { void xa_init_flags(struct xarray *, gfp_t flags); void *xa_load(struct xarray *, unsigned long index); +bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t); +void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); +void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); /** * xa_init() - Initialise an empty XArray. @@ -278,6 +296,19 @@ static inline bool xa_empty(const struct xarray *xa) return xa->xa_head == NULL; } +/** + * xa_marked() - Inquire whether any entry in this array has a mark set + * @xa: Array + * @mark: Mark value + * + * Context: Any context. + * Return: %true if any entry has this mark set. + */ +static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark) +{ + return xa->xa_flags & XA_FLAGS_MARK(mark); +} + #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) #define xa_unlock(xa) spin_unlock(&(xa)->xa_lock) @@ -290,6 +321,12 @@ static inline bool xa_empty(const struct xarray *xa) #define xa_unlock_irqrestore(xa, flags) \ spin_unlock_irqrestore(&(xa)->xa_lock, flags) +/* + * Versions of the normal API which require the caller to hold the xa_lock. + */ +void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); +void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); + /* Everything below here is the Advanced API. Proceed with caution. */ /* @@ -388,6 +425,22 @@ static inline void *xa_entry_locked(const struct xarray *xa, lockdep_is_held(&xa->xa_lock)); } +/* Private */ +static inline struct xa_node *xa_parent(const struct xarray *xa, + const struct xa_node *node) +{ + return rcu_dereference_check(node->parent, + lockdep_is_held(&xa->xa_lock)); +} + +/* Private */ +static inline struct xa_node *xa_parent_locked(const struct xarray *xa, + const struct xa_node *node) +{ + return rcu_dereference_protected(node->parent, + lockdep_is_held(&xa->xa_lock)); +} + /* Private */ static inline struct xa_node *xa_to_node(const void *entry) { @@ -588,6 +641,12 @@ static inline bool xas_valid(const struct xa_state *xas) return !xas_invalid(xas); } +/* True if the pointer is something other than a node */ +static inline bool xas_not_node(struct xa_node *node) +{ + return ((unsigned long)node & 3) || !node; +} + /** * xas_reset() - Reset an XArray operation state. * @xas: XArray operation state. @@ -625,6 +684,10 @@ static inline bool xas_retry(struct xa_state *xas, const void *entry) void *xas_load(struct xa_state *); +bool xas_get_mark(const struct xa_state *, xa_mark_t); +void xas_set_mark(const struct xa_state *, xa_mark_t); +void xas_clear_mark(const struct xa_state *, xa_mark_t); + /** * xas_reload() - Refetch an entry from the xarray. * @xas: XArray operation state. diff --git a/lib/test_xarray.c b/lib/test_xarray.c index a7248b87617f..f8b0e9ba80a4 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -67,11 +67,45 @@ static noinline void check_xa_load(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) +{ + /* NULL elements have no marks set */ + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); + xa_set_mark(xa, index, XA_MARK_0); + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); + + /* Storing a pointer will not make a mark appear */ + XA_BUG_ON(xa, xa_store_index(xa, index, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); + xa_set_mark(xa, index, XA_MARK_0); + XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0)); + + /* Setting one mark will not set another mark */ + XA_BUG_ON(xa, xa_get_mark(xa, index + 1, XA_MARK_0)); + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_1)); + + /* Storing NULL clears marks, and they can't be set again */ + xa_erase_index(xa, index); + XA_BUG_ON(xa, !xa_empty(xa)); + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); + xa_set_mark(xa, index, XA_MARK_0); + XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); +} + +static noinline void check_xa_mark(struct xarray *xa) +{ + unsigned long index; + + for (index = 0; index < 16384; index += 4) + check_xa_mark_1(xa, index); +} + static RADIX_TREE(array, GFP_KERNEL); static int xarray_checks(void) { check_xa_load(&array); + check_xa_mark(&array); printk("XArray: %u of %u tests passed\n", tests_passed, tests_run); return (tests_run == tests_passed) ? 0 : -EINVAL; diff --git a/lib/xarray.c b/lib/xarray.c index 19cfcbc69a68..aa86c47e532f 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -5,6 +5,7 @@ * Author: Matthew Wilcox */ +#include #include #include @@ -24,6 +25,48 @@ * @entry refers to something stored in a slot in the xarray */ +static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) +{ + if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) + xa->xa_flags |= XA_FLAGS_MARK(mark); +} + +static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark) +{ + if (xa->xa_flags & XA_FLAGS_MARK(mark)) + xa->xa_flags &= ~(XA_FLAGS_MARK(mark)); +} + +static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark) +{ + return node->marks[(__force unsigned)mark]; +} + +static inline bool node_get_mark(struct xa_node *node, + unsigned int offset, xa_mark_t mark) +{ + return test_bit(offset, node_marks(node, mark)); +} + +/* returns true if the bit was set */ +static inline bool node_set_mark(struct xa_node *node, unsigned int offset, + xa_mark_t mark) +{ + return __test_and_set_bit(offset, node_marks(node, mark)); +} + +/* returns true if the bit was set */ +static inline bool node_clear_mark(struct xa_node *node, unsigned int offset, + xa_mark_t mark) +{ + return __test_and_clear_bit(offset, node_marks(node, mark)); +} + +static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) +{ + return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); +} + /* extracts the offset within this node from the index */ static unsigned int get_offset(unsigned long index, struct xa_node *node) { @@ -118,6 +161,85 @@ void *xas_load(struct xa_state *xas) } EXPORT_SYMBOL_GPL(xas_load); +/** + * xas_get_mark() - Returns the state of this mark. + * @xas: XArray operation state. + * @mark: Mark number. + * + * Return: true if the mark is set, false if the mark is clear or @xas + * is in an error state. + */ +bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark) +{ + if (xas_invalid(xas)) + return false; + if (!xas->xa_node) + return xa_marked(xas->xa, mark); + return node_get_mark(xas->xa_node, xas->xa_offset, mark); +} +EXPORT_SYMBOL_GPL(xas_get_mark); + +/** + * xas_set_mark() - Sets the mark on this entry and its parents. + * @xas: XArray operation state. + * @mark: Mark number. + * + * Sets the specified mark on this entry, and walks up the tree setting it + * on all the ancestor entries. Does nothing if @xas has not been walked to + * an entry, or is in an error state. + */ +void xas_set_mark(const struct xa_state *xas, xa_mark_t mark) +{ + struct xa_node *node = xas->xa_node; + unsigned int offset = xas->xa_offset; + + if (xas_invalid(xas)) + return; + + while (node) { + if (node_set_mark(node, offset, mark)) + return; + offset = node->offset; + node = xa_parent_locked(xas->xa, node); + } + + if (!xa_marked(xas->xa, mark)) + xa_mark_set(xas->xa, mark); +} +EXPORT_SYMBOL_GPL(xas_set_mark); + +/** + * xas_clear_mark() - Clears the mark on this entry and its parents. + * @xas: XArray operation state. + * @mark: Mark number. + * + * Clears the specified mark on this entry, and walks back to the head + * attempting to clear it on all the ancestor entries. Does nothing if + * @xas has not been walked to an entry, or is in an error state. + */ +void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark) +{ + struct xa_node *node = xas->xa_node; + unsigned int offset = xas->xa_offset; + + if (xas_invalid(xas)) + return; + + while (node) { + if (!node_clear_mark(node, offset, mark)) + return; + if (node_any_mark(node, mark)) + return; + + offset = node->offset; + node = xa_parent_locked(xas->xa, node); + } + + if (xa_marked(xas->xa, mark)) + xa_mark_clear(xas->xa, mark); +} +EXPORT_SYMBOL_GPL(xas_clear_mark); + /** * xa_init_flags() - Initialise an empty XArray with flags. * @xa: XArray. @@ -160,6 +282,112 @@ void *xa_load(struct xarray *xa, unsigned long index) } EXPORT_SYMBOL(xa_load); +/** + * __xa_set_mark() - Set this mark on this entry while locked. + * @xa: XArray. + * @index: Index of entry. + * @mark: Mark number. + * + * Attempting to set a mark on a NULL entry does not succeed. + * + * Context: Any context. Expects xa_lock to be held on entry. + */ +void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark) +{ + XA_STATE(xas, xa, index); + void *entry = xas_load(&xas); + + if (entry) + xas_set_mark(&xas, mark); +} +EXPORT_SYMBOL_GPL(__xa_set_mark); + +/** + * __xa_clear_mark() - Clear this mark on this entry while locked. + * @xa: XArray. + * @index: Index of entry. + * @mark: Mark number. + * + * Context: Any context. Expects xa_lock to be held on entry. + */ +void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark) +{ + XA_STATE(xas, xa, index); + void *entry = xas_load(&xas); + + if (entry) + xas_clear_mark(&xas, mark); +} +EXPORT_SYMBOL_GPL(__xa_clear_mark); + +/** + * xa_get_mark() - Inquire whether this mark is set on this entry. + * @xa: XArray. + * @index: Index of entry. + * @mark: Mark number. + * + * This function uses the RCU read lock, so the result may be out of date + * by the time it returns. If you need the result to be stable, use a lock. + * + * Context: Any context. Takes and releases the RCU lock. + * Return: True if the entry at @index has this mark set, false if it doesn't. + */ +bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark) +{ + XA_STATE(xas, xa, index); + void *entry; + + rcu_read_lock(); + entry = xas_start(&xas); + while (xas_get_mark(&xas, mark)) { + if (!xa_is_node(entry)) + goto found; + entry = xas_descend(&xas, xa_to_node(entry)); + } + rcu_read_unlock(); + return false; + found: + rcu_read_unlock(); + return true; +} +EXPORT_SYMBOL(xa_get_mark); + +/** + * xa_set_mark() - Set this mark on this entry. + * @xa: XArray. + * @index: Index of entry. + * @mark: Mark number. + * + * Attempting to set a mark on a NULL entry does not succeed. + * + * Context: Process context. Takes and releases the xa_lock. + */ +void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark) +{ + xa_lock(xa); + __xa_set_mark(xa, index, mark); + xa_unlock(xa); +} +EXPORT_SYMBOL(xa_set_mark); + +/** + * xa_clear_mark() - Clear this mark on this entry. + * @xa: XArray. + * @index: Index of entry. + * @mark: Mark number. + * + * Clearing a mark always succeeds. + * + * Context: Process context. Takes and releases the xa_lock. + */ +void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark) +{ + xa_lock(xa); + __xa_clear_mark(xa, index, mark); + xa_unlock(xa); +} +EXPORT_SYMBOL(xa_clear_mark); + #ifdef XA_DEBUG void xa_dump_node(const struct xa_node *node) { @@ -230,8 +458,8 @@ void xa_dump(const struct xarray *xa) unsigned int shift = 0; pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry, - xa->xa_flags, radix_tree_tagged(xa, 0), - radix_tree_tagged(xa, 1), radix_tree_tagged(xa, 2)); + xa->xa_flags, xa_marked(xa, XA_MARK_0), + xa_marked(xa, XA_MARK_1), xa_marked(xa, XA_MARK_2)); if (xa_is_node(entry)) shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT; xa_dump_entry(entry, 0, shift); diff --git a/tools/include/asm-generic/bitops.h b/tools/include/asm-generic/bitops.h index 9bce3b56b5e7..5d2ab38965cc 100644 --- a/tools/include/asm-generic/bitops.h +++ b/tools/include/asm-generic/bitops.h @@ -27,5 +27,6 @@ #include #include +#include #endif /* __TOOLS_ASM_GENERIC_BITOPS_H */ diff --git a/tools/include/asm-generic/bitops/atomic.h b/tools/include/asm-generic/bitops/atomic.h index 21c41ccd1266..2f6ea28764a7 100644 --- a/tools/include/asm-generic/bitops/atomic.h +++ b/tools/include/asm-generic/bitops/atomic.h @@ -15,13 +15,4 @@ static inline void clear_bit(int nr, unsigned long *addr) addr[nr / __BITS_PER_LONG] &= ~(1UL << (nr % __BITS_PER_LONG)); } -static __always_inline int test_bit(unsigned int nr, const unsigned long *addr) -{ - return ((1UL << (nr % __BITS_PER_LONG)) & - (((unsigned long *)addr)[nr / __BITS_PER_LONG])) != 0; -} - -#define __set_bit(nr, addr) set_bit(nr, addr) -#define __clear_bit(nr, addr) clear_bit(nr, addr) - #endif /* _TOOLS_LINUX_ASM_GENERIC_BITOPS_ATOMIC_H_ */ diff --git a/tools/include/asm-generic/bitops/non-atomic.h b/tools/include/asm-generic/bitops/non-atomic.h new file mode 100644 index 000000000000..7e10c4b50c5d --- /dev/null +++ b/tools/include/asm-generic/bitops/non-atomic.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ +#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ + +#include + +/** + * __set_bit - Set a bit in memory + * @nr: the bit to set + * @addr: the address to start counting from + * + * Unlike set_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static inline void __set_bit(int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p |= mask; +} + +static inline void __clear_bit(int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p &= ~mask; +} + +/** + * __change_bit - Toggle a bit in memory + * @nr: the bit to change + * @addr: the address to start counting from + * + * Unlike change_bit(), this function is non-atomic and may be reordered. + * If it's called on the same region of memory simultaneously, the effect + * may be that only one operation succeeds. + */ +static inline void __change_bit(int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + + *p ^= mask; +} + +/** + * __test_and_set_bit - Set a bit and return its old value + * @nr: Bit to set + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old | mask; + return (old & mask) != 0; +} + +/** + * __test_and_clear_bit - Clear a bit and return its old value + * @nr: Bit to clear + * @addr: Address to count from + * + * This operation is non-atomic and can be reordered. + * If two examples of this operation race, one can appear to succeed + * but actually fail. You must protect multiple accesses with a lock. + */ +static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old & ~mask; + return (old & mask) != 0; +} + +/* WARNING: non atomic and it can be reordered! */ +static inline int __test_and_change_bit(int nr, + volatile unsigned long *addr) +{ + unsigned long mask = BIT_MASK(nr); + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + unsigned long old = *p; + + *p = old ^ mask; + return (old & mask) != 0; +} + +/** + * test_bit - Determine whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static inline int test_bit(int nr, const volatile unsigned long *addr) +{ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */ diff --git a/tools/include/linux/spinlock.h b/tools/include/linux/spinlock.h index 1738c0391da4..622266b197d0 100644 --- a/tools/include/linux/spinlock.h +++ b/tools/include/linux/spinlock.h @@ -8,8 +8,14 @@ #define spinlock_t pthread_mutex_t #define DEFINE_SPINLOCK(x) pthread_mutex_t x = PTHREAD_MUTEX_INITIALIZER #define __SPIN_LOCK_UNLOCKED(x) (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER -#define spin_lock_init(x) pthread_mutex_init(x, NULL) - +#define spin_lock_init(x) pthread_mutex_init(x, NULL) + +#define spin_lock(x) pthread_mutex_lock(x) +#define spin_unlock(x) pthread_mutex_unlock(x) +#define spin_lock_bh(x) pthread_mutex_lock(x) +#define spin_unlock_bh(x) pthread_mutex_unlock(x) +#define spin_lock_irq(x) pthread_mutex_lock(x) +#define spin_unlock_irq(x) pthread_mutex_unlock(x) #define spin_lock_irqsave(x, f) (void)f, pthread_mutex_lock(x) #define spin_unlock_irqrestore(x, f) (void)f, pthread_mutex_unlock(x) -- cgit v1.2.3 From 58d6ea3085f2e53714810a513c61629f6d2be0a6 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Fri, 10 Nov 2017 15:15:08 -0500 Subject: xarray: Add XArray unconditional store operations xa_store() differs from radix_tree_insert() in that it will overwrite an existing element in the array rather than returning an error. This is the behaviour which most users want, and those that want more complex behaviour generally want to use the xas family of routines anyway. For memory allocation, xa_store() will first attempt to request memory from the slab allocator; if memory is not immediately available, it will drop the xa_lock and allocate memory, keeping a pointer in the xa_state. It does not use the per-CPU cache, although those will continue to exist until all radix tree users are converted to the xarray. This patch also includes xa_erase() and __xa_erase() for a streamlined way to store NULL. Since there is no need to allocate memory in order to store a NULL in the XArray, we do not need to trouble the user with deciding what memory allocation flags to use. Signed-off-by: Matthew Wilcox --- include/linux/xarray.h | 147 ++++++- lib/radix-tree.c | 4 +- lib/test_xarray.c | 177 +++++++- lib/xarray.c | 693 +++++++++++++++++++++++++++++++ tools/include/linux/bitmap.h | 1 + tools/include/linux/spinlock.h | 2 + tools/testing/radix-tree/Makefile | 2 +- tools/testing/radix-tree/bitmap.c | 23 + tools/testing/radix-tree/linux/kernel.h | 4 + tools/testing/radix-tree/linux/lockdep.h | 11 + 10 files changed, 1055 insertions(+), 9 deletions(-) create mode 100644 tools/testing/radix-tree/bitmap.c create mode 100644 tools/testing/radix-tree/linux/lockdep.h (limited to 'tools') diff --git a/include/linux/xarray.h b/include/linux/xarray.h index 2de504ae9ba4..15eab63286ed 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -205,10 +205,17 @@ typedef unsigned __bitwise xa_mark_t; #define XA_PRESENT ((__force xa_mark_t)8U) #define XA_MARK_MAX XA_MARK_2 +enum xa_lock_type { + XA_LOCK_IRQ = 1, + XA_LOCK_BH = 2, +}; + /* * Values for xa_flags. The radix tree stores its GFP flags in the xa_flags, * and we remain compatible with that. */ +#define XA_FLAGS_LOCK_IRQ ((__force gfp_t)XA_LOCK_IRQ) +#define XA_FLAGS_LOCK_BH ((__force gfp_t)XA_LOCK_BH) #define XA_FLAGS_MARK(mark) ((__force gfp_t)((1U << __GFP_BITS_SHIFT) << \ (__force unsigned)(mark))) @@ -267,6 +274,7 @@ struct xarray { void xa_init_flags(struct xarray *, gfp_t flags); void *xa_load(struct xarray *, unsigned long index); +void *xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); bool xa_get_mark(struct xarray *, unsigned long index, xa_mark_t); void xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); void xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); @@ -309,6 +317,23 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark) return xa->xa_flags & XA_FLAGS_MARK(mark); } +/** + * xa_erase() - Erase this entry from the XArray. + * @xa: XArray. + * @index: Index of entry. + * + * This function is the equivalent of calling xa_store() with %NULL as + * the third argument. The XArray does not need to allocate memory, so + * the user does not need to provide GFP flags. + * + * Context: Process context. Takes and releases the xa_lock. + * Return: The entry which used to be at this index. + */ +static inline void *xa_erase(struct xarray *xa, unsigned long index) +{ + return xa_store(xa, index, NULL, 0); +} + #define xa_trylock(xa) spin_trylock(&(xa)->xa_lock) #define xa_lock(xa) spin_lock(&(xa)->xa_lock) #define xa_unlock(xa) spin_unlock(&(xa)->xa_lock) @@ -322,11 +347,65 @@ static inline bool xa_marked(const struct xarray *xa, xa_mark_t mark) spin_unlock_irqrestore(&(xa)->xa_lock, flags) /* - * Versions of the normal API which require the caller to hold the xa_lock. - */ + * Versions of the normal API which require the caller to hold the + * xa_lock. If the GFP flags allow it, they will drop the lock to + * allocate memory, then reacquire it afterwards. These functions + * may also re-enable interrupts if the XArray flags indicate the + * locking should be interrupt safe. + */ +void *__xa_erase(struct xarray *, unsigned long index); +void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void __xa_set_mark(struct xarray *, unsigned long index, xa_mark_t); void __xa_clear_mark(struct xarray *, unsigned long index, xa_mark_t); +/** + * xa_erase_bh() - Erase this entry from the XArray. + * @xa: XArray. + * @index: Index of entry. + * + * This function is the equivalent of calling xa_store() with %NULL as + * the third argument. The XArray does not need to allocate memory, so + * the user does not need to provide GFP flags. + * + * Context: Process context. Takes and releases the xa_lock while + * disabling softirqs. + * Return: The entry which used to be at this index. + */ +static inline void *xa_erase_bh(struct xarray *xa, unsigned long index) +{ + void *entry; + + xa_lock_bh(xa); + entry = __xa_erase(xa, index); + xa_unlock_bh(xa); + + return entry; +} + +/** + * xa_erase_irq() - Erase this entry from the XArray. + * @xa: XArray. + * @index: Index of entry. + * + * This function is the equivalent of calling xa_store() with %NULL as + * the third argument. The XArray does not need to allocate memory, so + * the user does not need to provide GFP flags. + * + * Context: Process context. Takes and releases the xa_lock while + * disabling interrupts. + * Return: The entry which used to be at this index. + */ +static inline void *xa_erase_irq(struct xarray *xa, unsigned long index) +{ + void *entry; + + xa_lock_irq(xa); + entry = __xa_erase(xa, index); + xa_unlock_irq(xa); + + return entry; +} + /* Everything below here is the Advanced API. Proceed with caution. */ /* @@ -441,6 +520,12 @@ static inline struct xa_node *xa_parent_locked(const struct xarray *xa, lockdep_is_held(&xa->xa_lock)); } +/* Private */ +static inline void *xa_mk_node(const struct xa_node *node) +{ + return (void *)((unsigned long)node | 2); +} + /* Private */ static inline struct xa_node *xa_to_node(const void *entry) { @@ -647,6 +732,12 @@ static inline bool xas_not_node(struct xa_node *node) return ((unsigned long)node & 3) || !node; } +/* True if the node represents head-of-tree, RESTART or BOUNDS */ +static inline bool xas_top(struct xa_node *node) +{ + return node <= XAS_RESTART; +} + /** * xas_reset() - Reset an XArray operation state. * @xas: XArray operation state. @@ -683,10 +774,14 @@ static inline bool xas_retry(struct xa_state *xas, const void *entry) } void *xas_load(struct xa_state *); +void *xas_store(struct xa_state *, void *entry); bool xas_get_mark(const struct xa_state *, xa_mark_t); void xas_set_mark(const struct xa_state *, xa_mark_t); void xas_clear_mark(const struct xa_state *, xa_mark_t); +void xas_init_marks(const struct xa_state *); + +bool xas_nomem(struct xa_state *, gfp_t); /** * xas_reload() - Refetch an entry from the xarray. @@ -711,4 +806,52 @@ static inline void *xas_reload(struct xa_state *xas) return xa_head(xas->xa); } +/** + * xas_set() - Set up XArray operation state for a different index. + * @xas: XArray operation state. + * @index: New index into the XArray. + * + * Move the operation state to refer to a different index. This will + * have the effect of starting a walk from the top; see xas_next() + * to move to an adjacent index. + */ +static inline void xas_set(struct xa_state *xas, unsigned long index) +{ + xas->xa_index = index; + xas->xa_node = XAS_RESTART; +} + +/** + * xas_set_order() - Set up XArray operation state for a multislot entry. + * @xas: XArray operation state. + * @index: Target of the operation. + * @order: Entry occupies 2^@order indices. + */ +static inline void xas_set_order(struct xa_state *xas, unsigned long index, + unsigned int order) +{ +#ifdef CONFIG_XARRAY_MULTI + xas->xa_index = order < BITS_PER_LONG ? (index >> order) << order : 0; + xas->xa_shift = order - (order % XA_CHUNK_SHIFT); + xas->xa_sibs = (1 << (order % XA_CHUNK_SHIFT)) - 1; + xas->xa_node = XAS_RESTART; +#else + BUG_ON(order > 0); + xas_set(xas, index); +#endif +} + +/** + * xas_set_update() - Set up XArray operation state for a callback. + * @xas: XArray operation state. + * @update: Function to call when updating a node. + * + * The XArray can notify a caller after it has updated an xa_node. + * This is advanced functionality and is only needed by the page cache. + */ +static inline void xas_set_update(struct xa_state *xas, xa_update_node_t update) +{ + xas->xa_update = update; +} + #endif /* _LINUX_XARRAY_H */ diff --git a/lib/radix-tree.c b/lib/radix-tree.c index b8e961428484..3479d93c32b9 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -47,7 +47,7 @@ static unsigned long height_to_maxnodes[RADIX_TREE_MAX_PATH + 1] __read_mostly; /* * Radix tree node cache. */ -static struct kmem_cache *radix_tree_node_cachep; +struct kmem_cache *radix_tree_node_cachep; /* * The radix tree is variable-height, so an insert operation not only has @@ -365,7 +365,7 @@ out: return ret; } -static void radix_tree_node_rcu_free(struct rcu_head *head) +void radix_tree_node_rcu_free(struct rcu_head *head) { struct radix_tree_node *node = container_of(head, struct radix_tree_node, rcu_head); diff --git a/lib/test_xarray.c b/lib/test_xarray.c index f8b0e9ba80a4..b711030fbc32 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -30,13 +30,49 @@ void xa_dump(const struct xarray *xa) { } static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp) { - radix_tree_insert(xa, index, xa_mk_value(index)); - return NULL; + return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp); } static void xa_erase_index(struct xarray *xa, unsigned long index) { - radix_tree_delete(xa, index); + XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX)); + XA_BUG_ON(xa, xa_load(xa, index) != NULL); +} + +/* + * If anyone needs this, please move it to xarray.c. We have no current + * users outside the test suite because all current multislot users want + * to use the advanced API. + */ +static void *xa_store_order(struct xarray *xa, unsigned long index, + unsigned order, void *entry, gfp_t gfp) +{ + XA_STATE_ORDER(xas, xa, index, order); + void *curr; + + do { + xas_lock(&xas); + curr = xas_store(&xas, entry); + xas_unlock(&xas); + } while (xas_nomem(&xas, gfp)); + + return curr; +} + +static noinline void check_xa_err(struct xarray *xa) +{ + XA_BUG_ON(xa, xa_err(xa_store_index(xa, 0, GFP_NOWAIT)) != 0); + XA_BUG_ON(xa, xa_err(xa_erase(xa, 0)) != 0); +#ifndef __KERNEL__ + /* The kernel does not fail GFP_NOWAIT allocations */ + XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM); + XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM); +#endif + XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_KERNEL)) != 0); + XA_BUG_ON(xa, xa_err(xa_store(xa, 1, xa_mk_value(0), GFP_KERNEL)) != 0); + XA_BUG_ON(xa, xa_err(xa_erase(xa, 1)) != 0); +// kills the test-suite :-( +// XA_BUG_ON(xa, xa_err(xa_store(xa, 0, xa_mk_internal(0), 0)) != -EINVAL); } static noinline void check_xa_load(struct xarray *xa) @@ -69,6 +105,9 @@ static noinline void check_xa_load(struct xarray *xa) static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) { + unsigned int order; + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 8 : 1; + /* NULL elements have no marks set */ XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); xa_set_mark(xa, index, XA_MARK_0); @@ -90,6 +129,37 @@ static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); xa_set_mark(xa, index, XA_MARK_0); XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0)); + + /* + * Storing a multi-index entry over entries with marks gives the + * entire entry the union of the marks + */ + BUG_ON((index % 4) != 0); + for (order = 2; order < max_order; order++) { + unsigned long base = round_down(index, 1UL << order); + unsigned long next = base + (1UL << order); + unsigned long i; + + XA_BUG_ON(xa, xa_store_index(xa, index + 1, GFP_KERNEL)); + xa_set_mark(xa, index + 1, XA_MARK_0); + XA_BUG_ON(xa, xa_store_index(xa, index + 2, GFP_KERNEL)); + xa_set_mark(xa, index + 2, XA_MARK_1); + XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL)); + xa_store_order(xa, index, order, xa_mk_value(index), + GFP_KERNEL); + for (i = base; i < next; i++) { + XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0)); + XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1)); + XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2)); + } + XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_0)); + XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_1)); + XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_2)); + xa_erase_index(xa, index); + xa_erase_index(xa, next); + XA_BUG_ON(xa, !xa_empty(xa)); + } + XA_BUG_ON(xa, !xa_empty(xa)); } static noinline void check_xa_mark(struct xarray *xa) @@ -100,12 +170,111 @@ static noinline void check_xa_mark(struct xarray *xa) check_xa_mark_1(xa, index); } -static RADIX_TREE(array, GFP_KERNEL); +static noinline void check_xa_shrink(struct xarray *xa) +{ + XA_STATE(xas, xa, 1); + struct xa_node *node; + + XA_BUG_ON(xa, !xa_empty(xa)); + XA_BUG_ON(xa, xa_store_index(xa, 0, GFP_KERNEL) != NULL); + XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL); + + /* + * Check that erasing the entry at 1 shrinks the tree and properly + * marks the node as being deleted. + */ + xas_lock(&xas); + XA_BUG_ON(xa, xas_load(&xas) != xa_mk_value(1)); + node = xas.xa_node; + XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != xa_mk_value(0)); + XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1)); + XA_BUG_ON(xa, xa_load(xa, 1) != NULL); + XA_BUG_ON(xa, xas.xa_node != XAS_BOUNDS); + XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != XA_RETRY_ENTRY); + XA_BUG_ON(xa, xas_load(&xas) != NULL); + xas_unlock(&xas); + XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0)); + xa_erase_index(xa, 0); + XA_BUG_ON(xa, !xa_empty(xa)); +} + +static noinline void check_multi_store(struct xarray *xa) +{ +#ifdef CONFIG_XARRAY_MULTI + unsigned long i, j, k; + unsigned int max_order = (sizeof(long) == 4) ? 30 : 60; + + /* Loading from any position returns the same value */ + xa_store_order(xa, 0, 1, xa_mk_value(0), GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0)); + XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0)); + XA_BUG_ON(xa, xa_load(xa, 2) != NULL); + rcu_read_lock(); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 2); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2); + rcu_read_unlock(); + + /* Storing adjacent to the value does not alter the value */ + xa_store(xa, 3, xa, GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0)); + XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0)); + XA_BUG_ON(xa, xa_load(xa, 2) != NULL); + rcu_read_lock(); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 3); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2); + rcu_read_unlock(); + + /* Overwriting multiple indexes works */ + xa_store_order(xa, 0, 2, xa_mk_value(1), GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(1)); + XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(1)); + XA_BUG_ON(xa, xa_load(xa, 2) != xa_mk_value(1)); + XA_BUG_ON(xa, xa_load(xa, 3) != xa_mk_value(1)); + XA_BUG_ON(xa, xa_load(xa, 4) != NULL); + rcu_read_lock(); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 4); + XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 4); + rcu_read_unlock(); + + /* We can erase multiple values with a single store */ + xa_store_order(xa, 0, 63, NULL, GFP_KERNEL); + XA_BUG_ON(xa, !xa_empty(xa)); + + /* Even when the first slot is empty but the others aren't */ + xa_store_index(xa, 1, GFP_KERNEL); + xa_store_index(xa, 2, GFP_KERNEL); + xa_store_order(xa, 0, 2, NULL, GFP_KERNEL); + XA_BUG_ON(xa, !xa_empty(xa)); + + for (i = 0; i < max_order; i++) { + for (j = 0; j < max_order; j++) { + xa_store_order(xa, 0, i, xa_mk_value(i), GFP_KERNEL); + xa_store_order(xa, 0, j, xa_mk_value(j), GFP_KERNEL); + + for (k = 0; k < max_order; k++) { + void *entry = xa_load(xa, (1UL << k) - 1); + if ((i < k) && (j < k)) + XA_BUG_ON(xa, entry != NULL); + else + XA_BUG_ON(xa, entry != xa_mk_value(j)); + } + + xa_erase(xa, 0); + XA_BUG_ON(xa, !xa_empty(xa)); + } + } +#endif +} + +static DEFINE_XARRAY(array); static int xarray_checks(void) { + check_xa_err(&array); check_xa_load(&array); check_xa_mark(&array); + check_xa_shrink(&array); + check_multi_store(&array); printk("XArray: %u of %u tests passed\n", tests_passed, tests_run); return (tests_run == tests_passed) ? 0 : -EINVAL; diff --git a/lib/xarray.c b/lib/xarray.c index aa86c47e532f..4596a95ed9cd 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -7,6 +7,8 @@ #include #include +#include +#include #include /* @@ -25,6 +27,31 @@ * @entry refers to something stored in a slot in the xarray */ +static inline unsigned int xa_lock_type(const struct xarray *xa) +{ + return (__force unsigned int)xa->xa_flags & 3; +} + +static inline void xas_lock_type(struct xa_state *xas, unsigned int lock_type) +{ + if (lock_type == XA_LOCK_IRQ) + xas_lock_irq(xas); + else if (lock_type == XA_LOCK_BH) + xas_lock_bh(xas); + else + xas_lock(xas); +} + +static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) +{ + if (lock_type == XA_LOCK_IRQ) + xas_unlock_irq(xas); + else if (lock_type == XA_LOCK_BH) + xas_unlock_bh(xas); + else + xas_unlock(xas); +} + static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark) { if (!(xa->xa_flags & XA_FLAGS_MARK(mark))) @@ -67,6 +94,34 @@ static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark) return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE); } +#define mark_inc(mark) do { \ + mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \ +} while (0) + +/* + * xas_squash_marks() - Merge all marks to the first entry + * @xas: Array operation state. + * + * Set a mark on the first entry if any entry has it set. Clear marks on + * all sibling entries. + */ +static void xas_squash_marks(const struct xa_state *xas) +{ + unsigned int mark = 0; + unsigned int limit = xas->xa_offset + xas->xa_sibs + 1; + + if (!xas->xa_sibs) + return; + + do { + unsigned long *marks = xas->xa_node->marks[mark]; + if (find_next_bit(marks, limit, xas->xa_offset + 1) == limit) + continue; + __set_bit(xas->xa_offset, marks); + bitmap_clear(marks, xas->xa_offset + 1, xas->xa_sibs); + } while (mark++ != (__force unsigned)XA_MARK_MAX); +} + /* extracts the offset within this node from the index */ static unsigned int get_offset(unsigned long index, struct xa_node *node) { @@ -161,6 +216,516 @@ void *xas_load(struct xa_state *xas) } EXPORT_SYMBOL_GPL(xas_load); +/* Move the radix tree node cache here */ +extern struct kmem_cache *radix_tree_node_cachep; +extern void radix_tree_node_rcu_free(struct rcu_head *head); + +#define XA_RCU_FREE ((struct xarray *)1) + +static void xa_node_free(struct xa_node *node) +{ + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); + node->array = XA_RCU_FREE; + call_rcu(&node->rcu_head, radix_tree_node_rcu_free); +} + +/* + * xas_destroy() - Free any resources allocated during the XArray operation. + * @xas: XArray operation state. + * + * This function is now internal-only. + */ +static void xas_destroy(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_alloc; + + if (!node) + return; + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); + kmem_cache_free(radix_tree_node_cachep, node); + xas->xa_alloc = NULL; +} + +/** + * xas_nomem() - Allocate memory if needed. + * @xas: XArray operation state. + * @gfp: Memory allocation flags. + * + * If we need to add new nodes to the XArray, we try to allocate memory + * with GFP_NOWAIT while holding the lock, which will usually succeed. + * If it fails, @xas is flagged as needing memory to continue. The caller + * should drop the lock and call xas_nomem(). If xas_nomem() succeeds, + * the caller should retry the operation. + * + * Forward progress is guaranteed as one node is allocated here and + * stored in the xa_state where it will be found by xas_alloc(). More + * nodes will likely be found in the slab allocator, but we do not tie + * them up here. + * + * Return: true if memory was needed, and was successfully allocated. + */ +bool xas_nomem(struct xa_state *xas, gfp_t gfp) +{ + if (xas->xa_node != XA_ERROR(-ENOMEM)) { + xas_destroy(xas); + return false; + } + xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + if (!xas->xa_alloc) + return false; + XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); + xas->xa_node = XAS_RESTART; + return true; +} +EXPORT_SYMBOL_GPL(xas_nomem); + +/* + * __xas_nomem() - Drop locks and allocate memory if needed. + * @xas: XArray operation state. + * @gfp: Memory allocation flags. + * + * Internal variant of xas_nomem(). + * + * Return: true if memory was needed, and was successfully allocated. + */ +static bool __xas_nomem(struct xa_state *xas, gfp_t gfp) + __must_hold(xas->xa->xa_lock) +{ + unsigned int lock_type = xa_lock_type(xas->xa); + + if (xas->xa_node != XA_ERROR(-ENOMEM)) { + xas_destroy(xas); + return false; + } + if (gfpflags_allow_blocking(gfp)) { + xas_unlock_type(xas, lock_type); + xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + xas_lock_type(xas, lock_type); + } else { + xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + } + if (!xas->xa_alloc) + return false; + XA_NODE_BUG_ON(xas->xa_alloc, !list_empty(&xas->xa_alloc->private_list)); + xas->xa_node = XAS_RESTART; + return true; +} + +static void xas_update(struct xa_state *xas, struct xa_node *node) +{ + if (xas->xa_update) + xas->xa_update(node); + else + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); +} + +static void *xas_alloc(struct xa_state *xas, unsigned int shift) +{ + struct xa_node *parent = xas->xa_node; + struct xa_node *node = xas->xa_alloc; + + if (xas_invalid(xas)) + return NULL; + + if (node) { + xas->xa_alloc = NULL; + } else { + node = kmem_cache_alloc(radix_tree_node_cachep, + GFP_NOWAIT | __GFP_NOWARN); + if (!node) { + xas_set_err(xas, -ENOMEM); + return NULL; + } + } + + if (parent) { + node->offset = xas->xa_offset; + parent->count++; + XA_NODE_BUG_ON(node, parent->count > XA_CHUNK_SIZE); + xas_update(xas, parent); + } + XA_NODE_BUG_ON(node, shift > BITS_PER_LONG); + XA_NODE_BUG_ON(node, !list_empty(&node->private_list)); + node->shift = shift; + node->count = 0; + node->nr_values = 0; + RCU_INIT_POINTER(node->parent, xas->xa_node); + node->array = xas->xa; + + return node; +} + +/* + * Use this to calculate the maximum index that will need to be created + * in order to add the entry described by @xas. Because we cannot store a + * multiple-index entry at index 0, the calculation is a little more complex + * than you might expect. + */ +static unsigned long xas_max(struct xa_state *xas) +{ + unsigned long max = xas->xa_index; + +#ifdef CONFIG_XARRAY_MULTI + if (xas->xa_shift || xas->xa_sibs) { + unsigned long mask; + mask = (((xas->xa_sibs + 1UL) << xas->xa_shift) - 1); + max |= mask; + if (mask == max) + max++; + } +#endif + + return max; +} + +/* The maximum index that can be contained in the array without expanding it */ +static unsigned long max_index(void *entry) +{ + if (!xa_is_node(entry)) + return 0; + return (XA_CHUNK_SIZE << xa_to_node(entry)->shift) - 1; +} + +static void xas_shrink(struct xa_state *xas) +{ + struct xarray *xa = xas->xa; + struct xa_node *node = xas->xa_node; + + for (;;) { + void *entry; + + XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE); + if (node->count != 1) + break; + entry = xa_entry_locked(xa, node, 0); + if (!entry) + break; + if (!xa_is_node(entry) && node->shift) + break; + xas->xa_node = XAS_BOUNDS; + + RCU_INIT_POINTER(xa->xa_head, entry); + + node->count = 0; + node->nr_values = 0; + if (!xa_is_node(entry)) + RCU_INIT_POINTER(node->slots[0], XA_RETRY_ENTRY); + xas_update(xas, node); + xa_node_free(node); + if (!xa_is_node(entry)) + break; + node = xa_to_node(entry); + node->parent = NULL; + } +} + +/* + * xas_delete_node() - Attempt to delete an xa_node + * @xas: Array operation state. + * + * Attempts to delete the @xas->xa_node. This will fail if xa->node has + * a non-zero reference count. + */ +static void xas_delete_node(struct xa_state *xas) +{ + struct xa_node *node = xas->xa_node; + + for (;;) { + struct xa_node *parent; + + XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE); + if (node->count) + break; + + parent = xa_parent_locked(xas->xa, node); + xas->xa_node = parent; + xas->xa_offset = node->offset; + xa_node_free(node); + + if (!parent) { + xas->xa->xa_head = NULL; + xas->xa_node = XAS_BOUNDS; + return; + } + + parent->slots[xas->xa_offset] = NULL; + parent->count--; + XA_NODE_BUG_ON(parent, parent->count > XA_CHUNK_SIZE); + node = parent; + xas_update(xas, node); + } + + if (!node->parent) + xas_shrink(xas); +} + +/** + * xas_free_nodes() - Free this node and all nodes that it references + * @xas: Array operation state. + * @top: Node to free + * + * This node has been removed from the tree. We must now free it and all + * of its subnodes. There may be RCU walkers with references into the tree, + * so we must replace all entries with retry markers. + */ +static void xas_free_nodes(struct xa_state *xas, struct xa_node *top) +{ + unsigned int offset = 0; + struct xa_node *node = top; + + for (;;) { + void *entry = xa_entry_locked(xas->xa, node, offset); + + if (xa_is_node(entry)) { + node = xa_to_node(entry); + offset = 0; + continue; + } + if (entry) + RCU_INIT_POINTER(node->slots[offset], XA_RETRY_ENTRY); + offset++; + while (offset == XA_CHUNK_SIZE) { + struct xa_node *parent; + + parent = xa_parent_locked(xas->xa, node); + offset = node->offset + 1; + node->count = 0; + node->nr_values = 0; + xas_update(xas, node); + xa_node_free(node); + if (node == top) + return; + node = parent; + } + } +} + +/* + * xas_expand adds nodes to the head of the tree until it has reached + * sufficient height to be able to contain @xas->xa_index + */ +static int xas_expand(struct xa_state *xas, void *head) +{ + struct xarray *xa = xas->xa; + struct xa_node *node = NULL; + unsigned int shift = 0; + unsigned long max = xas_max(xas); + + if (!head) { + if (max == 0) + return 0; + while ((max >> shift) >= XA_CHUNK_SIZE) + shift += XA_CHUNK_SHIFT; + return shift + XA_CHUNK_SHIFT; + } else if (xa_is_node(head)) { + node = xa_to_node(head); + shift = node->shift + XA_CHUNK_SHIFT; + } + xas->xa_node = NULL; + + while (max > max_index(head)) { + xa_mark_t mark = 0; + + XA_NODE_BUG_ON(node, shift > BITS_PER_LONG); + node = xas_alloc(xas, shift); + if (!node) + return -ENOMEM; + + node->count = 1; + if (xa_is_value(head)) + node->nr_values = 1; + RCU_INIT_POINTER(node->slots[0], head); + + /* Propagate the aggregated mark info to the new child */ + for (;;) { + if (xa_marked(xa, mark)) + node_set_mark(node, 0, mark); + if (mark == XA_MARK_MAX) + break; + mark_inc(mark); + } + + /* + * Now that the new node is fully initialised, we can add + * it to the tree + */ + if (xa_is_node(head)) { + xa_to_node(head)->offset = 0; + rcu_assign_pointer(xa_to_node(head)->parent, node); + } + head = xa_mk_node(node); + rcu_assign_pointer(xa->xa_head, head); + xas_update(xas, node); + + shift += XA_CHUNK_SHIFT; + } + + xas->xa_node = node; + return shift; +} + +/* + * xas_create() - Create a slot to store an entry in. + * @xas: XArray operation state. + * + * Most users will not need to call this function directly, as it is called + * by xas_store(). It is useful for doing conditional store operations + * (see the xa_cmpxchg() implementation for an example). + * + * Return: If the slot already existed, returns the contents of this slot. + * If the slot was newly created, returns NULL. If it failed to create the + * slot, returns NULL and indicates the error in @xas. + */ +static void *xas_create(struct xa_state *xas) +{ + struct xarray *xa = xas->xa; + void *entry; + void __rcu **slot; + struct xa_node *node = xas->xa_node; + int shift; + unsigned int order = xas->xa_shift; + + if (xas_top(node)) { + entry = xa_head_locked(xa); + xas->xa_node = NULL; + shift = xas_expand(xas, entry); + if (shift < 0) + return NULL; + entry = xa_head_locked(xa); + slot = &xa->xa_head; + } else if (xas_error(xas)) { + return NULL; + } else if (node) { + unsigned int offset = xas->xa_offset; + + shift = node->shift; + entry = xa_entry_locked(xa, node, offset); + slot = &node->slots[offset]; + } else { + shift = 0; + entry = xa_head_locked(xa); + slot = &xa->xa_head; + } + + while (shift > order) { + shift -= XA_CHUNK_SHIFT; + if (!entry) { + node = xas_alloc(xas, shift); + if (!node) + break; + rcu_assign_pointer(*slot, xa_mk_node(node)); + } else if (xa_is_node(entry)) { + node = xa_to_node(entry); + } else { + break; + } + entry = xas_descend(xas, node); + slot = &node->slots[xas->xa_offset]; + } + + return entry; +} + +static void update_node(struct xa_state *xas, struct xa_node *node, + int count, int values) +{ + if (!node || (!count && !values)) + return; + + node->count += count; + node->nr_values += values; + XA_NODE_BUG_ON(node, node->count > XA_CHUNK_SIZE); + XA_NODE_BUG_ON(node, node->nr_values > XA_CHUNK_SIZE); + xas_update(xas, node); + if (count < 0) + xas_delete_node(xas); +} + +/** + * xas_store() - Store this entry in the XArray. + * @xas: XArray operation state. + * @entry: New entry. + * + * If @xas is operating on a multi-index entry, the entry returned by this + * function is essentially meaningless (it may be an internal entry or it + * may be %NULL, even if there are non-NULL entries at some of the indices + * covered by the range). This is not a problem for any current users, + * and can be changed if needed. + * + * Return: The old entry at this index. + */ +void *xas_store(struct xa_state *xas, void *entry) +{ + struct xa_node *node; + void __rcu **slot = &xas->xa->xa_head; + unsigned int offset, max; + int count = 0; + int values = 0; + void *first, *next; + bool value = xa_is_value(entry); + + if (entry) + first = xas_create(xas); + else + first = xas_load(xas); + + if (xas_invalid(xas)) + return first; + node = xas->xa_node; + if (node && (xas->xa_shift < node->shift)) + xas->xa_sibs = 0; + if ((first == entry) && !xas->xa_sibs) + return first; + + next = first; + offset = xas->xa_offset; + max = xas->xa_offset + xas->xa_sibs; + if (node) { + slot = &node->slots[offset]; + if (xas->xa_sibs) + xas_squash_marks(xas); + } + if (!entry) + xas_init_marks(xas); + + for (;;) { + /* + * Must clear the marks before setting the entry to NULL, + * otherwise xas_for_each_marked may find a NULL entry and + * stop early. rcu_assign_pointer contains a release barrier + * so the mark clearing will appear to happen before the + * entry is set to NULL. + */ + rcu_assign_pointer(*slot, entry); + if (xa_is_node(next)) + xas_free_nodes(xas, xa_to_node(next)); + if (!node) + break; + count += !next - !entry; + values += !xa_is_value(first) - !value; + if (entry) { + if (offset == max) + break; + if (!xa_is_sibling(entry)) + entry = xa_mk_sibling(xas->xa_offset); + } else { + if (offset == XA_CHUNK_MASK) + break; + } + next = xa_entry_locked(xas->xa, node, ++offset); + if (!xa_is_sibling(next)) { + if (!entry && (offset > max)) + break; + first = next; + } + slot++; + } + + update_node(xas, node, count, values); + return first; +} +EXPORT_SYMBOL_GPL(xas_store); + /** * xas_get_mark() - Returns the state of this mark. * @xas: XArray operation state. @@ -240,6 +805,30 @@ void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark) } EXPORT_SYMBOL_GPL(xas_clear_mark); +/** + * xas_init_marks() - Initialise all marks for the entry + * @xas: Array operations state. + * + * Initialise all marks for the entry specified by @xas. If we're tracking + * free entries with a mark, we need to set it on all entries. All other + * marks are cleared. + * + * This implementation is not as efficient as it could be; we may walk + * up the tree multiple times. + */ +void xas_init_marks(const struct xa_state *xas) +{ + xa_mark_t mark = 0; + + for (;;) { + xas_clear_mark(xas, mark); + if (mark == XA_MARK_MAX) + break; + mark_inc(mark); + } +} +EXPORT_SYMBOL_GPL(xas_init_marks); + /** * xa_init_flags() - Initialise an empty XArray with flags. * @xa: XArray. @@ -253,9 +842,19 @@ EXPORT_SYMBOL_GPL(xas_clear_mark); */ void xa_init_flags(struct xarray *xa, gfp_t flags) { + unsigned int lock_type; + static struct lock_class_key xa_lock_irq; + static struct lock_class_key xa_lock_bh; + spin_lock_init(&xa->xa_lock); xa->xa_flags = flags; xa->xa_head = NULL; + + lock_type = xa_lock_type(xa); + if (lock_type == XA_LOCK_IRQ) + lockdep_set_class(&xa->xa_lock, &xa_lock_irq); + else if (lock_type == XA_LOCK_BH) + lockdep_set_class(&xa->xa_lock, &xa_lock_bh); } EXPORT_SYMBOL(xa_init_flags); @@ -282,6 +881,100 @@ void *xa_load(struct xarray *xa, unsigned long index) } EXPORT_SYMBOL(xa_load); +static void *xas_result(struct xa_state *xas, void *curr) +{ + XA_NODE_BUG_ON(xas->xa_node, xa_is_internal(curr)); + if (xas_error(xas)) + curr = xas->xa_node; + return curr; +} + +/** + * __xa_erase() - Erase this entry from the XArray while locked. + * @xa: XArray. + * @index: Index into array. + * + * If the entry at this index is a multi-index entry then all indices will + * be erased, and the entry will no longer be a multi-index entry. + * This function expects the xa_lock to be held on entry. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: The old entry at this index. + */ +void *__xa_erase(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + return xas_result(&xas, xas_store(&xas, NULL)); +} +EXPORT_SYMBOL_GPL(__xa_erase); + +/** + * xa_store() - Store this entry in the XArray. + * @xa: XArray. + * @index: Index into array. + * @entry: New entry. + * @gfp: Memory allocation flags. + * + * After this function returns, loads from this index will return @entry. + * Storing into an existing multislot entry updates the entry of every index. + * The marks associated with @index are unaffected unless @entry is %NULL. + * + * Context: Process context. Takes and releases the xa_lock. May sleep + * if the @gfp flags permit. + * Return: The old entry at this index on success, xa_err(-EINVAL) if @entry + * cannot be stored in an XArray, or xa_err(-ENOMEM) if memory allocation + * failed. + */ +void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) +{ + XA_STATE(xas, xa, index); + void *curr; + + if (WARN_ON_ONCE(xa_is_internal(entry))) + return XA_ERROR(-EINVAL); + + do { + xas_lock(&xas); + curr = xas_store(&xas, entry); + xas_unlock(&xas); + } while (xas_nomem(&xas, gfp)); + + return xas_result(&xas, curr); +} +EXPORT_SYMBOL(xa_store); + +/** + * __xa_store() - Store this entry in the XArray. + * @xa: XArray. + * @index: Index into array. + * @entry: New entry. + * @gfp: Memory allocation flags. + * + * You must already be holding the xa_lock when calling this function. + * It will drop the lock if needed to allocate memory, and then reacquire + * it afterwards. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: The old entry at this index or xa_err() if an error happened. + */ +void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) +{ + XA_STATE(xas, xa, index); + void *curr; + + if (WARN_ON_ONCE(xa_is_internal(entry))) + return XA_ERROR(-EINVAL); + + do { + curr = xas_store(&xas, entry); + } while (__xas_nomem(&xas, gfp)); + + return xas_result(&xas, curr); +} +EXPORT_SYMBOL(__xa_store); + /** * __xa_set_mark() - Set this mark on this entry while locked. * @xa: XArray. diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index e63662db131b..05dca5c203f3 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -15,6 +15,7 @@ void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); +void bitmap_clear(unsigned long *map, unsigned int start, int len); #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1))) diff --git a/tools/include/linux/spinlock.h b/tools/include/linux/spinlock.h index 622266b197d0..c934572d935c 100644 --- a/tools/include/linux/spinlock.h +++ b/tools/include/linux/spinlock.h @@ -37,4 +37,6 @@ static inline bool arch_spin_is_locked(arch_spinlock_t *mutex) return true; } +#include + #endif diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index 1379f1d78d0b..acf1afa01c5b 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -5,7 +5,7 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ LDFLAGS += -fsanitize=address -fsanitize=undefined LDLIBS+= -lpthread -lurcu TARGETS = main idr-test multiorder xarray -CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o +CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ tag_check.o multiorder.o idr-test.o iteration_check.o benchmark.o diff --git a/tools/testing/radix-tree/bitmap.c b/tools/testing/radix-tree/bitmap.c new file mode 100644 index 000000000000..66ec4a24a203 --- /dev/null +++ b/tools/testing/radix-tree/bitmap.c @@ -0,0 +1,23 @@ +/* lib/bitmap.c pulls in at least two other files. */ + +#include + +void bitmap_clear(unsigned long *map, unsigned int start, int len) +{ + unsigned long *p = map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + while (len - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + len -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + p++; + } + if (len) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *p &= ~mask_to_clear; + } +} diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h index 5d06ac75a14d..4568248222ae 100644 --- a/tools/testing/radix-tree/linux/kernel.h +++ b/tools/testing/radix-tree/linux/kernel.h @@ -18,4 +18,8 @@ #define pr_debug printk #define pr_cont printk +#define __acquires(x) +#define __releases(x) +#define __must_hold(x) + #endif /* _KERNEL_H */ diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h new file mode 100644 index 000000000000..565fccdfe6e9 --- /dev/null +++ b/tools/testing/radix-tree/linux/lockdep.h @@ -0,0 +1,11 @@ +#ifndef _LINUX_LOCKDEP_H +#define _LINUX_LOCKDEP_H +struct lock_class_key { + unsigned int a; +}; + +static inline void lockdep_set_class(spinlock_t *lock, + struct lock_class_key *key) +{ +} +#endif /* _LINUX_LOCKDEP_H */ -- cgit v1.2.3 From f32f004cddf86d63a9c42994bbce9f4e2f07c9fa Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 4 Jul 2018 15:42:46 -0400 Subject: ida: Convert to XArray Use the XA_TRACK_FREE ability to track which entries have a free bit, similarly to how it uses the radix tree's IDR_FREE tag. This eliminates the per-cpu ida_bitmap preload, and fixes the memory consumption regression I introduced when making the IDR able to store any pointer. Signed-off-by: Matthew Wilcox --- include/linux/idr.h | 18 +- lib/idr.c | 379 +++++++++++++++++++----------------- lib/radix-tree.c | 71 ------- tools/testing/radix-tree/idr-test.c | 8 +- 4 files changed, 213 insertions(+), 263 deletions(-) (limited to 'tools') diff --git a/include/linux/idr.h b/include/linux/idr.h index 3ec8628ce17f..60daf34b625d 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -214,8 +214,7 @@ static inline void idr_preload_end(void) ++id, (entry) = idr_get_next((idr), &(id))) /* - * IDA - IDR based id allocator, use when translation from id to - * pointer isn't necessary. + * IDA - ID Allocator, use when translation from id to pointer isn't necessary. */ #define IDA_CHUNK_SIZE 128 /* 128 bytes per chunk */ #define IDA_BITMAP_LONGS (IDA_CHUNK_SIZE / sizeof(long)) @@ -225,14 +224,14 @@ struct ida_bitmap { unsigned long bitmap[IDA_BITMAP_LONGS]; }; -DECLARE_PER_CPU(struct ida_bitmap *, ida_bitmap); - struct ida { - struct radix_tree_root ida_rt; + struct xarray xa; }; +#define IDA_INIT_FLAGS (XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC) + #define IDA_INIT(name) { \ - .ida_rt = RADIX_TREE_INIT(name, IDR_RT_MARKER | GFP_NOWAIT), \ + .xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \ } #define DEFINE_IDA(name) struct ida name = IDA_INIT(name) @@ -292,7 +291,7 @@ static inline int ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp) static inline void ida_init(struct ida *ida) { - INIT_RADIX_TREE(&ida->ida_rt, IDR_RT_MARKER | GFP_NOWAIT); + xa_init_flags(&ida->xa, IDA_INIT_FLAGS); } #define ida_simple_get(ida, start, end, gfp) \ @@ -301,9 +300,6 @@ static inline void ida_init(struct ida *ida) static inline bool ida_is_empty(const struct ida *ida) { - return radix_tree_empty(&ida->ida_rt); + return xa_empty(&ida->xa); } - -/* in lib/radix-tree.c */ -int ida_pre_get(struct ida *ida, gfp_t gfp_mask); #endif /* __IDR_H__ */ diff --git a/lib/idr.c b/lib/idr.c index 9a366b5be2c2..3c20ad9b0463 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -6,8 +6,6 @@ #include #include -DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap); - /** * idr_alloc_u32() - Allocate an ID. * @idr: IDR handle. @@ -320,6 +318,9 @@ EXPORT_SYMBOL(idr_replace); * free the individual IDs in it. You can use ida_is_empty() to find * out whether the IDA has any IDs currently allocated. * + * The IDA handles its own locking. It is safe to call any of the IDA + * functions without synchronisation in your code. + * * IDs are currently limited to the range [0-INT_MAX]. If this is an awkward * limitation, it should be quite straightforward to raise the maximum. */ @@ -327,151 +328,197 @@ EXPORT_SYMBOL(idr_replace); /* * Developer's notes: * - * The IDA uses the functionality provided by the IDR & radix tree to store - * bitmaps in each entry. The IDR_FREE tag means there is at least one bit - * free, unlike the IDR where it means at least one entry is free. + * The IDA uses the functionality provided by the XArray to store bitmaps in + * each entry. The XA_FREE_MARK is only cleared when all bits in the bitmap + * have been set. * - * I considered telling the radix tree that each slot is an order-10 node - * and storing the bit numbers in the radix tree, but the radix tree can't - * allow a single multiorder entry at index 0, which would significantly - * increase memory consumption for the IDA. So instead we divide the index - * by the number of bits in the leaf bitmap before doing a radix tree lookup. + * I considered telling the XArray that each slot is an order-10 node + * and indexing by bit number, but the XArray can't allow a single multi-index + * entry in the head, which would significantly increase memory consumption + * for the IDA. So instead we divide the index by the number of bits in the + * leaf bitmap before doing a radix tree lookup. * * As an optimisation, if there are only a few low bits set in any given * leaf, instead of allocating a 128-byte bitmap, we store the bits - * directly in the entry. - * - * We allow the radix tree 'exceptional' count to get out of date. Nothing - * in the IDA nor the radix tree code checks it. If it becomes important - * to maintain an accurate exceptional count, switch the rcu_assign_pointer() - * calls to radix_tree_iter_replace() which will correct the exceptional - * count. - * - * The IDA always requires a lock to alloc/free. If we add a 'test_bit' + * as a value entry. Value entries never have the XA_FREE_MARK cleared + * because we can always convert them into a bitmap entry. + * + * It would be possible to optimise further; once we've run out of a + * single 128-byte bitmap, we currently switch to a 576-byte node, put + * the 128-byte bitmap in the first entry and then start allocating extra + * 128-byte entries. We could instead use the 512 bytes of the node's + * data as a bitmap before moving to that scheme. I do not believe this + * is a worthwhile optimisation; Rasmus Villemoes surveyed the current + * users of the IDA and almost none of them use more than 1024 entries. + * Those that do use more than the 8192 IDs that the 512 bytes would + * provide. + * + * The IDA always uses a lock to alloc/free. If we add a 'test_bit' * equivalent, it will still need locking. Going to RCU lookup would require * using RCU to free bitmaps, and that's not trivial without embedding an * RCU head in the bitmap, which adds a 2-pointer overhead to each 128-byte * bitmap, which is excessive. */ -#define IDA_MAX (0x80000000U / IDA_BITMAP_BITS - 1) - -static int ida_get_new_above(struct ida *ida, int start) +/** + * ida_alloc_range() - Allocate an unused ID. + * @ida: IDA handle. + * @min: Lowest ID to allocate. + * @max: Highest ID to allocate. + * @gfp: Memory allocation flags. + * + * Allocate an ID between @min and @max, inclusive. The allocated ID will + * not exceed %INT_MAX, even if @max is larger. + * + * Context: Any context. + * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, + * or %-ENOSPC if there are no free IDs. + */ +int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max, + gfp_t gfp) { - struct radix_tree_root *root = &ida->ida_rt; - void __rcu **slot; - struct radix_tree_iter iter; - struct ida_bitmap *bitmap; - unsigned long index; - unsigned bit; - int new; - - index = start / IDA_BITMAP_BITS; - bit = start % IDA_BITMAP_BITS; - - slot = radix_tree_iter_init(&iter, index); - for (;;) { - if (slot) - slot = radix_tree_next_slot(slot, &iter, - RADIX_TREE_ITER_TAGGED); - if (!slot) { - slot = idr_get_free(root, &iter, GFP_NOWAIT, IDA_MAX); - if (IS_ERR(slot)) { - if (slot == ERR_PTR(-ENOMEM)) - return -EAGAIN; - return PTR_ERR(slot); + XA_STATE(xas, &ida->xa, min / IDA_BITMAP_BITS); + unsigned bit = min % IDA_BITMAP_BITS; + unsigned long flags; + struct ida_bitmap *bitmap, *alloc = NULL; + + if ((int)min < 0) + return -ENOSPC; + + if ((int)max < 0) + max = INT_MAX; + +retry: + xas_lock_irqsave(&xas, flags); +next: + bitmap = xas_find_marked(&xas, max / IDA_BITMAP_BITS, XA_FREE_MARK); + if (xas.xa_index > min / IDA_BITMAP_BITS) + bit = 0; + if (xas.xa_index * IDA_BITMAP_BITS + bit > max) + goto nospc; + + if (xa_is_value(bitmap)) { + unsigned long tmp = xa_to_value(bitmap); + + if (bit < BITS_PER_XA_VALUE) { + bit = find_next_zero_bit(&tmp, BITS_PER_XA_VALUE, bit); + if (xas.xa_index * IDA_BITMAP_BITS + bit > max) + goto nospc; + if (bit < BITS_PER_XA_VALUE) { + tmp |= 1UL << bit; + xas_store(&xas, xa_mk_value(tmp)); + goto out; } } - if (iter.index > index) - bit = 0; - new = iter.index * IDA_BITMAP_BITS; - bitmap = rcu_dereference_raw(*slot); - if (xa_is_value(bitmap)) { - unsigned long tmp = xa_to_value(bitmap); - int vbit = find_next_zero_bit(&tmp, BITS_PER_XA_VALUE, - bit); - if (vbit < BITS_PER_XA_VALUE) { - tmp |= 1UL << vbit; - rcu_assign_pointer(*slot, xa_mk_value(tmp)); - return new + vbit; - } - bitmap = this_cpu_xchg(ida_bitmap, NULL); - if (!bitmap) - return -EAGAIN; - bitmap->bitmap[0] = tmp; - rcu_assign_pointer(*slot, bitmap); + bitmap = alloc; + if (!bitmap) + bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT); + if (!bitmap) + goto alloc; + bitmap->bitmap[0] = tmp; + xas_store(&xas, bitmap); + if (xas_error(&xas)) { + bitmap->bitmap[0] = 0; + goto out; } + } - if (bitmap) { - bit = find_next_zero_bit(bitmap->bitmap, - IDA_BITMAP_BITS, bit); - new += bit; - if (new < 0) - return -ENOSPC; - if (bit == IDA_BITMAP_BITS) - continue; + if (bitmap) { + bit = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, bit); + if (xas.xa_index * IDA_BITMAP_BITS + bit > max) + goto nospc; + if (bit == IDA_BITMAP_BITS) + goto next; - __set_bit(bit, bitmap->bitmap); - if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS)) - radix_tree_iter_tag_clear(root, &iter, - IDR_FREE); + __set_bit(bit, bitmap->bitmap); + if (bitmap_full(bitmap->bitmap, IDA_BITMAP_BITS)) + xas_clear_mark(&xas, XA_FREE_MARK); + } else { + if (bit < BITS_PER_XA_VALUE) { + bitmap = xa_mk_value(1UL << bit); } else { - new += bit; - if (new < 0) - return -ENOSPC; - if (bit < BITS_PER_XA_VALUE) { - bitmap = xa_mk_value(1UL << bit); - } else { - bitmap = this_cpu_xchg(ida_bitmap, NULL); - if (!bitmap) - return -EAGAIN; - __set_bit(bit, bitmap->bitmap); - } - radix_tree_iter_replace(root, &iter, slot, bitmap); + bitmap = alloc; + if (!bitmap) + bitmap = kzalloc(sizeof(*bitmap), GFP_NOWAIT); + if (!bitmap) + goto alloc; + __set_bit(bit, bitmap->bitmap); } - - return new; + xas_store(&xas, bitmap); + } +out: + xas_unlock_irqrestore(&xas, flags); + if (xas_nomem(&xas, gfp)) { + xas.xa_index = min / IDA_BITMAP_BITS; + bit = min % IDA_BITMAP_BITS; + goto retry; } + if (bitmap != alloc) + kfree(alloc); + if (xas_error(&xas)) + return xas_error(&xas); + return xas.xa_index * IDA_BITMAP_BITS + bit; +alloc: + xas_unlock_irqrestore(&xas, flags); + alloc = kzalloc(sizeof(*bitmap), gfp); + if (!alloc) + return -ENOMEM; + xas_set(&xas, min / IDA_BITMAP_BITS); + bit = min % IDA_BITMAP_BITS; + goto retry; +nospc: + xas_unlock_irqrestore(&xas, flags); + return -ENOSPC; } +EXPORT_SYMBOL(ida_alloc_range); -static void ida_remove(struct ida *ida, int id) +/** + * ida_free() - Release an allocated ID. + * @ida: IDA handle. + * @id: Previously allocated ID. + * + * Context: Any context. + */ +void ida_free(struct ida *ida, unsigned int id) { - unsigned long index = id / IDA_BITMAP_BITS; - unsigned offset = id % IDA_BITMAP_BITS; + XA_STATE(xas, &ida->xa, id / IDA_BITMAP_BITS); + unsigned bit = id % IDA_BITMAP_BITS; struct ida_bitmap *bitmap; - unsigned long *btmp; - struct radix_tree_iter iter; - void __rcu **slot; + unsigned long flags; - slot = radix_tree_iter_lookup(&ida->ida_rt, &iter, index); - if (!slot) - goto err; + BUG_ON((int)id < 0); + + xas_lock_irqsave(&xas, flags); + bitmap = xas_load(&xas); - bitmap = rcu_dereference_raw(*slot); if (xa_is_value(bitmap)) { - btmp = (unsigned long *)slot; - offset += 1; /* Intimate knowledge of the value encoding */ - if (offset >= BITS_PER_LONG) + unsigned long v = xa_to_value(bitmap); + if (bit >= BITS_PER_XA_VALUE) + goto err; + if (!(v & (1UL << bit))) goto err; + v &= ~(1UL << bit); + if (!v) + goto delete; + xas_store(&xas, xa_mk_value(v)); } else { - btmp = bitmap->bitmap; - } - if (!test_bit(offset, btmp)) - goto err; - - __clear_bit(offset, btmp); - radix_tree_iter_tag_set(&ida->ida_rt, &iter, IDR_FREE); - if (xa_is_value(bitmap)) { - if (xa_to_value(rcu_dereference_raw(*slot)) == 0) - radix_tree_iter_delete(&ida->ida_rt, &iter, slot); - } else if (bitmap_empty(btmp, IDA_BITMAP_BITS)) { - kfree(bitmap); - radix_tree_iter_delete(&ida->ida_rt, &iter, slot); + if (!test_bit(bit, bitmap->bitmap)) + goto err; + __clear_bit(bit, bitmap->bitmap); + xas_set_mark(&xas, XA_FREE_MARK); + if (bitmap_empty(bitmap->bitmap, IDA_BITMAP_BITS)) { + kfree(bitmap); +delete: + xas_store(&xas, NULL); + } } + xas_unlock_irqrestore(&xas, flags); return; err: + xas_unlock_irqrestore(&xas, flags); WARN(1, "ida_free called for id=%d which is not allocated.\n", id); } +EXPORT_SYMBOL(ida_free); /** * ida_destroy() - Free all IDs. @@ -486,80 +533,60 @@ static void ida_remove(struct ida *ida, int id) */ void ida_destroy(struct ida *ida) { + XA_STATE(xas, &ida->xa, 0); + struct ida_bitmap *bitmap; unsigned long flags; - struct radix_tree_iter iter; - void __rcu **slot; - xa_lock_irqsave(&ida->ida_rt, flags); - radix_tree_for_each_slot(slot, &ida->ida_rt, &iter, 0) { - struct ida_bitmap *bitmap = rcu_dereference_raw(*slot); + xas_lock_irqsave(&xas, flags); + xas_for_each(&xas, bitmap, ULONG_MAX) { if (!xa_is_value(bitmap)) kfree(bitmap); - radix_tree_iter_delete(&ida->ida_rt, &iter, slot); + xas_store(&xas, NULL); } - xa_unlock_irqrestore(&ida->ida_rt, flags); + xas_unlock_irqrestore(&xas, flags); } EXPORT_SYMBOL(ida_destroy); -/** - * ida_alloc_range() - Allocate an unused ID. - * @ida: IDA handle. - * @min: Lowest ID to allocate. - * @max: Highest ID to allocate. - * @gfp: Memory allocation flags. - * - * Allocate an ID between @min and @max, inclusive. The allocated ID will - * not exceed %INT_MAX, even if @max is larger. - * - * Context: Any context. - * Return: The allocated ID, or %-ENOMEM if memory could not be allocated, - * or %-ENOSPC if there are no free IDs. - */ -int ida_alloc_range(struct ida *ida, unsigned int min, unsigned int max, - gfp_t gfp) -{ - int id = 0; - unsigned long flags; - - if ((int)min < 0) - return -ENOSPC; +#ifndef __KERNEL__ +extern void xa_dump_index(unsigned long index, unsigned int shift); +#define IDA_CHUNK_SHIFT ilog2(IDA_BITMAP_BITS) - if ((int)max < 0) - max = INT_MAX; - -again: - xa_lock_irqsave(&ida->ida_rt, flags); - id = ida_get_new_above(ida, min); - if (id > (int)max) { - ida_remove(ida, id); - id = -ENOSPC; - } - xa_unlock_irqrestore(&ida->ida_rt, flags); +static void ida_dump_entry(void *entry, unsigned long index) +{ + unsigned long i; + + if (!entry) + return; + + if (xa_is_node(entry)) { + struct xa_node *node = xa_to_node(entry); + unsigned int shift = node->shift + IDA_CHUNK_SHIFT + + XA_CHUNK_SHIFT; + + xa_dump_index(index * IDA_BITMAP_BITS, shift); + xa_dump_node(node); + for (i = 0; i < XA_CHUNK_SIZE; i++) + ida_dump_entry(node->slots[i], + index | (i << node->shift)); + } else if (xa_is_value(entry)) { + xa_dump_index(index * IDA_BITMAP_BITS, ilog2(BITS_PER_LONG)); + pr_cont("value: data %lx [%px]\n", xa_to_value(entry), entry); + } else { + struct ida_bitmap *bitmap = entry; - if (unlikely(id == -EAGAIN)) { - if (!ida_pre_get(ida, gfp)) - return -ENOMEM; - goto again; + xa_dump_index(index * IDA_BITMAP_BITS, IDA_CHUNK_SHIFT); + pr_cont("bitmap: %p data", bitmap); + for (i = 0; i < IDA_BITMAP_LONGS; i++) + pr_cont(" %lx", bitmap->bitmap[i]); + pr_cont("\n"); } - - return id; } -EXPORT_SYMBOL(ida_alloc_range); -/** - * ida_free() - Release an allocated ID. - * @ida: IDA handle. - * @id: Previously allocated ID. - * - * Context: Any context. - */ -void ida_free(struct ida *ida, unsigned int id) +static void ida_dump(struct ida *ida) { - unsigned long flags; - - BUG_ON((int)id < 0); - xa_lock_irqsave(&ida->ida_rt, flags); - ida_remove(ida, id); - xa_unlock_irqrestore(&ida->ida_rt, flags); + struct xarray *xa = &ida->xa; + pr_debug("ida: %p node %p free %d\n", ida, xa->xa_head, + xa->xa_flags >> ROOT_TAG_SHIFT); + ida_dump_entry(xa->xa_head, 0); } -EXPORT_SYMBOL(ida_free); +#endif diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 3479d93c32b9..68702061464f 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -255,54 +255,6 @@ static unsigned long next_index(unsigned long index, return (index & ~node_maxindex(node)) + (offset << node->shift); } -#ifndef __KERNEL__ -static void dump_ida_node(void *entry, unsigned long index) -{ - unsigned long i; - - if (!entry) - return; - - if (radix_tree_is_internal_node(entry)) { - struct radix_tree_node *node = entry_to_node(entry); - - pr_debug("ida node: %p offset %d indices %lu-%lu parent %p free %lx shift %d count %d\n", - node, node->offset, index * IDA_BITMAP_BITS, - ((index | node_maxindex(node)) + 1) * - IDA_BITMAP_BITS - 1, - node->parent, node->tags[0][0], node->shift, - node->count); - for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) - dump_ida_node(node->slots[i], - index | (i << node->shift)); - } else if (xa_is_value(entry)) { - pr_debug("ida excp: %p offset %d indices %lu-%lu data %lx\n", - entry, (int)(index & RADIX_TREE_MAP_MASK), - index * IDA_BITMAP_BITS, - index * IDA_BITMAP_BITS + BITS_PER_XA_VALUE, - xa_to_value(entry)); - } else { - struct ida_bitmap *bitmap = entry; - - pr_debug("ida btmp: %p offset %d indices %lu-%lu data", bitmap, - (int)(index & RADIX_TREE_MAP_MASK), - index * IDA_BITMAP_BITS, - (index + 1) * IDA_BITMAP_BITS - 1); - for (i = 0; i < IDA_BITMAP_LONGS; i++) - pr_cont(" %lx", bitmap->bitmap[i]); - pr_cont("\n"); - } -} - -static void ida_dump(struct ida *ida) -{ - struct radix_tree_root *root = &ida->ida_rt; - pr_debug("ida: %p node %p free %d\n", ida, root->xa_head, - root->xa_flags >> ROOT_TAG_SHIFT); - dump_ida_node(root->xa_head, 0); -} -#endif - /* * This assumes that the caller has performed appropriate preallocation, and * that the caller has pinned this thread of control to the current CPU. @@ -2039,27 +1991,6 @@ void idr_preload(gfp_t gfp_mask) } EXPORT_SYMBOL(idr_preload); -int ida_pre_get(struct ida *ida, gfp_t gfp) -{ - /* - * The IDA API has no preload_end() equivalent. Instead, - * ida_get_new() can return -EAGAIN, prompting the caller - * to return to the ida_pre_get() step. - */ - if (!__radix_tree_preload(gfp, IDA_PRELOAD_SIZE)) - preempt_enable(); - - if (!this_cpu_read(ida_bitmap)) { - struct ida_bitmap *bitmap = kzalloc(sizeof(*bitmap), gfp); - if (!bitmap) - return 0; - if (this_cpu_cmpxchg(ida_bitmap, NULL, bitmap)) - kfree(bitmap); - } - - return 1; -} - void __rcu **idr_get_free(struct radix_tree_root *root, struct radix_tree_iter *iter, gfp_t gfp, unsigned long max) @@ -2201,8 +2132,6 @@ static int radix_tree_cpu_dead(unsigned int cpu) kmem_cache_free(radix_tree_node_cachep, node); rtp->nr--; } - kfree(per_cpu(ida_bitmap, cpu)); - per_cpu(ida_bitmap, cpu) = NULL; return 0; } diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c index a5a4494922a6..1b63bdb7688f 100644 --- a/tools/testing/radix-tree/idr-test.c +++ b/tools/testing/radix-tree/idr-test.c @@ -402,16 +402,15 @@ void ida_check_nomem(void) */ void ida_check_conv_user(void) { -#if 0 DEFINE_IDA(ida); unsigned long i; - radix_tree_cpu_dead(1); for (i = 0; i < 1000000; i++) { int id = ida_alloc(&ida, GFP_NOWAIT); if (id == -ENOMEM) { - IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) != - BITS_PER_XA_VALUE); + IDA_BUG_ON(&ida, ((i % IDA_BITMAP_BITS) != + BITS_PER_XA_VALUE) && + ((i % IDA_BITMAP_BITS) != 0)); id = ida_alloc(&ida, GFP_KERNEL); } else { IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) == @@ -420,7 +419,6 @@ void ida_check_conv_user(void) IDA_BUG_ON(&ida, id != i); } ida_destroy(&ida); -#endif } void ida_check_random(void) -- cgit v1.2.3 From 3ece58a270cd1e5026282abe778bd50db7a11d08 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 16 May 2018 18:00:33 -0400 Subject: page cache: Convert find_get_pages_contig to XArray There's no direct replacement for radix_tree_for_each_contig() in the XArray API as it's an unusual thing to do. Instead, open-code a loop using xas_next(). This removes the only user of radix_tree_for_each_contig() so delete the iterator from the API and the test suite code for it. Signed-off-by: Matthew Wilcox --- .clang-format | 1 - include/linux/radix-tree.h | 17 ----------- mm/filemap.c | 53 ++++++++++++++-------------------- tools/testing/radix-tree/regression3.c | 23 --------------- 4 files changed, 22 insertions(+), 72 deletions(-) (limited to 'tools') diff --git a/.clang-format b/.clang-format index 1d5da22e0ba5..e6080f5834a3 100644 --- a/.clang-format +++ b/.clang-format @@ -323,7 +323,6 @@ ForEachMacros: - 'protocol_for_each_card' - 'protocol_for_each_dev' - 'queue_for_each_hw_ctx' - - 'radix_tree_for_each_contig' - 'radix_tree_for_each_slot' - 'radix_tree_for_each_tagged' - 'rbtree_postorder_for_each_entry_safe' diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index d55de428a589..023e888e1163 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -522,23 +522,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \ slot = radix_tree_next_slot(slot, iter, 0)) -/** - * radix_tree_for_each_contig - iterate over contiguous slots - * - * @slot: the void** variable for pointer to slot - * @root: the struct radix_tree_root pointer - * @iter: the struct radix_tree_iter pointer - * @start: iteration starting index - * - * @slot points to radix tree slot, @iter->index contains its index. - */ -#define radix_tree_for_each_contig(slot, root, iter, start) \ - for (slot = radix_tree_iter_init(iter, start) ; \ - slot || (slot = radix_tree_next_chunk(root, iter, \ - RADIX_TREE_ITER_CONTIG)) ; \ - slot = radix_tree_next_slot(slot, iter, \ - RADIX_TREE_ITER_CONTIG)) - /** * radix_tree_for_each_tagged - iterate over tagged slots * diff --git a/mm/filemap.c b/mm/filemap.c index b72c39fe61c2..089b67598100 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1721,57 +1721,43 @@ out: unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, unsigned int nr_pages, struct page **pages) { - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, &mapping->i_pages, index); + struct page *page; unsigned int ret = 0; if (unlikely(!nr_pages)) return 0; rcu_read_lock(); - radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) { - struct page *head, *page; -repeat: - page = radix_tree_deref_slot(slot); - /* The hole, there no reason to continue */ - if (unlikely(!page)) - break; - - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - /* - * A shadow entry of a recently evicted page, - * or a swap entry from shmem/tmpfs. Stop - * looking for contiguous pages. - */ + for (page = xas_load(&xas); page; page = xas_next(&xas)) { + struct page *head; + if (xas_retry(&xas, page)) + continue; + /* + * If the entry has been swapped out, we can stop looking. + * No current caller is looking for DAX entries. + */ + if (xa_is_value(page)) break; - } head = compound_head(page); if (!page_cache_get_speculative(head)) - goto repeat; + goto retry; /* The page was split under us? */ - if (compound_head(page) != head) { - put_page(head); - goto repeat; - } + if (compound_head(page) != head) + goto put_page; /* Has the page moved? */ - if (unlikely(page != *slot)) { - put_page(head); - goto repeat; - } + if (unlikely(page != xas_reload(&xas))) + goto put_page; /* * must check mapping and index after taking the ref. * otherwise we can get both false positives and false * negatives, which is just confusing to the caller. */ - if (page->mapping == NULL || page_to_pgoff(page) != iter.index) { + if (!page->mapping || page_to_pgoff(page) != xas.xa_index) { put_page(page); break; } @@ -1779,6 +1765,11 @@ repeat: pages[ret] = page; if (++ret == nr_pages) break; + continue; +put_page: + put_page(head); +retry: + xas_reset(&xas); } rcu_read_unlock(); return ret; diff --git a/tools/testing/radix-tree/regression3.c b/tools/testing/radix-tree/regression3.c index ace2543c3eda..9f9a3b280f56 100644 --- a/tools/testing/radix-tree/regression3.c +++ b/tools/testing/radix-tree/regression3.c @@ -69,21 +69,6 @@ void regression3_test(void) continue; } } - radix_tree_delete(&root, 1); - - first = true; - radix_tree_for_each_contig(slot, &root, &iter, 0) { - printv(2, "contig %ld %p\n", iter.index, *slot); - if (first) { - radix_tree_insert(&root, 1, ptr); - first = false; - } - if (radix_tree_deref_retry(*slot)) { - printv(2, "retry at %ld\n", iter.index); - slot = radix_tree_iter_retry(&iter); - continue; - } - } radix_tree_for_each_slot(slot, &root, &iter, 0) { printv(2, "slot %ld %p\n", iter.index, *slot); @@ -93,14 +78,6 @@ void regression3_test(void) } } - radix_tree_for_each_contig(slot, &root, &iter, 0) { - printv(2, "contig %ld %p\n", iter.index, *slot); - if (!iter.index) { - printv(2, "next at %ld\n", iter.index); - slot = radix_tree_iter_resume(slot, &iter); - } - } - radix_tree_tag_set(&root, 0, 0); radix_tree_tag_set(&root, 1, 0); radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) { -- cgit v1.2.3 From a332125fc3223e1092f765db442b7afb9fd4ecde Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 17 May 2018 00:13:27 -0400 Subject: radix tree test suite: Convert regression1 to XArray Now the page cache lookup is using the XArray, let's convert this regression test from the radix tree API to the XArray so it's testing roughly the same thing it was testing before. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/regression1.c | 58 +++++++++++----------------------- 1 file changed, 19 insertions(+), 39 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/regression1.c b/tools/testing/radix-tree/regression1.c index 0aece092f40e..b4a4a7168986 100644 --- a/tools/testing/radix-tree/regression1.c +++ b/tools/testing/radix-tree/regression1.c @@ -53,12 +53,12 @@ struct page { unsigned long index; }; -static struct page *page_alloc(void) +static struct page *page_alloc(int index) { struct page *p; p = malloc(sizeof(struct page)); p->count = 1; - p->index = 1; + p->index = index; pthread_mutex_init(&p->lock, NULL); return p; @@ -80,53 +80,33 @@ static void page_free(struct page *p) static unsigned find_get_pages(unsigned long start, unsigned int nr_pages, struct page **pages) { - unsigned int i; - unsigned int ret; - unsigned int nr_found; + XA_STATE(xas, &mt_tree, start); + struct page *page; + unsigned int ret = 0; rcu_read_lock(); -restart: - nr_found = radix_tree_gang_lookup_slot(&mt_tree, - (void ***)pages, NULL, start, nr_pages); - ret = 0; - for (i = 0; i < nr_found; i++) { - struct page *page; -repeat: - page = radix_tree_deref_slot((void **)pages[i]); - if (unlikely(!page)) + xas_for_each(&xas, page, ULONG_MAX) { + if (xas_retry(&xas, page)) continue; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) { - /* - * Transient condition which can only trigger - * when entry at index 0 moves out of or back - * to root: none yet gotten, safe to restart. - */ - assert((start | i) == 0); - goto restart; - } - /* - * No exceptional entries are inserted in this test. - */ - assert(0); - } - pthread_mutex_lock(&page->lock); - if (!page->count) { - pthread_mutex_unlock(&page->lock); - goto repeat; - } + if (!page->count) + goto unlock; + /* don't actually update page refcount */ pthread_mutex_unlock(&page->lock); /* Has the page moved? */ - if (unlikely(page != *((void **)pages[i]))) { - goto repeat; - } + if (unlikely(page != xas_reload(&xas))) + goto put_page; pages[ret] = page; ret++; + continue; +unlock: + pthread_mutex_unlock(&page->lock); +put_page: + xas_reset(&xas); } rcu_read_unlock(); return ret; @@ -145,12 +125,12 @@ static void *regression1_fn(void *arg) for (j = 0; j < 1000000; j++) { struct page *p; - p = page_alloc(); + p = page_alloc(0); pthread_mutex_lock(&mt_lock); radix_tree_insert(&mt_tree, 0, p); pthread_mutex_unlock(&mt_lock); - p = page_alloc(); + p = page_alloc(1); pthread_mutex_lock(&mt_lock); radix_tree_insert(&mt_tree, 1, p); pthread_mutex_unlock(&mt_lock); -- cgit v1.2.3 From e21a29552fa3f44ea41c53488875015ae70fd7f8 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 22 Nov 2017 08:36:00 -0500 Subject: shmem: Convert find_swap_entry to XArray This is a 1:1 conversion. The major part of this patch is converting the test framework from userspace to kernel space and mirroring the algorithm now used in find_swap_entry(). Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 56 +++++++++++++++++++++++++++++++++++++ mm/shmem.c | 27 +++++++----------- tools/testing/radix-tree/main.c | 61 ----------------------------------------- tools/testing/radix-tree/test.c | 22 --------------- tools/testing/radix-tree/test.h | 1 - 5 files changed, 66 insertions(+), 101 deletions(-) (limited to 'tools') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 128c6489082f..815daffdd8c9 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -631,6 +631,61 @@ static noinline void check_find(struct xarray *xa) check_multi_find_2(xa); } +/* See find_swap_entry() in mm/shmem.c */ +static noinline unsigned long xa_find_entry(struct xarray *xa, void *item) +{ + XA_STATE(xas, xa, 0); + unsigned int checked = 0; + void *entry; + + rcu_read_lock(); + xas_for_each(&xas, entry, ULONG_MAX) { + if (xas_retry(&xas, entry)) + continue; + if (entry == item) + break; + checked++; + if ((checked % 4) != 0) + continue; + xas_pause(&xas); + } + rcu_read_unlock(); + + return entry ? xas.xa_index : -1; +} + +static noinline void check_find_entry(struct xarray *xa) +{ +#ifdef CONFIG_XARRAY_MULTI + unsigned int order; + unsigned long offset, index; + + for (order = 0; order < 20; order++) { + for (offset = 0; offset < (1UL << (order + 3)); + offset += (1UL << order)) { + for (index = 0; index < (1UL << (order + 5)); + index += (1UL << order)) { + xa_store_order(xa, index, order, + xa_mk_value(index), GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, index) != + xa_mk_value(index)); + XA_BUG_ON(xa, xa_find_entry(xa, + xa_mk_value(index)) != index); + } + XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1); + xa_destroy(xa); + } + } +#endif + + XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1); + xa_store_index(xa, ULONG_MAX, GFP_KERNEL); + XA_BUG_ON(xa, xa_find_entry(xa, xa) != -1); + XA_BUG_ON(xa, xa_find_entry(xa, xa_mk_value(LONG_MAX)) != -1); + xa_erase_index(xa, ULONG_MAX); + XA_BUG_ON(xa, !xa_empty(xa)); +} + static noinline void check_move_small(struct xarray *xa, unsigned long idx) { XA_STATE(xas, xa, 0); @@ -972,6 +1027,7 @@ static int xarray_checks(void) check_multi_store(&array); check_xa_alloc(); check_find(&array); + check_find_entry(&array); check_destroy(&array); check_move(&array); check_create_range(&array); diff --git a/mm/shmem.c b/mm/shmem.c index ce91569426f3..a305529d6b89 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1100,34 +1100,27 @@ static void shmem_evict_inode(struct inode *inode) clear_inode(inode); } -static unsigned long find_swap_entry(struct radix_tree_root *root, void *item) +static unsigned long find_swap_entry(struct xarray *xa, void *item) { - struct radix_tree_iter iter; - void __rcu **slot; - unsigned long found = -1; + XA_STATE(xas, xa, 0); unsigned int checked = 0; + void *entry; rcu_read_lock(); - radix_tree_for_each_slot(slot, root, &iter, 0) { - void *entry = radix_tree_deref_slot(slot); - - if (radix_tree_deref_retry(entry)) { - slot = radix_tree_iter_retry(&iter); + xas_for_each(&xas, entry, ULONG_MAX) { + if (xas_retry(&xas, entry)) continue; - } - if (entry == item) { - found = iter.index; + if (entry == item) break; - } checked++; - if ((checked % 4096) != 0) + if ((checked % XA_CHECK_SCHED) != 0) continue; - slot = radix_tree_iter_resume(slot, &iter); + xas_pause(&xas); cond_resched_rcu(); } - rcu_read_unlock(); - return found; + + return entry ? xas.xa_index : -1; } /* diff --git a/tools/testing/radix-tree/main.c b/tools/testing/radix-tree/main.c index 09deaf4f0959..79589ea570ab 100644 --- a/tools/testing/radix-tree/main.c +++ b/tools/testing/radix-tree/main.c @@ -236,63 +236,6 @@ void copy_tag_check(void) item_kill_tree(&tree); } -static void __locate_check(struct radix_tree_root *tree, unsigned long index, - unsigned order) -{ - struct item *item; - unsigned long index2; - - item_insert_order(tree, index, order); - item = item_lookup(tree, index); - index2 = find_item(tree, item); - if (index != index2) { - printv(2, "index %ld order %d inserted; found %ld\n", - index, order, index2); - abort(); - } -} - -static void __order_0_locate_check(void) -{ - RADIX_TREE(tree, GFP_KERNEL); - int i; - - for (i = 0; i < 50; i++) - __locate_check(&tree, rand() % INT_MAX, 0); - - item_kill_tree(&tree); -} - -static void locate_check(void) -{ - RADIX_TREE(tree, GFP_KERNEL); - unsigned order; - unsigned long offset, index; - - __order_0_locate_check(); - - for (order = 0; order < 20; order++) { - for (offset = 0; offset < (1 << (order + 3)); - offset += (1UL << order)) { - for (index = 0; index < (1UL << (order + 5)); - index += (1UL << order)) { - __locate_check(&tree, index + offset, order); - } - if (find_item(&tree, &tree) != -1) - abort(); - - item_kill_tree(&tree); - } - } - - if (find_item(&tree, &tree) != -1) - abort(); - __locate_check(&tree, -1, 0); - if (find_item(&tree, &tree) != -1) - abort(); - item_kill_tree(&tree); -} - static void single_thread_tests(bool long_run) { int i; @@ -303,10 +246,6 @@ static void single_thread_tests(bool long_run) rcu_barrier(); printv(2, "after multiorder_check: %d allocated, preempt %d\n", nr_allocated, preempt_count); - locate_check(); - rcu_barrier(); - printv(2, "after locate_check: %d allocated, preempt %d\n", - nr_allocated, preempt_count); tag_check(); rcu_barrier(); printv(2, "after tag_check: %d allocated, preempt %d\n", diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 70ddf964d51c..470419bfd49d 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -209,28 +209,6 @@ int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock, return tagged; } -/* Use the same pattern as find_swap_entry() in mm/shmem.c */ -unsigned long find_item(struct radix_tree_root *root, void *item) -{ - struct radix_tree_iter iter; - void **slot; - unsigned long found = -1; - unsigned long checked = 0; - - radix_tree_for_each_slot(slot, root, &iter, 0) { - if (*slot == item) { - found = iter.index; - break; - } - checked++; - if ((checked % 4) != 0) - continue; - slot = radix_tree_iter_resume(slot, &iter); - } - - return found; -} - static int verify_node(struct radix_tree_node *slot, unsigned int tag, int tagged) { diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index e3dc7a16f09b..9532c18c6cb1 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -32,7 +32,6 @@ void item_kill_tree(struct radix_tree_root *root); int tag_tagged_items(struct radix_tree_root *, pthread_mutex_t *, unsigned long start, unsigned long end, unsigned batch, unsigned iftag, unsigned thentag); -unsigned long find_item(struct radix_tree_root *, void *item); void xarray_tests(void); void tag_check(void); -- cgit v1.2.3 From 1cf56f9d670b88b2e947a7ccdb8ba32e6477915d Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 9 Apr 2018 16:24:45 -0400 Subject: radix tree: Remove radix_tree_update_node_t The only user of this functionality was the workingset code, and it's now been converted to the XArray. Remove __radix_tree_delete_node() entirely as it was also only used by the workingset code. Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 7 +----- lib/idr.c | 2 +- lib/radix-tree.c | 42 +++++++---------------------------- tools/testing/radix-tree/multiorder.c | 2 +- 4 files changed, 11 insertions(+), 42 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 8a4280bc350f..7513d0df984b 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -242,17 +242,12 @@ void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index, void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, unsigned long index); -typedef void (*radix_tree_update_node_t)(struct radix_tree_node *); void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, - void __rcu **slot, void *entry, - radix_tree_update_node_t update_node); + void __rcu **slot, void *entry); void radix_tree_iter_replace(struct radix_tree_root *, const struct radix_tree_iter *, void __rcu **slot, void *entry); void radix_tree_replace_slot(struct radix_tree_root *, void __rcu **slot, void *entry); -void __radix_tree_delete_node(struct radix_tree_root *, - struct radix_tree_node *, - radix_tree_update_node_t update_node); void radix_tree_iter_delete(struct radix_tree_root *, struct radix_tree_iter *iter, void __rcu **slot); void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); diff --git a/lib/idr.c b/lib/idr.c index 3c20ad9b0463..cb1db9b8d3f6 100644 --- a/lib/idr.c +++ b/lib/idr.c @@ -297,7 +297,7 @@ void *idr_replace(struct idr *idr, void *ptr, unsigned long id) if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) return ERR_PTR(-ENOENT); - __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL); + __radix_tree_replace(&idr->idr_rt, node, slot, ptr); return entry; } diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 2f9c0e45eeb7..c4c252185734 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -562,8 +562,7 @@ out: * radix_tree_shrink - shrink radix tree to minimum height * @root radix tree root */ -static inline bool radix_tree_shrink(struct radix_tree_root *root, - radix_tree_update_node_t update_node) +static inline bool radix_tree_shrink(struct radix_tree_root *root) { bool shrunk = false; @@ -631,8 +630,6 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, node->count = 0; if (!radix_tree_is_internal_node(child)) { node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; - if (update_node) - update_node(node); } WARN_ON_ONCE(!list_empty(&node->private_list)); @@ -644,8 +641,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, } static bool delete_node(struct radix_tree_root *root, - struct radix_tree_node *node, - radix_tree_update_node_t update_node) + struct radix_tree_node *node) { bool deleted = false; @@ -655,7 +651,7 @@ static bool delete_node(struct radix_tree_root *root, if (node->count) { if (node_to_entry(node) == rcu_dereference_raw(root->xa_head)) - deleted |= radix_tree_shrink(root, update_node); + deleted |= radix_tree_shrink(root); return deleted; } @@ -1059,15 +1055,13 @@ static int calculate_count(struct radix_tree_root *root, * @node: pointer to tree node * @slot: pointer to slot in @node * @item: new item to store in the slot. - * @update_node: callback for changing leaf nodes * * For use with __radix_tree_lookup(). Caller must hold tree write locked * across slot lookup and replacement. */ void __radix_tree_replace(struct radix_tree_root *root, struct radix_tree_node *node, - void __rcu **slot, void *item, - radix_tree_update_node_t update_node) + void __rcu **slot, void *item) { void *old = rcu_dereference_raw(*slot); int values = !!xa_is_value(item) - !!xa_is_value(old); @@ -1085,10 +1079,7 @@ void __radix_tree_replace(struct radix_tree_root *root, if (!node) return; - if (update_node) - update_node(node); - - delete_node(root, node, update_node); + delete_node(root, node); } /** @@ -1110,7 +1101,7 @@ void __radix_tree_replace(struct radix_tree_root *root, void radix_tree_replace_slot(struct radix_tree_root *root, void __rcu **slot, void *item) { - __radix_tree_replace(root, NULL, slot, item, NULL); + __radix_tree_replace(root, NULL, slot, item); } EXPORT_SYMBOL(radix_tree_replace_slot); @@ -1127,7 +1118,7 @@ void radix_tree_iter_replace(struct radix_tree_root *root, const struct radix_tree_iter *iter, void __rcu **slot, void *item) { - __radix_tree_replace(root, iter->node, slot, item, NULL); + __radix_tree_replace(root, iter->node, slot, item); } #ifdef CONFIG_RADIX_TREE_MULTIORDER @@ -1807,23 +1798,6 @@ radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *root, } EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot); -/** - * __radix_tree_delete_node - try to free node after clearing a slot - * @root: radix tree root - * @node: node containing @index - * @update_node: callback for changing leaf nodes - * - * After clearing the slot at @index in @node from radix tree - * rooted at @root, call this function to attempt freeing the - * node and shrinking the tree. - */ -void __radix_tree_delete_node(struct radix_tree_root *root, - struct radix_tree_node *node, - radix_tree_update_node_t update_node) -{ - delete_node(root, node, update_node); -} - static bool __radix_tree_delete(struct radix_tree_root *root, struct radix_tree_node *node, void __rcu **slot) { @@ -1839,7 +1813,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, node_tag_clear(root, node, tag, offset); replace_slot(slot, NULL, node, -1, values); - return node && delete_node(root, node, NULL); + return node && delete_node(root, node); } /** diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 60786fa55302..0e0ff26c9bcb 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -618,7 +618,7 @@ static void multiorder_account(void) __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); __radix_tree_lookup(&tree, 1 << 5, &node, &slot); assert(node->count == node->nr_values * 2); - __radix_tree_replace(&tree, node, slot, NULL, NULL); + __radix_tree_replace(&tree, node, slot, NULL); assert(node->nr_values == 0); item_kill_tree(&tree); -- cgit v1.2.3 From 2956c6644bfd9aab9f6b21a12e1bd75876d9dd73 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 19 May 2018 16:47:47 -0400 Subject: radix tree: Remove split/join code radix_tree_split and radix_tree_join were never used upstream. Remove them; if they're needed in future they will be replaced by XArray equivalents. Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 6 - lib/radix-tree.c | 171 +---------------------- tools/testing/radix-tree/benchmark.c | 91 ------------- tools/testing/radix-tree/multiorder.c | 247 ---------------------------------- 4 files changed, 2 insertions(+), 513 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 7513d0df984b..44f9abdcb5ab 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -284,12 +284,6 @@ static inline void radix_tree_preload_end(void) preempt_enable(); } -int radix_tree_split_preload(unsigned old_order, unsigned new_order, gfp_t); -int radix_tree_split(struct radix_tree_root *, unsigned long index, - unsigned new_order); -int radix_tree_join(struct radix_tree_root *, unsigned long index, - unsigned new_order, void *); - void __rcu **idr_get_free(struct radix_tree_root *root, struct radix_tree_iter *iter, gfp_t gfp, unsigned long max); diff --git a/lib/radix-tree.c b/lib/radix-tree.c index c4c252185734..c52e0bef5264 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -415,28 +415,6 @@ int radix_tree_maybe_preload(gfp_t gfp_mask) } EXPORT_SYMBOL(radix_tree_maybe_preload); -#ifdef CONFIG_RADIX_TREE_MULTIORDER -/* - * Preload with enough objects to ensure that we can split a single entry - * of order @old_order into many entries of size @new_order - */ -int radix_tree_split_preload(unsigned int old_order, unsigned int new_order, - gfp_t gfp_mask) -{ - unsigned top = 1 << (old_order % RADIX_TREE_MAP_SHIFT); - unsigned layers = (old_order / RADIX_TREE_MAP_SHIFT) - - (new_order / RADIX_TREE_MAP_SHIFT); - unsigned nr = 0; - - WARN_ON_ONCE(!gfpflags_allow_blocking(gfp_mask)); - BUG_ON(new_order >= old_order); - - while (layers--) - nr = nr * RADIX_TREE_MAP_SIZE + 1; - return __radix_tree_preload(gfp_mask, top * nr); -} -#endif - /* * The same as function above, but preload number of nodes required to insert * (1 << order) continuous naturally-aligned elements. @@ -1111,8 +1089,8 @@ EXPORT_SYMBOL(radix_tree_replace_slot); * @slot: pointer to slot * @item: new item to store in the slot. * - * For use with radix_tree_split() and radix_tree_for_each_slot(). - * Caller must hold tree write locked across split and replacement. + * For use with radix_tree_for_each_slot(). + * Caller must hold tree write locked. */ void radix_tree_iter_replace(struct radix_tree_root *root, const struct radix_tree_iter *iter, @@ -1121,151 +1099,6 @@ void radix_tree_iter_replace(struct radix_tree_root *root, __radix_tree_replace(root, iter->node, slot, item); } -#ifdef CONFIG_RADIX_TREE_MULTIORDER -/** - * radix_tree_join - replace multiple entries with one multiorder entry - * @root: radix tree root - * @index: an index inside the new entry - * @order: order of the new entry - * @item: new entry - * - * Call this function to replace several entries with one larger entry. - * The existing entries are presumed to not need freeing as a result of - * this call. - * - * The replacement entry will have all the tags set on it that were set - * on any of the entries it is replacing. - */ -int radix_tree_join(struct radix_tree_root *root, unsigned long index, - unsigned order, void *item) -{ - struct radix_tree_node *node; - void __rcu **slot; - int error; - - BUG_ON(radix_tree_is_internal_node(item)); - - error = __radix_tree_create(root, index, order, &node, &slot); - if (!error) - error = insert_entries(node, slot, item, order, true); - if (error > 0) - error = 0; - - return error; -} - -/** - * radix_tree_split - Split an entry into smaller entries - * @root: radix tree root - * @index: An index within the large entry - * @order: Order of new entries - * - * Call this function as the first step in replacing a multiorder entry - * with several entries of lower order. After this function returns, - * loop over the relevant portion of the tree using radix_tree_for_each_slot() - * and call radix_tree_iter_replace() to set up each new entry. - * - * The tags from this entry are replicated to all the new entries. - * - * The radix tree should be locked against modification during the entire - * replacement operation. Lock-free lookups will see RADIX_TREE_RETRY which - * should prompt RCU walkers to restart the lookup from the root. - */ -int radix_tree_split(struct radix_tree_root *root, unsigned long index, - unsigned order) -{ - struct radix_tree_node *parent, *node, *child; - void __rcu **slot; - unsigned int offset, end; - unsigned n, tag, tags = 0; - gfp_t gfp = root_gfp_mask(root); - - if (!__radix_tree_lookup(root, index, &parent, &slot)) - return -ENOENT; - if (!parent) - return -ENOENT; - - offset = get_slot_offset(parent, slot); - - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tag_get(parent, tag, offset)) - tags |= 1 << tag; - - for (end = offset + 1; end < RADIX_TREE_MAP_SIZE; end++) { - if (!xa_is_sibling(rcu_dereference_raw(parent->slots[end]))) - break; - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tags & (1 << tag)) - tag_set(parent, tag, end); - /* rcu_assign_pointer ensures tags are set before RETRY */ - rcu_assign_pointer(parent->slots[end], RADIX_TREE_RETRY); - } - rcu_assign_pointer(parent->slots[offset], RADIX_TREE_RETRY); - parent->nr_values -= (end - offset); - - if (order == parent->shift) - return 0; - if (order > parent->shift) { - while (offset < end) - offset += insert_entries(parent, &parent->slots[offset], - RADIX_TREE_RETRY, order, true); - return 0; - } - - node = parent; - - for (;;) { - if (node->shift > order) { - child = radix_tree_node_alloc(gfp, node, root, - node->shift - RADIX_TREE_MAP_SHIFT, - offset, 0, 0); - if (!child) - goto nomem; - if (node != parent) { - node->count++; - rcu_assign_pointer(node->slots[offset], - node_to_entry(child)); - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tags & (1 << tag)) - tag_set(node, tag, offset); - } - - node = child; - offset = 0; - continue; - } - - n = insert_entries(node, &node->slots[offset], - RADIX_TREE_RETRY, order, false); - BUG_ON(n > RADIX_TREE_MAP_SIZE); - - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tags & (1 << tag)) - tag_set(node, tag, offset); - offset += n; - - while (offset == RADIX_TREE_MAP_SIZE) { - if (node == parent) - break; - offset = node->offset; - child = node; - node = node->parent; - rcu_assign_pointer(node->slots[offset], - node_to_entry(child)); - offset++; - } - if ((node == parent) && (offset == end)) - return 0; - } - - nomem: - /* Shouldn't happen; did user forget to preload? */ - /* TODO: free all the allocated nodes */ - WARN_ON(1); - return -ENOMEM; -} -#endif - static void node_tag_set(struct radix_tree_root *root, struct radix_tree_node *node, unsigned int tag, unsigned int offset) diff --git a/tools/testing/radix-tree/benchmark.c b/tools/testing/radix-tree/benchmark.c index 99c40f3ed133..35741b9c2a4a 100644 --- a/tools/testing/radix-tree/benchmark.c +++ b/tools/testing/radix-tree/benchmark.c @@ -146,90 +146,6 @@ static void benchmark_size(unsigned long size, unsigned long step, int order) rcu_barrier(); } -static long long __benchmark_split(unsigned long index, - int old_order, int new_order) -{ - struct timespec start, finish; - long long nsec; - RADIX_TREE(tree, GFP_ATOMIC); - - item_insert_order(&tree, index, old_order); - - clock_gettime(CLOCK_MONOTONIC, &start); - radix_tree_split(&tree, index, new_order); - clock_gettime(CLOCK_MONOTONIC, &finish); - nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + - (finish.tv_nsec - start.tv_nsec); - - item_kill_tree(&tree); - - return nsec; - -} - -static void benchmark_split(unsigned long size, unsigned long step) -{ - int i, j, idx; - long long nsec = 0; - - - for (idx = 0; idx < size; idx += step) { - for (i = 3; i < 11; i++) { - for (j = 0; j < i; j++) { - nsec += __benchmark_split(idx, i, j); - } - } - } - - printv(2, "Size %8ld, step %8ld, split time %10lld ns\n", - size, step, nsec); - -} - -static long long __benchmark_join(unsigned long index, - unsigned order1, unsigned order2) -{ - unsigned long loc; - struct timespec start, finish; - long long nsec; - void *item, *item2 = item_create(index + 1, order1); - RADIX_TREE(tree, GFP_KERNEL); - - item_insert_order(&tree, index, order2); - item = radix_tree_lookup(&tree, index); - - clock_gettime(CLOCK_MONOTONIC, &start); - radix_tree_join(&tree, index + 1, order1, item2); - clock_gettime(CLOCK_MONOTONIC, &finish); - nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + - (finish.tv_nsec - start.tv_nsec); - - loc = find_item(&tree, item); - if (loc == -1) - free(item); - - item_kill_tree(&tree); - - return nsec; -} - -static void benchmark_join(unsigned long step) -{ - int i, j, idx; - long long nsec = 0; - - for (idx = 0; idx < 1 << 10; idx += step) { - for (i = 1; i < 15; i++) { - for (j = 0; j < i; j++) { - nsec += __benchmark_join(idx, i, j); - } - } - } - - printv(2, "Size %8d, step %8ld, join time %10lld ns\n", - 1 << 10, step, nsec); -} - void benchmark(void) { unsigned long size[] = {1 << 10, 1 << 20, 0}; @@ -247,11 +163,4 @@ void benchmark(void) for (c = 0; size[c]; c++) for (s = 0; step[s]; s++) benchmark_size(size[c], step[s] << 9, 9); - - for (c = 0; size[c]; c++) - for (s = 0; step[s]; s++) - benchmark_split(size[c], step[s]); - - for (s = 0; step[s]; s++) - benchmark_join(step[s]); } diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 0e0ff26c9bcb..221e042d1b89 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -356,251 +356,6 @@ void multiorder_tagged_iteration(void) item_kill_tree(&tree); } -/* - * Basic join checks: make sure we can't find an entry in the tree after - * a larger entry has replaced it - */ -static void multiorder_join1(unsigned long index, - unsigned order1, unsigned order2) -{ - unsigned long loc; - void *item, *item2 = item_create(index + 1, order1); - RADIX_TREE(tree, GFP_KERNEL); - - item_insert_order(&tree, index, order2); - item = radix_tree_lookup(&tree, index); - radix_tree_join(&tree, index + 1, order1, item2); - loc = find_item(&tree, item); - if (loc == -1) - free(item); - item = radix_tree_lookup(&tree, index + 1); - assert(item == item2); - item_kill_tree(&tree); -} - -/* - * Check that the accounting of value entries is handled correctly - * by joining a value entry to a normal pointer. - */ -static void multiorder_join2(unsigned order1, unsigned order2) -{ - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_node *node; - void *item1 = item_create(0, order1); - void *item2; - - item_insert_order(&tree, 0, order2); - radix_tree_insert(&tree, 1 << order2, xa_mk_value(5)); - item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); - assert(item2 == xa_mk_value(5)); - assert(node->nr_values == 1); - - item2 = radix_tree_lookup(&tree, 0); - free(item2); - - radix_tree_join(&tree, 0, order1, item1); - item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL); - assert(item2 == item1); - assert(node->nr_values == 0); - item_kill_tree(&tree); -} - -/* - * This test revealed an accounting bug for value entries at one point. - * Nodes were being freed back into the pool with an elevated exception count - * by radix_tree_join() and then radix_tree_split() was failing to zero the - * count of value entries. - */ -static void multiorder_join3(unsigned int order) -{ - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_node *node; - void **slot; - struct radix_tree_iter iter; - unsigned long i; - - for (i = 0; i < (1 << order); i++) { - radix_tree_insert(&tree, i, xa_mk_value(5)); - } - - radix_tree_join(&tree, 0, order, xa_mk_value(7)); - rcu_barrier(); - - radix_tree_split(&tree, 0, 0); - - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(5)); - } - - __radix_tree_lookup(&tree, 0, &node, NULL); - assert(node->nr_values == node->count); - - item_kill_tree(&tree); -} - -static void multiorder_join(void) -{ - int i, j, idx; - - for (idx = 0; idx < 1024; idx = idx * 2 + 3) { - for (i = 1; i < 15; i++) { - for (j = 0; j < i; j++) { - multiorder_join1(idx, i, j); - } - } - } - - for (i = 1; i < 15; i++) { - for (j = 0; j < i; j++) { - multiorder_join2(i, j); - } - } - - for (i = 3; i < 10; i++) { - multiorder_join3(i); - } -} - -static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc) -{ - struct radix_tree_preload *rtp = &radix_tree_preloads; - if (rtp->nr != 0) - printv(2, "split(%u %u) remaining %u\n", old_order, new_order, - rtp->nr); - /* - * Can't check for equality here as some nodes may have been - * RCU-freed while we ran. But we should never finish with more - * nodes allocated since they should have all been preloaded. - */ - if (nr_allocated > alloc) - printv(2, "split(%u %u) allocated %u %u\n", old_order, new_order, - alloc, nr_allocated); -} - -static void __multiorder_split(int old_order, int new_order) -{ - RADIX_TREE(tree, GFP_ATOMIC); - void **slot; - struct radix_tree_iter iter; - unsigned alloc; - struct item *item; - - radix_tree_preload(GFP_KERNEL); - assert(item_insert_order(&tree, 0, old_order) == 0); - radix_tree_preload_end(); - - /* Wipe out the preloaded cache or it'll confuse check_mem() */ - radix_tree_cpu_dead(0); - - item = radix_tree_tag_set(&tree, 0, 2); - - radix_tree_split_preload(old_order, new_order, GFP_KERNEL); - alloc = nr_allocated; - radix_tree_split(&tree, 0, new_order); - check_mem(old_order, new_order, alloc); - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, - item_create(iter.index, new_order)); - } - radix_tree_preload_end(); - - item_kill_tree(&tree); - free(item); -} - -static void __multiorder_split2(int old_order, int new_order) -{ - RADIX_TREE(tree, GFP_KERNEL); - void **slot; - struct radix_tree_iter iter; - struct radix_tree_node *node; - void *item; - - __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); - - item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == xa_mk_value(5)); - assert(node->nr_values > 0); - - radix_tree_split(&tree, 0, new_order); - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, - item_create(iter.index, new_order)); - } - - item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item != xa_mk_value(5)); - assert(node->nr_values == 0); - - item_kill_tree(&tree); -} - -static void __multiorder_split3(int old_order, int new_order) -{ - RADIX_TREE(tree, GFP_KERNEL); - void **slot; - struct radix_tree_iter iter; - struct radix_tree_node *node; - void *item; - - __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); - - item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == xa_mk_value(5)); - assert(node->nr_values > 0); - - radix_tree_split(&tree, 0, new_order); - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_iter_replace(&tree, &iter, slot, xa_mk_value(7)); - } - - item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == xa_mk_value(7)); - assert(node->nr_values > 0); - - item_kill_tree(&tree); - - __radix_tree_insert(&tree, 0, old_order, xa_mk_value(5)); - - item = __radix_tree_lookup(&tree, 0, &node, NULL); - assert(item == xa_mk_value(5)); - assert(node->nr_values > 0); - - radix_tree_split(&tree, 0, new_order); - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - if (iter.index == (1 << new_order)) - radix_tree_iter_replace(&tree, &iter, slot, - xa_mk_value(7)); - else - radix_tree_iter_replace(&tree, &iter, slot, NULL); - } - - item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL); - assert(item == xa_mk_value(7)); - assert(node->count == node->nr_values); - do { - node = node->parent; - if (!node) - break; - assert(node->count == 1); - assert(node->nr_values == 0); - } while (1); - - item_kill_tree(&tree); -} - -static void multiorder_split(void) -{ - int i, j; - - for (i = 3; i < 11; i++) - for (j = 0; j < i; j++) { - __multiorder_split(i, j); - __multiorder_split2(i, j); - __multiorder_split3(i, j); - } -} - static void multiorder_account(void) { RADIX_TREE(tree, GFP_KERNEL); @@ -702,8 +457,6 @@ void multiorder_checks(void) multiorder_tag_tests(); multiorder_iteration(); multiorder_tagged_iteration(); - multiorder_join(); - multiorder_split(); multiorder_account(); multiorder_iteration_race(); -- cgit v1.2.3 From adb9d9c4ccb1ff0bf1d376e65f36aa5573c75c1a Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 9 Apr 2018 16:52:21 -0400 Subject: radix tree: Remove radix_tree_clear_tags The page cache was the only user of this interface and it has now been converted to the XArray. Transform the test into a test of xas_init_marks(). Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 2 -- lib/radix-tree.c | 13 ------------ lib/test_xarray.c | 40 ++++++++++++++++++++++++++++++++++++ tools/testing/radix-tree/tag_check.c | 29 -------------------------- 4 files changed, 40 insertions(+), 44 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 664d50c5c2ff..7b009af3bb1e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -252,8 +252,6 @@ void radix_tree_iter_delete(struct radix_tree_root *, struct radix_tree_iter *iter, void __rcu **slot); void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); void *radix_tree_delete(struct radix_tree_root *, unsigned long); -void radix_tree_clear_tags(struct radix_tree_root *, struct radix_tree_node *, - void __rcu **slot); unsigned int radix_tree_gang_lookup(const struct radix_tree_root *, void **results, unsigned long first_index, unsigned int max_items); diff --git a/lib/radix-tree.c b/lib/radix-tree.c index b57ddc3dbbbd..e92f2b6ae9c5 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -1670,19 +1670,6 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index) } EXPORT_SYMBOL(radix_tree_delete); -void radix_tree_clear_tags(struct radix_tree_root *root, - struct radix_tree_node *node, - void __rcu **slot) -{ - if (node) { - unsigned int tag, offset = get_slot_offset(node, slot); - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - node_tag_clear(root, node, tag, offset); - } else { - root_tag_clear_all(root); - } -} - /** * radix_tree_tagged - test whether any items in the tree are tagged * @root: radix tree root diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 815daffdd8c9..ca86141641cb 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -213,12 +213,52 @@ static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_xa_mark_2(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + unsigned long index; + unsigned int count = 0; + void *entry; + + xa_store_index(xa, 0, GFP_KERNEL); + xa_set_mark(xa, 0, XA_MARK_0); + xas_lock(&xas); + xas_load(&xas); + xas_init_marks(&xas); + xas_unlock(&xas); + XA_BUG_ON(xa, !xa_get_mark(xa, 0, XA_MARK_0) == 0); + + for (index = 3500; index < 4500; index++) { + xa_store_index(xa, index, GFP_KERNEL); + xa_set_mark(xa, index, XA_MARK_0); + } + + xas_reset(&xas); + rcu_read_lock(); + xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0) + count++; + rcu_read_unlock(); + XA_BUG_ON(xa, count != 1000); + + xas_lock(&xas); + xas_for_each(&xas, entry, ULONG_MAX) { + xas_init_marks(&xas); + XA_BUG_ON(xa, !xa_get_mark(xa, xas.xa_index, XA_MARK_0)); + XA_BUG_ON(xa, !xas_get_mark(&xas, XA_MARK_0)); + } + xas_unlock(&xas); + + xa_destroy(xa); +} + static noinline void check_xa_mark(struct xarray *xa) { unsigned long index; for (index = 0; index < 16384; index += 4) check_xa_mark_1(xa, index); + + check_xa_mark_2(xa); } static noinline void check_xa_shrink(struct xarray *xa) diff --git a/tools/testing/radix-tree/tag_check.c b/tools/testing/radix-tree/tag_check.c index 543181e4847b..56a42f1c5ab0 100644 --- a/tools/testing/radix-tree/tag_check.c +++ b/tools/testing/radix-tree/tag_check.c @@ -331,34 +331,6 @@ static void single_check(void) item_kill_tree(&tree); } -void radix_tree_clear_tags_test(void) -{ - unsigned long index; - struct radix_tree_node *node; - struct radix_tree_iter iter; - void **slot; - - RADIX_TREE(tree, GFP_KERNEL); - - item_insert(&tree, 0); - item_tag_set(&tree, 0, 0); - __radix_tree_lookup(&tree, 0, &node, &slot); - radix_tree_clear_tags(&tree, node, slot); - assert(item_tag_get(&tree, 0, 0) == 0); - - for (index = 0; index < 1000; index++) { - item_insert(&tree, index); - item_tag_set(&tree, index, 0); - } - - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - radix_tree_clear_tags(&tree, iter.node, slot); - assert(item_tag_get(&tree, iter.index, 0) == 0); - } - - item_kill_tree(&tree); -} - void tag_check(void) { single_check(); @@ -376,5 +348,4 @@ void tag_check(void) thrash_tags(); rcu_barrier(); printv(2, "after thrash_tags: %d allocated\n", nr_allocated); - radix_tree_clear_tags_test(); } -- cgit v1.2.3 From 372266ba0267803564824b1c09f1bb7f3f3fc761 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 18 Aug 2018 07:09:22 -0400 Subject: radix tree test suite: Convert tag_tagged_items to XArray The tag_tagged_items() function is supposed to test the page-writeback tagging code. Since that has been converted to the XArray, there's not much point in testing the radix tree's tagging code. This requires using the pthread mutex embedded in the xarray instead of an external lock, so remove the pthread mutexes which protect xarrays/radix trees. Also remove radix_tree_iter_tag_set() as this was the last user. Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 2 -- lib/radix-tree.c | 12 ---------- tools/testing/radix-tree/iteration_check.c | 16 ++++++------- tools/testing/radix-tree/main.c | 4 ++-- tools/testing/radix-tree/multiorder.c | 12 +++++----- tools/testing/radix-tree/regression1.c | 17 +++++++------- tools/testing/radix-tree/regression2.c | 8 +++---- tools/testing/radix-tree/tag_check.c | 4 ++-- tools/testing/radix-tree/test.c | 37 ++++++++++++------------------ tools/testing/radix-tree/test.h | 5 ++-- 10 files changed, 46 insertions(+), 71 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 7b009af3bb1e..9a1460488163 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -264,8 +264,6 @@ void *radix_tree_tag_clear(struct radix_tree_root *, unsigned long index, unsigned int tag); int radix_tree_tag_get(const struct radix_tree_root *, unsigned long index, unsigned int tag); -void radix_tree_iter_tag_set(struct radix_tree_root *, - const struct radix_tree_iter *iter, unsigned int tag); void radix_tree_iter_tag_clear(struct radix_tree_root *, const struct radix_tree_iter *iter, unsigned int tag); unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *, diff --git a/lib/radix-tree.c b/lib/radix-tree.c index e92f2b6ae9c5..f107dd2698e3 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -1108,18 +1108,6 @@ void *radix_tree_tag_set(struct radix_tree_root *root, } EXPORT_SYMBOL(radix_tree_tag_set); -/** - * radix_tree_iter_tag_set - set a tag on the current iterator entry - * @root: radix tree root - * @iter: iterator state - * @tag: tag to set - */ -void radix_tree_iter_tag_set(struct radix_tree_root *root, - const struct radix_tree_iter *iter, unsigned int tag) -{ - node_tag_set(root, iter->node, tag, iter_offset(iter)); -} - static void node_tag_clear(struct radix_tree_root *root, struct radix_tree_node *node, unsigned int tag, unsigned int offset) diff --git a/tools/testing/radix-tree/iteration_check.c b/tools/testing/radix-tree/iteration_check.c index a92bab513701..d047327bb9ef 100644 --- a/tools/testing/radix-tree/iteration_check.c +++ b/tools/testing/radix-tree/iteration_check.c @@ -18,10 +18,9 @@ #define NUM_THREADS 5 #define MAX_IDX 100 -#define TAG 0 -#define NEW_TAG 1 +#define TAG XA_MARK_0 +#define NEW_TAG XA_MARK_1 -static pthread_mutex_t tree_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_t threads[NUM_THREADS]; static unsigned int seeds[3]; static RADIX_TREE(tree, GFP_KERNEL); @@ -38,7 +37,7 @@ static void *add_entries_fn(void *arg) int order; for (pgoff = 0; pgoff < MAX_IDX; pgoff++) { - pthread_mutex_lock(&tree_lock); + xa_lock(&tree); for (order = max_order; order >= 0; order--) { if (item_insert_order(&tree, pgoff, order) == 0) { @@ -46,7 +45,7 @@ static void *add_entries_fn(void *arg) break; } } - pthread_mutex_unlock(&tree_lock); + xa_unlock(&tree); } } @@ -150,9 +149,9 @@ static void *remove_entries_fn(void *arg) pgoff = rand_r(&seeds[2]) % MAX_IDX; - pthread_mutex_lock(&tree_lock); + xa_lock(&tree); item_delete(&tree, pgoff); - pthread_mutex_unlock(&tree_lock); + xa_unlock(&tree); } rcu_unregister_thread(); @@ -165,8 +164,7 @@ static void *tag_entries_fn(void *arg) rcu_register_thread(); while (!test_complete) { - tag_tagged_items(&tree, &tree_lock, 0, MAX_IDX, 10, TAG, - NEW_TAG); + tag_tagged_items(&tree, 0, MAX_IDX, 10, TAG, NEW_TAG); } rcu_unregister_thread(); return NULL; diff --git a/tools/testing/radix-tree/main.c b/tools/testing/radix-tree/main.c index 79589ea570ab..77a44c54998f 100644 --- a/tools/testing/radix-tree/main.c +++ b/tools/testing/radix-tree/main.c @@ -214,7 +214,7 @@ void copy_tag_check(void) } // printf("\ncopying tags...\n"); - tagged = tag_tagged_items(&tree, NULL, start, end, ITEMS, 0, 1); + tagged = tag_tagged_items(&tree, start, end, ITEMS, XA_MARK_0, XA_MARK_1); // printf("checking copied tags\n"); assert(tagged == count); @@ -223,7 +223,7 @@ void copy_tag_check(void) /* Copy tags in several rounds */ // printf("\ncopying tags...\n"); tmp = rand() % (count / 10 + 2); - tagged = tag_tagged_items(&tree, NULL, start, end, tmp, 0, 2); + tagged = tag_tagged_items(&tree, start, end, tmp, XA_MARK_0, XA_MARK_2); assert(tagged == count); // printf("%lu %lu %lu\n", tagged, tmp, count); diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 221e042d1b89..0436554a099a 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -59,7 +59,7 @@ static void __multiorder_tag_test(int index, int order) assert(!radix_tree_tag_get(&tree, i, 1)); } - assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1); + assert(tag_tagged_items(&tree, 0, ~0UL, 10, XA_MARK_0, XA_MARK_1) == 1); assert(radix_tree_tag_clear(&tree, index, 0)); for_each_index(i, base, order) { @@ -87,7 +87,7 @@ static void __multiorder_tag_test2(unsigned order, unsigned long index2) assert(radix_tree_tag_set(&tree, 0, 0)); assert(radix_tree_tag_set(&tree, index2, 0)); - assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2); + assert(tag_tagged_items(&tree, 0, ~0UL, 10, XA_MARK_0, XA_MARK_1) == 2); item_kill_tree(&tree); } @@ -318,8 +318,8 @@ void multiorder_tagged_iteration(void) } } - assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) == - TAG_ENTRIES); + assert(tag_tagged_items(&tree, 0, ~0UL, TAG_ENTRIES, XA_MARK_1, + XA_MARK_2) == TAG_ENTRIES); for (j = 0; j < 256; j++) { int mask, k; @@ -345,8 +345,8 @@ void multiorder_tagged_iteration(void) } } - assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0) - == TAG_ENTRIES); + assert(tag_tagged_items(&tree, 1, ~0UL, MT_NUM_ENTRIES * 2, XA_MARK_1, + XA_MARK_0) == TAG_ENTRIES); i = 0; radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) { assert(iter.index == tag_index[i]); diff --git a/tools/testing/radix-tree/regression1.c b/tools/testing/radix-tree/regression1.c index b4a4a7168986..a61c7bcbc72d 100644 --- a/tools/testing/radix-tree/regression1.c +++ b/tools/testing/radix-tree/regression1.c @@ -44,7 +44,6 @@ #include "regression.h" static RADIX_TREE(mt_tree, GFP_KERNEL); -static pthread_mutex_t mt_lock = PTHREAD_MUTEX_INITIALIZER; struct page { pthread_mutex_t lock; @@ -126,29 +125,29 @@ static void *regression1_fn(void *arg) struct page *p; p = page_alloc(0); - pthread_mutex_lock(&mt_lock); + xa_lock(&mt_tree); radix_tree_insert(&mt_tree, 0, p); - pthread_mutex_unlock(&mt_lock); + xa_unlock(&mt_tree); p = page_alloc(1); - pthread_mutex_lock(&mt_lock); + xa_lock(&mt_tree); radix_tree_insert(&mt_tree, 1, p); - pthread_mutex_unlock(&mt_lock); + xa_unlock(&mt_tree); - pthread_mutex_lock(&mt_lock); + xa_lock(&mt_tree); p = radix_tree_delete(&mt_tree, 1); pthread_mutex_lock(&p->lock); p->count--; pthread_mutex_unlock(&p->lock); - pthread_mutex_unlock(&mt_lock); + xa_unlock(&mt_tree); page_free(p); - pthread_mutex_lock(&mt_lock); + xa_lock(&mt_tree); p = radix_tree_delete(&mt_tree, 0); pthread_mutex_lock(&p->lock); p->count--; pthread_mutex_unlock(&p->lock); - pthread_mutex_unlock(&mt_lock); + xa_unlock(&mt_tree); page_free(p); } } else { diff --git a/tools/testing/radix-tree/regression2.c b/tools/testing/radix-tree/regression2.c index 424b91c77831..f2c7e640a919 100644 --- a/tools/testing/radix-tree/regression2.c +++ b/tools/testing/radix-tree/regression2.c @@ -53,9 +53,9 @@ #include "regression.h" #include "test.h" -#define PAGECACHE_TAG_DIRTY 0 -#define PAGECACHE_TAG_WRITEBACK 1 -#define PAGECACHE_TAG_TOWRITE 2 +#define PAGECACHE_TAG_DIRTY XA_MARK_0 +#define PAGECACHE_TAG_WRITEBACK XA_MARK_1 +#define PAGECACHE_TAG_TOWRITE XA_MARK_2 static RADIX_TREE(mt_tree, GFP_KERNEL); unsigned long page_count = 0; @@ -92,7 +92,7 @@ void regression2_test(void) /* 1. */ start = 0; end = max_slots - 2; - tag_tagged_items(&mt_tree, NULL, start, end, 1, + tag_tagged_items(&mt_tree, start, end, 1, PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE); /* 2. */ diff --git a/tools/testing/radix-tree/tag_check.c b/tools/testing/radix-tree/tag_check.c index 56a42f1c5ab0..f898957b1a19 100644 --- a/tools/testing/radix-tree/tag_check.c +++ b/tools/testing/radix-tree/tag_check.c @@ -24,7 +24,7 @@ __simple_checks(struct radix_tree_root *tree, unsigned long index, int tag) item_tag_set(tree, index, tag); ret = item_tag_get(tree, index, tag); assert(ret != 0); - ret = tag_tagged_items(tree, NULL, first, ~0UL, 10, tag, !tag); + ret = tag_tagged_items(tree, first, ~0UL, 10, tag, !tag); assert(ret == 1); ret = item_tag_get(tree, index, !tag); assert(ret != 0); @@ -321,7 +321,7 @@ static void single_check(void) assert(ret == 0); verify_tag_consistency(&tree, 0); verify_tag_consistency(&tree, 1); - ret = tag_tagged_items(&tree, NULL, first, 10, 10, 0, 1); + ret = tag_tagged_items(&tree, first, 10, 10, XA_MARK_0, XA_MARK_1); assert(ret == 1); ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 1); assert(ret == 1); diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 470419bfd49d..d70adcd03d35 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -176,35 +176,28 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start, } /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */ -int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock, - unsigned long start, unsigned long end, unsigned batch, - unsigned iftag, unsigned thentag) +int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end, + unsigned batch, xa_mark_t iftag, xa_mark_t thentag) { - unsigned long tagged = 0; - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, xa, start); + unsigned int tagged = 0; + struct item *item; if (batch == 0) batch = 1; - if (lock) - pthread_mutex_lock(lock); - radix_tree_for_each_tagged(slot, root, &iter, start, iftag) { - if (iter.index > end) - break; - radix_tree_iter_tag_set(root, &iter, thentag); - tagged++; - if ((tagged % batch) != 0) + xas_lock_irq(&xas); + xas_for_each_marked(&xas, item, end, iftag) { + xas_set_mark(&xas, thentag); + if (++tagged % batch) continue; - slot = radix_tree_iter_resume(slot, &iter); - if (lock) { - pthread_mutex_unlock(lock); - rcu_barrier(); - pthread_mutex_lock(lock); - } + + xas_pause(&xas); + xas_unlock_irq(&xas); + rcu_barrier(); + xas_lock_irq(&xas); } - if (lock) - pthread_mutex_unlock(lock); + xas_unlock_irq(&xas); return tagged; } diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index 9532c18c6cb1..100e9a456f91 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -29,9 +29,8 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start, unsigned long nr, int chunk); void item_kill_tree(struct radix_tree_root *root); -int tag_tagged_items(struct radix_tree_root *, pthread_mutex_t *, - unsigned long start, unsigned long end, unsigned batch, - unsigned iftag, unsigned thentag); +int tag_tagged_items(struct xarray *, unsigned long start, unsigned long end, + unsigned batch, xa_mark_t iftag, xa_mark_t thentag); void xarray_tests(void); void tag_check(void); -- cgit v1.2.3 From 47e0fab2b15155e33fdff777c791bebfd5855bbc Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Mon, 20 Aug 2018 15:48:46 -0400 Subject: radix tree test suite: Convert iteration test to XArray With no code left in the kernel using the multiorder radix tree, convert the iteration test from the radix tree to the XArray. It's unlikely to suffer the same bug as the radix tree, but this test will prevent that bug from ever creeping into the XArray implementation. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/iteration_check.c | 103 ++++++++++++++--------------- tools/testing/radix-tree/test.c | 17 +++-- tools/testing/radix-tree/test.h | 1 + 3 files changed, 63 insertions(+), 58 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/iteration_check.c b/tools/testing/radix-tree/iteration_check.c index d047327bb9ef..238db187aa15 100644 --- a/tools/testing/radix-tree/iteration_check.c +++ b/tools/testing/radix-tree/iteration_check.c @@ -1,5 +1,5 @@ /* - * iteration_check.c: test races having to do with radix tree iteration + * iteration_check.c: test races having to do with xarray iteration * Copyright (c) 2016 Intel Corporation * Author: Ross Zwisler * @@ -12,7 +12,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. */ -#include #include #include "test.h" @@ -23,29 +22,44 @@ static pthread_t threads[NUM_THREADS]; static unsigned int seeds[3]; -static RADIX_TREE(tree, GFP_KERNEL); +static DEFINE_XARRAY(array); static bool test_complete; static int max_order; -/* relentlessly fill the tree with tagged entries */ +void my_item_insert(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + struct item *item = item_create(index, 0); + int order; + +retry: + xas_lock(&xas); + for (order = max_order; order >= 0; order--) { + xas_set_order(&xas, index, order); + item->order = order; + if (xas_find_conflict(&xas)) + continue; + xas_store(&xas, item); + xas_set_mark(&xas, TAG); + break; + } + xas_unlock(&xas); + if (xas_nomem(&xas, GFP_KERNEL)) + goto retry; + if (order < 0) + free(item); +} + +/* relentlessly fill the array with tagged entries */ static void *add_entries_fn(void *arg) { rcu_register_thread(); while (!test_complete) { unsigned long pgoff; - int order; for (pgoff = 0; pgoff < MAX_IDX; pgoff++) { - xa_lock(&tree); - for (order = max_order; order >= 0; order--) { - if (item_insert_order(&tree, pgoff, order) - == 0) { - item_tag_set(&tree, pgoff, TAG); - break; - } - } - xa_unlock(&tree); + my_item_insert(&array, pgoff); } } @@ -55,33 +69,25 @@ static void *add_entries_fn(void *arg) } /* - * Iterate over the tagged entries, doing a radix_tree_iter_retry() as we find - * things that have been removed and randomly resetting our iteration to the - * next chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and - * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a - * NULL 'slot' variable. + * Iterate over tagged entries, retrying when we find ourselves in a deleted + * node and randomly pausing the iteration. */ static void *tagged_iteration_fn(void *arg) { - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, &array, 0); + void *entry; rcu_register_thread(); while (!test_complete) { + xas_set(&xas, 0); rcu_read_lock(); - radix_tree_for_each_tagged(slot, &tree, &iter, 0, TAG) { - void *entry = radix_tree_deref_slot(slot); - if (unlikely(!entry)) + xas_for_each_marked(&xas, entry, ULONG_MAX, TAG) { + if (xas_retry(&xas, entry)) continue; - if (radix_tree_deref_retry(entry)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - if (rand_r(&seeds[0]) % 50 == 0) { - slot = radix_tree_iter_resume(slot, &iter); + xas_pause(&xas); rcu_read_unlock(); rcu_barrier(); rcu_read_lock(); @@ -96,33 +102,25 @@ static void *tagged_iteration_fn(void *arg) } /* - * Iterate over the entries, doing a radix_tree_iter_retry() as we find things - * that have been removed and randomly resetting our iteration to the next - * chunk with radix_tree_iter_resume(). Both radix_tree_iter_retry() and - * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a - * NULL 'slot' variable. + * Iterate over the entries, retrying when we find ourselves in a deleted + * node and randomly pausing the iteration. */ static void *untagged_iteration_fn(void *arg) { - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, &array, 0); + void *entry; rcu_register_thread(); while (!test_complete) { + xas_set(&xas, 0); rcu_read_lock(); - radix_tree_for_each_slot(slot, &tree, &iter, 0) { - void *entry = radix_tree_deref_slot(slot); - if (unlikely(!entry)) + xas_for_each(&xas, entry, ULONG_MAX) { + if (xas_retry(&xas, entry)) continue; - if (radix_tree_deref_retry(entry)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - if (rand_r(&seeds[1]) % 50 == 0) { - slot = radix_tree_iter_resume(slot, &iter); + xas_pause(&xas); rcu_read_unlock(); rcu_barrier(); rcu_read_lock(); @@ -137,7 +135,7 @@ static void *untagged_iteration_fn(void *arg) } /* - * Randomly remove entries to help induce radix_tree_iter_retry() calls in the + * Randomly remove entries to help induce retries in the * two iteration functions. */ static void *remove_entries_fn(void *arg) @@ -146,12 +144,13 @@ static void *remove_entries_fn(void *arg) while (!test_complete) { int pgoff; + struct item *item; pgoff = rand_r(&seeds[2]) % MAX_IDX; - xa_lock(&tree); - item_delete(&tree, pgoff); - xa_unlock(&tree); + item = xa_erase(&array, pgoff); + if (item) + item_free(item, pgoff); } rcu_unregister_thread(); @@ -164,7 +163,7 @@ static void *tag_entries_fn(void *arg) rcu_register_thread(); while (!test_complete) { - tag_tagged_items(&tree, 0, MAX_IDX, 10, TAG, NEW_TAG); + tag_tagged_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG); } rcu_unregister_thread(); return NULL; @@ -215,5 +214,5 @@ void iteration_test(unsigned order, unsigned test_duration) } } - item_kill_tree(&tree); + item_kill_tree(&array); } diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index d70adcd03d35..32973dd51ec5 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -63,16 +63,21 @@ void item_sanity(struct item *item, unsigned long index) assert((item->index | mask) == (index | mask)); } +void item_free(struct item *item, unsigned long index) +{ + item_sanity(item, index); + free(item); +} + int item_delete(struct radix_tree_root *root, unsigned long index) { struct item *item = radix_tree_delete(root, index); - if (item) { - item_sanity(item, index); - free(item); - return 1; - } - return 0; + if (!item) + return 0; + + item_free(item, index); + return 1; } static void item_free_rcu(struct rcu_head *head) diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index 100e9a456f91..a2f53c889a31 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -14,6 +14,7 @@ struct item *item_create(unsigned long index, unsigned int order); int __item_insert(struct radix_tree_root *root, struct item *item); int item_insert(struct radix_tree_root *root, unsigned long index); void item_sanity(struct item *item, unsigned long index); +void item_free(struct item *item, unsigned long index); int item_insert_order(struct radix_tree_root *root, unsigned long index, unsigned order); int item_delete(struct radix_tree_root *root, unsigned long index); -- cgit v1.2.3 From d6427f8179b5dd65eb468c61fc8cc24657c336c9 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Tue, 28 Aug 2018 16:13:16 -0400 Subject: xarray: Move multiorder account test in-kernel Move this test to the in-kernel test suite, and enhance it to test several different orders. Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 32 ++++++++++++++++++++++++++++++++ tools/testing/radix-tree/multiorder.c | 24 ------------------------ 2 files changed, 32 insertions(+), 24 deletions(-) (limited to 'tools') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index ca86141641cb..38cab4ccb24e 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1019,6 +1019,37 @@ static noinline void check_workingset(struct xarray *xa, unsigned long index) XA_BUG_ON(xa, !xa_empty(xa)); } +/* + * Check that the pointer / value / sibling entries are accounted the + * way we expect them to be. + */ +static noinline void check_account(struct xarray *xa) +{ +#ifdef CONFIG_XARRAY_MULTI + unsigned int order; + + for (order = 1; order < 12; order++) { + XA_STATE(xas, xa, 1 << order); + + xa_store_order(xa, 0, order, xa, GFP_KERNEL); + xas_load(&xas); + XA_BUG_ON(xa, xas.xa_node->count == 0); + XA_BUG_ON(xa, xas.xa_node->count > (1 << order)); + XA_BUG_ON(xa, xas.xa_node->nr_values != 0); + + xa_store_order(xa, 1 << order, order, xa_mk_value(1 << order), + GFP_KERNEL); + XA_BUG_ON(xa, xas.xa_node->count != xas.xa_node->nr_values * 2); + + xa_erase(xa, 1 << order); + XA_BUG_ON(xa, xas.xa_node->nr_values != 0); + + xa_erase(xa, 0); + XA_BUG_ON(xa, !xa_empty(xa)); + } +#endif +} + static noinline void check_destroy(struct xarray *xa) { unsigned long index; @@ -1068,6 +1099,7 @@ static int xarray_checks(void) check_xa_alloc(); check_find(&array); check_find_entry(&array); + check_account(&array); check_destroy(&array); check_move(&array); check_create_range(&array); diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 0436554a099a..dc27a3da210a 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -356,29 +356,6 @@ void multiorder_tagged_iteration(void) item_kill_tree(&tree); } -static void multiorder_account(void) -{ - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_node *node; - void **slot; - - item_insert_order(&tree, 0, 5); - - __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); - __radix_tree_lookup(&tree, 0, &node, NULL); - assert(node->count == node->nr_values * 2); - radix_tree_delete(&tree, 1 << 5); - assert(node->nr_values == 0); - - __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5)); - __radix_tree_lookup(&tree, 1 << 5, &node, &slot); - assert(node->count == node->nr_values * 2); - __radix_tree_replace(&tree, node, slot, NULL); - assert(node->nr_values == 0); - - item_kill_tree(&tree); -} - bool stop_iteration = false; static void *creator_func(void *ptr) @@ -457,7 +434,6 @@ void multiorder_checks(void) multiorder_tag_tests(); multiorder_iteration(); multiorder_tagged_iteration(); - multiorder_account(); multiorder_iteration_race(); radix_tree_cpu_dead(0); -- cgit v1.2.3 From 93eb07f72c8d86f8fe5e90907df1cc037f6ffbb7 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 8 Sep 2018 12:09:52 -0400 Subject: xarray: Move multiorder_shrink to kernel tests Test this functionality inside the kernel as well as in userspace. Also remove insert_bug() as there's no comparable thing to test in the XArray code. Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 37 ++++++++ tools/testing/radix-tree/multiorder.c | 173 ---------------------------------- 2 files changed, 37 insertions(+), 173 deletions(-) (limited to 'tools') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 38cab4ccb24e..ff94b54a926d 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -199,9 +199,25 @@ static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index) xa_store_order(xa, index, order, xa_mk_value(index), GFP_KERNEL); for (i = base; i < next; i++) { + XA_STATE(xas, xa, i); + unsigned int seen = 0; + void *entry; + XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0)); XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1)); XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2)); + + /* We should see two elements in the array */ + xas_for_each(&xas, entry, ULONG_MAX) + seen++; + XA_BUG_ON(xa, seen != 2); + + /* One of which is marked */ + xas_set(&xas, 0); + seen = 0; + xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0) + seen++; + XA_BUG_ON(xa, seen != 1); } XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_0)); XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_1)); @@ -265,6 +281,8 @@ static noinline void check_xa_shrink(struct xarray *xa) { XA_STATE(xas, xa, 1); struct xa_node *node; + unsigned int order; + unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 15 : 1; XA_BUG_ON(xa, !xa_empty(xa)); XA_BUG_ON(xa, xa_store_index(xa, 0, GFP_KERNEL) != NULL); @@ -287,6 +305,25 @@ static noinline void check_xa_shrink(struct xarray *xa) XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0)); xa_erase_index(xa, 0); XA_BUG_ON(xa, !xa_empty(xa)); + + for (order = 0; order < max_order; order++) { + unsigned long max = (1UL << order) - 1; + xa_store_order(xa, 0, order, xa_mk_value(0), GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, max) != xa_mk_value(0)); + XA_BUG_ON(xa, xa_load(xa, max + 1) != NULL); + rcu_read_lock(); + node = xa_head(xa); + rcu_read_unlock(); + XA_BUG_ON(xa, xa_store_index(xa, ULONG_MAX, GFP_KERNEL) != + NULL); + rcu_read_lock(); + XA_BUG_ON(xa, xa_head(xa) == node); + rcu_read_unlock(); + XA_BUG_ON(xa, xa_load(xa, max + 1) != NULL); + xa_erase_index(xa, ULONG_MAX); + XA_BUG_ON(xa, xa->xa_head != node); + xa_erase_index(xa, 0); + } } static noinline void check_cmpxchg(struct xarray *xa) diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index dc27a3da210a..a60c03287e9c 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -20,127 +20,6 @@ #include "test.h" -#define for_each_index(i, base, order) \ - for (i = base; i < base + (1 << order); i++) - -static void __multiorder_tag_test(int index, int order) -{ - RADIX_TREE(tree, GFP_KERNEL); - int base, err, i; - - /* our canonical entry */ - base = index & ~((1 << order) - 1); - - printv(2, "Multiorder tag test with index %d, canonical entry %d\n", - index, base); - - err = item_insert_order(&tree, index, order); - assert(!err); - - /* - * Verify we get collisions for covered indices. We try and fail to - * insert a value entry so we don't leak memory via - * item_insert_order(). - */ - for_each_index(i, base, order) { - err = __radix_tree_insert(&tree, i, order, xa_mk_value(0xA0)); - assert(err == -EEXIST); - } - - for_each_index(i, base, order) { - assert(!radix_tree_tag_get(&tree, i, 0)); - assert(!radix_tree_tag_get(&tree, i, 1)); - } - - assert(radix_tree_tag_set(&tree, index, 0)); - - for_each_index(i, base, order) { - assert(radix_tree_tag_get(&tree, i, 0)); - assert(!radix_tree_tag_get(&tree, i, 1)); - } - - assert(tag_tagged_items(&tree, 0, ~0UL, 10, XA_MARK_0, XA_MARK_1) == 1); - assert(radix_tree_tag_clear(&tree, index, 0)); - - for_each_index(i, base, order) { - assert(!radix_tree_tag_get(&tree, i, 0)); - assert(radix_tree_tag_get(&tree, i, 1)); - } - - assert(radix_tree_tag_clear(&tree, index, 1)); - - assert(!radix_tree_tagged(&tree, 0)); - assert(!radix_tree_tagged(&tree, 1)); - - item_kill_tree(&tree); -} - -static void __multiorder_tag_test2(unsigned order, unsigned long index2) -{ - RADIX_TREE(tree, GFP_KERNEL); - unsigned long index = (1 << order); - index2 += index; - - assert(item_insert_order(&tree, 0, order) == 0); - assert(item_insert(&tree, index2) == 0); - - assert(radix_tree_tag_set(&tree, 0, 0)); - assert(radix_tree_tag_set(&tree, index2, 0)); - - assert(tag_tagged_items(&tree, 0, ~0UL, 10, XA_MARK_0, XA_MARK_1) == 2); - - item_kill_tree(&tree); -} - -static void multiorder_tag_tests(void) -{ - int i, j; - - /* test multi-order entry for indices 0-7 with no sibling pointers */ - __multiorder_tag_test(0, 3); - __multiorder_tag_test(5, 3); - - /* test multi-order entry for indices 8-15 with no sibling pointers */ - __multiorder_tag_test(8, 3); - __multiorder_tag_test(15, 3); - - /* - * Our order 5 entry covers indices 0-31 in a tree with height=2. - * This is broken up as follows: - * 0-7: canonical entry - * 8-15: sibling 1 - * 16-23: sibling 2 - * 24-31: sibling 3 - */ - __multiorder_tag_test(0, 5); - __multiorder_tag_test(29, 5); - - /* same test, but with indices 32-63 */ - __multiorder_tag_test(32, 5); - __multiorder_tag_test(44, 5); - - /* - * Our order 8 entry covers indices 0-255 in a tree with height=3. - * This is broken up as follows: - * 0-63: canonical entry - * 64-127: sibling 1 - * 128-191: sibling 2 - * 192-255: sibling 3 - */ - __multiorder_tag_test(0, 8); - __multiorder_tag_test(190, 8); - - /* same test, but with indices 256-511 */ - __multiorder_tag_test(256, 8); - __multiorder_tag_test(300, 8); - - __multiorder_tag_test(0x12345678UL, 8); - - for (i = 1; i < 10; i++) - for (j = 0; j < (10 << i); j++) - __multiorder_tag_test2(i, j); -} - static void multiorder_check(unsigned long index, int order) { unsigned long i; @@ -181,53 +60,6 @@ static void multiorder_check(unsigned long index, int order) item_check_absent(&tree, i); } -static void multiorder_shrink(unsigned long index, int order) -{ - unsigned long i; - unsigned long max = 1 << order; - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_node *node; - - printv(2, "Multiorder shrink index %ld, order %d\n", index, order); - - assert(item_insert_order(&tree, 0, order) == 0); - - node = tree.xa_head; - - assert(item_insert(&tree, index) == 0); - assert(node != tree.xa_head); - - assert(item_delete(&tree, index) != 0); - assert(node == tree.xa_head); - - for (i = 0; i < max; i++) { - struct item *item = item_lookup(&tree, i); - assert(item != 0); - assert(item->index == 0); - } - for (i = max; i < 2*max; i++) - item_check_absent(&tree, i); - - if (!item_delete(&tree, 0)) { - printv(2, "failed to delete index %ld (order %d)\n", index, order); - abort(); - } - - for (i = 0; i < 2*max; i++) - item_check_absent(&tree, i); -} - -static void multiorder_insert_bug(void) -{ - RADIX_TREE(tree, GFP_KERNEL); - - item_insert(&tree, 0); - radix_tree_tag_set(&tree, 0, 0); - item_insert_order(&tree, 3 << 6, 6); - - item_kill_tree(&tree); -} - void multiorder_iteration(void) { RADIX_TREE(tree, GFP_KERNEL); @@ -427,11 +259,6 @@ void multiorder_checks(void) multiorder_check((1UL << i) + 1, i); } - for (i = 0; i < 15; i++) - multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i); - - multiorder_insert_bug(); - multiorder_tag_tests(); multiorder_iteration(); multiorder_tagged_iteration(); multiorder_iteration_race(); -- cgit v1.2.3 From 4f06d6302da682157890f72c0573e12a73536814 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sun, 9 Sep 2018 01:52:17 -0400 Subject: xarray: Move multiorder_check to in-kernel tests This version is a little less thorough in order to be a little quicker, but tests the important edge cases. Also test adding a multiorder entry at a non-canonical index, and erasing it. Signed-off-by: Matthew Wilcox --- lib/test_xarray.c | 44 ++++++++++++++++++++++++++++++++ tools/testing/radix-tree/multiorder.c | 48 ----------------------------------- 2 files changed, 44 insertions(+), 48 deletions(-) (limited to 'tools') diff --git a/lib/test_xarray.c b/lib/test_xarray.c index ff94b54a926d..0f06a93b4d0e 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -422,6 +422,43 @@ static noinline void check_xas_erase(struct xarray *xa) } } +#ifdef CONFIG_XARRAY_MULTI +static noinline void check_multi_store_1(struct xarray *xa, unsigned long index, + unsigned int order) +{ + XA_STATE(xas, xa, index); + unsigned long min = index & ~((1UL << order) - 1); + unsigned long max = min + (1UL << order); + + xa_store_order(xa, index, order, xa_mk_value(index), GFP_KERNEL); + XA_BUG_ON(xa, xa_load(xa, min) != xa_mk_value(index)); + XA_BUG_ON(xa, xa_load(xa, max - 1) != xa_mk_value(index)); + XA_BUG_ON(xa, xa_load(xa, max) != NULL); + XA_BUG_ON(xa, xa_load(xa, min - 1) != NULL); + + XA_BUG_ON(xa, xas_store(&xas, xa_mk_value(min)) != xa_mk_value(index)); + XA_BUG_ON(xa, xa_load(xa, min) != xa_mk_value(min)); + XA_BUG_ON(xa, xa_load(xa, max - 1) != xa_mk_value(min)); + XA_BUG_ON(xa, xa_load(xa, max) != NULL); + XA_BUG_ON(xa, xa_load(xa, min - 1) != NULL); + + xa_erase_index(xa, min); + XA_BUG_ON(xa, !xa_empty(xa)); +} + +static noinline void check_multi_store_2(struct xarray *xa, unsigned long index, + unsigned int order) +{ + XA_STATE(xas, xa, index); + xa_store_order(xa, index, order, xa_mk_value(0), GFP_KERNEL); + + XA_BUG_ON(xa, xas_store(&xas, xa_mk_value(1)) != xa_mk_value(0)); + XA_BUG_ON(xa, xas.xa_index != index); + XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1)); + XA_BUG_ON(xa, !xa_empty(xa)); +} +#endif + static noinline void check_multi_store(struct xarray *xa) { #ifdef CONFIG_XARRAY_MULTI @@ -487,6 +524,13 @@ static noinline void check_multi_store(struct xarray *xa) XA_BUG_ON(xa, !xa_empty(xa)); } } + + for (i = 0; i < 20; i++) { + check_multi_store_1(xa, 200, i); + check_multi_store_1(xa, 0, i); + check_multi_store_1(xa, (1UL << i) + 1, i); + } + check_multi_store_2(xa, 4095, 9); #endif } diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index a60c03287e9c..6e8d66c2aa89 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -20,46 +20,6 @@ #include "test.h" -static void multiorder_check(unsigned long index, int order) -{ - unsigned long i; - unsigned long min = index & ~((1UL << order) - 1); - unsigned long max = min + (1UL << order); - void **slot; - struct item *item2 = item_create(min, order); - RADIX_TREE(tree, GFP_KERNEL); - - printv(2, "Multiorder index %ld, order %d\n", index, order); - - assert(item_insert_order(&tree, index, order) == 0); - - for (i = min; i < max; i++) { - struct item *item = item_lookup(&tree, i); - assert(item != 0); - assert(item->index == index); - } - for (i = 0; i < min; i++) - item_check_absent(&tree, i); - for (i = max; i < 2*max; i++) - item_check_absent(&tree, i); - for (i = min; i < max; i++) - assert(radix_tree_insert(&tree, i, item2) == -EEXIST); - - slot = radix_tree_lookup_slot(&tree, index); - free(*slot); - radix_tree_replace_slot(&tree, slot, item2); - for (i = min; i < max; i++) { - struct item *item = item_lookup(&tree, i); - assert(item != 0); - assert(item->index == min); - } - - assert(item_delete(&tree, min) != 0); - - for (i = 0; i < 2*max; i++) - item_check_absent(&tree, i); -} - void multiorder_iteration(void) { RADIX_TREE(tree, GFP_KERNEL); @@ -251,14 +211,6 @@ static void multiorder_iteration_race(void) void multiorder_checks(void) { - int i; - - for (i = 0; i < 20; i++) { - multiorder_check(200, i); - multiorder_check(0, i); - multiorder_check((1UL << i) + 1, i); - } - multiorder_iteration(); multiorder_tagged_iteration(); multiorder_iteration_race(); -- cgit v1.2.3 From 9076b33d7dad8e316c90918845417e85347f682c Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 12 Sep 2018 12:52:45 -0400 Subject: radix tree test suite: Remove __item_insert Inline it into its one caller Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/test.c | 7 +------ tools/testing/radix-tree/test.h | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 32973dd51ec5..5991cfd34f2b 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -25,11 +25,6 @@ int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag) return radix_tree_tag_get(root, index, tag); } -int __item_insert(struct radix_tree_root *root, struct item *item) -{ - return __radix_tree_insert(root, item->index, item->order, item); -} - struct item *item_create(unsigned long index, unsigned int order) { struct item *ret = malloc(sizeof(*ret)); @@ -43,7 +38,7 @@ int item_insert_order(struct radix_tree_root *root, unsigned long index, unsigned order) { struct item *item = item_create(index, order); - int err = __item_insert(root, item); + int err = __radix_tree_insert(root, item->index, item->order, item); if (err) free(item); return err; diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index a2f53c889a31..28961a08828e 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -11,7 +11,6 @@ struct item { }; struct item *item_create(unsigned long index, unsigned int order); -int __item_insert(struct radix_tree_root *root, struct item *item); int item_insert(struct radix_tree_root *root, unsigned long index); void item_sanity(struct item *item, unsigned long index); void item_free(struct item *item, unsigned long index); -- cgit v1.2.3 From 879a9ae7b5bc046f195a725d62bbc96258e5d0c8 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 12 Sep 2018 23:12:47 -0400 Subject: radix tree test suite: Remove multiorder benchmarking The multiorder radix tree code is being removed, so remove the benchmarking of its performance. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/benchmark.c | 50 +++++++++++++++--------------------- 1 file changed, 21 insertions(+), 29 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/benchmark.c b/tools/testing/radix-tree/benchmark.c index 35741b9c2a4a..7e195ed8e92d 100644 --- a/tools/testing/radix-tree/benchmark.c +++ b/tools/testing/radix-tree/benchmark.c @@ -17,9 +17,6 @@ #include #include "test.h" -#define for_each_index(i, base, order) \ - for (i = base; i < base + (1 << order); i++) - #define NSEC_PER_SEC 1000000000L static long long benchmark_iter(struct radix_tree_root *root, bool tagged) @@ -61,7 +58,7 @@ again: } static void benchmark_insert(struct radix_tree_root *root, - unsigned long size, unsigned long step, int order) + unsigned long size, unsigned long step) { struct timespec start, finish; unsigned long index; @@ -70,19 +67,19 @@ static void benchmark_insert(struct radix_tree_root *root, clock_gettime(CLOCK_MONOTONIC, &start); for (index = 0 ; index < size ; index += step) - item_insert_order(root, index, order); + item_insert(root, index); clock_gettime(CLOCK_MONOTONIC, &finish); nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + (finish.tv_nsec - start.tv_nsec); - printv(2, "Size: %8ld, step: %8ld, order: %d, insertion: %15lld ns\n", - size, step, order, nsec); + printv(2, "Size: %8ld, step: %8ld, insertion: %15lld ns\n", + size, step, nsec); } static void benchmark_tagging(struct radix_tree_root *root, - unsigned long size, unsigned long step, int order) + unsigned long size, unsigned long step) { struct timespec start, finish; unsigned long index; @@ -98,49 +95,48 @@ static void benchmark_tagging(struct radix_tree_root *root, nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + (finish.tv_nsec - start.tv_nsec); - printv(2, "Size: %8ld, step: %8ld, order: %d, tagging: %17lld ns\n", - size, step, order, nsec); + printv(2, "Size: %8ld, step: %8ld, tagging: %17lld ns\n", + size, step, nsec); } static void benchmark_delete(struct radix_tree_root *root, - unsigned long size, unsigned long step, int order) + unsigned long size, unsigned long step) { struct timespec start, finish; - unsigned long index, i; + unsigned long index; long long nsec; clock_gettime(CLOCK_MONOTONIC, &start); for (index = 0 ; index < size ; index += step) - for_each_index(i, index, order) - item_delete(root, i); + item_delete(root, index); clock_gettime(CLOCK_MONOTONIC, &finish); nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + (finish.tv_nsec - start.tv_nsec); - printv(2, "Size: %8ld, step: %8ld, order: %d, deletion: %16lld ns\n", - size, step, order, nsec); + printv(2, "Size: %8ld, step: %8ld, deletion: %16lld ns\n", + size, step, nsec); } -static void benchmark_size(unsigned long size, unsigned long step, int order) +static void benchmark_size(unsigned long size, unsigned long step) { RADIX_TREE(tree, GFP_KERNEL); long long normal, tagged; - benchmark_insert(&tree, size, step, order); - benchmark_tagging(&tree, size, step, order); + benchmark_insert(&tree, size, step); + benchmark_tagging(&tree, size, step); tagged = benchmark_iter(&tree, true); normal = benchmark_iter(&tree, false); - printv(2, "Size: %8ld, step: %8ld, order: %d, tagged iteration: %8lld ns\n", - size, step, order, tagged); - printv(2, "Size: %8ld, step: %8ld, order: %d, normal iteration: %8lld ns\n", - size, step, order, normal); + printv(2, "Size: %8ld, step: %8ld, tagged iteration: %8lld ns\n", + size, step, tagged); + printv(2, "Size: %8ld, step: %8ld, normal iteration: %8lld ns\n", + size, step, normal); - benchmark_delete(&tree, size, step, order); + benchmark_delete(&tree, size, step); item_kill_tree(&tree); rcu_barrier(); @@ -158,9 +154,5 @@ void benchmark(void) for (c = 0; size[c]; c++) for (s = 0; step[s]; s++) - benchmark_size(size[c], step[s], 0); - - for (c = 0; size[c]; c++) - for (s = 0; step[s]; s++) - benchmark_size(size[c], step[s] << 9, 9); + benchmark_size(size[c], step[s]); } -- cgit v1.2.3 From 4bb53bdda0d1e061035774ed4868bdeb4d889044 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Wed, 12 Sep 2018 23:29:32 -0400 Subject: radix tree tests: Move item_insert_order The remaining tests are not suitable for moving in-kernel, so move item_insert_order() into multiorder.c, make it static and make it use the XArray. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/multiorder.c | 19 +++++++++++++++++++ tools/testing/radix-tree/test.c | 12 +++--------- tools/testing/radix-tree/test.h | 2 -- 3 files changed, 22 insertions(+), 11 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 6e8d66c2aa89..8c41dca272b1 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -20,6 +20,25 @@ #include "test.h" +static int item_insert_order(struct xarray *xa, unsigned long index, + unsigned order) +{ + XA_STATE_ORDER(xas, xa, index, order); + struct item *item = item_create(index, order); + + do { + xas_lock(&xas); + xas_store(&xas, item); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + + if (!xas_error(&xas)) + return 0; + + free(item); + return xas_error(&xas); +} + void multiorder_iteration(void) { RADIX_TREE(tree, GFP_KERNEL); diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 5991cfd34f2b..5376b8c5d8d6 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -34,21 +34,15 @@ struct item *item_create(unsigned long index, unsigned int order) return ret; } -int item_insert_order(struct radix_tree_root *root, unsigned long index, - unsigned order) +int item_insert(struct radix_tree_root *root, unsigned long index) { - struct item *item = item_create(index, order); - int err = __radix_tree_insert(root, item->index, item->order, item); + struct item *item = item_create(index, 0); + int err = radix_tree_insert(root, item->index, item); if (err) free(item); return err; } -int item_insert(struct radix_tree_root *root, unsigned long index) -{ - return item_insert_order(root, index, 0); -} - void item_sanity(struct item *item, unsigned long index) { unsigned long mask; diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index 28961a08828e..e259c0839d5d 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -14,8 +14,6 @@ struct item *item_create(unsigned long index, unsigned int order); int item_insert(struct radix_tree_root *root, unsigned long index); void item_sanity(struct item *item, unsigned long index); void item_free(struct item *item, unsigned long index); -int item_insert_order(struct radix_tree_root *root, unsigned long index, - unsigned order); int item_delete(struct radix_tree_root *root, unsigned long index); int item_delete_rcu(struct radix_tree_root *root, unsigned long index); struct item *item_lookup(struct radix_tree_root *root, unsigned long index); -- cgit v1.2.3 From ccc89e30fac790e3a175dbfc863c67286fce96b0 Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Thu, 13 Sep 2018 10:15:41 -0400 Subject: radix tree tests: Convert item_kill_tree to XArray In preparation for the removal of the multiorder radix tree code, convert item_kill_tree() to use the XArray so it can still be called for XArrays containing multi-index entries. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/test.c | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 5376b8c5d8d6..19045ce3bd23 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -252,31 +252,19 @@ void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag) verify_node(node, tag, !!root_tag_get(root, tag)); } -void item_kill_tree(struct radix_tree_root *root) +void item_kill_tree(struct xarray *xa) { - struct radix_tree_iter iter; - void **slot; - struct item *items[32]; - int nfound; - - radix_tree_for_each_slot(slot, root, &iter, 0) { - if (xa_is_value(*slot)) - radix_tree_delete(root, iter.index); - } - - while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) { - int i; - - for (i = 0; i < nfound; i++) { - void *ret; + XA_STATE(xas, xa, 0); + void *entry; - ret = radix_tree_delete(root, items[i]->index); - assert(ret == items[i]); - free(items[i]); + xas_for_each(&xas, entry, ULONG_MAX) { + if (!xa_is_value(entry)) { + item_free(entry, xas.xa_index); } + xas_store(&xas, NULL); } - assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0); - assert(root->xa_head == NULL); + + assert(xa_empty(xa)); } void tree_verify_min_height(struct radix_tree_root *root, int maxindex) -- cgit v1.2.3 From b66b5a48b8a0e43dc114573da11c1a9c586a2d4f Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 22 Sep 2018 15:34:28 -0400 Subject: radix tree tests: Convert item_delete_rcu to XArray In preparation for the removal of the multiorder radix tree code, convert item_delete_rcu() to use the XArray so it can still be called for XArrays containing multi-index entries. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/test.c | 4 ++-- tools/testing/radix-tree/test.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c index 19045ce3bd23..a15d0512e633 100644 --- a/tools/testing/radix-tree/test.c +++ b/tools/testing/radix-tree/test.c @@ -76,9 +76,9 @@ static void item_free_rcu(struct rcu_head *head) free(item); } -int item_delete_rcu(struct radix_tree_root *root, unsigned long index) +int item_delete_rcu(struct xarray *xa, unsigned long index) { - struct item *item = radix_tree_delete(root, index); + struct item *item = xa_erase(xa, index); if (item) { item_sanity(item, index); diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h index e259c0839d5d..1ee4b2c0ad10 100644 --- a/tools/testing/radix-tree/test.h +++ b/tools/testing/radix-tree/test.h @@ -15,7 +15,7 @@ int item_insert(struct radix_tree_root *root, unsigned long index); void item_sanity(struct item *item, unsigned long index); void item_free(struct item *item, unsigned long index); int item_delete(struct radix_tree_root *root, unsigned long index); -int item_delete_rcu(struct radix_tree_root *root, unsigned long index); +int item_delete_rcu(struct xarray *xa, unsigned long index); struct item *item_lookup(struct radix_tree_root *root, unsigned long index); void item_check_present(struct radix_tree_root *root, unsigned long index); -- cgit v1.2.3 From 542980aa9318edcfb68aa7bf6eacf2814dc137dd Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 22 Sep 2018 16:12:41 -0400 Subject: radix tree test: Convert multiorder tests to XArray This is the last remaining user of the multiorder functionality of the radix tree. Test the XArray instead. Signed-off-by: Matthew Wilcox --- tools/testing/radix-tree/multiorder.c | 105 ++++++++++++++++------------------ 1 file changed, 49 insertions(+), 56 deletions(-) (limited to 'tools') diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 8c41dca272b1..ff27a74d9762 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c @@ -39,21 +39,20 @@ static int item_insert_order(struct xarray *xa, unsigned long index, return xas_error(&xas); } -void multiorder_iteration(void) +void multiorder_iteration(struct xarray *xa) { - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, xa, 0); + struct item *item; int i, j, err; - printv(1, "Multiorder iteration test\n"); - #define NUM_ENTRIES 11 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128}; int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7}; + printv(1, "Multiorder iteration test\n"); + for (i = 0; i < NUM_ENTRIES; i++) { - err = item_insert_order(&tree, index[i], order[i]); + err = item_insert_order(xa, index[i], order[i]); assert(!err); } @@ -62,14 +61,14 @@ void multiorder_iteration(void) if (j <= (index[i] | ((1 << order[i]) - 1))) break; - radix_tree_for_each_slot(slot, &tree, &iter, j) { - int height = order[i] / RADIX_TREE_MAP_SHIFT; - int shift = height * RADIX_TREE_MAP_SHIFT; + xas_set(&xas, j); + xas_for_each(&xas, item, ULONG_MAX) { + int height = order[i] / XA_CHUNK_SHIFT; + int shift = height * XA_CHUNK_SHIFT; unsigned long mask = (1UL << order[i]) - 1; - struct item *item = *slot; - assert((iter.index | mask) == (index[i] | mask)); - assert(iter.shift == shift); + assert((xas.xa_index | mask) == (index[i] | mask)); + assert(xas.xa_node->shift == shift); assert(!radix_tree_is_internal_node(item)); assert((item->index | mask) == (index[i] | mask)); assert(item->order == order[i]); @@ -77,18 +76,15 @@ void multiorder_iteration(void) } } - item_kill_tree(&tree); + item_kill_tree(xa); } -void multiorder_tagged_iteration(void) +void multiorder_tagged_iteration(struct xarray *xa) { - RADIX_TREE(tree, GFP_KERNEL); - struct radix_tree_iter iter; - void **slot; + XA_STATE(xas, xa, 0); + struct item *item; int i, j; - printv(1, "Multiorder tagged iteration test\n"); - #define MT_NUM_ENTRIES 9 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128}; int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7}; @@ -96,13 +92,15 @@ void multiorder_tagged_iteration(void) #define TAG_ENTRIES 7 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128}; + printv(1, "Multiorder tagged iteration test\n"); + for (i = 0; i < MT_NUM_ENTRIES; i++) - assert(!item_insert_order(&tree, index[i], order[i])); + assert(!item_insert_order(xa, index[i], order[i])); - assert(!radix_tree_tagged(&tree, 1)); + assert(!xa_marked(xa, XA_MARK_1)); for (i = 0; i < TAG_ENTRIES; i++) - assert(radix_tree_tag_set(&tree, tag_index[i], 1)); + xa_set_mark(xa, tag_index[i], XA_MARK_1); for (j = 0; j < 256; j++) { int k; @@ -114,22 +112,22 @@ void multiorder_tagged_iteration(void) break; } - radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) { + xas_set(&xas, j); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_1) { unsigned long mask; - struct item *item = *slot; for (k = i; index[k] < tag_index[i]; k++) ; mask = (1UL << order[k]) - 1; - assert((iter.index | mask) == (tag_index[i] | mask)); - assert(!radix_tree_is_internal_node(item)); + assert((xas.xa_index | mask) == (tag_index[i] | mask)); + assert(!xa_is_internal(item)); assert((item->index | mask) == (tag_index[i] | mask)); assert(item->order == order[k]); i++; } } - assert(tag_tagged_items(&tree, 0, ~0UL, TAG_ENTRIES, XA_MARK_1, + assert(tag_tagged_items(xa, 0, ULONG_MAX, TAG_ENTRIES, XA_MARK_1, XA_MARK_2) == TAG_ENTRIES); for (j = 0; j < 256; j++) { @@ -142,29 +140,31 @@ void multiorder_tagged_iteration(void) break; } - radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) { - struct item *item = *slot; + xas_set(&xas, j); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_2) { for (k = i; index[k] < tag_index[i]; k++) ; mask = (1 << order[k]) - 1; - assert((iter.index | mask) == (tag_index[i] | mask)); - assert(!radix_tree_is_internal_node(item)); + assert((xas.xa_index | mask) == (tag_index[i] | mask)); + assert(!xa_is_internal(item)); assert((item->index | mask) == (tag_index[i] | mask)); assert(item->order == order[k]); i++; } } - assert(tag_tagged_items(&tree, 1, ~0UL, MT_NUM_ENTRIES * 2, XA_MARK_1, + assert(tag_tagged_items(xa, 1, ULONG_MAX, MT_NUM_ENTRIES * 2, XA_MARK_1, XA_MARK_0) == TAG_ENTRIES); i = 0; - radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) { - assert(iter.index == tag_index[i]); + xas_set(&xas, 0); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_0) { + assert(xas.xa_index == tag_index[i]); i++; } + assert(i == TAG_ENTRIES); - item_kill_tree(&tree); + item_kill_tree(xa); } bool stop_iteration = false; @@ -187,52 +187,45 @@ static void *creator_func(void *ptr) static void *iterator_func(void *ptr) { - struct radix_tree_root *tree = ptr; - struct radix_tree_iter iter; + XA_STATE(xas, ptr, 0); struct item *item; - void **slot; while (!stop_iteration) { rcu_read_lock(); - radix_tree_for_each_slot(slot, tree, &iter, 0) { - item = radix_tree_deref_slot(slot); - - if (!item) + xas_for_each(&xas, item, ULONG_MAX) { + if (xas_retry(&xas, item)) continue; - if (radix_tree_deref_retry(item)) { - slot = radix_tree_iter_retry(&iter); - continue; - } - item_sanity(item, iter.index); + item_sanity(item, xas.xa_index); } rcu_read_unlock(); } return NULL; } -static void multiorder_iteration_race(void) +static void multiorder_iteration_race(struct xarray *xa) { const int num_threads = sysconf(_SC_NPROCESSORS_ONLN); pthread_t worker_thread[num_threads]; - RADIX_TREE(tree, GFP_KERNEL); int i; - pthread_create(&worker_thread[0], NULL, &creator_func, &tree); + pthread_create(&worker_thread[0], NULL, &creator_func, xa); for (i = 1; i < num_threads; i++) - pthread_create(&worker_thread[i], NULL, &iterator_func, &tree); + pthread_create(&worker_thread[i], NULL, &iterator_func, xa); for (i = 0; i < num_threads; i++) pthread_join(worker_thread[i], NULL); - item_kill_tree(&tree); + item_kill_tree(xa); } +static DEFINE_XARRAY(array); + void multiorder_checks(void) { - multiorder_iteration(); - multiorder_tagged_iteration(); - multiorder_iteration_race(); + multiorder_iteration(&array); + multiorder_tagged_iteration(&array); + multiorder_iteration_race(&array); radix_tree_cpu_dead(0); } -- cgit v1.2.3 From 3a08cd52c37c793ffc199f6fc2ecfc368e284b2d Mon Sep 17 00:00:00 2001 From: Matthew Wilcox Date: Sat, 22 Sep 2018 16:14:30 -0400 Subject: radix tree: Remove multiorder support All users have now been converted to the XArray. Removing the support reduces code size and ensures new users will use the XArray instead. Signed-off-by: Matthew Wilcox --- include/linux/radix-tree.h | 40 +---- lib/Kconfig | 4 - lib/radix-tree.c | 215 ++------------------------ mm/Kconfig | 4 +- tools/testing/radix-tree/generated/autoconf.h | 1 - 5 files changed, 19 insertions(+), 245 deletions(-) (limited to 'tools') diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 9a1460488163..06c4c7a6c09c 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -96,7 +96,6 @@ static inline bool radix_tree_empty(const struct radix_tree_root *root) * @next_index: one beyond the last index for this chunk * @tags: bit-mask for tag-iterating * @node: node that contains current slot - * @shift: shift for the node that holds our slots * * This radix tree iterator works in terms of "chunks" of slots. A chunk is a * subinterval of slots contained within one radix tree leaf node. It is @@ -110,20 +109,8 @@ struct radix_tree_iter { unsigned long next_index; unsigned long tags; struct radix_tree_node *node; -#ifdef CONFIG_RADIX_TREE_MULTIORDER - unsigned int shift; -#endif }; -static inline unsigned int iter_shift(const struct radix_tree_iter *iter) -{ -#ifdef CONFIG_RADIX_TREE_MULTIORDER - return iter->shift; -#else - return 0; -#endif -} - /** * Radix-tree synchronization * @@ -230,13 +217,8 @@ static inline int radix_tree_exception(void *arg) return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK); } -int __radix_tree_insert(struct radix_tree_root *, unsigned long index, - unsigned order, void *); -static inline int radix_tree_insert(struct radix_tree_root *root, - unsigned long index, void *entry) -{ - return __radix_tree_insert(root, index, 0, entry); -} +int radix_tree_insert(struct radix_tree_root *, unsigned long index, + void *); void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index, struct radix_tree_node **nodep, void __rcu ***slotp); void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); @@ -384,7 +366,7 @@ void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter) static inline unsigned long __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots) { - return iter->index + (slots << iter_shift(iter)); + return iter->index + slots; } /** @@ -409,20 +391,8 @@ void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot, static __always_inline long radix_tree_chunk_size(struct radix_tree_iter *iter) { - return (iter->next_index - iter->index) >> iter_shift(iter); -} - -#ifdef CONFIG_RADIX_TREE_MULTIORDER -void __rcu **__radix_tree_next_slot(void __rcu **slot, - struct radix_tree_iter *iter, unsigned flags); -#else -/* Can't happen without sibling entries, but the compiler can't tell that */ -static inline void __rcu **__radix_tree_next_slot(void __rcu **slot, - struct radix_tree_iter *iter, unsigned flags) -{ - return slot; + return iter->next_index - iter->index; } -#endif /** * radix_tree_next_slot - find next slot in chunk @@ -482,8 +452,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, return NULL; found: - if (unlikely(radix_tree_is_internal_node(rcu_dereference_raw(*slot)))) - return __radix_tree_next_slot(slot, iter, flags); return slot; } diff --git a/lib/Kconfig b/lib/Kconfig index 40bfa6ccd294..a9965f4af4dd 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -405,10 +405,6 @@ config XARRAY_MULTI Support entries which occupy multiple consecutive indices in the XArray. -config RADIX_TREE_MULTIORDER - bool - select XARRAY_MULTI - config ASSOCIATIVE_ARRAY bool help diff --git a/lib/radix-tree.c b/lib/radix-tree.c index f107dd2698e3..1106bb6aa01e 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -110,11 +110,6 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK; void __rcu **entry = rcu_dereference_raw(parent->slots[offset]); - if (xa_is_sibling(entry)) { - offset = xa_to_sibling(entry); - entry = rcu_dereference_raw(parent->slots[offset]); - } - *nodep = (void *)entry; return offset; } @@ -229,7 +224,7 @@ radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag, static unsigned int iter_offset(const struct radix_tree_iter *iter) { - return (iter->index >> iter_shift(iter)) & RADIX_TREE_MAP_MASK; + return iter->index & RADIX_TREE_MAP_MASK; } /* @@ -506,16 +501,13 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root) /* * The candidate node has more than one child, or its child - * is not at the leftmost slot, or the child is a multiorder - * entry, we cannot shrink. + * is not at the leftmost slot, we cannot shrink. */ if (node->count != 1) break; child = rcu_dereference_raw(node->slots[0]); if (!child) break; - if (!radix_tree_is_internal_node(child) && node->shift) - break; /* * For an IDR, we must not shrink entry 0 into the root in @@ -613,7 +605,6 @@ static bool delete_node(struct radix_tree_root *root, * __radix_tree_create - create a slot in a radix tree * @root: radix tree root * @index: index key - * @order: index occupies 2^order aligned slots * @nodep: returns node * @slotp: returns slot * @@ -627,21 +618,19 @@ static bool delete_node(struct radix_tree_root *root, * Returns -ENOMEM, or 0 for success. */ static int __radix_tree_create(struct radix_tree_root *root, - unsigned long index, unsigned order, - struct radix_tree_node **nodep, void __rcu ***slotp) + unsigned long index, struct radix_tree_node **nodep, + void __rcu ***slotp) { struct radix_tree_node *node = NULL, *child; void __rcu **slot = (void __rcu **)&root->xa_head; unsigned long maxindex; unsigned int shift, offset = 0; - unsigned long max = index | ((1UL << order) - 1); + unsigned long max = index; gfp_t gfp = root_gfp_mask(root); shift = radix_tree_load_root(root, &child, &maxindex); /* Make sure the tree is high enough. */ - if (order > 0 && max == ((1UL << order) - 1)) - max++; if (max > maxindex) { int error = radix_tree_extend(root, gfp, max, shift); if (error < 0) @@ -650,7 +639,7 @@ static int __radix_tree_create(struct radix_tree_root *root, child = rcu_dereference_raw(root->xa_head); } - while (shift > order) { + while (shift > 0) { shift -= RADIX_TREE_MAP_SHIFT; if (child == NULL) { /* Have to add a child node. */ @@ -711,70 +700,8 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) } } -#ifdef CONFIG_RADIX_TREE_MULTIORDER -static inline int insert_entries(struct radix_tree_node *node, - void __rcu **slot, void *item, unsigned order, bool replace) -{ - void *sibling; - unsigned i, n, tag, offset, tags = 0; - - if (node) { - if (order > node->shift) - n = 1 << (order - node->shift); - else - n = 1; - offset = get_slot_offset(node, slot); - } else { - n = 1; - offset = 0; - } - - if (n > 1) { - offset = offset & ~(n - 1); - slot = &node->slots[offset]; - } - sibling = xa_mk_sibling(offset); - - for (i = 0; i < n; i++) { - if (slot[i]) { - if (replace) { - node->count--; - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tag_get(node, tag, offset + i)) - tags |= 1 << tag; - } else - return -EEXIST; - } - } - - for (i = 0; i < n; i++) { - struct radix_tree_node *old = rcu_dereference_raw(slot[i]); - if (i) { - rcu_assign_pointer(slot[i], sibling); - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tags & (1 << tag)) - tag_clear(node, tag, offset + i); - } else { - rcu_assign_pointer(slot[i], item); - for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) - if (tags & (1 << tag)) - tag_set(node, tag, offset); - } - if (xa_is_node(old)) - radix_tree_free_nodes(old); - if (xa_is_value(old)) - node->nr_values--; - } - if (node) { - node->count += n; - if (xa_is_value(item)) - node->nr_values += n; - } - return n; -} -#else static inline int insert_entries(struct radix_tree_node *node, - void __rcu **slot, void *item, unsigned order, bool replace) + void __rcu **slot, void *item, bool replace) { if (*slot) return -EEXIST; @@ -786,19 +713,17 @@ static inline int insert_entries(struct radix_tree_node *node, } return 1; } -#endif /** * __radix_tree_insert - insert into a radix tree * @root: radix tree root * @index: index key - * @order: key covers the 2^order indices around index * @item: item to insert * * Insert an item into the radix tree at position @index. */ -int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, - unsigned order, void *item) +int radix_tree_insert(struct radix_tree_root *root, unsigned long index, + void *item) { struct radix_tree_node *node; void __rcu **slot; @@ -806,11 +731,11 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, BUG_ON(radix_tree_is_internal_node(item)); - error = __radix_tree_create(root, index, order, &node, &slot); + error = __radix_tree_create(root, index, &node, &slot); if (error) return error; - error = insert_entries(node, slot, item, order, false); + error = insert_entries(node, slot, item, false); if (error < 0) return error; @@ -825,7 +750,7 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, return 0; } -EXPORT_SYMBOL(__radix_tree_insert); +EXPORT_SYMBOL(radix_tree_insert); /** * __radix_tree_lookup - lookup an item in a radix tree @@ -917,32 +842,12 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) } EXPORT_SYMBOL(radix_tree_lookup); -static inline void replace_sibling_entries(struct radix_tree_node *node, - void __rcu **slot, int count, int values) -{ -#ifdef CONFIG_RADIX_TREE_MULTIORDER - unsigned offset = get_slot_offset(node, slot); - void *ptr = xa_mk_sibling(offset); - - while (++offset < RADIX_TREE_MAP_SIZE) { - if (rcu_dereference_raw(node->slots[offset]) != ptr) - break; - if (count < 0) { - node->slots[offset] = NULL; - node->count--; - } - node->nr_values += values; - } -#endif -} - static void replace_slot(void __rcu **slot, void *item, struct radix_tree_node *node, int count, int values) { if (node && (count || values)) { node->count += count; node->nr_values += values; - replace_sibling_entries(node, slot, count, values); } rcu_assign_pointer(*slot, item); @@ -1223,14 +1128,6 @@ int radix_tree_tag_get(const struct radix_tree_root *root, } EXPORT_SYMBOL(radix_tree_tag_get); -static inline void __set_iter_shift(struct radix_tree_iter *iter, - unsigned int shift) -{ -#ifdef CONFIG_RADIX_TREE_MULTIORDER - iter->shift = shift; -#endif -} - /* Construct iter->tags bit-mask from node->tags[tag] array */ static void set_iter_tags(struct radix_tree_iter *iter, struct radix_tree_node *node, unsigned offset, @@ -1257,92 +1154,11 @@ static void set_iter_tags(struct radix_tree_iter *iter, } } -#ifdef CONFIG_RADIX_TREE_MULTIORDER -static void __rcu **skip_siblings(struct radix_tree_node **nodep, - void __rcu **slot, struct radix_tree_iter *iter) -{ - while (iter->index < iter->next_index) { - *nodep = rcu_dereference_raw(*slot); - if (*nodep && !xa_is_sibling(*nodep)) - return slot; - slot++; - iter->index = __radix_tree_iter_add(iter, 1); - iter->tags >>= 1; - } - - *nodep = NULL; - return NULL; -} - -void __rcu **__radix_tree_next_slot(void __rcu **slot, - struct radix_tree_iter *iter, unsigned flags) -{ - unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK; - struct radix_tree_node *node; - - slot = skip_siblings(&node, slot, iter); - - while (radix_tree_is_internal_node(node)) { - unsigned offset; - unsigned long next_index; - - if (node == RADIX_TREE_RETRY) - return slot; - node = entry_to_node(node); - iter->node = node; - iter->shift = node->shift; - - if (flags & RADIX_TREE_ITER_TAGGED) { - offset = radix_tree_find_next_bit(node, tag, 0); - if (offset == RADIX_TREE_MAP_SIZE) - return NULL; - slot = &node->slots[offset]; - iter->index = __radix_tree_iter_add(iter, offset); - set_iter_tags(iter, node, offset, tag); - node = rcu_dereference_raw(*slot); - } else { - offset = 0; - slot = &node->slots[0]; - for (;;) { - node = rcu_dereference_raw(*slot); - if (node) - break; - slot++; - offset++; - if (offset == RADIX_TREE_MAP_SIZE) - return NULL; - } - iter->index = __radix_tree_iter_add(iter, offset); - } - if ((flags & RADIX_TREE_ITER_CONTIG) && (offset > 0)) - goto none; - next_index = (iter->index | shift_maxindex(iter->shift)) + 1; - if (next_index < iter->next_index) - iter->next_index = next_index; - } - - return slot; - none: - iter->next_index = 0; - return NULL; -} -EXPORT_SYMBOL(__radix_tree_next_slot); -#else -static void __rcu **skip_siblings(struct radix_tree_node **nodep, - void __rcu **slot, struct radix_tree_iter *iter) -{ - return slot; -} -#endif - void __rcu **radix_tree_iter_resume(void __rcu **slot, struct radix_tree_iter *iter) { - struct radix_tree_node *node; - slot++; iter->index = __radix_tree_iter_add(iter, 1); - skip_siblings(&node, slot, iter); iter->next_index = iter->index; iter->tags = 0; return NULL; @@ -1393,7 +1209,6 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, iter->next_index = maxindex + 1; iter->tags = 1; iter->node = NULL; - __set_iter_shift(iter, 0); return (void __rcu **)&root->xa_head; } @@ -1414,8 +1229,6 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, while (++offset < RADIX_TREE_MAP_SIZE) { void *slot = rcu_dereference_raw( node->slots[offset]); - if (xa_is_sibling(slot)) - continue; if (slot) break; } @@ -1436,10 +1249,9 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, } while (node->shift && radix_tree_is_internal_node(child)); /* Update the iterator state */ - iter->index = (index &~ node_maxindex(node)) | (offset << node->shift); + iter->index = (index &~ node_maxindex(node)) | offset; iter->next_index = (index | node_maxindex(node)) + 1; iter->node = node; - __set_iter_shift(iter, node->shift); if (flags & RADIX_TREE_ITER_TAGGED) set_iter_tags(iter, node, offset, tag); @@ -1750,7 +1562,6 @@ void __rcu **idr_get_free(struct radix_tree_root *root, else iter->next_index = 1; iter->node = node; - __set_iter_shift(iter, shift); set_iter_tags(iter, node, offset, IDR_FREE); return slot; diff --git a/mm/Kconfig b/mm/Kconfig index de64ea658716..02301a89089e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -379,7 +379,7 @@ config TRANSPARENT_HUGEPAGE bool "Transparent Hugepage Support" depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE select COMPACTION - select RADIX_TREE_MULTIORDER + select XARRAY_MULTI help Transparent Hugepages allows the kernel to use huge pages and huge tlb transparently to the applications whenever possible. @@ -671,7 +671,7 @@ config ZONE_DEVICE depends on MEMORY_HOTREMOVE depends on SPARSEMEM_VMEMMAP depends on ARCH_HAS_ZONE_DEVICE - select RADIX_TREE_MULTIORDER + select XARRAY_MULTI help Device memory hotplug support allows for establishing pmem, diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h index ca8e03ad19ac..2218b3cc184e 100644 --- a/tools/testing/radix-tree/generated/autoconf.h +++ b/tools/testing/radix-tree/generated/autoconf.h @@ -1,2 +1 @@ -#define CONFIG_RADIX_TREE_MULTIORDER 1 #define CONFIG_XARRAY_MULTI 1 -- cgit v1.2.3 From 9d954f9519707bc0af29ead8b1dedc660306da1b Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:12:55 +0900 Subject: selftests/ftrace: Add --stop-fail hidden option for debug Add --stop-fail option for debugging the ftracetest. With this option, ftracetest stops right after a testcase fails instead of finish running all testcases. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index f9a9d424c980..c9c7fa8dc440 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -69,6 +69,10 @@ parse_opts() { # opts DEBUG=1 shift 1 ;; + --stop-fail) + STOP_FAILURE=1 + shift 1 + ;; --fail-unsupported) UNSUPPORTED_RESULT=1 shift 1 @@ -117,6 +121,7 @@ KEEP_LOG=0 DEBUG=0 VERBOSE=0 UNSUPPORTED_RESULT=0 +STOP_FAILURE=0 # Parse command-line options parse_opts $* @@ -304,6 +309,10 @@ run_test() { # testfile # Main loop for t in $TEST_CASES; do run_test $t + if [ $STOP_FAILURE -ne 0 -a $TOTAL_RESULT -ne 0 ]; then + echo "A failure detected. Stop test." + exit 1 + fi done # Test on instance loop @@ -315,6 +324,10 @@ for t in $TEST_CASES; do run_test $t rmdir $TRACING_DIR TRACING_DIR=$SAVED_TRACING_DIR + if [ $STOP_FAILURE -ne 0 -a $TOTAL_RESULT -ne 0 ]; then + echo "A failure detected. Stop test." + exit 1 + fi done prlog "" -- cgit v1.2.3 From 5d1c6580d616dfc864ffdec587e645e50a21c072 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:13:23 +0900 Subject: selftests/ftrace: Add --console hidden option Add --console hidden option for debug test cases. This option allows to put "sh" or something else when the test case hits a bug. For example, if you find a testcase which doesn't pass, you can insert sh for interactive debug as below ----- #!/bin/sh # description: sample test case good-command suspicious-wrong-command sh # <- add this for interactive debug ----- Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index c9c7fa8dc440..8debd37038e4 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -60,11 +60,21 @@ parse_opts() { # opts shift 1 ;; --verbose|-v|-vv|-vvv) + if [ $VERBOSE -eq -1 ]; then + usage "--console can not use with --verbose" + fi VERBOSE=$((VERBOSE + 1)) [ $1 = '-vv' ] && VERBOSE=$((VERBOSE + 1)) [ $1 = '-vvv' ] && VERBOSE=$((VERBOSE + 2)) shift 1 ;; + --console) + if [ $VERBOSE -ne 0 ]; then + usage "--console can not use with --verbose" + fi + VERBOSE=-1 + shift 1 + ;; --debug|-d) DEBUG=1 shift 1 @@ -283,7 +293,9 @@ run_test() { # testfile testcase $1 echo "execute$INSTANCE: "$1 > $testlog SIG_RESULT=0 - if [ -z "$LOG_FILE" ]; then + if [ $VERBOSE -eq -1 ]; then + __run_test $1 + elif [ -z "$LOG_FILE" ]; then __run_test $1 2>&1 elif [ $VERBOSE -ge 3 ]; then __run_test $1 | tee -a $testlog 2>&1 -- cgit v1.2.3 From 7d0abba4293e555a5b1a6ea060cddd05f4ee357f Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:13:51 +0900 Subject: selftests/ftrace: Add case number prefix to logfile Add a case number prefix to each logfile. This makes it easier to find which logfile is corresponding to which failure. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 8debd37038e4..3ca596255eff 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -284,13 +284,13 @@ __run_test() { # testfile # Run one test case run_test() { # testfile local testname=`basename $1` + testcase $1 if [ ! -z "$LOG_FILE" ] ; then - local testlog=`mktemp $LOG_DIR/${testname}-log.XXXXXX` + local testlog=`mktemp $LOG_DIR/${CASENO}-${testname}-log.XXXXXX` else local testlog=/proc/self/fd/1 fi export TMPDIR=`mktemp -d /tmp/ftracetest-dir.XXXXXX` - testcase $1 echo "execute$INSTANCE: "$1 > $testlog SIG_RESULT=0 if [ $VERBOSE -eq -1 ]; then -- cgit v1.2.3 From 7590472e78c8fbe1383aba30450655fd6415fa7b Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:14:19 +0900 Subject: selftests/ftrace: More initialize features in initialize_ftrace Clear pid filter, synthetic_events, snapshots, ftrace filter, and trace log in initialize_ftrace(), since those are used in test cases. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/functions | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions index e4645d5e3126..b2bfa4dd43f7 100644 --- a/tools/testing/selftests/ftrace/test.d/functions +++ b/tools/testing/selftests/ftrace/test.d/functions @@ -89,12 +89,17 @@ initialize_ftrace() { # Reset ftrace to initial-state reset_tracer reset_trigger reset_events_filter + reset_ftrace_filter disable_events echo > set_event_pid # event tracer is always on + echo > set_ftrace_pid [ -f set_ftrace_filter ] && echo | tee set_ftrace_* [ -f set_graph_function ] && echo | tee set_graph_* [ -f stack_trace_filter ] && echo > stack_trace_filter [ -f kprobe_events ] && echo > kprobe_events [ -f uprobe_events ] && echo > uprobe_events + [ -f synthetic_events ] && echo > synthetic_events + [ -f snapshot ] && echo 0 > snapshot + clear_trace enable_tracing } -- cgit v1.2.3 From 3a96a9d0a4e985e80280357393d86ff4401eeff7 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:14:48 +0900 Subject: selftests/ftrace: Add SPDX License Identifier to template Add SPDX License Identifier line to template file so that someone who makes new testcase from the template does not forgot it. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/template | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/template b/tools/testing/selftests/ftrace/test.d/template index 5c39ceb18a0d..799da7e0b3c9 100644 --- a/tools/testing/selftests/ftrace/test.d/template +++ b/tools/testing/selftests/ftrace/test.d/template @@ -1,4 +1,5 @@ #!/bin/sh +# SPDX-License-Identifier: GPL2.0 # description: %HERE DESCRIBE WHAT THIS DOES% # you have to add ".tc" extention for your testcase file # Note that all tests are run with "errexit" option. -- cgit v1.2.3 From 9768cc46d39a029f7da47beea449d023b377103d Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:15:16 +0900 Subject: selftests/ftrace: Cleanup ftrace after running test Cleanup ftrace by initialize_ftrace() after running all test cases. This means we also don't need cleanup ftrace on each test case, except for some special options. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 3ca596255eff..5c71d58febb2 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -341,6 +341,7 @@ for t in $TEST_CASES; do exit 1 fi done +(cd $TRACING_DIR; initialize_ftrace) # for cleanup prlog "" prlog "# of passed: " `echo $PASSED_CASES | wc -w` -- cgit v1.2.3 From e527c47081c7e2795ffc3201809de9545cc25830 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:15:44 +0900 Subject: selftests/ftrace: Remove unneeded per-test init/cleanup ftrace Since ftracetest framework calls initialize_ftrace() right before each test and after all tests, we don't need to init/cleanup ftrace for each test case. Just remove such unneeded init/cleanup code because it can increase logfile size. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/event/event-enable.tc | 6 ------ tools/testing/selftests/ftrace/test.d/event/event-pid.tc | 3 +-- .../selftests/ftrace/test.d/event/subsystem-enable.tc | 6 ------ .../selftests/ftrace/test.d/event/toplevel-enable.tc | 6 ------ .../ftrace/test.d/ftrace/fgraph-filter-stack.tc | 4 ---- .../selftests/ftrace/test.d/ftrace/fgraph-filter.tc | 9 --------- .../selftests/ftrace/test.d/ftrace/func-filter-pid.tc | 6 ------ .../ftrace/test.d/ftrace/func_event_triggers.tc | 3 --- .../selftests/ftrace/test.d/ftrace/func_profiler.tc | 4 ---- .../ftrace/test.d/ftrace/func_set_ftrace_file.tc | 13 ------------- .../ftrace/test.d/ftrace/func_traceonoff_triggers.tc | 11 ----------- .../selftests/ftrace/test.d/kprobe/add_and_remove.tc | 3 --- .../testing/selftests/ftrace/test.d/kprobe/busy_check.tc | 3 --- .../selftests/ftrace/test.d/kprobe/kprobe_args.tc | 2 -- .../selftests/ftrace/test.d/kprobe/kprobe_args_string.tc | 5 ----- .../selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc | 5 ----- .../selftests/ftrace/test.d/kprobe/kprobe_args_type.tc | 4 ---- .../selftests/ftrace/test.d/kprobe/kprobe_eventname.tc | 5 ----- .../selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc | 9 --------- .../selftests/ftrace/test.d/kprobe/kprobe_module.tc | 5 ----- .../selftests/ftrace/test.d/kprobe/kretprobe_args.tc | 2 -- .../ftrace/test.d/kprobe/kretprobe_maxactive.tc | 4 ---- .../selftests/ftrace/test.d/kprobe/multiple_kprobes.tc | 5 ----- .../testing/selftests/ftrace/test.d/kprobe/probepoint.tc | 4 ---- .../inter-event/trigger-extended-error-support.tc | 12 ------------ .../inter-event/trigger-field-variable-support.tc | 13 ------------- .../inter-event/trigger-inter-event-combined-hist.tc | 13 ------------- .../trigger/inter-event/trigger-multi-actions-accept.tc | 14 -------------- .../trigger/inter-event/trigger-onmatch-action-hist.tc | 13 ------------- .../inter-event/trigger-onmatch-onmax-action-hist.tc | 13 ------------- .../trigger/inter-event/trigger-onmax-action-hist.tc | 13 ------------- .../inter-event/trigger-synthetic-event-createremove.tc | 12 ------------ .../ftrace/test.d/trigger/trigger-eventonoff.tc | 12 ------------ .../selftests/ftrace/test.d/trigger/trigger-filter.tc | 14 -------------- .../selftests/ftrace/test.d/trigger/trigger-hist-mod.tc | 12 ------------ .../selftests/ftrace/test.d/trigger/trigger-hist.tc | 12 ------------ .../selftests/ftrace/test.d/trigger/trigger-multihist.tc | 16 ---------------- .../selftests/ftrace/test.d/trigger/trigger-snapshot.tc | 12 ------------ .../ftrace/test.d/trigger/trigger-stacktrace.tc | 12 ------------ .../ftrace/test.d/trigger/trigger-trace-marker-hist.tc | 11 ----------- .../test.d/trigger/trigger-trace-marker-snapshot.tc | 12 ------------ .../trigger/trigger-trace-marker-synthetic-kernel.tc | 12 ------------ .../test.d/trigger/trigger-trace-marker-synthetic.tc | 12 ------------ .../ftrace/test.d/trigger/trigger-traceonoff.tc | 12 ------------ 44 files changed, 1 insertion(+), 378 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/event/event-enable.tc b/tools/testing/selftests/ftrace/test.d/event/event-enable.tc index 9daf034186f5..386dbddbdc11 100644 --- a/tools/testing/selftests/ftrace/test.d/event/event-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/event-enable.tc @@ -9,7 +9,6 @@ do_reset() { } fail() { #msg - do_reset echo $1 exit_fail } @@ -23,9 +22,6 @@ if [ ! -f set_event -o ! -d events/sched ]; then exit_unsupported fi -reset_tracer -do_reset - echo 'sched:sched_switch' > set_event yield @@ -57,6 +53,4 @@ if [ $count -ne 0 ]; then fail "sched_switch events should not be recorded" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc index 132478b305c2..6dbf67544fde 100644 --- a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc @@ -30,8 +30,7 @@ if [ ! -f set_event_pid ]; then exit_unsupported fi -reset_tracer -do_reset +echo 0 > options/event-fork echo 1 > events/sched/sched_switch/enable diff --git a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc index 6a37a8642ee6..10eb17f161b7 100644 --- a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc @@ -9,7 +9,6 @@ do_reset() { } fail() { #msg - do_reset echo $1 exit_fail } @@ -23,9 +22,6 @@ if [ ! -f set_event -o ! -d events/sched ]; then exit_unsupported fi -reset_tracer -do_reset - echo 'sched:*' > set_event yield @@ -57,6 +53,4 @@ if [ $count -ne 0 ]; then fail "any of scheduler events should not be recorded" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc b/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc index 4e9b6e2c0219..8d55a233c1a4 100644 --- a/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc @@ -8,7 +8,6 @@ do_reset() { } fail() { #msg - do_reset echo $1 exit_fail } @@ -22,9 +21,6 @@ if [ ! -f available_events -o ! -f set_event -o ! -d events ]; then exit_unsupported fi -reset_tracer -do_reset - echo '*:*' > set_event yield @@ -60,6 +56,4 @@ if [ $count -ne 0 ]; then fail "any of events should not be recorded" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter-stack.tc b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter-stack.tc index 1aec99d108eb..aefab0c66d54 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter-stack.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter-stack.tc @@ -16,13 +16,9 @@ if [ ! -f set_ftrace_filter ]; then fi do_reset() { - reset_tracer if [ -e /proc/sys/kernel/stack_tracer_enabled ]; then echo 0 > /proc/sys/kernel/stack_tracer_enabled fi - enable_tracing - clear_trace - echo > set_ftrace_filter } fail() { # msg diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter.tc b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter.tc index 9f8d27ca39cf..c8a5209f2119 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-filter.tc @@ -9,14 +9,7 @@ if ! grep -q function_graph available_tracers; then exit_unsupported fi -do_reset() { - reset_tracer - enable_tracing - clear_trace -} - fail() { # msg - do_reset echo $1 exit_fail } @@ -48,6 +41,4 @@ if [ $count -eq 0 ]; then fail "No schedule traces found?" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc index 524ce24b3c22..970903cb7834 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc @@ -35,12 +35,6 @@ if [ $do_function_fork -eq 1 ]; then fi do_reset() { - reset_tracer - clear_trace - enable_tracing - echo > set_ftrace_filter - echo > set_ftrace_pid - if [ $do_function_fork -eq 0 ]; then return fi diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc index 6fed4cf2db81..ca2ffd7957f9 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc @@ -25,15 +25,12 @@ do_reset() { } fail() { # mesg - do_reset echo $1 exit_fail } SLEEP_TIME=".1" -do_reset - echo "Testing function probes with events:" EVENT="sched:sched_switch" diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_profiler.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_profiler.tc index b2d5a8febfe8..dfbae637c60c 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_profiler.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_profiler.tc @@ -29,8 +29,6 @@ if [ ! -f function_profile_enabled ]; then fi fail() { # mesg - reset_tracer - echo > set_ftrace_filter echo $1 exit_fail } @@ -76,6 +74,4 @@ if ! grep -v -e '^#' -e 'schedule' trace > /dev/null; then fail "no other functions besides schedule was found" fi -reset_tracer - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc index 68e7a48f5828..51f6e6146bd9 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_set_ftrace_file.tc @@ -15,22 +15,11 @@ if [ ! -f set_ftrace_filter ]; then exit_unsupported fi -do_reset() { - reset_tracer - reset_ftrace_filter - disable_events - clear_trace - enable_tracing -} - fail() { # mesg - do_reset echo $1 exit_fail } -do_reset - FILTER=set_ftrace_filter FUNC1="schedule" FUNC2="do_softirq" @@ -165,6 +154,4 @@ test_actual rm $TMPDIR/expected rm $TMPDIR/actual -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc index f6d9ac73268a..0c04282d33dd 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_traceonoff_triggers.tc @@ -16,24 +16,13 @@ if [ ! -f set_ftrace_filter ]; then exit_unsupported fi -do_reset() { - reset_ftrace_filter - reset_tracer - disable_events - clear_trace - enable_tracing -} - fail() { # mesg - do_reset echo $1 exit_fail } SLEEP_TIME=".1" -do_reset - echo "Testing function probes with enabling disabling tracing:" cnt_trace() { diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc b/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc index 4604d2103c89..bb1eb5a7c64e 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/add_and_remove.tc @@ -4,10 +4,7 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -echo 0 > events/enable -echo > kprobe_events echo p:myevent _do_fork > kprobe_events grep myevent kprobe_events test -d events/kprobes/myevent echo > kprobe_events -clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc index bbc443a9190c..442c1a8c5edf 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/busy_check.tc @@ -4,12 +4,9 @@ [ -f kprobe_events ] || exit_unsupported -echo 0 > events/enable -echo > kprobe_events echo p:myevent _do_fork > kprobe_events test -d events/kprobes/myevent echo 1 > events/kprobes/myevent/enable echo > kprobe_events && exit_fail # this must fail echo 0 > events/kprobes/myevent/enable echo > kprobe_events # this must succeed -clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc index 8b43c6804fc3..9578535ccd67 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc @@ -4,8 +4,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -echo 0 > events/enable -echo > kprobe_events echo 'p:testprobe _do_fork $stack $stack0 +0($stack)' > kprobe_events grep testprobe kprobe_events test -d events/kprobes/testprobe diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index 1ad70cdaf442..46e7744f8358 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -4,9 +4,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -echo 0 > events/enable -echo > kprobe_events - case `uname -m` in x86_64) ARG1=%di @@ -44,5 +41,3 @@ echo 1 > events/kprobes/testprobe/enable echo "p:test _do_fork" >> kprobe_events grep -qe "testprobe.* arg1=\"test\" arg2=\"test\"" trace -echo 0 > events/enable -echo > kprobe_events diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc index d026ff4e562f..d783a8ce0692 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc @@ -6,9 +6,6 @@ grep "x8/16/32/64" README > /dev/null || exit_unsupported # version issue -echo 0 > events/enable -echo > kprobe_events - PROBEFUNC="vfs_read" GOODREG= BADREG= @@ -100,5 +97,3 @@ test_badarg "${GOODREG}::${GOODTYPE}" "${GOODREG}:${BADTYPE}" \ test_goodarg "\$comm:string" "+0(\$stack):string" test_badarg "\$comm:x64" "\$stack:string" "${GOODREG}:string" - -echo > kprobe_events diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc index 2a1755bfc290..37443dd23d15 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc @@ -6,10 +6,6 @@ grep "x8/16/32/64" README > /dev/null || exit_unsupported # version issue -echo 0 > events/enable -echo > kprobe_events -enable_tracing - echo 'p:testprobe _do_fork $stack0:s32 $stack0:u32 $stack0:x32 $stack0:b8@4/32' > kprobe_events grep testprobe kprobe_events test -d events/kprobes/testprobe diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_eventname.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_eventname.tc index 2724a1068cb1..3fb70e01b1fe 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_eventname.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_eventname.tc @@ -4,9 +4,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -disable_events -echo > kprobe_events - :;: "Add an event on function without name" ;: FUNC=`grep " [tT] .*vfs_read$" /proc/kallsyms | tail -n 1 | cut -f 3 -d " "` @@ -33,5 +30,3 @@ echo "p $FUNC" > kprobe_events EVENT=`grep $FUNC kprobe_events | cut -f 1 -d " " | cut -f 2 -d:` [ "x" != "x$EVENT" ] || exit_failure test -d events/$EVENT || exit_failure - -echo > kprobe_events diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc index cc4cac0e60f2..492426e95e09 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_ftrace.tc @@ -8,8 +8,6 @@ grep function available_tracers || exit_unsupported # this is configurable # prepare echo nop > current_tracer echo _do_fork > set_ftrace_filter -echo 0 > events/enable -echo > kprobe_events echo 'p:testprobe _do_fork' > kprobe_events # kprobe on / ftrace off @@ -47,10 +45,3 @@ echo > trace ( echo "forked") grep testprobe trace ! grep '_do_fork <-' trace - -# cleanup -echo nop > current_tracer -echo > set_ftrace_filter -echo 0 > events/kprobes/testprobe/enable -echo > kprobe_events -echo > trace diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc index 1e9f75f7a30f..ec6c5fffef1e 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc @@ -4,9 +4,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -disable_events -echo > kprobe_events - :;: "Add an event on a module function without specifying event name" ;: MOD=`lsmod | head -n 2 | tail -n 1 | cut -f1 -d" "` @@ -25,5 +22,3 @@ test -d events/kprobes/event1 || exit_failure echo "p:kprobes1/event1 $MOD:$FUNC" > kprobe_events test -d events/kprobes1/event1 || exit_failure - -echo > kprobe_events diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc index 321954683aaa..4c6b21e30b02 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc @@ -4,8 +4,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable -echo 0 > events/enable -echo > kprobe_events echo 'r:testprobe2 _do_fork $retval' > kprobe_events grep testprobe2 kprobe_events test -d events/kprobes/testprobe2 diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_maxactive.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_maxactive.tc index 7c0290684c43..8e05b178519a 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_maxactive.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_maxactive.tc @@ -5,8 +5,6 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable grep -q 'r\[maxactive\]' README || exit_unsupported # this is older version -echo > kprobe_events - # Test if we successfully reject unknown messages if echo 'a:myprobeaccept inet_csk_accept' > kprobe_events; then false; else true; fi @@ -37,5 +35,3 @@ echo > kprobe_events echo 'r10 inet_csk_accept' > kprobe_events grep inet_csk_accept kprobe_events echo > kprobe_events - -clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/multiple_kprobes.tc b/tools/testing/selftests/ftrace/test.d/kprobe/multiple_kprobes.tc index ce361b9d62cf..5862eee91e1d 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/multiple_kprobes.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/multiple_kprobes.tc @@ -12,11 +12,6 @@ case `uname -m` in *) OFFS=0;; esac -if [ -d events/kprobes ]; then - echo 0 > events/kprobes/enable - echo > kprobe_events -fi - N=0 echo "Setup up kprobes on first available 256 text symbols" grep -i " t " /proc/kallsyms | cut -f3 -d" " | grep -v .*\\..* | \ diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/probepoint.tc b/tools/testing/selftests/ftrace/test.d/kprobe/probepoint.tc index 519d2763f5d2..a902aa0aaabc 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/probepoint.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/probepoint.tc @@ -30,8 +30,6 @@ if [ `printf "%x" -1 | wc -c` != 9 ]; then UINT_TEST=yes fi -echo 0 > events/enable -echo > kprobe_events echo "p:testprobe ${TARGET_FUNC}" > kprobe_events echo "p:testprobe ${TARGET}" > kprobe_events echo "p:testprobe ${TARGET_FUNC}${NEXT}" > kprobe_events @@ -39,5 +37,3 @@ echo "p:testprobe ${TARGET_FUNC}${NEXT}" > kprobe_events if [ "${UINT_TEST}" = yes ]; then ! echo "p:testprobe ${TARGET_FUNC}${OVERFLOW}" > kprobe_events fi -echo > kprobe_events -clear_trace diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc index 2aabab363cfb..401104344593 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc @@ -2,14 +2,7 @@ # description: event trigger - test extended error support -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -24,9 +17,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -reset_tracer -do_reset - echo "Test extended error support" echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' > events/sched/sched_wakeup/trigger ! echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' >> events/sched/sched_wakeup/trigger 2> /dev/null @@ -34,6 +24,4 @@ if ! grep -q "ERROR:" events/sched/sched_wakeup/hist; then fail "Failed to generate extended error in histogram" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc index 7fd5b4a8f060..575af23df04e 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc @@ -1,14 +1,7 @@ #!/bin/sh # description: event trigger - test field variable support -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -23,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test field variable support" echo 'wakeup_latency u64 lat; pid_t pid; int prio; char comm[16]' > synthetic_events @@ -49,6 +38,4 @@ if grep -q "synthetic_prio=prio" events/sched/sched_waking/hist; then fail "Failed to remove histogram with field variable" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc index c93dbe38b5df..92aef26e2631 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc @@ -1,14 +1,7 @@ #!/bin/sh # description: event trigger - test inter-event combined histogram trigger -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -23,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -reset_tracer -do_reset -clear_synthetic_events - echo "Test create synthetic event" echo 'waking_latency u64 lat pid_t pid' > synthetic_events @@ -53,6 +42,4 @@ if ! grep -q "pid:" events/synthetic/waking+wakeup_latency/hist; then fail "Failed to create combined histogram" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-multi-actions-accept.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-multi-actions-accept.tc index c193dce611a2..4ddc546771b5 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-multi-actions-accept.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-multi-actions-accept.tc @@ -1,15 +1,7 @@ #!/bin/sh # description: event trigger - test multiple actions on hist trigger - -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -24,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test multiple actions on hist trigger" echo 'wakeup_latency u64 lat; pid_t pid' >> synthetic_events TRIGGER1=events/sched/sched_wakeup/trigger @@ -39,6 +27,4 @@ echo 'hist:keys=next_pid:onmatch(sched.sched_wakeup).wakeup_latency(sched.sched_ echo 'hist:keys=next_pid:onmatch(sched.sched_wakeup).wakeup_latency(sched.sched_switch.$wakeup_lat,prev_pid) if next_comm=="cyclictest"' >> $TRIGGER2 echo 'hist:keys=next_pid if next_comm=="cyclictest"' >> $TRIGGER2 -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc index e84e7d048566..1ef5dd4521f0 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc @@ -1,14 +1,7 @@ #!/bin/sh # description: event trigger - test inter-event histogram trigger onmatch action -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -23,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test create synthetic event" echo 'wakeup_latency u64 lat pid_t pid char comm[16]' > synthetic_events @@ -45,6 +34,4 @@ if ! grep -q "ping" events/synthetic/wakeup_latency/hist; then fail "Failed to create onmatch action inter-event histogram" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc index 7907d8aacde3..7234683ee5e5 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc @@ -1,14 +1,7 @@ #!/bin/sh # description: event trigger - test inter-event histogram trigger onmatch-onmax action -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -23,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test create synthetic event" echo 'wakeup_latency u64 lat pid_t pid char comm[16]' > synthetic_events @@ -45,6 +34,4 @@ if [ ! grep -q "ping" events/synthetic/wakeup_latency/hist -o ! grep -q "max:" e fail "Failed to create onmatch-onmax action inter-event histogram" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc index 38b7ed6242b2..55b9630a1d70 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc @@ -1,14 +1,7 @@ #!/bin/sh # description: event trigger - test inter-event histogram trigger onmax action -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -23,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test create synthetic event" echo 'wakeup_latency u64 lat pid_t pid char comm[16]' > synthetic_events @@ -43,6 +32,4 @@ if ! grep -q "max:" events/sched/sched_switch/hist; then fail "Failed to create onmax action inter-event histogram" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc index cef11377dcbd..8d647fb572dd 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc @@ -1,13 +1,7 @@ #!/bin/sh # description: event trigger - test synthetic event create remove -do_reset() { - reset_trigger - echo > set_event - clear_trace -} fail() { #msg - do_reset echo $1 exit_fail } @@ -22,10 +16,6 @@ if [ ! -f synthetic_events ]; then exit_unsupported fi -clear_synthetic_events -reset_tracer -do_reset - echo "Test create synthetic event" echo 'wakeup_latency u64 lat pid_t pid char comm[16]' > synthetic_events @@ -49,6 +39,4 @@ if [ -d events/synthetic/wakeup_latency ]; then fail "Failed to delete wakeup_latency synthetic event" fi -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-eventonoff.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-eventonoff.tc index 28cc355a3a7b..eddb51e1fbf7 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-eventonoff.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-eventonoff.tc @@ -3,14 +3,7 @@ # description: event trigger - test event enable/disable trigger # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -25,9 +18,6 @@ if [ ! -f events/sched/sched_process_fork/trigger ]; then exit_unsupported fi -reset_tracer -do_reset - FEATURE=`grep enable_event events/sched/sched_process_fork/trigger` if [ -z "$FEATURE" ]; then echo "event enable/disable trigger is not supported" @@ -61,6 +51,4 @@ echo 'enable_event:sched:sched_switch' > events/sched/sched_process_fork/trigger ! echo 'enable_event:sched:sched_switch' > events/sched/sched_process_fork/trigger ! echo 'disable_event:sched:sched_switch' > events/sched/sched_process_fork/trigger -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-filter.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-filter.tc index a48e23eb8a8b..2dcc2296ebdd 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-filter.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-filter.tc @@ -3,14 +3,7 @@ # description: event trigger - test trigger filter # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -25,9 +18,6 @@ if [ ! -f events/sched/sched_process_fork/trigger ]; then exit_unsupported fi -reset_tracer -do_reset - echo "Test trigger filter" echo 1 > tracing_on echo 'traceoff if child_pid == 0' > events/sched/sched_process_fork/trigger @@ -54,8 +44,4 @@ echo '!traceoff' > events/sched/sched_process_fork/trigger echo 'traceoff if parent_pid >= 0 || child_pid >= 0' > events/sched/sched_process_fork/trigger echo '!traceoff' > events/sched/sched_process_fork/trigger - - -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc index 8da80efc44d8..fab4431639d3 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc @@ -3,14 +3,7 @@ # description: event trigger - test histogram modifiers # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -30,9 +23,6 @@ if [ ! -f events/sched/sched_process_fork/hist ]; then exit_unsupported fi -reset_tracer -do_reset - echo "Test histogram with execname modifier" echo 'hist:keys=common_pid.execname' > events/sched/sched_process_fork/trigger @@ -71,6 +61,4 @@ for i in `seq 1 10` ; do ( echo "forked" > /dev/null); done grep 'bytes_req: ~ 2^[0-9]*' events/kmem/kmalloc/hist > /dev/null || \ fail "log2 modifier on kmem/kmalloc did not work" -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist.tc index 449fe9ff91a2..177e8d4c4744 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist.tc @@ -3,14 +3,7 @@ # description: event trigger - test histogram trigger # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -30,9 +23,6 @@ if [ ! -f events/sched/sched_process_fork/hist ]; then exit_unsupported fi -reset_tracer -do_reset - echo "Test histogram basic tigger" echo 'hist:keys=parent_pid:vals=child_pid' > events/sched/sched_process_fork/trigger @@ -79,6 +69,4 @@ check_inc `grep -o "child_pid:[[:space:]]*[[:digit:]]*" \ events/sched/sched_process_fork/hist | cut -d: -f2 ` || fail "sort param on sched_process_fork did not work" -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-multihist.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-multihist.tc index c5ef8b9d02b3..18fdaab9f570 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-multihist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-multihist.tc @@ -3,14 +3,7 @@ # description: event trigger - test multiple histogram triggers # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -30,11 +23,6 @@ if [ ! -f events/sched/sched_process_fork/hist ]; then exit_unsupported fi -reset_tracer -do_reset - -reset_trigger - echo "Test histogram multiple tiggers" echo 'hist:keys=parent_pid:vals=child_pid' > events/sched/sched_process_fork/trigger @@ -67,8 +55,4 @@ grep test_hist events/sched/sched_process_exit/hist > /dev/null || \ diffs=`diff events/sched/sched_process_exit/hist events/sched/sched_process_fork/hist | wc -l` test $diffs -eq 0 || fail "Same name histograms are not same" -reset_trigger - -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc index ed38f0050d77..7717c0a09686 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc @@ -2,14 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # description: event trigger - test snapshot-trigger -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -29,9 +22,6 @@ if [ ! -f snapshot ]; then exit_unsupported fi -reset_tracer -do_reset - FEATURE=`grep snapshot events/sched/sched_process_fork/trigger` if [ -z "$FEATURE" ]; then echo "snapshot trigger is not supported" @@ -57,6 +47,4 @@ echo "Test snapshot semantic errors" echo "snapshot" > events/sched/sched_process_fork/trigger ! echo "snapshot" > events/sched/sched_process_fork/trigger -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-stacktrace.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-stacktrace.tc index 3121d795a868..398c05c4d2a7 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-stacktrace.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-stacktrace.tc @@ -2,14 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # description: event trigger - test stacktrace-trigger -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -24,9 +17,6 @@ if [ ! -f events/sched/sched_process_fork/trigger ]; then exit_unsupported fi -reset_tracer -do_reset - FEATURE=`grep stacktrace events/sched/sched_process_fork/trigger` if [ -z "$FEATURE" ]; then echo "stacktrace trigger is not supported" @@ -49,6 +39,4 @@ echo "Test stacktrace semantic errors" echo "stacktrace" > events/sched/sched_process_fork/trigger ! echo "stacktrace" > events/sched/sched_process_fork/trigger -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-hist.tc index 2acbfe2c0c0c..ab6bedb25736 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-hist.tc @@ -3,14 +3,7 @@ # description: trace_marker trigger - test histogram trigger # flags: instance -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -35,8 +28,6 @@ if [ ! -f events/ftrace/print/hist ]; then exit_unsupported fi -do_reset - echo "Test histogram trace_marker tigger" echo 'hist:keys=common_pid' > events/ftrace/print/trigger @@ -44,6 +35,4 @@ for i in `seq 1 10` ; do echo "hello" > trace_marker; done grep 'hitcount: *10$' events/ftrace/print/hist > /dev/null || \ fail "hist trigger did not trigger correct times on trace_marker" -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc index 6748e8cb42d0..79ce7d51350b 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc @@ -3,15 +3,7 @@ # description: trace_marker trigger - test snapshot trigger # flags: instance -do_reset() { - reset_trigger - echo > set_event - echo 0 > snapshot - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -54,8 +46,6 @@ test_trace() { done } -do_reset - echo "Test snapshot trace_marker tigger" echo 'snapshot' > events/ftrace/print/trigger @@ -69,6 +59,4 @@ for i in `seq 1 10` ; do echo "hello >$i<" > trace_marker; done test_trace trace 1 test_trace snapshot 2 -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic-kernel.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic-kernel.tc index 0a69c5d1cda8..18b4d1c2807e 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic-kernel.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic-kernel.tc @@ -3,15 +3,7 @@ # description: trace_marker trigger - test histogram with synthetic event against kernel event # flags: -do_reset() { - reset_trigger - echo > set_event - echo > synthetic_events - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -46,8 +38,6 @@ if [ ! -f events/ftrace/print/hist ]; then exit_unsupported fi -do_reset - echo "Test histogram kernel event to trace_marker latency histogram trigger" echo 'latency u64 lat' > synthetic_events @@ -63,6 +53,4 @@ grep 'hitcount: *1$' events/ftrace/print/hist > /dev/null || \ grep 'hitcount: *1$' events/synthetic/latency/hist > /dev/null || \ fail "hist trigger did not trigger " -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic.tc index 3666dd6ab02a..dd262d6d0db6 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-synthetic.tc @@ -3,15 +3,7 @@ # description: trace_marker trigger - test histogram with synthetic event # flags: -do_reset() { - reset_trigger - echo > set_event - echo > synthetic_events - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -41,8 +33,6 @@ if [ ! -f events/ftrace/print/hist ]; then exit_unsupported fi -do_reset - echo "Test histogram trace_marker to trace_marker latency histogram trigger" echo 'latency u64 lat' > synthetic_events @@ -61,6 +51,4 @@ fi grep 'hitcount: *1$' events/synthetic/latency/hist > /dev/null || \ fail "hist trigger did not trigger " -do_reset - exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-traceonoff.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-traceonoff.tc index c59d9eb546da..d5d2dcbc9cab 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-traceonoff.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-traceonoff.tc @@ -2,14 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # description: event trigger - test traceon/off trigger -do_reset() { - reset_trigger - echo > set_event - clear_trace -} - fail() { #msg - do_reset echo $1 exit_fail } @@ -24,9 +17,6 @@ if [ ! -f events/sched/sched_process_fork/trigger ]; then exit_unsupported fi -reset_tracer -do_reset - echo "Test traceoff trigger" echo 1 > tracing_on echo 'traceoff' > events/sched/sched_process_fork/trigger @@ -54,6 +44,4 @@ echo 'traceon' > events/sched/sched_process_fork/trigger ! echo 'traceon' > events/sched/sched_process_fork/trigger ! echo 'traceoff' > events/sched/sched_process_fork/trigger -do_reset - exit 0 -- cgit v1.2.3 From 2452c96e617a0ff6fb2692e55217a3fa57a7322c Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:16:13 +0900 Subject: selftests/ftrace: Fix to test kprobe $comm arg only if available Test $comm in kprobe-event argument syntax testcase only if it is supported on the kernel because $comm has been introduced 4.8 kernel. So on older stable kernel, it should be skipped. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc index d783a8ce0692..6f0f19953193 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc @@ -75,8 +75,11 @@ test_badarg "\$stackp" "\$stack0+10" "\$stack1-10" echo "r ${PROBEFUNC} \$retval" > kprobe_events ! echo "p ${PROBEFUNC} \$retval" > kprobe_events +# $comm was introduced in 4.8, older kernels reject it. +if grep -A1 "fetcharg:" README | grep -q '\$comm' ; then : "Comm access" test_goodarg "\$comm" +fi : "Indirect memory access" test_goodarg "+0(${GOODREG})" "-0(${GOODREG})" "+10(\$stack)" \ -- cgit v1.2.3 From 72ce3daf92ba4f5bae6e91095d40e67b367c6b2f Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:16:41 +0900 Subject: selftests/ftrace: Fix checkbashisms errors Fix a test case to make checkbashisms clean. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc index 79ce7d51350b..df246e505af7 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-trace-marker-snapshot.tc @@ -39,10 +39,10 @@ test_trace() { fi echo "testing $line for >$x<" match=`echo $line | sed -e "s/>$x$x< in it" fi - let x=$x+2 + x=$((x+2)) done } -- cgit v1.2.3 From cec3adf5f57160d829a033bcb578c3dd0a90c693 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:17:10 +0900 Subject: selftests/ftrace: Use loopback address instead of localhost Use raw loopback address instead of localhost, because "localhost" can depend on nsswitch and in some case we can not resolve the localhost. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/event/event-enable.tc | 4 ---- tools/testing/selftests/ftrace/test.d/event/event-pid.tc | 4 ---- tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc | 4 ---- tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc | 4 ---- tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc | 4 ---- tools/testing/selftests/ftrace/test.d/functions | 6 ++++++ .../test.d/trigger/inter-event/trigger-field-variable-support.tc | 2 +- .../test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc | 2 +- .../test.d/trigger/inter-event/trigger-onmatch-action-hist.tc | 3 ++- .../test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc | 3 ++- .../ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc | 3 ++- 11 files changed, 14 insertions(+), 25 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/event/event-enable.tc b/tools/testing/selftests/ftrace/test.d/event/event-enable.tc index 386dbddbdc11..dfb0d5122f7b 100644 --- a/tools/testing/selftests/ftrace/test.d/event/event-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/event-enable.tc @@ -13,10 +13,6 @@ fail() { #msg exit_fail } -yield() { - ping localhost -c 1 || sleep .001 || usleep 1 || sleep 1 -} - if [ ! -f set_event -o ! -d events/sched ]; then echo "event tracing is not supported" exit_unsupported diff --git a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc index 6dbf67544fde..e28897738500 100644 --- a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc @@ -16,10 +16,6 @@ fail() { #msg exit_fail } -yield() { - ping localhost -c 1 || sleep .001 || usleep 1 || sleep 1 -} - if [ ! -f set_event -o ! -d events/sched ]; then echo "event tracing is not supported" exit_unsupported diff --git a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc index 10eb17f161b7..83a8c571e93a 100644 --- a/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/subsystem-enable.tc @@ -13,10 +13,6 @@ fail() { #msg exit_fail } -yield() { - ping localhost -c 1 || sleep .001 || usleep 1 || sleep 1 -} - if [ ! -f set_event -o ! -d events/sched ]; then echo "event tracing is not supported" exit_unsupported diff --git a/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc b/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc index 8d55a233c1a4..84d7bda08d2a 100644 --- a/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc +++ b/tools/testing/selftests/ftrace/test.d/event/toplevel-enable.tc @@ -12,10 +12,6 @@ fail() { #msg exit_fail } -yield() { - ping localhost -c 1 || sleep .001 || usleep 1 || sleep 1 -} - if [ ! -f available_events -o ! -f set_event -o ! -d events ]; then echo "event tracing is not supported" exit_unsupported diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc index 970903cb7834..64cfcc75e3c1 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc @@ -48,10 +48,6 @@ fail() { # msg exit_fail } -yield() { - ping localhost -c 1 || sleep .001 || usleep 1 || sleep 1 -} - do_test() { disable_tracing diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions index b2bfa4dd43f7..7b96e80e6b8a 100644 --- a/tools/testing/selftests/ftrace/test.d/functions +++ b/tools/testing/selftests/ftrace/test.d/functions @@ -103,3 +103,9 @@ initialize_ftrace() { # Reset ftrace to initial-state clear_trace enable_tracing } + +LOCALHOST=127.0.0.1 + +yield() { + ping $LOCALHOST -c 1 || sleep .001 || usleep 1 || sleep 1 +} diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc index 575af23df04e..f59b2a9a1f22 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-field-variable-support.tc @@ -23,7 +23,7 @@ echo 'hist:keys=comm:ts0=common_timestamp.usecs if comm=="ping"' > events/sched/ echo 'hist:keys=next_comm:wakeup_lat=common_timestamp.usecs-$ts0:onmatch(sched.sched_waking).wakeup_latency($wakeup_lat,next_pid,sched.sched_waking.prio,next_comm) if next_comm=="ping"' > events/sched/sched_switch/trigger echo 'hist:keys=pid,prio,comm:vals=lat:sort=pid,prio' > events/synthetic/wakeup_latency/trigger -ping localhost -c 3 +ping $LOCALHOST -c 3 if ! grep -q "ping" events/synthetic/wakeup_latency/hist; then fail "Failed to create inter-event histogram" fi diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc index 92aef26e2631..524d9ce361e2 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-inter-event-combined-hist.tc @@ -37,7 +37,7 @@ echo 'waking+wakeup_latency u64 lat; pid_t pid' >> synthetic_events echo 'hist:keys=pid,lat:sort=pid,lat:ww_lat=$waking_lat+$wakeup_lat:onmatch(synthetic.wakeup_latency).waking+wakeup_latency($ww_lat,pid)' >> events/synthetic/wakeup_latency/trigger echo 'hist:keys=pid,lat:sort=pid,lat' >> events/synthetic/waking+wakeup_latency/trigger -ping localhost -c 3 +ping $LOCALHOST -c 3 if ! grep -q "pid:" events/synthetic/waking+wakeup_latency/hist; then fail "Failed to create combined histogram" fi diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc index 1ef5dd4521f0..39fb65b0cd9f 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-action-hist.tc @@ -29,7 +29,8 @@ echo "Test histogram variables,simple expression support and onmatch action" echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' > events/sched/sched_wakeup/trigger echo 'hist:keys=next_pid:wakeup_lat=common_timestamp.usecs-$ts0:onmatch(sched.sched_wakeup).wakeup_latency($wakeup_lat,next_pid,next_comm) if next_comm=="ping"' > events/sched/sched_switch/trigger echo 'hist:keys=comm,pid,lat:wakeup_lat=lat:sort=lat' > events/synthetic/wakeup_latency/trigger -ping localhost -c 5 + +ping $LOCALHOST -c 5 if ! grep -q "ping" events/synthetic/wakeup_latency/hist; then fail "Failed to create onmatch action inter-event histogram" fi diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc index 7234683ee5e5..81ab3939c96a 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmatch-onmax-action-hist.tc @@ -29,7 +29,8 @@ echo "Test histogram variables,simple expression support and onmatch-onmax actio echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' > events/sched/sched_wakeup/trigger echo 'hist:keys=next_pid:wakeup_lat=common_timestamp.usecs-$ts0:onmatch(sched.sched_wakeup).wakeup_latency($wakeup_lat,next_pid,next_comm):onmax($wakeup_lat).save(next_comm,prev_pid,prev_prio,prev_comm) if next_comm=="ping"' >> events/sched/sched_switch/trigger echo 'hist:keys=comm,pid,lat:wakeup_lat=lat:sort=lat' > events/synthetic/wakeup_latency/trigger -ping localhost -c 5 + +ping $LOCALHOST -c 5 if [ ! grep -q "ping" events/synthetic/wakeup_latency/hist -o ! grep -q "max:" events/sched/sched_switch/hist]; then fail "Failed to create onmatch-onmax action inter-event histogram" fi diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc index 55b9630a1d70..1180ab5f0845 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-onmax-action-hist.tc @@ -27,7 +27,8 @@ echo "Test onmax action" echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' >> events/sched/sched_waking/trigger echo 'hist:keys=next_pid:wakeup_lat=common_timestamp.usecs-$ts0:onmax($wakeup_lat).save(next_comm,prev_pid,prev_prio,prev_comm) if next_comm=="ping"' >> events/sched/sched_switch/trigger -ping localhost -c 3 + +ping $LOCALHOST -c 3 if ! grep -q "max:" events/sched/sched_switch/hist; then fail "Failed to create onmax action inter-event histogram" fi -- cgit v1.2.3 From af2a0750f374935c07b8beab78d6032b1ca2218d Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:17:38 +0900 Subject: selftests/ftrace: Improve kprobe on module testcase to load/unload module Improve kprobe events on module testcase to check module load/unload with disabled/enabled events. This also change the target module to trace_printk.ko, so it depends on CONFIG_SAMPLE_TRACE_PRINTK=m. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/config | 4 +++ .../ftrace/test.d/kprobe/kprobe_module.tc | 35 ++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/config b/tools/testing/selftests/ftrace/config index 07db5ab09cc7..32cf61b6bb54 100644 --- a/tools/testing/selftests/ftrace/config +++ b/tools/testing/selftests/ftrace/config @@ -7,3 +7,7 @@ CONFIG_HIST_TRIGGERS=y CONFIG_PREEMPT_TRACER=y CONFIG_IRQSOFF_TRACER=y CONFIG_PREEMPTIRQ_DELAY_TEST=m +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_SAMPLES=y +CONFIG_SAMPLE_TRACE_PRINTK=m diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc index ec6c5fffef1e..d861bd776c5e 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_module.tc @@ -4,11 +4,18 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable +rmmod trace-printk ||: +if ! modprobe trace-printk ; then + echo "No trace-printk sample module - please make CONFIG_SAMPLE_TRACE_PRINTK= +m" + exit_unresolved; +fi + +MOD=trace_printk +FUNC=trace_printk_irq_work + :;: "Add an event on a module function without specifying event name" ;: -MOD=`lsmod | head -n 2 | tail -n 1 | cut -f1 -d" "` -FUNC=`grep -m 1 ".* t .*\\[$MOD\\]" /proc/kallsyms | xargs | cut -f3 -d" "` -[ "x" != "x$MOD" -a "y" != "y$FUNC" ] || exit_unresolved echo "p $MOD:$FUNC" > kprobe_events PROBE_NAME=`echo $MOD:$FUNC | tr ".:" "_"` test -d events/kprobes/p_${PROBE_NAME}_0 || exit_failure @@ -22,3 +29,25 @@ test -d events/kprobes/event1 || exit_failure echo "p:kprobes1/event1 $MOD:$FUNC" > kprobe_events test -d events/kprobes1/event1 || exit_failure + +:;: "Remove target module, but event still be there" ;: +if ! rmmod trace-printk ; then + echo "Failed to unload module - please enable CONFIG_MODULE_UNLOAD" + exit_unresolved; +fi +test -d events/kprobes1/event1 + +:;: "Check posibility to defining events on unloaded module";: +echo "p:event2 $MOD:$FUNC" >> kprobe_events + +:;: "Target is gone, but we can prepare for next time";: +echo 1 > events/kprobes1/event1/enable + +:;: "Load module again, which means the event1 should be recorded";: +modprobe trace-printk +grep "event1:" trace + +:;: "Remove the module again and check the event is not locked" +rmmod trace-printk +echo 0 > events/kprobes1/event1/enable +echo "-:kprobes1/event1" >> kprobe_events -- cgit v1.2.3 From e42e5c46b07d09b6b985ceea954eae165103c7ed Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:18:06 +0900 Subject: selftests/ftrace: Improve kprobe testcase to check log data Improve kprobe testcase to check the log data correctness and ensure the event definition is corrctly including argument definition. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc index 9578535ccd67..bcdecf80a8f1 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args.tc @@ -5,10 +5,14 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable echo 'p:testprobe _do_fork $stack $stack0 +0($stack)' > kprobe_events -grep testprobe kprobe_events +grep testprobe kprobe_events | grep -q 'arg1=\$stack arg2=\$stack0 arg3=+0(\$stack)' test -d events/kprobes/testprobe + echo 1 > events/kprobes/testprobe/enable ( echo "forked") +grep testprobe trace | grep '_do_fork' | \ + grep -q 'arg1=0x[[:xdigit:]]* arg2=0x[[:xdigit:]]* arg3=0x[[:xdigit:]]*$' + echo 0 > events/kprobes/testprobe/enable echo "-:testprobe" >> kprobe_events clear_trace -- cgit v1.2.3 From 1e51263ef4e637199b725df6811f0d2bea70156c Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:18:35 +0900 Subject: selftests/ftrace: Improve kretprobe testcase to check log data Improve kretprobe testcase to check the log data correctness and ensure the event definition is corrctly including argument definition. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc index 4c6b21e30b02..ac9ab4a12e53 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kretprobe_args.tc @@ -4,11 +4,16 @@ [ -f kprobe_events ] || exit_unsupported # this is configurable +# Add new kretprobe event echo 'r:testprobe2 _do_fork $retval' > kprobe_events -grep testprobe2 kprobe_events +grep testprobe2 kprobe_events | grep -q 'arg1=\$retval' test -d events/kprobes/testprobe2 + echo 1 > events/kprobes/testprobe2/enable ( echo "forked") + +cat trace | grep testprobe2 | grep -q '<- _do_fork' + echo 0 > events/kprobes/testprobe2/enable echo '-:testprobe2' >> kprobe_events clear_trace -- cgit v1.2.3 From 00b2f2bc3aacb658c44dcb76e004423b6a6380eb Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:19:03 +0900 Subject: selftests/ftrace: Test kprobe-event argument with various bitsize Improve the kprobe-event with argument types testcase to test it with various bitsize. kprobe-event argument can be recorded in given types with various bitsize (8, 16, 32, 64), thus the type testcase should test the different bitsize too. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../ftrace/test.d/kprobe/kprobe_args_type.tc | 48 ++++++++++++++-------- 1 file changed, 32 insertions(+), 16 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc index 37443dd23d15..1bcb67dcae26 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc @@ -6,29 +6,45 @@ grep "x8/16/32/64" README > /dev/null || exit_unsupported # version issue -echo 'p:testprobe _do_fork $stack0:s32 $stack0:u32 $stack0:x32 $stack0:b8@4/32' > kprobe_events -grep testprobe kprobe_events -test -d events/kprobes/testprobe - -echo 1 > events/kprobes/testprobe/enable -( echo "forked") -echo 0 > events/kprobes/testprobe/enable -ARGS=`tail -n 1 trace | sed -e 's/.* arg1=\(.*\) arg2=\(.*\) arg3=\(.*\) arg4=\(.*\)/\1 \2 \3 \4/'` +gen_event() { # Bitsize + echo "p:testprobe _do_fork \$stack0:s$1 \$stack0:u$1 \$stack0:x$1 \$stack0:b4@4/$1" +} -check_types() { - X1=`printf "%x" $1 | tail -c 8` +check_types() { # s-type u-type x-type bf-type width + test $# -eq 5 + CW=$5 + CW=$((CW / 4)) + X1=`printf "%x" $1 | tail -c ${CW}` X2=`printf "%x" $2` X3=`printf "%x" $3` test $X1 = $X2 test $X2 = $X3 test 0x$X3 = $3 - B4=`printf "%02x" $4` - B3=`echo -n $X3 | tail -c 3 | head -c 2` + B4=`printf "%1x" $4` + B3=`printf "%03x" 0x$X3 | tail -c 2 | head -c 1` test $B3 = $B4 } -check_types $ARGS -echo "-:testprobe" >> kprobe_events -clear_trace -test -d events/kprobes/testprobe && exit_fail || exit_pass +for width in 64 32 16 8; do + : "Add new event with basic types" + gen_event $width > kprobe_events + grep testprobe kprobe_events + test -d events/kprobes/testprobe + + : "Trace the event" + echo 1 > events/kprobes/testprobe/enable + ( echo "forked") + echo 0 > events/kprobes/testprobe/enable + + : "Confirm the arguments is recorded in given types correctly" + ARGS=`grep "testprobe" trace | sed -e 's/.* arg1=\(.*\) arg2=\(.*\) arg3=\(.*\) arg4=\(.*\)/\1 \2 \3 \4/'` + check_types $ARGS $width + + : "Clear event for next loop" + echo "-:testprobe" >> kprobe_events + clear_trace + +done + +exit_pass -- cgit v1.2.3 From ddbc31e63aef5e7d12f28f3ac4742420bd5b274c Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:19:31 +0900 Subject: selftests/ftrace: Check set_event_pid result Ensure the set_event_pid shows set pid list. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/event/event-pid.tc | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc index e28897738500..f9cb214220b1 100644 --- a/tools/testing/selftests/ftrace/test.d/event/event-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/event/event-pid.tc @@ -42,6 +42,7 @@ do_reset read mypid rest < /proc/self/stat echo $mypid > set_event_pid +grep -q $mypid set_event_pid echo 'sched:sched_switch' > set_event yield -- cgit v1.2.3 From b0898e84851c96248c376897c96f3fde3ff7e3b5 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:19:59 +0900 Subject: selftests/ftrace: Add kprobe event with $comm argument testcase Add kprobe-event with $comm argument testcase to ftracetest. This not only checks syntax but also checks log file. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/kprobe/kprobe_args_comm.tc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_comm.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_comm.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_comm.tc new file mode 100644 index 000000000000..15c1f70fcaf9 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_comm.tc @@ -0,0 +1,17 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe event with comm arguments + +[ -f kprobe_events ] || exit_unsupported # this is configurable + +grep -A1 "fetcharg:" README | grep -q "\$comm" || exit_unsupported # this is too old + +echo 'p:testprobe _do_fork comm=$comm ' > kprobe_events +grep testprobe kprobe_events | grep -q 'comm=$comm' +test -d events/kprobes/testprobe + +echo 1 > events/kprobes/testprobe/enable +( echo "forked") +grep testprobe trace | grep -q 'comm=".*"' + +exit 0 -- cgit v1.2.3 From 32f6e5da83c7f06db7e4efbdacdbc59c16fd6ecb Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:20:28 +0900 Subject: selftests/ftrace: Add kprobe profile testcase Add a testcase for testing kprobe_profile interface which provides per-kprobe event hit/misshit counts. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/test.d/kprobe/profile.tc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/profile.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc b/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc new file mode 100644 index 000000000000..0384b525cdee --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/profile.tc @@ -0,0 +1,15 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe dynamic event - adding and removing + +[ -f kprobe_events ] || exit_unsupported # this is configurable + +! grep -q 'myevent' kprobe_profile +echo p:myevent _do_fork > kprobe_events +grep -q 'myevent[[:space:]]*0[[:space:]]*0$' kprobe_profile +echo 1 > events/kprobes/myevent/enable +( echo "forked" ) +grep -q 'myevent[[:space:]]*[[:digit:]]*[[:space:]]*0$' kprobe_profile +echo 0 > events/kprobes/myevent/enable +echo > kprobe_events +! grep -q 'myevent' kprobe_profile -- cgit v1.2.3 From 873b4af2103fbd1bb7895be202a58878254d89bc Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:20:56 +0900 Subject: selftests/ftrace: Add kprobe-event with symbol argument test Add a testcase for kprobe-event with @symbol argument. Since @symbol needs to refer the kernel data symbol (linux_proc_banner), it requires CONFIG_KALLSYMS_ALL. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/config | 1 + .../ftrace/test.d/kprobe/kprobe_args_symbol.tc | 39 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_symbol.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/config b/tools/testing/selftests/ftrace/config index 32cf61b6bb54..b7ac74fcb0fa 100644 --- a/tools/testing/selftests/ftrace/config +++ b/tools/testing/selftests/ftrace/config @@ -11,3 +11,4 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_SAMPLES=y CONFIG_SAMPLE_TRACE_PRINTK=m +CONFIG_KALLSYMS_ALL=y diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_symbol.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_symbol.tc new file mode 100644 index 000000000000..2b6dd33f9076 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_symbol.tc @@ -0,0 +1,39 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe event symbol argument + +[ -f kprobe_events ] || exit_unsupported # this is configurable + +SYMBOL="linux_proc_banner" + +if [ ! -f /proc/kallsyms ]; then + echo "Can not check the target symbol - please enable CONFIG_KALLSYMS" + exit_unresolved +elif ! grep "$SYMBOL\$" /proc/kallsyms; then + echo "Linux banner is not exported - please enable CONFIG_KALLSYMS_ALL" + exit_unresolved +fi + +: "Test get basic types symbol argument" +echo "p:testprobe_u _do_fork arg1=@linux_proc_banner:u64 arg2=@linux_proc_banner:u32 arg3=@linux_proc_banner:u16 arg4=@linux_proc_banner:u8" > kprobe_events +echo "p:testprobe_s _do_fork arg1=@linux_proc_banner:s64 arg2=@linux_proc_banner:s32 arg3=@linux_proc_banner:s16 arg4=@linux_proc_banner:s8" >> kprobe_events +if grep -q "x8/16/32/64" README; then + echo "p:testprobe_x _do_fork arg1=@linux_proc_banner:x64 arg2=@linux_proc_banner:x32 arg3=@linux_proc_banner:x16 arg4=@linux_proc_banner:x8" >> kprobe_events +fi +echo "p:testprobe_bf _do_fork arg1=@linux_proc_banner:b8@4/32" >> kprobe_events +echo 1 > events/kprobes/enable +(echo "forked") +echo 0 > events/kprobes/enable +grep "testprobe_[usx]:.* arg1=.* arg2=.* arg3=.* arg4=.*" trace +grep "testprobe_bf:.* arg1=.*" trace + +: "Test get string symbol argument" +echo "p:testprobe_str _do_fork arg1=@linux_proc_banner:string" > kprobe_events +echo 1 > events/kprobes/enable +(echo "forked") +echo 0 > events/kprobes/enable +RESULT=`grep "testprobe_str" trace | sed -e 's/.* arg1=\(.*\)/\1/'` + +RESULT=`echo $RESULT | sed -e 's/.* \((.*)\) \((.*)\) .*/\1 \2/'` +ORIG=`cat /proc/version | sed -e 's/.* \((.*)\) \((.*)\) .*/\1 \2/'` +test "$RESULT" = "$ORIG" -- cgit v1.2.3 From 4d23e9b4fd2e44f10d22b9d2ba1dfb3fff8cd9d5 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:21:25 +0900 Subject: selftests/ftrace: Add trace_printk sample module test Add trace_printk sample module test. This requires to enable trace_printk.ko module for test. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/event/trace_printk.tc | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/event/trace_printk.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/event/trace_printk.tc b/tools/testing/selftests/ftrace/test.d/event/trace_printk.tc new file mode 100644 index 000000000000..b02550b42be9 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/event/trace_printk.tc @@ -0,0 +1,27 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Test trace_printk from module + +rmmod trace-printk ||: +if ! modprobe trace-printk ; then + echo "No trace-printk sample module - please make CONFIG_SAMPLE_TRACE_PRINTK=m" + exit_unresolved; +fi + +echo "Waiting for irq work" +sleep 1 + +grep -q ": This .* trace_bputs" trace +grep -q ": This .* trace_puts" trace +grep -q ": This .* trace_bprintk" trace +grep -q ": This .* trace_printk" trace + +grep -q ": (irq) .* trace_bputs" trace +grep -q ": (irq) .* trace_puts" trace +grep -q ": (irq) .* trace_bprintk" trace +grep -q ": (irq) .* trace_printk" trace + +grep -q "This is a %s that will use trace_bprintk" printk_formats +grep -q "(irq) This is a static string that will use trace_bputs" printk_formats + +rmmod trace-printk ||: -- cgit v1.2.3 From c21ceceae94dc22f5b42c8174efa4597a0c990de Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:21:53 +0900 Subject: selftests/ftrace: Add ringbuffer size changing testcase Add a testcase for changing ringbuffer size. This tests not only ringbuffer size but also tests the imbalance per-cpu buffer size change too. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- .../ftrace/test.d/00basic/ringbuffer_size.tc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/00basic/ringbuffer_size.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/00basic/ringbuffer_size.tc b/tools/testing/selftests/ftrace/test.d/00basic/ringbuffer_size.tc new file mode 100644 index 000000000000..ab70f0077c35 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/00basic/ringbuffer_size.tc @@ -0,0 +1,22 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Change the ringbuffer size +# flags: instance + +rb_size_test() { +ORIG=`cat buffer_size_kb` + +expr $ORIG / 2 > buffer_size_kb + +expr $ORIG \* 2 > buffer_size_kb + +echo $ORIG > buffer_size_kb +} + +rb_size_test + +: "If per-cpu buffer is supported, imbalance it" +if [ -d per_cpu/cpu0 ]; then + cd per_cpu/cpu0 + rb_size_test +fi -- cgit v1.2.3 From 5b56a07b9c461bb4db26797c82d6ac3be0f83a94 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:22:21 +0900 Subject: selftests/ftrace: Add function profiling stat testcase Add a testcase for function profiling per-cpu statistics interface. There is already func_profile.tc, but that is mainly focusing on the combination of function-profiler and function tracer. This testcase ensures trace_stat per-cpu function statistics is correctly updated. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- .../ftrace/test.d/ftrace/func_profile_stat.tc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/func_profile_stat.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_profile_stat.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_profile_stat.tc new file mode 100644 index 000000000000..0d501058aa75 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_profile_stat.tc @@ -0,0 +1,22 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: ftrace - function profiling + +[ ! -f function_profile_enabled ] && exit_unsupported + +: "Enable function profile" +echo 1 > function_profile_enabled + +: "Profile must be updated" +cp trace_stat/function0 $TMPDIR/ +( echo "forked"; sleep 1 ) +: "diff returns 0 if there is no difference" +! diff trace_stat/function0 $TMPDIR/function0 + +echo 0 > function_profile_enabled + +: "Profile must NOT be updated" +cp trace_stat/function0 $TMPDIR/ +( echo "forked"; sleep 1 ) +: "diff returns 0 if there is no difference" +diff trace_stat/function0 $TMPDIR/function0 -- cgit v1.2.3 From 47c509d30db4cef11a406f87279633cd21380163 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:22:49 +0900 Subject: selftests/ftrace: Add max stack tracer testcase Add a testcase for max stack tracer, which checks basic max stack usage tracing and its filter feature. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../ftrace/test.d/ftrace/func_stack_tracer.tc | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/func_stack_tracer.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_stack_tracer.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_stack_tracer.tc new file mode 100644 index 000000000000..b414f0e3c646 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_stack_tracer.tc @@ -0,0 +1,39 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: ftrace - Max stack tracer +# Test the basic function of max-stack usage tracing + +if [ ! -f stack_trace ]; then + echo "Max stack tracer is not supported - please make CONFIG_STACK_TRACER=y" + exit_unsupported +fi + +echo > stack_trace_filter +echo 0 > stack_max_size +echo 1 > /proc/sys/kernel/stack_tracer_enabled + +: "Fork and wait for the first entry become !lock" +timeout=10 +while [ $timeout -ne 0 ]; do + ( echo "forked" ) + FL=`grep " 0)" stack_trace` + echo $FL | grep -q "lock" || break; + timeout=$((timeout - 1)) +done +echo 0 > /proc/sys/kernel/stack_tracer_enabled + +echo '*lock*' > stack_trace_filter +test `cat stack_trace_filter | wc -l` -eq `grep lock stack_trace_filter | wc -l` + +echo 0 > stack_max_size +echo 1 > /proc/sys/kernel/stack_tracer_enabled + +: "Fork and always the first entry including lock" +timeout=10 +while [ $timeout -ne 0 ]; do + ( echo "forked" ) + FL=`grep " 0)" stack_trace` + echo $FL | grep -q "lock" + timeout=$((timeout - 1)) +done +echo 0 > /proc/sys/kernel/stack_tracer_enabled -- cgit v1.2.3 From 7bc026d6c0328f49469a56168b50ff7a498a5810 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:23:18 +0900 Subject: selftests/ftrace: Add function filter on module testcase Add a testcase for function filter on module. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../ftrace/test.d/ftrace/func_mod_trace.tc | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/func_mod_trace.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_mod_trace.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_mod_trace.tc new file mode 100644 index 000000000000..9330c873f9fe --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_mod_trace.tc @@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: ftrace - function trace on module + +[ ! -f set_ftrace_filter ] && exit_unsupported + +: "mod: allows to filter a non exist function" +echo 'non_exist_func:mod:non_exist_module' > set_ftrace_filter +grep -q "non_exist_func" set_ftrace_filter + +: "mod: on exist module" +echo '*:mod:trace_printk' > set_ftrace_filter +if ! modprobe trace-printk ; then + echo "No trace-printk sample module - please make CONFIG_SAMPLE_TRACE_PRINTK= +m" + exit_unresolved; +fi + +: "Wildcard should be resolved after loading module" +grep -q "trace_printk_irq_work" set_ftrace_filter + +: "After removing the filter becomes empty" +rmmod trace_printk +test `cat set_ftrace_filter | wc -l` -eq 0 -- cgit v1.2.3 From 174fd823397a2f19905a20bbcf47907e30e58444 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:23:46 +0900 Subject: selftests/ftrace: Add trace_pipe testcase Add a simple testcase for trace_pipe which can consume ringbuffer. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/00basic/trace_pipe.tc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/00basic/trace_pipe.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/00basic/trace_pipe.tc b/tools/testing/selftests/ftrace/test.d/00basic/trace_pipe.tc new file mode 100644 index 000000000000..5058fbcfd90f --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/00basic/trace_pipe.tc @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: trace_pipe and trace_marker +# flags: instance + +[ ! -f trace_marker ] && exit_unsupported + +echo "test input 1" > trace_marker + +: "trace interface never consume the ring buffer" +grep -q "test input 1" trace +grep -q "test input 1" trace + +: "trace interface never consume the ring buffer" +head -n 1 trace_pipe | grep -q "test input 1" +! grep -q "test input 1" trace -- cgit v1.2.3 From 91ae08a265142d9378f7d972e00de29d5c91a2de Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:24:14 +0900 Subject: selftests/ftrace: Add stacktrace ftrace filter command testcase Add a test case for stacktrace filter command for ftrace. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/ftrace/func-filter-stacktrace.tc | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/func-filter-stacktrace.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-stacktrace.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-stacktrace.tc new file mode 100644 index 000000000000..bf72e783d014 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-stacktrace.tc @@ -0,0 +1,12 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL2.0 +# description: ftrace - stacktrace filter command +# flags: instance + +echo _do_fork:stacktrace >> set_ftrace_filter + +grep -q "_do_fork:stacktrace:unlimited" set_ftrace_filter + +(echo "forked"; sleep 1) + +grep -q "" trace -- cgit v1.2.3 From c1edd060b41383bb6fa76008aacf4b0b3b1c46e7 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:24:42 +0900 Subject: selftests/ftrace: Add wakeup tracer testcase Add a testcase for wakeup tracer. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/config | 1 + .../selftests/ftrace/test.d/tracer/wakeup.tc | 25 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/tracer/wakeup.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/config b/tools/testing/selftests/ftrace/config index b7ac74fcb0fa..c2c8de4fafff 100644 --- a/tools/testing/selftests/ftrace/config +++ b/tools/testing/selftests/ftrace/config @@ -4,6 +4,7 @@ CONFIG_FUNCTION_PROFILER=y CONFIG_TRACER_SNAPSHOT=y CONFIG_STACK_TRACER=y CONFIG_HIST_TRIGGERS=y +CONFIG_SCHED_TRACER=y CONFIG_PREEMPT_TRACER=y CONFIG_IRQSOFF_TRACER=y CONFIG_PREEMPTIRQ_DELAY_TEST=m diff --git a/tools/testing/selftests/ftrace/test.d/tracer/wakeup.tc b/tools/testing/selftests/ftrace/test.d/tracer/wakeup.tc new file mode 100644 index 000000000000..e3005fa785f0 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/tracer/wakeup.tc @@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL2.0 +# description: Test wakeup tracer + +if ! which chrt ; then + echo "chrt is not found. This test requires nice command." + exit_unresolved +fi + +if ! grep -wq "wakeup" available_tracers ; then + echo "wakeup tracer is not supported" + exit_unsupported +fi + +echo wakeup > current_tracer +echo 1 > tracing_on +echo 0 > tracing_max_latency + +: "Wakeup higher priority task" +chrt -f 5 sleep 1 + +echo 0 > tracing_on +grep '+ \[[[:digit:]]*\]' trace +grep '==> \[[[:digit:]]*\]' trace + -- cgit v1.2.3 From c575dea2c1a50306d382615fad06767564734e0b Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:25:11 +0900 Subject: selftests/ftrace: Add wakeup_rt tracer testcase Add a testcase for wakeup_rt tracer. This requires chrt command to test. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/tracer/wakeup_rt.tc | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/tracer/wakeup_rt.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/tracer/wakeup_rt.tc b/tools/testing/selftests/ftrace/test.d/tracer/wakeup_rt.tc new file mode 100644 index 000000000000..f99b5178e00a --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/tracer/wakeup_rt.tc @@ -0,0 +1,25 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL2.0 +# description: Test wakeup RT tracer + +if ! which chrt ; then + echo "chrt is not found. This test requires chrt command." + exit_unresolved +fi + +if ! grep -wq "wakeup_rt" available_tracers ; then + echo "wakeup_rt tracer is not supported" + exit_unsupported +fi + +echo wakeup_rt > current_tracer +echo 1 > tracing_on +echo 0 > tracing_max_latency + +: "Wakeup a realtime task" +chrt -f 5 sleep 1 + +echo 0 > tracing_on +grep "+ \[[[:digit:]]*\]" trace +grep "==> \[[[:digit:]]*\]" trace + -- cgit v1.2.3 From 8ffade754bdf8350f63792c6125346d77f975b41 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 30 Aug 2018 23:25:39 +0900 Subject: selftests/ftrace: Add ftrace cpumask testcase Add a testcase for tracing_cpumask with function tracer. Signed-off-by: Masami Hiramatsu Signed-off-by: Shuah Khan (Samsung OSG) --- .../selftests/ftrace/test.d/ftrace/func_cpumask.tc | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/func_cpumask.tc (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_cpumask.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_cpumask.tc new file mode 100644 index 000000000000..0e6810743576 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_cpumask.tc @@ -0,0 +1,42 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL2.0 +# description: ftrace - function trace with cpumask + +if ! which nproc ; then + nproc() { + ls -d /sys/devices/system/cpu/cpu[0-9]* | wc -l + } +fi + +NP=`nproc` + +if [ $NP -eq 1 ] ;then + echo "We can not test cpumask on UP environment" + exit_unresolved +fi + +ORIG_CPUMASK=`cat tracing_cpumask` + +do_reset() { + echo $ORIG_CPUMASK > tracing_cpumask +} + +echo 0 > tracing_on +echo > trace +: "Bitmask only record on CPU1" +echo 2 > tracing_cpumask +MASK=0x`cat tracing_cpumask` +test `printf "%d" $MASK` -eq 2 || do_reset + +echo function > current_tracer +echo 1 > tracing_on +(echo "forked") +echo 0 > tracing_on + +: "Check CPU1 events are recorded" +grep -q -e "\[001\]" trace || do_reset + +: "There should be No other cpu events" +! grep -qv -e "\[001\]" -e "^#" trace || do_reset + +do_reset -- cgit v1.2.3 From 9a244229a4b850b11952a0df79607c69b18fd8df Mon Sep 17 00:00:00 2001 From: "Shuah Khan (Samsung OSG)" Date: Wed, 26 Sep 2018 13:07:11 -0600 Subject: selftests: watchdog: fix message when /dev/watchdog open fails When /dev/watchdog open fails, watchdog exits with "watchdog not enabled" message. This is incorrect when open fails due to insufficient privilege. Fix message to clearly state the reason when open fails with EACCESS when a non-root user runs it. Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/watchdog/watchdog-test.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index 6e290874b70e..e029e2017280 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -89,7 +89,13 @@ int main(int argc, char *argv[]) fd = open("/dev/watchdog", O_WRONLY); if (fd == -1) { - printf("Watchdog device not enabled.\n"); + if (errno == ENOENT) + printf("Watchdog device not enabled.\n"); + else if (errno == EACCES) + printf("Run watchdog as root.\n"); + else + printf("Watchdog device open failed %s\n", + strerror(errno)); exit(-1); } -- cgit v1.2.3 From 04d5e4bd37516ad60854eb74592c7dbddd75d277 Mon Sep 17 00:00:00 2001 From: Jerry Hoemann Date: Wed, 26 Sep 2018 15:23:08 -0600 Subject: selftests: watchdog: Fix error message. Printf's say errno but print the string version of error. Make consistent. Signed-off-by: Jerry Hoemann Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/watchdog/watchdog-test.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index e029e2017280..f1c6e025cbe5 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) printf("Last boot is caused by: %s.\n", (flags != 0) ? "Watchdog" : "Power-On-Reset"); else - printf("WDIOC_GETBOOTSTATUS errno '%s'\n", strerror(errno)); + printf("WDIOC_GETBOOTSTATUS error '%s'\n", strerror(errno)); break; case 'd': flags = WDIOS_DISABLECARD; @@ -117,7 +117,7 @@ int main(int argc, char *argv[]) if (!ret) printf("Watchdog card disabled.\n"); else - printf("WDIOS_DISABLECARD errno '%s'\n", strerror(errno)); + printf("WDIOS_DISABLECARD error '%s'\n", strerror(errno)); break; case 'e': flags = WDIOS_ENABLECARD; @@ -125,7 +125,7 @@ int main(int argc, char *argv[]) if (!ret) printf("Watchdog card enabled.\n"); else - printf("WDIOS_ENABLECARD errno '%s'\n", strerror(errno)); + printf("WDIOS_ENABLECARD error '%s'\n", strerror(errno)); break; case 'p': ping_rate = strtoul(optarg, NULL, 0); @@ -139,7 +139,7 @@ int main(int argc, char *argv[]) if (!ret) printf("Watchdog timeout set to %u seconds.\n", flags); else - printf("WDIOC_SETTIMEOUT errno '%s'\n", strerror(errno)); + printf("WDIOC_SETTIMEOUT error '%s'\n", strerror(errno)); break; default: usage(argv[0]); -- cgit v1.2.3 From 7514c39cf1d9433203c9b7c6674dbfa429db804b Mon Sep 17 00:00:00 2001 From: Jerry Hoemann Date: Wed, 26 Sep 2018 15:23:09 -0600 Subject: selftests: watchdog: Add gettimeout and get|set pretimeout Add command line arguments to call ioctl WDIOC_GETTIMEOUT, WDIOC_GETPRETIMEOUT and WDIOC_SETPRETIMEOUT. Signed-off-by: Jerry Hoemann Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/watchdog/watchdog-test.c | 33 +++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index f1c6e025cbe5..e0f1600dc1b3 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -19,7 +19,7 @@ int fd; const char v = 'V'; -static const char sopts[] = "bdehp:t:"; +static const char sopts[] = "bdehp:t:Tn:N"; static const struct option lopts[] = { {"bootstatus", no_argument, NULL, 'b'}, {"disable", no_argument, NULL, 'd'}, @@ -27,6 +27,9 @@ static const struct option lopts[] = { {"help", no_argument, NULL, 'h'}, {"pingrate", required_argument, NULL, 'p'}, {"timeout", required_argument, NULL, 't'}, + {"gettimeout", no_argument, NULL, 'T'}, + {"pretimeout", required_argument, NULL, 'n'}, + {"getpretimeout", no_argument, NULL, 'N'}, {NULL, no_argument, NULL, 0x0} }; @@ -71,9 +74,13 @@ static void usage(char *progname) printf(" -h, --help Print the help message\n"); printf(" -p, --pingrate=P Set ping rate to P seconds (default %d)\n", DEFAULT_PING_RATE); printf(" -t, --timeout=T Set timeout to T seconds\n"); + printf(" -T, --gettimeout Get the timeout\n"); + printf(" -n, --pretimeout=T Set the pretimeout to T seconds\n"); + printf(" -N, --getpretimeout Get the pretimeout\n"); printf("\n"); printf("Parameters are parsed left-to-right in real-time.\n"); printf("Example: %s -d -t 10 -p 5 -e\n", progname); + printf("Example: %s -t 12 -T -n 7 -N\n", progname); } int main(int argc, char *argv[]) @@ -141,6 +148,30 @@ int main(int argc, char *argv[]) else printf("WDIOC_SETTIMEOUT error '%s'\n", strerror(errno)); break; + case 'T': + oneshot = 1; + ret = ioctl(fd, WDIOC_GETTIMEOUT, &flags); + if (!ret) + printf("WDIOC_GETTIMEOUT returns %u seconds.\n", flags); + else + printf("WDIOC_GETTIMEOUT error '%s'\n", strerror(errno)); + break; + case 'n': + flags = strtoul(optarg, NULL, 0); + ret = ioctl(fd, WDIOC_SETPRETIMEOUT, &flags); + if (!ret) + printf("Watchdog pretimeout set to %u seconds.\n", flags); + else + printf("WDIOC_SETPRETIMEOUT error '%s'\n", strerror(errno)); + break; + case 'N': + oneshot = 1; + ret = ioctl(fd, WDIOC_GETPRETIMEOUT, &flags); + if (!ret) + printf("WDIOC_GETPRETIMEOUT returns %u seconds.\n", flags); + else + printf("WDIOC_GETPRETIMEOUT error '%s'\n", strerror(errno)); + break; default: usage(argv[0]); goto end; -- cgit v1.2.3 From d89d08ffd2c5a9043c8df7333d9bee6c09904a8f Mon Sep 17 00:00:00 2001 From: "Shuah Khan (Samsung OSG)" Date: Wed, 26 Sep 2018 16:37:43 -0600 Subject: selftests: watchdog: Fix ioctl SET* error paths to take oneshot exit path When ioctls for WDIOC_SETOPTIONS (WDIOS_DISABLECARD or WDIOS_ENABLECARD), WDIOC_SETTIMEOUT, and WDIOC_SETPRETIMEOUT fail, the error path continues to handler watchdog timer until user terminates it. When ioctl returns error, it might not be safe to let the watchdog tick. The error could be due an unsupported ioctl command or some other error. Fix it to handle error paths as oneshot to stop the watchdog ticks and exit right away. Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/watchdog/watchdog-test.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/watchdog/watchdog-test.c b/tools/testing/selftests/watchdog/watchdog-test.c index e0f1600dc1b3..c6bd9a68306b 100644 --- a/tools/testing/selftests/watchdog/watchdog-test.c +++ b/tools/testing/selftests/watchdog/watchdog-test.c @@ -123,16 +123,20 @@ int main(int argc, char *argv[]) ret = ioctl(fd, WDIOC_SETOPTIONS, &flags); if (!ret) printf("Watchdog card disabled.\n"); - else + else { printf("WDIOS_DISABLECARD error '%s'\n", strerror(errno)); + oneshot = 1; + } break; case 'e': flags = WDIOS_ENABLECARD; ret = ioctl(fd, WDIOC_SETOPTIONS, &flags); if (!ret) printf("Watchdog card enabled.\n"); - else + else { printf("WDIOS_ENABLECARD error '%s'\n", strerror(errno)); + oneshot = 1; + } break; case 'p': ping_rate = strtoul(optarg, NULL, 0); @@ -145,8 +149,10 @@ int main(int argc, char *argv[]) ret = ioctl(fd, WDIOC_SETTIMEOUT, &flags); if (!ret) printf("Watchdog timeout set to %u seconds.\n", flags); - else + else { printf("WDIOC_SETTIMEOUT error '%s'\n", strerror(errno)); + oneshot = 1; + } break; case 'T': oneshot = 1; @@ -161,8 +167,10 @@ int main(int argc, char *argv[]) ret = ioctl(fd, WDIOC_SETPRETIMEOUT, &flags); if (!ret) printf("Watchdog pretimeout set to %u seconds.\n", flags); - else + else { printf("WDIOC_SETPRETIMEOUT error '%s'\n", strerror(errno)); + oneshot = 1; + } break; case 'N': oneshot = 1; -- cgit v1.2.3 From a4f222e000b3f75bd1ab763dcc289d004ff6596e Mon Sep 17 00:00:00 2001 From: Fathi Boudra Date: Fri, 7 Sep 2018 09:34:54 -0500 Subject: selftests: gpio: restructure Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch cleans up the Makefile by restructuring a couple of things, namely: 1) change explicit paths in targets for variables 2) substitute a variable (BINARIES) for another, part of the selftests build system (TEST_PROGS_EXTENDED) 3) proper cleaning up of the EXTRA objects The resulting Makefile is much more readable and manageable. Signed-off-by: Fathi Boudra Signed-off-by: Daniel Díaz Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/gpio/Makefile | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/gpio/Makefile b/tools/testing/selftests/gpio/Makefile index 4665cdbf1a8d..a4a01ec8b2a5 100644 --- a/tools/testing/selftests/gpio/Makefile +++ b/tools/testing/selftests/gpio/Makefile @@ -1,28 +1,26 @@ # SPDX-License-Identifier: GPL-2.0 +CFLAGS += -O2 -g -std=gnu99 -Wall -I../../../../usr/include/ +LDLIBS += -lmount -I/usr/include/libmount + TEST_PROGS := gpio-mockup.sh -TEST_FILES := gpio-mockup-sysfs.sh $(BINARIES) -BINARIES := gpio-mockup-chardev -EXTRA_PROGS := ../gpiogpio-event-mon ../gpiogpio-hammer ../gpiolsgpio -EXTRA_DIRS := ../gpioinclude/ -EXTRA_OBJS := ../gpiogpio-event-mon-in.o ../gpiogpio-event-mon.o -EXTRA_OBJS += ../gpiogpio-hammer-in.o ../gpiogpio-utils.o ../gpiolsgpio-in.o -EXTRA_OBJS += ../gpiolsgpio.o +TEST_FILES := gpio-mockup-sysfs.sh +TEST_PROGS_EXTENDED := gpio-mockup-chardev + +GPIODIR := ../../../gpio +GPIOOBJ := gpio-utils.o include ../lib.mk -all: $(BINARIES) +all: $(TEST_PROGS_EXTENDED) override define CLEAN - $(RM) $(BINARIES) $(EXTRA_PROGS) $(EXTRA_OBJS) - $(RM) -r $(EXTRA_DIRS) + $(RM) $(TEST_PROGS_EXTENDED) + $(MAKE) -C $(GPIODIR) clean endef -CFLAGS += -O2 -g -std=gnu99 -Wall -I../../../../usr/include/ -LDLIBS += -lmount -I/usr/include/libmount - -$(BINARIES):| khdr -$(BINARIES): ../../../gpio/gpio-utils.o +$(TEST_PROGS_EXTENDED):| khdr +$(TEST_PROGS_EXTENDED): $(GPIODIR)/$(GPIOOBJ) -../../../gpio/gpio-utils.o: - make ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) -C ../../../gpio +$(GPIODIR)/$(GPIOOBJ): + $(MAKE) -C $(GPIODIR) -- cgit v1.2.3 From 8c7f6316fccf38ca45d33ed658cc4813837b6294 Mon Sep 17 00:00:00 2001 From: Daniel Díaz Date: Fri, 7 Sep 2018 09:34:55 -0500 Subject: selftests: gpio: Fix OUTPUT directory in Makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When simply running `make' from the selftests top dir, this error shows up: cc -O2 -g -std=gnu99 -Wall -I../../../../usr/include/ -I/usr/include/libmount -I/usr/include/blkid -I/usr/include/uuid gpio-mockup-chardev.c ../../../gpio/gpio-utils.o -lmount -o gpio-mockup-chardev cc: error: ../../../gpio/gpio-utils.o: No such file or directory : recipe for target 'gpio-mockup-chardev' failed make[1]: *** [gpio-mockup-chardev] Error 1 because the output directory is set to "selftests/gpio" and all binaries built from ../../../gpio/ end up there. In fact, they appear as, exempli gratia: * gpiogpio-event-mon * gpiogpio-hammer * gpioinclude/ * gpiolsgpio which is wrong, as it's missing a directory separator somewhere. This patch sets straight the output directory when building ../../../gpio/ so that binaries don't cross paths. Signed-off-by: Daniel Díaz Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/gpio/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/gpio/Makefile b/tools/testing/selftests/gpio/Makefile index a4a01ec8b2a5..46648427d537 100644 --- a/tools/testing/selftests/gpio/Makefile +++ b/tools/testing/selftests/gpio/Makefile @@ -7,7 +7,7 @@ TEST_PROGS := gpio-mockup.sh TEST_FILES := gpio-mockup-sysfs.sh TEST_PROGS_EXTENDED := gpio-mockup-chardev -GPIODIR := ../../../gpio +GPIODIR := $(realpath ../../../gpio) GPIOOBJ := gpio-utils.o include ../lib.mk @@ -16,11 +16,11 @@ all: $(TEST_PROGS_EXTENDED) override define CLEAN $(RM) $(TEST_PROGS_EXTENDED) - $(MAKE) -C $(GPIODIR) clean + $(MAKE) -C $(GPIODIR) OUTPUT=$(GPIODIR)/ clean endef $(TEST_PROGS_EXTENDED):| khdr $(TEST_PROGS_EXTENDED): $(GPIODIR)/$(GPIOOBJ) $(GPIODIR)/$(GPIOOBJ): - $(MAKE) -C $(GPIODIR) + $(MAKE) OUTPUT=$(GPIODIR)/ -C $(GPIODIR) -- cgit v1.2.3 From 717da97e94a9ba590632624992a9f79d53d3579b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 13 Oct 2018 16:41:05 +0100 Subject: kvm: selftests: fix spelling mistake "Insufficent" -> "Insufficient" Trivial fix to spelling mistake in TEST_ASSERT message text Signed-off-by: Colin Ian King Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/kvm/lib/kvm_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 6fd8c089cafc..a56ef6b187bb 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -123,7 +123,7 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) /* Allocate memory. */ vm = calloc(1, sizeof(*vm)); - TEST_ASSERT(vm != NULL, "Insufficent Memory"); + TEST_ASSERT(vm != NULL, "Insufficient Memory"); vm->mode = mode; vm_open(vm, perm); -- cgit v1.2.3 From 8f381ac4d3210025891c5397e744fd4c771481d7 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Fri, 12 Oct 2018 22:21:06 -0400 Subject: selftests/ftrace: Add color to the PASS / FAIL results Now that ftracetest has over 80 tests, it is difficult to simply scroll up the console window to find the failed tests when it reports just two tests have failed. In order to make this stand out better, have the color of the word "PASS" be green, "FAIL" and "XFAIL" be red, and all other results be blue. This helps tremendously in quickly spotting the failed tests by just scrolling up the console window. Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 5c71d58febb2..4946b2edfcff 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -153,7 +153,7 @@ else fi prlog() { # messages - [ -z "$LOG_FILE" ] && echo "$@" || echo "$@" | tee -a $LOG_FILE + [ -z "$LOG_FILE" ] && echo -e "$@" || echo -e "$@" | tee -a $LOG_FILE } catlog() { #file [ -z "$LOG_FILE" ] && cat $1 || cat $1 | tee -a $LOG_FILE @@ -195,37 +195,37 @@ test_on_instance() { # testfile eval_result() { # sigval case $1 in $PASS) - prlog " [PASS]" + prlog " [\e[32mPASS\e[30m]" PASSED_CASES="$PASSED_CASES $CASENO" return 0 ;; $FAIL) - prlog " [FAIL]" + prlog " [\e[31mFAIL\e[30m]" FAILED_CASES="$FAILED_CASES $CASENO" return 1 # this is a bug. ;; $UNRESOLVED) - prlog " [UNRESOLVED]" + prlog " [\e[34mUNRESOLVED\e[30m]" UNRESOLVED_CASES="$UNRESOLVED_CASES $CASENO" return 1 # this is a kind of bug.. something happened. ;; $UNTESTED) - prlog " [UNTESTED]" + prlog " [\e[34mUNTESTED\e[30m]" UNTESTED_CASES="$UNTESTED_CASES $CASENO" return 0 ;; $UNSUPPORTED) - prlog " [UNSUPPORTED]" + prlog " [\e[34mUNSUPPORTED\e[30m]" UNSUPPORTED_CASES="$UNSUPPORTED_CASES $CASENO" return $UNSUPPORTED_RESULT # depends on use case ;; $XFAIL) - prlog " [XFAIL]" + prlog " [\e[31mXFAIL\e[30m]" XFAILED_CASES="$XFAILED_CASES $CASENO" return 0 ;; *) - prlog " [UNDEFINED]" + prlog " [\e[34mUNDEFINED\e[30m]" UNDEFINED_CASES="$UNDEFINED_CASES $CASENO" return 1 # this must be a test bug ;; -- cgit v1.2.3 From fb363e2d20351e1d16629df19e7bce1a31b3227a Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Tue, 16 Oct 2018 16:13:46 +0200 Subject: selftests: kvm: Fix -Wformat warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the following warnings: dirty_log_test.c: In function ‘help’: dirty_log_test.c:216:9: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 2 has type ‘int’ [-Wformat=] printf(" -i: specify iteration counts (default: %"PRIu64")\n", ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In file included from include/test_util.h:18:0, from dirty_log_test.c:16: /usr/include/inttypes.h:105:34: note: format string is defined here # define PRIu64 __PRI64_PREFIX "u" dirty_log_test.c:218:9: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 2 has type ‘int’ [-Wformat=] printf(" -I: specify interval in ms (default: %"PRIu64" ms)\n", ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In file included from include/test_util.h:18:0, from dirty_log_test.c:16: /usr/include/inttypes.h:105:34: note: format string is defined here # define PRIu64 __PRI64_PREFIX "u" Signed-off-by: Andrea Parri Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/kvm/dirty_log_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 0c2cdc105f96..a9c4b5e21d7e 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -31,9 +31,9 @@ /* How many pages to dirty for each guest loop */ #define TEST_PAGES_PER_LOOP 1024 /* How many host loops to run (one KVM_GET_DIRTY_LOG for each loop) */ -#define TEST_HOST_LOOP_N 32 +#define TEST_HOST_LOOP_N 32UL /* Interval for each host loop (ms) */ -#define TEST_HOST_LOOP_INTERVAL 10 +#define TEST_HOST_LOOP_INTERVAL 10UL /* * Guest variables. We use these variables to share data between host -- cgit v1.2.3 From 0387662d1b6c5ad2950d8e94d5e380af3f15c05c Mon Sep 17 00:00:00 2001 From: Peng Hao Date: Fri, 12 Oct 2018 03:53:50 +0800 Subject: selftests: fix warning: "_GNU_SOURCE" redefined Makefile contains -D_GNU_SOURCE. remove define "_GNU_SOURCE" in c files. Signed-off-by: Peng Hao Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/proc/fd-001-lookup.c | 2 +- tools/testing/selftests/proc/fd-003-kthread.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/proc/fd-001-lookup.c b/tools/testing/selftests/proc/fd-001-lookup.c index a2010dfb2110..60d7948e7124 100644 --- a/tools/testing/selftests/proc/fd-001-lookup.c +++ b/tools/testing/selftests/proc/fd-001-lookup.c @@ -14,7 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ // Test /proc/*/fd lookup. -#define _GNU_SOURCE + #undef NDEBUG #include #include diff --git a/tools/testing/selftests/proc/fd-003-kthread.c b/tools/testing/selftests/proc/fd-003-kthread.c index 1d659d55368c..dc591f97b63d 100644 --- a/tools/testing/selftests/proc/fd-003-kthread.c +++ b/tools/testing/selftests/proc/fd-003-kthread.c @@ -14,7 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ // Test that /proc/$KERNEL_THREAD/fd/ is empty. -#define _GNU_SOURCE + #undef NDEBUG #include #include -- cgit v1.2.3 From 8096fbcf55c0da535bdca9adab982ec0c5affb67 Mon Sep 17 00:00:00 2001 From: Daniel Díaz Date: Tue, 16 Oct 2018 12:02:20 -0500 Subject: selftests/ftrace: Use colored output when available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If test is being directly executed (with stdout opened on the terminal) and the terminal capabilities indicate enough colors, then use the existing scheme of green, red, and blue to show when tests pass, fail or end in a different way. When running the tests redirecting the stdout, for instance, to a file, then colors are not shown, thus producing a more readable output. Signed-off-by: Daniel Díaz Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 4946b2edfcff..d987bbec675f 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -152,6 +152,21 @@ else date > $LOG_FILE fi +# Define text colors +# Check available colors on the terminal, if any +ncolors=`tput colors 2>/dev/null` +color_reset= +color_red= +color_green= +color_blue= +# If stdout exists and number of colors is eight or more, use them +if [ -t 1 -a "$ncolors" -a "$ncolors" -ge 8 ]; then + color_reset="\e[0m" + color_red="\e[31m" + color_green="\e[32m" + color_blue="\e[34m" +fi + prlog() { # messages [ -z "$LOG_FILE" ] && echo -e "$@" || echo -e "$@" | tee -a $LOG_FILE } @@ -195,37 +210,37 @@ test_on_instance() { # testfile eval_result() { # sigval case $1 in $PASS) - prlog " [\e[32mPASS\e[30m]" + prlog " [${color_green}PASS${color_reset}]" PASSED_CASES="$PASSED_CASES $CASENO" return 0 ;; $FAIL) - prlog " [\e[31mFAIL\e[30m]" + prlog " [${color_red}FAIL${color_reset}]" FAILED_CASES="$FAILED_CASES $CASENO" return 1 # this is a bug. ;; $UNRESOLVED) - prlog " [\e[34mUNRESOLVED\e[30m]" + prlog " [${color_blue}UNRESOLVED${color_reset}]" UNRESOLVED_CASES="$UNRESOLVED_CASES $CASENO" return 1 # this is a kind of bug.. something happened. ;; $UNTESTED) - prlog " [\e[34mUNTESTED\e[30m]" + prlog " [${color_blue}UNTESTED${color_reset}]" UNTESTED_CASES="$UNTESTED_CASES $CASENO" return 0 ;; $UNSUPPORTED) - prlog " [\e[34mUNSUPPORTED\e[30m]" + prlog " [${color_blue}UNSUPPORTED${color_reset}]" UNSUPPORTED_CASES="$UNSUPPORTED_CASES $CASENO" return $UNSUPPORTED_RESULT # depends on use case ;; $XFAIL) - prlog " [\e[31mXFAIL\e[30m]" + prlog " [${color_red}XFAIL${color_reset}]" XFAILED_CASES="$XFAILED_CASES $CASENO" return 0 ;; *) - prlog " [\e[34mUNDEFINED\e[30m]" + prlog " [${color_blue}UNDEFINED${color_reset}]" UNDEFINED_CASES="$UNDEFINED_CASES $CASENO" return 1 # this must be a test bug ;; -- cgit v1.2.3 From f73581f8d9a367459f36e6a4ef0065997ab14c75 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 17 Oct 2018 12:33:23 +0900 Subject: selftests/ftrace: Strip escape sequences for log file Strip escape sequences from the stream to the ftracetest summary log file. Note that all test-case results are dumped raw in each file. Signed-off-by: Masami Hiramatsu Acked-by: Steven Rostedt (VMware) Signed-off-by: Shuah Khan (Samsung OSG) --- tools/testing/selftests/ftrace/ftracetest | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index d987bbec675f..75244db70331 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -167,11 +167,18 @@ if [ -t 1 -a "$ncolors" -a "$ncolors" -ge 8 ]; then color_blue="\e[34m" fi +strip_esc() { + # busybox sed implementation doesn't accept "\x1B", so use [:cntrl:] instead. + sed -E "s/[[:cntrl:]]\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" +} + prlog() { # messages - [ -z "$LOG_FILE" ] && echo -e "$@" || echo -e "$@" | tee -a $LOG_FILE + echo -e "$@" + [ "$LOG_FILE" ] && echo -e "$@" | strip_esc >> $LOG_FILE } catlog() { #file - [ -z "$LOG_FILE" ] && cat $1 || cat $1 | tee -a $LOG_FILE + cat $1 + [ "$LOG_FILE" ] && cat $1 | strip_esc >> $LOG_FILE } prlog "=== Ftrace unit tests ===" -- cgit v1.2.3 From a3f49d97c22022666fb94101454520a7adacaf08 Mon Sep 17 00:00:00 2001 From: Naresh Kamboju Date: Thu, 25 Oct 2018 20:17:28 +0530 Subject: selftests/bpf: add config fragments BPF_STREAM_PARSER and XDP_SOCKETS BPF sockmap and hashmap are dependent on CONFIG_BPF_STREAM_PARSER and xskmap is dependent on CONFIG_XDP_SOCKETS Signed-off-by: Naresh Kamboju Signed-off-by: Daniel Borkmann --- tools/testing/selftests/bpf/config | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools') diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index dd49df5e2df4..7f90d3645af8 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -20,3 +20,5 @@ CONFIG_VXLAN=y CONFIG_GENEVE=y CONFIG_NET_CLS_FLOWER=m CONFIG_LWTUNNEL=y +CONFIG_BPF_STREAM_PARSER=y +CONFIG_XDP_SOCKETS=y -- cgit v1.2.3 From 36641ad61db5ce9befd5eb0071abb36eaff16cfc Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 24 Oct 2018 22:05:43 +0200 Subject: bpf: fix test suite to enable all unpriv program types Given BPF_PROG_TYPE_CGROUP_SKB program types are also valid in an unprivileged setting, lets not omit these tests and potentially have issues fall through the cracks. Make this more obvious by adding a small test_as_unpriv() helper. Signed-off-by: Daniel Borkmann Acked-by: Alexei Starovoitov Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 769d68a48f30..8e1a79d2792c 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -4891,6 +4891,8 @@ static struct bpf_test tests[] = { BPF_EXIT_INSN(), }, .result = ACCEPT, + .result_unpriv = REJECT, + .errstr_unpriv = "R3 pointer comparison prohibited", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { @@ -5146,6 +5148,7 @@ static struct bpf_test tests[] = { .fixup_cgroup_storage = { 1 }, .result = REJECT, .errstr = "get_local_storage() doesn't support non-zero flags", + .errstr_unpriv = "R2 leaks addr into helper function", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { @@ -5261,6 +5264,7 @@ static struct bpf_test tests[] = { .fixup_percpu_cgroup_storage = { 1 }, .result = REJECT, .errstr = "get_local_storage() doesn't support non-zero flags", + .errstr_unpriv = "R2 leaks addr into helper function", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { @@ -14050,6 +14054,13 @@ static void get_unpriv_disabled() fclose(fd); } +static bool test_as_unpriv(struct bpf_test *test) +{ + return !test->prog_type || + test->prog_type == BPF_PROG_TYPE_SOCKET_FILTER || + test->prog_type == BPF_PROG_TYPE_CGROUP_SKB; +} + static int do_test(bool unpriv, unsigned int from, unsigned int to) { int i, passes = 0, errors = 0, skips = 0; @@ -14060,10 +14071,10 @@ static int do_test(bool unpriv, unsigned int from, unsigned int to) /* Program types that are not supported by non-root we * skip right away. */ - if (!test->prog_type && unpriv_disabled) { + if (test_as_unpriv(test) && unpriv_disabled) { printf("#%d/u %s SKIP\n", i, test->descr); skips++; - } else if (!test->prog_type) { + } else if (test_as_unpriv(test)) { if (!unpriv) set_admin(false); printf("#%d/u %s ", i, test->descr); -- cgit v1.2.3 From ab21c1b5f799395232b838e98981cfed6d647905 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 24 Oct 2018 22:05:44 +0200 Subject: bpf: disallow direct packet access for unpriv in cg_skb Commit b39b5f411dcf ("bpf: add cg_skb_is_valid_access for BPF_PROG_TYPE_CGROUP_SKB") added support for returning pkt pointers for direct packet access. Given this program type is allowed for both unprivileged and privileged users, we shouldn't allow unprivileged ones to use it, e.g. besides others one reason would be to avoid any potential speculation on the packet test itself, thus guard this for root only. Fixes: b39b5f411dcf ("bpf: add cg_skb_is_valid_access for BPF_PROG_TYPE_CGROUP_SKB") Signed-off-by: Daniel Borkmann Acked-by: Alexei Starovoitov Cc: Song Liu Signed-off-by: Alexei Starovoitov --- net/core/filter.c | 6 ++++++ tools/testing/selftests/bpf/test_verifier.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/net/core/filter.c b/net/core/filter.c index 35c6933c2622..3fdddfa9a0fd 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5496,7 +5496,13 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, data_meta): case bpf_ctx_range(struct __sk_buff, flow_keys): return false; + case bpf_ctx_range(struct __sk_buff, data): + case bpf_ctx_range(struct __sk_buff, data_end): + if (!capable(CAP_SYS_ADMIN)) + return false; + break; } + if (type == BPF_WRITE) { switch (off) { case bpf_ctx_range(struct __sk_buff, mark): diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 8e1a79d2792c..36f3d3009d1a 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -4892,7 +4892,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .result_unpriv = REJECT, - .errstr_unpriv = "R3 pointer comparison prohibited", + .errstr_unpriv = "invalid bpf_context access off=76 size=4", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { -- cgit v1.2.3 From 0d0352d8b3d6d7ca9a710b40e194cbbaeb841c88 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Mon, 22 Oct 2018 00:08:48 +0900 Subject: selftests/ftrace: Fix synthetic event test to delete event correctly Fix the synthetic event test case to remove event correctly. If redirecting command to synthetic_event file without append mode, it cleans up all existing events and execute (parse) the command. This means "delete event" always fails to find the target event. Since previous synthetic event has a bug which doesn't return -ENOENT even if it fails to find the deleting event, this test passed. But fixing that bug, this test fails because this test itself has a bug. This fixes that bug by trying to delete event right after adding an event, and use append mode redirection ('>>') instead of normal redirection ('>'). Link: http://lkml.kernel.org/r/154013452832.25576.2305459545429386517.stgit@devbox Acked-by: Shuah Khan Acked-by: Tom Zanussi Tested-by: Tom Zanussi Cc: Tom Zanussi Cc: Tom Zanussi Cc: Rajvi Jingar Cc: Shuah Khan Cc: stable@vger.kernel.org Fixes: f06eec4d0f2c ('selftests: ftrace: Add inter-event hist triggers testcases') Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- .../inter-event/trigger-synthetic-event-createremove.tc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc index cef11377dcbd..c604438df13b 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-event-createremove.tc @@ -35,18 +35,18 @@ fi reset_trigger -echo "Test create synthetic event with an error" -echo 'wakeup_latency u64 lat pid_t pid char' > synthetic_events > /dev/null +echo "Test remove synthetic event" +echo '!wakeup_latency u64 lat pid_t pid char comm[16]' >> synthetic_events if [ -d events/synthetic/wakeup_latency ]; then - fail "Created wakeup_latency synthetic event with an invalid format" + fail "Failed to delete wakeup_latency synthetic event" fi reset_trigger -echo "Test remove synthetic event" -echo '!wakeup_latency u64 lat pid_t pid char comm[16]' > synthetic_events +echo "Test create synthetic event with an error" +echo 'wakeup_latency u64 lat pid_t pid char' > synthetic_events > /dev/null if [ -d events/synthetic/wakeup_latency ]; then - fail "Failed to delete wakeup_latency synthetic event" + fail "Created wakeup_latency synthetic event with an invalid format" fi do_reset -- cgit v1.2.3 From bcb6fb5da77c2a228adf07cc9cb1a0c2aa2001c6 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 31 Oct 2018 21:57:30 -0500 Subject: objtool: Support GCC 9 cold subfunction naming scheme Starting with GCC 8, a lot of unlikely code was moved out of line to "cold" subfunctions in .text.unlikely. For example, the unlikely bits of: irq_do_set_affinity() are moved out to the following subfunction: irq_do_set_affinity.cold.49() Starting with GCC 9, the numbered suffix has been removed. So in the above example, the cold subfunction is instead: irq_do_set_affinity.cold() Tweak the objtool subfunction detection logic so that it detects both GCC 8 and GCC 9 naming schemes. Reported-by: Peter Zijlstra (Intel) Signed-off-by: Josh Poimboeuf Signed-off-by: Thomas Gleixner Tested-by: Peter Zijlstra (Intel) Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/015e9544b1f188d36a7f02fa31e9e95629aa5f50.1541040800.git.jpoimboe@redhat.com --- tools/objtool/elf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 7ec85d567598..f8cef271dfc7 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -301,7 +301,7 @@ static int read_symbols(struct elf *elf) if (sym->type != STT_FUNC) continue; sym->pfunc = sym->cfunc = sym; - coldstr = strstr(sym->name, ".cold."); + coldstr = strstr(sym->name, ".cold"); if (!coldstr) continue; -- cgit v1.2.3