diff options
Diffstat (limited to 'tools/testing')
86 files changed, 5614 insertions, 5061 deletions
diff --git a/tools/testing/memblock/.gitignore b/tools/testing/memblock/.gitignore index 654338e0be52..4cc7cd5aac2b 100644 --- a/tools/testing/memblock/.gitignore +++ b/tools/testing/memblock/.gitignore @@ -1,4 +1,5 @@ main memblock.c linux/memblock.h +asm/asm.h asm/cmpxchg.h diff --git a/tools/testing/memblock/Makefile b/tools/testing/memblock/Makefile index 2310ac4d080e..7a1ca694a982 100644 --- a/tools/testing/memblock/Makefile +++ b/tools/testing/memblock/Makefile @@ -29,13 +29,14 @@ include: ../../../include/linux/memblock.h ../../include/linux/*.h \ @mkdir -p linux test -L linux/memblock.h || ln -s ../../../../include/linux/memblock.h linux/memblock.h + test -L asm/asm.h || ln -s ../../../arch/x86/include/asm/asm.h asm/asm.h test -L asm/cmpxchg.h || ln -s ../../../arch/x86/include/asm/cmpxchg.h asm/cmpxchg.h memblock.c: $(EXTR_SRC) test -L memblock.c || ln -s $(EXTR_SRC) memblock.c clean: - $(RM) $(TARGETS) $(OFILES) linux/memblock.h memblock.c asm/cmpxchg.h + $(RM) $(TARGETS) $(OFILES) linux/memblock.h memblock.c asm/asm.h asm/cmpxchg.h help: @echo 'Memblock simulator' diff --git a/tools/testing/selftests/amd-pstate/Makefile b/tools/testing/selftests/amd-pstate/Makefile index 5f195ee756d6..5fd1424db37d 100644 --- a/tools/testing/selftests/amd-pstate/Makefile +++ b/tools/testing/selftests/amd-pstate/Makefile @@ -7,11 +7,6 @@ all: uname_M := $(shell uname -m 2>/dev/null || echo not) ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) -ifeq (x86,$(ARCH)) -TEST_GEN_FILES += ../../../power/x86/amd_pstate_tracer/amd_pstate_trace.py -TEST_GEN_FILES += ../../../power/x86/intel_pstate_tracer/intel_pstate_tracer.py -endif - TEST_PROGS := run.sh TEST_FILES := basic.sh tbench.sh gitsource.sh diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 07d2d0a8c5cb..401a75844cc0 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -36,6 +36,7 @@ test_cpp *.lskel.h /no_alu32 /bpf_gcc +/host-tools /tools /runqslower /bench diff --git a/tools/testing/selftests/bpf/DENYLIST.s390x b/tools/testing/selftests/bpf/DENYLIST.s390x index 585fcf73c731..3fc3e54b19aa 100644 --- a/tools/testing/selftests/bpf/DENYLIST.s390x +++ b/tools/testing/selftests/bpf/DENYLIST.s390x @@ -14,6 +14,7 @@ cgrp_kfunc # JIT does not support calling kernel f cgrp_local_storage # prog_attach unexpected error: -524 (trampoline) core_read_macros # unknown func bpf_probe_read#4 (overlapping) d_path # failed to auto-attach program 'prog_stat': -524 (trampoline) +decap_sanity # JIT does not support calling kernel function (kfunc) deny_namespace # failed to attach: ERROR: strerror_r(-524)=22 (trampoline) dummy_st_ops # test_run unexpected error: -524 (errno 524) (trampoline) fentry_fexit # fentry attach failed: -524 (trampoline) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index 6f8ed61fc4b4..3af6450763e9 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -1465,6 +1465,77 @@ out: bpf_iter_task_vma__destroy(skel); } +static void test_task_vma_dead_task(void) +{ + struct bpf_iter_task_vma *skel; + int wstatus, child_pid = -1; + time_t start_tm, cur_tm; + int err, iter_fd = -1; + int wait_sec = 3; + + skel = bpf_iter_task_vma__open(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vma__open")) + return; + + skel->bss->pid = getpid(); + + err = bpf_iter_task_vma__load(skel); + if (!ASSERT_OK(err, "bpf_iter_task_vma__load")) + goto out; + + skel->links.proc_maps = bpf_program__attach_iter( + skel->progs.proc_maps, NULL); + + if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { + skel->links.proc_maps = NULL; + goto out; + } + + start_tm = time(NULL); + cur_tm = start_tm; + + child_pid = fork(); + if (child_pid == 0) { + /* Fork short-lived processes in the background. */ + while (cur_tm < start_tm + wait_sec) { + system("echo > /dev/null"); + cur_tm = time(NULL); + } + exit(0); + } + + if (!ASSERT_GE(child_pid, 0, "fork_child")) + goto out; + + while (cur_tm < start_tm + wait_sec) { + iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps)); + if (!ASSERT_GE(iter_fd, 0, "create_iter")) + goto out; + + /* Drain all data from iter_fd. */ + while (cur_tm < start_tm + wait_sec) { + err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE); + if (!ASSERT_GE(err, 0, "read_iter_fd")) + goto out; + + cur_tm = time(NULL); + + if (err == 0) + break; + } + + close(iter_fd); + iter_fd = -1; + } + + check_bpf_link_info(skel->progs.proc_maps); + +out: + waitpid(child_pid, &wstatus, 0); + close(iter_fd); + bpf_iter_task_vma__destroy(skel); +} + void test_bpf_sockmap_map_iter_fd(void) { struct bpf_iter_sockmap *skel; @@ -1586,6 +1657,8 @@ void test_bpf_iter(void) test_task_file(); if (test__start_subtest("task_vma")) test_task_vma(); + if (test__start_subtest("task_vma_dead_task")) + test_task_vma_dead_task(); if (test__start_subtest("task_btf")) test_task_btf(); if (test__start_subtest("tcp4")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 0ba2e8b9c6ac..e9ea38aa8248 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -801,7 +801,7 @@ static void test_btf_dump_struct_data(struct btf *btf, struct btf_dump *d, static void test_btf_dump_var_data(struct btf *btf, struct btf_dump *d, char *str) { -#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) +#if 0 TEST_BTF_DUMP_VAR(btf, d, NULL, str, "cpu_number", int, BTF_F_COMPACT, "int cpu_number = (int)100", 100); #endif diff --git a/tools/testing/selftests/bpf/prog_tests/decap_sanity.c b/tools/testing/selftests/bpf/prog_tests/decap_sanity.c new file mode 100644 index 000000000000..0b2f73b88c53 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/decap_sanity.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include <sys/types.h> +#include <sys/socket.h> +#include <net/if.h> +#include <linux/in6.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "decap_sanity.skel.h" + +#define SYS(fmt, ...) \ + ({ \ + char cmd[1024]; \ + snprintf(cmd, sizeof(cmd), fmt, ##__VA_ARGS__); \ + if (!ASSERT_OK(system(cmd), cmd)) \ + goto fail; \ + }) + +#define NS_TEST "decap_sanity_ns" +#define IPV6_IFACE_ADDR "face::1" +#define UDP_TEST_PORT 7777 + +void test_decap_sanity(void) +{ + LIBBPF_OPTS(bpf_tc_hook, qdisc_hook, .attach_point = BPF_TC_EGRESS); + LIBBPF_OPTS(bpf_tc_opts, tc_attach); + struct nstoken *nstoken = NULL; + struct decap_sanity *skel; + struct sockaddr_in6 addr; + socklen_t addrlen; + char buf[128] = {}; + int sockfd, err; + + skel = decap_sanity__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + SYS("ip netns add %s", NS_TEST); + SYS("ip -net %s -6 addr add %s/128 dev lo nodad", NS_TEST, IPV6_IFACE_ADDR); + SYS("ip -net %s link set dev lo up", NS_TEST); + + nstoken = open_netns(NS_TEST); + if (!ASSERT_OK_PTR(nstoken, "open_netns")) + goto fail; + + qdisc_hook.ifindex = if_nametoindex("lo"); + if (!ASSERT_GT(qdisc_hook.ifindex, 0, "if_nametoindex lo")) + goto fail; + + err = bpf_tc_hook_create(&qdisc_hook); + if (!ASSERT_OK(err, "create qdisc hook")) + goto fail; + + tc_attach.prog_fd = bpf_program__fd(skel->progs.decap_sanity); + err = bpf_tc_attach(&qdisc_hook, &tc_attach); + if (!ASSERT_OK(err, "attach filter")) + goto fail; + + addrlen = sizeof(addr); + err = make_sockaddr(AF_INET6, IPV6_IFACE_ADDR, UDP_TEST_PORT, + (void *)&addr, &addrlen); + if (!ASSERT_OK(err, "make_sockaddr")) + goto fail; + sockfd = socket(AF_INET6, SOCK_DGRAM, 0); + if (!ASSERT_NEQ(sockfd, -1, "socket")) + goto fail; + err = sendto(sockfd, buf, sizeof(buf), 0, (void *)&addr, addrlen); + close(sockfd); + if (!ASSERT_EQ(err, sizeof(buf), "send")) + goto fail; + + ASSERT_TRUE(skel->bss->init_csum_partial, "init_csum_partial"); + ASSERT_TRUE(skel->bss->final_csum_none, "final_csum_none"); + ASSERT_FALSE(skel->bss->broken_csum_start, "broken_csum_start"); + +fail: + if (nstoken) { + bpf_tc_hook_destroy(&qdisc_hook); + close_netns(nstoken); + } + system("ip netns del " NS_TEST " >& /dev/null"); + decap_sanity__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/jeq_infer_not_null.c b/tools/testing/selftests/bpf/prog_tests/jeq_infer_not_null.c new file mode 100644 index 000000000000..3add34df5767 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/jeq_infer_not_null.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "jeq_infer_not_null_fail.skel.h" + +void test_jeq_infer_not_null(void) +{ + RUN_TESTS(jeq_infer_not_null_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 2cf0c7a3fe23..567e07c19ecc 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -30,6 +30,8 @@ #define MAX_STRERR_LEN 256 #define MAX_TEST_NAME 80 +#define __always_unused __attribute__((__unused__)) + #define _FAIL(errnum, fmt...) \ ({ \ error_at_line(0, (errnum), __func__, __LINE__, fmt); \ @@ -321,7 +323,8 @@ static int socket_loopback(int family, int sotype) return socket_loopback_reuseport(family, sotype, -1); } -static void test_insert_invalid(int family, int sotype, int mapfd) +static void test_insert_invalid(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u32 key = 0; u64 value; @@ -338,7 +341,8 @@ static void test_insert_invalid(int family, int sotype, int mapfd) FAIL_ERRNO("map_update: expected EBADF"); } -static void test_insert_opened(int family, int sotype, int mapfd) +static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u32 key = 0; u64 value; @@ -359,7 +363,8 @@ static void test_insert_opened(int family, int sotype, int mapfd) xclose(s); } -static void test_insert_bound(int family, int sotype, int mapfd) +static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { struct sockaddr_storage addr; socklen_t len; @@ -386,7 +391,8 @@ close: xclose(s); } -static void test_insert(int family, int sotype, int mapfd) +static void test_insert(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u64 value; u32 key; @@ -402,7 +408,8 @@ static void test_insert(int family, int sotype, int mapfd) xclose(s); } -static void test_delete_after_insert(int family, int sotype, int mapfd) +static void test_delete_after_insert(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u64 value; u32 key; @@ -419,7 +426,8 @@ static void test_delete_after_insert(int family, int sotype, int mapfd) xclose(s); } -static void test_delete_after_close(int family, int sotype, int mapfd) +static void test_delete_after_close(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { int err, s; u64 value; @@ -442,7 +450,8 @@ static void test_delete_after_close(int family, int sotype, int mapfd) FAIL_ERRNO("map_delete: expected EINVAL/EINVAL"); } -static void test_lookup_after_insert(int family, int sotype, int mapfd) +static void test_lookup_after_insert(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u64 cookie, value; socklen_t len; @@ -470,7 +479,8 @@ static void test_lookup_after_insert(int family, int sotype, int mapfd) xclose(s); } -static void test_lookup_after_delete(int family, int sotype, int mapfd) +static void test_lookup_after_delete(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { int err, s; u64 value; @@ -493,7 +503,8 @@ static void test_lookup_after_delete(int family, int sotype, int mapfd) xclose(s); } -static void test_lookup_32_bit_value(int family, int sotype, int mapfd) +static void test_lookup_32_bit_value(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { u32 key, value32; int err, s; @@ -523,7 +534,8 @@ close: xclose(s); } -static void test_update_existing(int family, int sotype, int mapfd) +static void test_update_existing(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { int s1, s2; u64 value; @@ -551,7 +563,7 @@ close_s1: /* Exercise the code path where we destroy child sockets that never * got accept()'ed, aka orphans, when parent socket gets closed. */ -static void test_destroy_orphan_child(int family, int sotype, int mapfd) +static void do_destroy_orphan_child(int family, int sotype, int mapfd) { struct sockaddr_storage addr; socklen_t len; @@ -582,10 +594,38 @@ close_srv: xclose(s); } +static void test_destroy_orphan_child(struct test_sockmap_listen *skel, + int family, int sotype, int mapfd) +{ + int msg_verdict = bpf_program__fd(skel->progs.prog_msg_verdict); + int skb_verdict = bpf_program__fd(skel->progs.prog_skb_verdict); + const struct test { + int progfd; + enum bpf_attach_type atype; + } tests[] = { + { -1, -1 }, + { msg_verdict, BPF_SK_MSG_VERDICT }, + { skb_verdict, BPF_SK_SKB_VERDICT }, + }; + const struct test *t; + + for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { + if (t->progfd != -1 && + xbpf_prog_attach(t->progfd, mapfd, t->atype, 0) != 0) + return; + + do_destroy_orphan_child(family, sotype, mapfd); + + if (t->progfd != -1) + xbpf_prog_detach2(t->progfd, mapfd, t->atype); + } +} + /* Perform a passive open after removing listening socket from SOCKMAP * to ensure that callbacks get restored properly. */ -static void test_clone_after_delete(int family, int sotype, int mapfd) +static void test_clone_after_delete(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { struct sockaddr_storage addr; socklen_t len; @@ -621,7 +661,8 @@ close_srv: * SOCKMAP, but got accept()'ed only after the parent has been removed * from SOCKMAP, gets cloned without parent psock state or callbacks. */ -static void test_accept_after_delete(int family, int sotype, int mapfd) +static void test_accept_after_delete(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { struct sockaddr_storage addr; const u32 zero = 0; @@ -675,7 +716,8 @@ close_srv: /* Check that child socket that got created and accepted while parent * was in a SOCKMAP is cloned without parent psock state or callbacks. */ -static void test_accept_before_delete(int family, int sotype, int mapfd) +static void test_accept_before_delete(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { struct sockaddr_storage addr; const u32 zero = 0, one = 1; @@ -784,7 +826,8 @@ done: return NULL; } -static void test_syn_recv_insert_delete(int family, int sotype, int mapfd) +static void test_syn_recv_insert_delete(struct test_sockmap_listen *skel __always_unused, + int family, int sotype, int mapfd) { struct connect_accept_ctx ctx = { 0 }; struct sockaddr_storage addr; @@ -847,7 +890,8 @@ static void *listen_thread(void *arg) return NULL; } -static void test_race_insert_listen(int family, int socktype, int mapfd) +static void test_race_insert_listen(struct test_sockmap_listen *skel __always_unused, + int family, int socktype, int mapfd) { struct connect_accept_ctx ctx = { 0 }; const u32 zero = 0; @@ -1473,7 +1517,8 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, int family, int sotype) { const struct op_test { - void (*fn)(int family, int sotype, int mapfd); + void (*fn)(struct test_sockmap_listen *skel, + int family, int sotype, int mapfd); const char *name; int sotype; } tests[] = { @@ -1520,7 +1565,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, if (!test__start_subtest(s)) continue; - t->fn(family, sotype, map_fd); + t->fn(skel, family, sotype, map_fd); test_ops_cleanup(map); } } diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index b394817126cf..cfed4df490f3 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -50,6 +50,12 @@ #define ICSK_TIME_LOSS_PROBE 5 #define ICSK_TIME_REO_TIMEOUT 6 +#define ETH_HLEN 14 +#define ETH_P_IPV6 0x86DD + +#define CHECKSUM_NONE 0 +#define CHECKSUM_PARTIAL 3 + #define IFNAMSIZ 16 #define RTF_GATEWAY 0x0002 diff --git a/tools/testing/selftests/bpf/progs/decap_sanity.c b/tools/testing/selftests/bpf/progs/decap_sanity.c new file mode 100644 index 000000000000..bd3c657c58a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/decap_sanity.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include "bpf_tracing_net.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +#define UDP_TEST_PORT 7777 + +void *bpf_cast_to_kern_ctx(void *) __ksym; +bool init_csum_partial = false; +bool final_csum_none = false; +bool broken_csum_start = false; + +static unsigned int skb_headlen(const struct sk_buff *skb) +{ + return skb->len - skb->data_len; +} + +static unsigned int skb_headroom(const struct sk_buff *skb) +{ + return skb->data - skb->head; +} + +static int skb_checksum_start_offset(const struct sk_buff *skb) +{ + return skb->csum_start - skb_headroom(skb); +} + +SEC("tc") +int decap_sanity(struct __sk_buff *skb) +{ + struct sk_buff *kskb; + struct ipv6hdr ip6h; + struct udphdr udph; + int err; + + if (skb->protocol != __bpf_constant_htons(ETH_P_IPV6)) + return TC_ACT_SHOT; + + if (bpf_skb_load_bytes(skb, ETH_HLEN, &ip6h, sizeof(ip6h))) + return TC_ACT_SHOT; + + if (ip6h.nexthdr != IPPROTO_UDP) + return TC_ACT_SHOT; + + if (bpf_skb_load_bytes(skb, ETH_HLEN + sizeof(ip6h), &udph, sizeof(udph))) + return TC_ACT_SHOT; + + if (udph.dest != __bpf_constant_htons(UDP_TEST_PORT)) + return TC_ACT_SHOT; + + kskb = bpf_cast_to_kern_ctx(skb); + init_csum_partial = (kskb->ip_summed == CHECKSUM_PARTIAL); + err = bpf_skb_adjust_room(skb, -(s32)(ETH_HLEN + sizeof(ip6h) + sizeof(udph)), + 1, BPF_F_ADJ_ROOM_FIXED_GSO); + if (err) + return TC_ACT_SHOT; + final_csum_none = (kskb->ip_summed == CHECKSUM_NONE); + if (kskb->ip_summed == CHECKSUM_PARTIAL && + (unsigned int)skb_checksum_start_offset(kskb) >= skb_headlen(kskb)) + broken_csum_start = true; + + return TC_ACT_SHOT; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/jeq_infer_not_null_fail.c b/tools/testing/selftests/bpf/progs/jeq_infer_not_null_fail.c new file mode 100644 index 000000000000..f46965053acb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/jeq_infer_not_null_fail.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, u64); + __type(value, u64); +} m_hash SEC(".maps"); + +SEC("?raw_tp") +__failure __msg("R8 invalid mem access 'map_value_or_null") +int jeq_infer_not_null_ptr_to_btfid(void *ctx) +{ + struct bpf_map *map = (struct bpf_map *)&m_hash; + struct bpf_map *inner_map = map->inner_map_meta; + u64 key = 0, ret = 0, *val; + + val = bpf_map_lookup_elem(map, &key); + /* Do not mark ptr as non-null if one of them is + * PTR_TO_BTF_ID (R9), reject because of invalid + * access to map value (R8). + * + * Here, we need to inline those insns to access + * R8 directly, since compiler may use other reg + * once it figures out val==inner_map. + */ + asm volatile("r8 = %[val];\n" + "r9 = %[inner_map];\n" + "if r8 != r9 goto +1;\n" + "%[ret] = *(u64 *)(r8 +0);\n" + : [ret] "+r"(ret) + : [inner_map] "r"(inner_map), [val] "r"(val) + : "r8", "r9"); + + return ret; +} diff --git a/tools/testing/selftests/bpf/progs/profiler.inc.h b/tools/testing/selftests/bpf/progs/profiler.inc.h index 92331053dba3..7bd76b9e0f98 100644 --- a/tools/testing/selftests/bpf/progs/profiler.inc.h +++ b/tools/testing/selftests/bpf/progs/profiler.inc.h @@ -826,7 +826,7 @@ out: SEC("kprobe/vfs_link") int BPF_KPROBE(kprobe__vfs_link, - struct dentry* old_dentry, struct user_namespace *mnt_userns, + struct dentry* old_dentry, struct mnt_idmap *idmap, struct inode* dir, struct dentry* new_dentry, struct inode** delegated_inode) { diff --git a/tools/testing/selftests/bpf/verifier/search_pruning.c b/tools/testing/selftests/bpf/verifier/search_pruning.c index 68b14fdfebdb..d63fd8991b03 100644 --- a/tools/testing/selftests/bpf/verifier/search_pruning.c +++ b/tools/testing/selftests/bpf/verifier/search_pruning.c @@ -225,3 +225,39 @@ .result_unpriv = ACCEPT, .insn_processed = 15, }, +/* The test performs a conditional 64-bit write to a stack location + * fp[-8], this is followed by an unconditional 8-bit write to fp[-8], + * then data is read from fp[-8]. This sequence is unsafe. + * + * The test would be mistakenly marked as safe w/o dst register parent + * preservation in verifier.c:copy_register_state() function. + * + * Note the usage of BPF_F_TEST_STATE_FREQ to force creation of the + * checkpoint state after conditional 64-bit assignment. + */ +{ + "write tracking and register parent chain bug", + .insns = { + /* r6 = ktime_get_ns() */ + BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + /* r0 = ktime_get_ns() */ + BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns), + /* if r0 > r6 goto +1 */ + BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_6, 1), + /* *(u64 *)(r10 - 8) = 0xdeadbeef */ + BPF_ST_MEM(BPF_DW, BPF_REG_FP, -8, 0xdeadbeef), + /* r1 = 42 */ + BPF_MOV64_IMM(BPF_REG_1, 42), + /* *(u8 *)(r10 - 8) = r1 */ + BPF_STX_MEM(BPF_B, BPF_REG_FP, BPF_REG_1, -8), + /* r2 = *(u64 *)(r10 - 8) */ + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_FP, -8), + /* exit(0) */ + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .flags = BPF_F_TEST_STATE_FREQ, + .errstr = "invalid read from stack off -8+1 size 8", + .result = REJECT, +}, diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 186e1c26867e..75c100de90ff 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -268,6 +268,7 @@ TEST_MATRIX=( # Taking away all CPUs from parent or itself if there are tasks # will make the partition invalid. " S+ C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1" + " S+ C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1" " S+ $SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" " S+ $SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" diff --git a/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh index 9c79bbcce5a8..aff0a59f92d9 100755 --- a/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh +++ b/tools/testing/selftests/drivers/net/ocelot/tc_flower_chains.sh @@ -246,7 +246,7 @@ test_vlan_ingress_modify() bridge vlan add dev $swp2 vid 300 tc filter add dev $swp1 ingress chain $(IS1 2) pref 3 \ - protocol 802.1Q flower skip_sw vlan_id 200 \ + protocol 802.1Q flower skip_sw vlan_id 200 src_mac $h1_mac \ action vlan modify id 300 \ action goto chain $(IS2 0 0) diff --git a/tools/testing/selftests/filesystems/fat/run_fat_tests.sh b/tools/testing/selftests/filesystems/fat/run_fat_tests.sh index 7f35dc3d15df..7f35dc3d15df 100644..100755 --- a/tools/testing/selftests/filesystems/fat/run_fat_tests.sh +++ b/tools/testing/selftests/filesystems/fat/run_fat_tests.sh diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 6ce8c488d62e..6d9381d60172 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -1,86 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only -/aarch64/aarch32_id_regs -/aarch64/arch_timer -/aarch64/debug-exceptions -/aarch64/get-reg-list -/aarch64/hypercalls -/aarch64/page_fault_test -/aarch64/psci_test -/aarch64/vcpu_width_config -/aarch64/vgic_init -/aarch64/vgic_irq -/s390x/memop -/s390x/resets -/s390x/sync_regs_test -/s390x/tprot -/x86_64/amx_test -/x86_64/cpuid_test -/x86_64/cr4_cpuid_sync_test -/x86_64/debug_regs -/x86_64/exit_on_emulation_failure_test -/x86_64/fix_hypercall_test -/x86_64/get_msr_index_features -/x86_64/kvm_clock_test -/x86_64/kvm_pv_test -/x86_64/hyperv_clock -/x86_64/hyperv_cpuid -/x86_64/hyperv_evmcs -/x86_64/hyperv_features -/x86_64/hyperv_ipi -/x86_64/hyperv_svm_test -/x86_64/hyperv_tlb_flush -/x86_64/max_vcpuid_cap_test -/x86_64/mmio_warning_test -/x86_64/monitor_mwait_test -/x86_64/nested_exceptions_test -/x86_64/nx_huge_pages_test -/x86_64/platform_info_test -/x86_64/pmu_event_filter_test -/x86_64/set_boot_cpu_id -/x86_64/set_sregs_test -/x86_64/sev_migrate_tests -/x86_64/smaller_maxphyaddr_emulation_test -/x86_64/smm_test -/x86_64/state_test -/x86_64/svm_vmcall_test -/x86_64/svm_int_ctl_test -/x86_64/svm_nested_soft_inject_test -/x86_64/svm_nested_shutdown_test -/x86_64/sync_regs_test -/x86_64/tsc_msrs_test -/x86_64/tsc_scaling_sync -/x86_64/ucna_injection_test -/x86_64/userspace_io_test -/x86_64/userspace_msr_exit_test -/x86_64/vmx_apic_access_test -/x86_64/vmx_close_while_nested_test -/x86_64/vmx_dirty_log_test -/x86_64/vmx_exception_with_invalid_guest_state -/x86_64/vmx_invalid_nested_guest_state -/x86_64/vmx_msrs_test -/x86_64/vmx_preemption_timer_test -/x86_64/vmx_set_nested_state_test -/x86_64/vmx_tsc_adjust_test -/x86_64/vmx_nested_tsc_scaling_test -/x86_64/xapic_ipi_test -/x86_64/xapic_state_test -/x86_64/xen_shinfo_test -/x86_64/xen_vmcall_test -/x86_64/xss_msr_test -/x86_64/vmx_pmu_caps_test -/x86_64/triple_fault_event_test -/access_tracking_perf_test -/demand_paging_test -/dirty_log_test -/dirty_log_perf_test -/hardware_disable_test -/kvm_create_max_vcpus -/kvm_page_table_test -/max_guest_memory_test -/memslot_modification_stress_test -/memslot_perf_test -/rseq_test -/set_memory_region_test -/steal_time -/kvm_binary_stats_test -/system_counter_offset_test +* +!/**/ +!*.c +!*.h +!*.S +!*.sh diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 947676983da1..1750f91dd936 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -7,35 +7,14 @@ top_srcdir = ../../../.. include $(top_srcdir)/scripts/subarch.include ARCH ?= $(SUBARCH) -# For cross-builds to work, UNAME_M has to map to ARCH and arch specific -# directories and targets in this Makefile. "uname -m" doesn't map to -# arch specific sub-directory names. -# -# UNAME_M variable to used to run the compiles pointing to the right arch -# directories and build the right targets for these supported architectures. -# -# TEST_GEN_PROGS and LIBKVM are set using UNAME_M variable. -# LINUX_TOOL_ARCH_INCLUDE is set using ARCH variable. -# -# x86_64 targets are named to include x86_64 as a suffix and directories -# for includes are in x86_64 sub-directory. s390x and aarch64 follow the -# same convention. "uname -m" doesn't result in the correct mapping for -# s390x and aarch64. -# -# No change necessary for x86_64 -UNAME_M := $(shell uname -m) - -# Set UNAME_M for arm64 compile/install to work -ifeq ($(ARCH),arm64) - UNAME_M := aarch64 -endif -# Set UNAME_M s390x compile/install to work -ifeq ($(ARCH),s390) - UNAME_M := s390x -endif -# Set UNAME_M riscv compile/install to work -ifeq ($(ARCH),riscv) - UNAME_M := riscv +ifeq ($(ARCH),x86) + ARCH_DIR := x86_64 +else ifeq ($(ARCH),arm64) + ARCH_DIR := aarch64 +else ifeq ($(ARCH),s390) + ARCH_DIR := s390x +else + ARCH_DIR := $(ARCH) endif LIBKVM += lib/assert.c @@ -196,10 +175,15 @@ TEST_GEN_PROGS_riscv += kvm_page_table_test TEST_GEN_PROGS_riscv += set_memory_region_test TEST_GEN_PROGS_riscv += kvm_binary_stats_test -TEST_PROGS += $(TEST_PROGS_$(UNAME_M)) -TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M)) -TEST_GEN_PROGS_EXTENDED += $(TEST_GEN_PROGS_EXTENDED_$(UNAME_M)) -LIBKVM += $(LIBKVM_$(UNAME_M)) +TEST_PROGS += $(TEST_PROGS_$(ARCH_DIR)) +TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(ARCH_DIR)) +TEST_GEN_PROGS_EXTENDED += $(TEST_GEN_PROGS_EXTENDED_$(ARCH_DIR)) +LIBKVM += $(LIBKVM_$(ARCH_DIR)) + +# lib.mak defines $(OUTPUT), prepends $(OUTPUT)/ to $(TEST_GEN_PROGS), and most +# importantly defines, i.e. overwrites, $(CC) (unless `make -e` or `make CC=`, +# which causes the environment variable to override the makefile). +include ../lib.mk INSTALL_HDR_PATH = $(top_srcdir)/usr LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/ @@ -210,25 +194,23 @@ else LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include endif CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \ + -Wno-gnu-variable-sized-type-not-at-end \ + -fno-builtin-memcmp -fno-builtin-memcpy -fno-builtin-memset \ -fno-stack-protector -fno-PIE -I$(LINUX_TOOL_INCLUDE) \ -I$(LINUX_TOOL_ARCH_INCLUDE) -I$(LINUX_HDR_PATH) -Iinclude \ - -I$(<D) -Iinclude/$(UNAME_M) -I ../rseq -I.. $(EXTRA_CFLAGS) \ + -I$(<D) -Iinclude/$(ARCH_DIR) -I ../rseq -I.. $(EXTRA_CFLAGS) \ $(KHDR_INCLUDES) -no-pie-option := $(call try-run, echo 'int main() { return 0; }' | \ - $(CC) -Werror -no-pie -x c - -o "$$TMP", -no-pie) +no-pie-option := $(call try-run, echo 'int main(void) { return 0; }' | \ + $(CC) -Werror $(CFLAGS) -no-pie -x c - -o "$$TMP", -no-pie) # On s390, build the testcases KVM-enabled -pgste-option = $(call try-run, echo 'int main() { return 0; }' | \ +pgste-option = $(call try-run, echo 'int main(void) { return 0; }' | \ $(CC) -Werror -Wl$(comma)--s390-pgste -x c - -o "$$TMP",-Wl$(comma)--s390-pgste) LDLIBS += -ldl LDFLAGS += -pthread $(no-pie-option) $(pgste-option) -# After inclusion, $(OUTPUT) is defined and -# $(TEST_GEN_PROGS) starts with $(OUTPUT)/ -include ../lib.mk - LIBKVM_C := $(filter %.c,$(LIBKVM)) LIBKVM_S := $(filter %.S,$(LIBKVM)) LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C)) diff --git a/tools/testing/selftests/kvm/aarch64/page_fault_test.c b/tools/testing/selftests/kvm/aarch64/page_fault_test.c index 95d22cfb7b41..54680dc5887f 100644 --- a/tools/testing/selftests/kvm/aarch64/page_fault_test.c +++ b/tools/testing/selftests/kvm/aarch64/page_fault_test.c @@ -117,7 +117,7 @@ static void guest_cas(void) GUEST_ASSERT(guest_check_lse()); asm volatile(".arch_extension lse\n" "casal %0, %1, [%2]\n" - :: "r" (0), "r" (TEST_DATA), "r" (guest_test_memory)); + :: "r" (0ul), "r" (TEST_DATA), "r" (guest_test_memory)); val = READ_ONCE(*guest_test_memory); GUEST_ASSERT_EQ(val, TEST_DATA); } @@ -237,6 +237,11 @@ static void guest_check_s1ptw_wr_in_dirty_log(void) GUEST_SYNC(CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG); } +static void guest_check_no_s1ptw_wr_in_dirty_log(void) +{ + GUEST_SYNC(CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG); +} + static void guest_exec(void) { int (*code)(void) = (int (*)(void))TEST_EXEC_GVA; @@ -304,7 +309,7 @@ static struct uffd_args { /* Returns true to continue the test, and false if it should be skipped. */ static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg, - struct uffd_args *args, bool expect_write) + struct uffd_args *args) { uint64_t addr = msg->arg.pagefault.address; uint64_t flags = msg->arg.pagefault.flags; @@ -313,7 +318,6 @@ static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg, TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING, "The only expected UFFD mode is MISSING"); - ASSERT_EQ(!!(flags & UFFD_PAGEFAULT_FLAG_WRITE), expect_write); ASSERT_EQ(addr, (uint64_t)args->hva); pr_debug("uffd fault: addr=%p write=%d\n", @@ -337,19 +341,14 @@ static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg, return 0; } -static int uffd_pt_write_handler(int mode, int uffd, struct uffd_msg *msg) -{ - return uffd_generic_handler(mode, uffd, msg, &pt_args, true); -} - -static int uffd_data_write_handler(int mode, int uffd, struct uffd_msg *msg) +static int uffd_pt_handler(int mode, int uffd, struct uffd_msg *msg) { - return uffd_generic_handler(mode, uffd, msg, &data_args, true); + return uffd_generic_handler(mode, uffd, msg, &pt_args); } -static int uffd_data_read_handler(int mode, int uffd, struct uffd_msg *msg) +static int uffd_data_handler(int mode, int uffd, struct uffd_msg *msg) { - return uffd_generic_handler(mode, uffd, msg, &data_args, false); + return uffd_generic_handler(mode, uffd, msg, &data_args); } static void setup_uffd_args(struct userspace_mem_region *region, @@ -471,9 +470,12 @@ static bool handle_cmd(struct kvm_vm *vm, int cmd) { struct userspace_mem_region *data_region, *pt_region; bool continue_test = true; + uint64_t pte_gpa, pte_pg; data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA); pt_region = vm_get_mem_region(vm, MEM_REGION_PT); + pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA)); + pte_pg = (pte_gpa - pt_region->region.guest_phys_addr) / getpagesize(); if (cmd == CMD_SKIP_TEST) continue_test = false; @@ -486,13 +488,13 @@ static bool handle_cmd(struct kvm_vm *vm, int cmd) TEST_ASSERT(check_write_in_dirty_log(vm, data_region, 0), "Missing write in dirty log"); if (cmd & CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG) - TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, 0), + TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, pte_pg), "Missing s1ptw write in dirty log"); if (cmd & CMD_CHECK_NO_WRITE_IN_DIRTY_LOG) TEST_ASSERT(!check_write_in_dirty_log(vm, data_region, 0), "Unexpected write in dirty log"); if (cmd & CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG) - TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, 0), + TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, pte_pg), "Unexpected s1ptw write in dirty log"); return continue_test; @@ -797,7 +799,7 @@ static void help(char *name) .expected_events = { .uffd_faults = _uffd_faults, }, \ } -#define TEST_DIRTY_LOG(_access, _with_af, _test_check) \ +#define TEST_DIRTY_LOG(_access, _with_af, _test_check, _pt_check) \ { \ .name = SCAT3(dirty_log, _access, _with_af), \ .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ @@ -805,13 +807,12 @@ static void help(char *name) .guest_prepare = { _PREPARE(_with_af), \ _PREPARE(_access) }, \ .guest_test = _access, \ - .guest_test_check = { _CHECK(_with_af), _test_check, \ - guest_check_s1ptw_wr_in_dirty_log}, \ + .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \ .expected_events = { 0 }, \ } #define TEST_UFFD_AND_DIRTY_LOG(_access, _with_af, _uffd_data_handler, \ - _uffd_faults, _test_check) \ + _uffd_faults, _test_check, _pt_check) \ { \ .name = SCAT3(uffd_and_dirty_log, _access, _with_af), \ .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ @@ -820,16 +821,17 @@ static void help(char *name) _PREPARE(_access) }, \ .guest_test = _access, \ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ - .guest_test_check = { _CHECK(_with_af), _test_check }, \ + .guest_test_check = { _CHECK(_with_af), _test_check, _pt_check }, \ .uffd_data_handler = _uffd_data_handler, \ - .uffd_pt_handler = uffd_pt_write_handler, \ + .uffd_pt_handler = uffd_pt_handler, \ .expected_events = { .uffd_faults = _uffd_faults, }, \ } #define TEST_RO_MEMSLOT(_access, _mmio_handler, _mmio_exits) \ { \ - .name = SCAT3(ro_memslot, _access, _with_af), \ + .name = SCAT2(ro_memslot, _access), \ .data_memslot_flags = KVM_MEM_READONLY, \ + .pt_memslot_flags = KVM_MEM_READONLY, \ .guest_prepare = { _PREPARE(_access) }, \ .guest_test = _access, \ .mmio_handler = _mmio_handler, \ @@ -840,6 +842,7 @@ static void help(char *name) { \ .name = SCAT2(ro_memslot_no_syndrome, _access), \ .data_memslot_flags = KVM_MEM_READONLY, \ + .pt_memslot_flags = KVM_MEM_READONLY, \ .guest_test = _access, \ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ .expected_events = { .fail_vcpu_runs = 1 }, \ @@ -848,9 +851,9 @@ static void help(char *name) #define TEST_RO_MEMSLOT_AND_DIRTY_LOG(_access, _mmio_handler, _mmio_exits, \ _test_check) \ { \ - .name = SCAT3(ro_memslot, _access, _with_af), \ + .name = SCAT2(ro_memslot, _access), \ .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ - .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ + .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ .guest_prepare = { _PREPARE(_access) }, \ .guest_test = _access, \ .guest_test_check = { _test_check }, \ @@ -862,7 +865,7 @@ static void help(char *name) { \ .name = SCAT2(ro_memslot_no_syn_and_dlog, _access), \ .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ - .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \ + .pt_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \ .guest_test = _access, \ .guest_test_check = { _test_check }, \ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ @@ -874,11 +877,12 @@ static void help(char *name) { \ .name = SCAT2(ro_memslot_uffd, _access), \ .data_memslot_flags = KVM_MEM_READONLY, \ + .pt_memslot_flags = KVM_MEM_READONLY, \ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ .guest_prepare = { _PREPARE(_access) }, \ .guest_test = _access, \ .uffd_data_handler = _uffd_data_handler, \ - .uffd_pt_handler = uffd_pt_write_handler, \ + .uffd_pt_handler = uffd_pt_handler, \ .mmio_handler = _mmio_handler, \ .expected_events = { .mmio_exits = _mmio_exits, \ .uffd_faults = _uffd_faults }, \ @@ -889,10 +893,11 @@ static void help(char *name) { \ .name = SCAT2(ro_memslot_no_syndrome, _access), \ .data_memslot_flags = KVM_MEM_READONLY, \ + .pt_memslot_flags = KVM_MEM_READONLY, \ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \ .guest_test = _access, \ .uffd_data_handler = _uffd_data_handler, \ - .uffd_pt_handler = uffd_pt_write_handler, \ + .uffd_pt_handler = uffd_pt_handler, \ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \ .expected_events = { .fail_vcpu_runs = 1, \ .uffd_faults = _uffd_faults }, \ @@ -933,44 +938,51 @@ static struct test_desc tests[] = { * (S1PTW). */ TEST_UFFD(guest_read64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 2), - /* no_af should also lead to a PT write. */ + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_read64, no_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 2), - /* Note how that cas invokes the read handler. */ + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_cas, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), /* * Can't test guest_at with_af as it's IMPDEF whether the AF is set. * The S1PTW fault should still be marked as a write. */ TEST_UFFD(guest_at, no_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 1), + uffd_no_handler, uffd_pt_handler, 1), TEST_UFFD(guest_ld_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_write64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_write_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_dc_zva, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_write_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_st_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_write_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), TEST_UFFD(guest_exec, with_af, CMD_HOLE_DATA | CMD_HOLE_PT, - uffd_data_read_handler, uffd_pt_write_handler, 2), + uffd_data_handler, uffd_pt_handler, 2), /* * Try accesses when the data and PT memory regions are both * tracked for dirty logging. */ - TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log), - /* no_af should also lead to a PT write. */ - TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log), - TEST_DIRTY_LOG(guest_ld_preidx, with_af, guest_check_no_write_in_dirty_log), - TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log), - TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log), - TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log), - TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log), - TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log), - TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log), + TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log, + guest_check_no_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_ld_preidx, with_af, + guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log, + guest_check_no_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), /* * Access when the data and PT memory regions are both marked for @@ -980,29 +992,43 @@ static struct test_desc tests[] = { * fault, and nothing in the dirty log. Any S1PTW should result in * a write in the dirty log and a userfaultfd write. */ - TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af, uffd_data_read_handler, 2, - guest_check_no_write_in_dirty_log), - /* no_af should also lead to a PT write. */ - TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af, uffd_data_read_handler, 2, - guest_check_no_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af, uffd_data_read_handler, - 2, guest_check_no_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, 0, 1, - guest_check_no_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af, uffd_data_read_handler, 2, - guest_check_no_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af, uffd_data_write_handler, - 2, guest_check_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af, uffd_data_read_handler, 2, - guest_check_write_in_dirty_log), - TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af, uffd_data_write_handler, - 2, guest_check_write_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af, + uffd_data_handler, 2, + guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af, + uffd_data_handler, 2, + guest_check_no_write_in_dirty_log, + guest_check_no_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af, + uffd_data_handler, + 2, guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, uffd_no_handler, 1, + guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af, + uffd_data_handler, 2, + guest_check_no_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af, + uffd_data_handler, + 2, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af, + uffd_data_handler, 2, + guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), + TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af, + uffd_data_handler, + 2, guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), TEST_UFFD_AND_DIRTY_LOG(guest_st_preidx, with_af, - uffd_data_write_handler, 2, - guest_check_write_in_dirty_log), - + uffd_data_handler, 2, + guest_check_write_in_dirty_log, + guest_check_s1ptw_wr_in_dirty_log), /* - * Try accesses when the data memory region is marked read-only + * Access when both the PT and data regions are marked read-only * (with KVM_MEM_READONLY). Writes with a syndrome result in an * MMIO exit, writes with no syndrome (e.g., CAS) result in a * failed vcpu run, and reads/execs with and without syndroms do @@ -1018,7 +1044,7 @@ static struct test_desc tests[] = { TEST_RO_MEMSLOT_NO_SYNDROME(guest_st_preidx), /* - * Access when both the data region is both read-only and marked + * The PT and data regions are both read-only and marked * for dirty logging at the same time. The expected result is that * for writes there should be no write in the dirty log. The * readonly handling is the same as if the memslot was not marked @@ -1043,7 +1069,7 @@ static struct test_desc tests[] = { guest_check_no_write_in_dirty_log), /* - * Access when the data region is both read-only and punched with + * The PT and data regions are both read-only and punched with * holes tracked with userfaultfd. The expected result is the * union of both userfaultfd and read-only behaviors. For example, * write accesses result in a userfaultfd write fault and an MMIO @@ -1051,22 +1077,15 @@ static struct test_desc tests[] = { * no userfaultfd write fault. Reads result in userfaultfd getting * triggered. */ - TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0, - uffd_data_read_handler, 2), - TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0, - uffd_data_read_handler, 2), - TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0, - uffd_no_handler, 1), - TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0, - uffd_data_read_handler, 2), + TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0, uffd_data_handler, 2), + TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0, uffd_data_handler, 2), + TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0, uffd_no_handler, 1), + TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0, uffd_data_handler, 2), TEST_RO_MEMSLOT_AND_UFFD(guest_write64, mmio_on_test_gpa_handler, 1, - uffd_data_write_handler, 2), - TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas, - uffd_data_read_handler, 2), - TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva, - uffd_no_handler, 1), - TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx, - uffd_no_handler, 1), + uffd_data_handler, 2), + TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas, uffd_data_handler, 2), + TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva, uffd_no_handler, 1), + TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx, uffd_no_handler, 1), { 0 } }; diff --git a/tools/testing/selftests/kvm/lib/aarch64/ucall.c b/tools/testing/selftests/kvm/lib/aarch64/ucall.c index 562c16dfbb00..f212bd8ab93d 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/ucall.c +++ b/tools/testing/selftests/kvm/lib/aarch64/ucall.c @@ -14,11 +14,13 @@ static vm_vaddr_t *ucall_exit_mmio_addr; void ucall_arch_init(struct kvm_vm *vm, vm_paddr_t mmio_gpa) { - virt_pg_map(vm, mmio_gpa, mmio_gpa); + vm_vaddr_t mmio_gva = vm_vaddr_unused_gap(vm, vm->page_size, KVM_UTIL_MIN_VADDR); + + virt_map(vm, mmio_gva, mmio_gpa, 1); vm->ucall_mmio_addr = mmio_gpa; - write_guest_global(vm, ucall_exit_mmio_addr, (vm_vaddr_t *)mmio_gpa); + write_guest_global(vm, ucall_exit_mmio_addr, (vm_vaddr_t *)mmio_gva); } void ucall_arch_do_ucall(vm_vaddr_t uc) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index c88c3ace16d2..56d5ea949cbb 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -186,6 +186,15 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES, "Missing new mode params?"); +/* + * Initializes vm->vpages_valid to match the canonical VA space of the + * architecture. + * + * The default implementation is valid for architectures which split the + * range addressed by a single page table into a low and high region + * based on the MSB of the VA. On architectures with this behavior + * the VA region spans [0, 2^(va_bits - 1)), [-(2^(va_bits - 1), -1]. + */ __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm) { sparsebit_set_num(vm->vpages_valid, @@ -1416,10 +1425,10 @@ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, while (npages--) { virt_pg_map(vm, vaddr, paddr); + sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift); + vaddr += page_size; paddr += page_size; - - sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift); } } diff --git a/tools/testing/selftests/kvm/lib/ucall_common.c b/tools/testing/selftests/kvm/lib/ucall_common.c index 0cc0971ce60e..2f0e2ea941cc 100644 --- a/tools/testing/selftests/kvm/lib/ucall_common.c +++ b/tools/testing/selftests/kvm/lib/ucall_common.c @@ -4,6 +4,8 @@ #include "linux/bitmap.h" #include "linux/atomic.h" +#define GUEST_UCALL_FAILED -1 + struct ucall_header { DECLARE_BITMAP(in_use, KVM_MAX_VCPUS); struct ucall ucalls[KVM_MAX_VCPUS]; @@ -41,7 +43,8 @@ static struct ucall *ucall_alloc(void) struct ucall *uc; int i; - GUEST_ASSERT(ucall_pool); + if (!ucall_pool) + goto ucall_failed; for (i = 0; i < KVM_MAX_VCPUS; ++i) { if (!test_and_set_bit(i, ucall_pool->in_use)) { @@ -51,7 +54,13 @@ static struct ucall *ucall_alloc(void) } } - GUEST_ASSERT(0); +ucall_failed: + /* + * If the vCPU cannot grab a ucall structure, make a bare ucall with a + * magic value to signal to get_ucall() that things went sideways. + * GUEST_ASSERT() depends on ucall_alloc() and so cannot be used here. + */ + ucall_arch_do_ucall(GUEST_UCALL_FAILED); return NULL; } @@ -93,6 +102,9 @@ uint64_t get_ucall(struct kvm_vcpu *vcpu, struct ucall *uc) addr = ucall_arch_get_ucall(vcpu); if (addr) { + TEST_ASSERT(addr != (void *)GUEST_UCALL_FAILED, + "Guest failed to allocate ucall struct"); + memcpy(uc, addr, sizeof(*uc)); vcpu_run_complete_io(vcpu); } else { diff --git a/tools/testing/selftests/kvm/lib/x86_64/processor.c b/tools/testing/selftests/kvm/lib/x86_64/processor.c index c4d368d56cfe..acfa1d01e7df 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/processor.c +++ b/tools/testing/selftests/kvm/lib/x86_64/processor.c @@ -1031,7 +1031,7 @@ bool is_amd_cpu(void) void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits) { if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) { - *pa_bits == kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32; + *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32; *va_bits = 32; } else { *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR); diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index e698306bf49d..e6587e193490 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -265,6 +265,9 @@ static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size) slots = data->nslots; while (--slots > 1) { pages_per_slot = mempages / slots; + if (!pages_per_slot) + continue; + rempages = mempages % pages_per_slot; if (check_slot_pages(host_page_size, guest_page_size, pages_per_slot, rempages)) diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c index 3045fdf9bdf5..f74e76d03b7e 100644 --- a/tools/testing/selftests/kvm/rseq_test.c +++ b/tools/testing/selftests/kvm/rseq_test.c @@ -41,18 +41,6 @@ static void guest_code(void) GUEST_SYNC(0); } -/* - * We have to perform direct system call for getcpu() because it's - * not available until glic 2.29. - */ -static void sys_getcpu(unsigned *cpu) -{ - int r; - - r = syscall(__NR_getcpu, cpu, NULL, NULL); - TEST_ASSERT(!r, "getcpu failed, errno = %d (%s)", errno, strerror(errno)); -} - static int next_cpu(int cpu) { /* @@ -249,7 +237,9 @@ int main(int argc, char *argv[]) * across the seq_cnt reads. */ smp_rmb(); - sys_getcpu(&cpu); + r = sys_getcpu(&cpu, NULL); + TEST_ASSERT(!r, "getcpu failed, errno = %d (%s)", + errno, strerror(errno)); rseq_cpu = rseq_current_cpu_raw(); smp_rmb(); } while (snapshot != atomic_read(&seq_cnt)); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c b/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c index 8b791eac7d5a..0cbb0e646ef8 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_ipi.c @@ -193,8 +193,9 @@ static void sender_guest_code(void *hcall_page, vm_vaddr_t pgs_gpa) GUEST_SYNC(stage++); /* * 'XMM Fast' HvCallSendSyntheticClusterIpiEx to HV_GENERIC_SET_ALL. - * Nothing to write anything to XMM regs. */ + ipi_ex->vp_set.valid_bank_mask = 0; + hyperv_write_xmm_input(&ipi_ex->vp_set.valid_bank_mask, 2); hyperv_hypercall(HVCALL_SEND_IPI_EX | HV_HYPERCALL_FAST_BIT, IPI_VECTOR, HV_GENERIC_SET_ALL); nop_loop(); diff --git a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c index ea0978f22db8..251794f83719 100644 --- a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c @@ -241,7 +241,7 @@ int main(int argc, char **argv) while ((opt = getopt(argc, argv, "hp:t:r")) != -1) { switch (opt) { case 'p': - reclaim_period_ms = atoi_non_negative("Reclaim period", optarg); + reclaim_period_ms = atoi_positive("Reclaim period", optarg); break; case 't': token = atoi_paranoid(optarg); diff --git a/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c b/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c index e497ace629c1..b34980d45648 100644 --- a/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c +++ b/tools/testing/selftests/kvm/x86_64/svm_nested_soft_inject_test.c @@ -41,8 +41,17 @@ static void guest_int_handler(struct ex_regs *regs) static void l2_guest_code_int(void) { GUEST_ASSERT_1(int_fired == 1, int_fired); - vmmcall(); - ud2(); + + /* + * Same as the vmmcall() function, but with a ud2 sneaked after the + * vmmcall. The caller injects an exception with the return address + * increased by 2, so the "pop rbp" must be after the ud2 and we cannot + * use vmmcall() directly. + */ + __asm__ __volatile__("push %%rbp; vmmcall; ud2; pop %%rbp" + : : "a"(0xdeadbeef), "c"(0xbeefdead) + : "rbx", "rdx", "rsi", "rdi", "r8", "r9", + "r10", "r11", "r12", "r13", "r14", "r15"); GUEST_ASSERT_1(bp_fired == 1, bp_fired); hlt(); diff --git a/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c index 5943187e8594..ff8ecdf32ae0 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_tsc_adjust_test.c @@ -49,11 +49,6 @@ enum { NUM_VMX_PAGES, }; -struct kvm_single_msr { - struct kvm_msrs header; - struct kvm_msr_entry entry; -} __attribute__((packed)); - /* The virtual machine object. */ static struct kvm_vm *vm; diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 721f6a693799..13c75dc18c10 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -434,6 +434,7 @@ static void *juggle_shinfo_state(void *arg) int main(int argc, char *argv[]) { struct timespec min_ts, max_ts, vm_ts; + struct kvm_xen_hvm_attr evt_reset; struct kvm_vm *vm; pthread_t thread; bool verbose; @@ -962,6 +963,10 @@ int main(int argc, char *argv[]) } done: + evt_reset.type = KVM_XEN_ATTR_TYPE_EVTCHN; + evt_reset.u.evtchn.flags = KVM_XEN_EVTCHN_RESET; + vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &evt_reset); + alarm(0); clock_gettime(CLOCK_REALTIME, &max_ts); diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 291144c284fb..f7900e75d230 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -20,7 +20,7 @@ CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) ifeq ($(CROSS_COMPILE),) ifeq ($(CLANG_TARGET_FLAGS),) -$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk +$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk) else CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) endif # CLANG_TARGET_FLAGS diff --git a/tools/testing/selftests/membarrier/membarrier_test_impl.h b/tools/testing/selftests/membarrier/membarrier_test_impl.h index 186be69f0a59..af89855adb7b 100644 --- a/tools/testing/selftests/membarrier/membarrier_test_impl.h +++ b/tools/testing/selftests/membarrier/membarrier_test_impl.h @@ -9,11 +9,38 @@ #include "../kselftest.h" +static int registrations; + static int sys_membarrier(int cmd, int flags) { return syscall(__NR_membarrier, cmd, flags); } +static int test_membarrier_get_registrations(int cmd) +{ + int ret, flags = 0; + const char *test_name = + "sys membarrier MEMBARRIER_CMD_GET_REGISTRATIONS"; + + registrations |= cmd; + + ret = sys_membarrier(MEMBARRIER_CMD_GET_REGISTRATIONS, 0); + if (ret < 0) { + ksft_exit_fail_msg( + "%s test: flags = %d, errno = %d\n", + test_name, flags, errno); + } else if (ret != registrations) { + ksft_exit_fail_msg( + "%s test: flags = %d, ret = %d, registrations = %d\n", + test_name, flags, ret, registrations); + } + ksft_test_result_pass( + "%s test: flags = %d, ret = %d, registrations = %d\n", + test_name, flags, ret, registrations); + + return 0; +} + static int test_membarrier_cmd_fail(void) { int cmd = -1, flags = 0; @@ -113,6 +140,8 @@ static int test_membarrier_register_private_expedited_success(void) ksft_test_result_pass( "%s test: flags = %d\n", test_name, flags); + + test_membarrier_get_registrations(cmd); return 0; } @@ -170,6 +199,8 @@ static int test_membarrier_register_private_expedited_sync_core_success(void) ksft_test_result_pass( "%s test: flags = %d\n", test_name, flags); + + test_membarrier_get_registrations(cmd); return 0; } @@ -204,6 +235,8 @@ static int test_membarrier_register_global_expedited_success(void) ksft_test_result_pass( "%s test: flags = %d\n", test_name, flags); + + test_membarrier_get_registrations(cmd); return 0; } diff --git a/tools/testing/selftests/membarrier/membarrier_test_multi_thread.c b/tools/testing/selftests/membarrier/membarrier_test_multi_thread.c index ac5613e5b0eb..a9cc17facfb3 100644 --- a/tools/testing/selftests/membarrier/membarrier_test_multi_thread.c +++ b/tools/testing/selftests/membarrier/membarrier_test_multi_thread.c @@ -62,7 +62,7 @@ static int test_mt_membarrier(void) int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(13); + ksft_set_plan(16); test_membarrier_query(); diff --git a/tools/testing/selftests/membarrier/membarrier_test_single_thread.c b/tools/testing/selftests/membarrier/membarrier_test_single_thread.c index c1c963902854..4cdc8b1d124c 100644 --- a/tools/testing/selftests/membarrier/membarrier_test_single_thread.c +++ b/tools/testing/selftests/membarrier/membarrier_test_single_thread.c @@ -12,7 +12,9 @@ int main(int argc, char **argv) { ksft_print_header(); - ksft_set_plan(13); + ksft_set_plan(18); + + test_membarrier_get_registrations(/*cmd=*/0); test_membarrier_query(); @@ -20,5 +22,7 @@ int main(int argc, char **argv) test_membarrier_success(); + test_membarrier_get_registrations(/*cmd=*/0); + return ksft_exit_pass(); } diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index 9cc84114741d..a6911cae368c 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only bind_bhash +bind_timewait csum cmsg_sender diag_uid diff --git a/tools/testing/selftests/net/af_unix/test_unix_oob.c b/tools/testing/selftests/net/af_unix/test_unix_oob.c index b57e91e1c3f2..532459a15067 100644 --- a/tools/testing/selftests/net/af_unix/test_unix_oob.c +++ b/tools/testing/selftests/net/af_unix/test_unix_oob.c @@ -124,7 +124,7 @@ void producer(struct sockaddr_un *consumer_addr) wait_for_signal(pipefd[0]); if (connect(cfd, (struct sockaddr *)consumer_addr, - sizeof(struct sockaddr)) != 0) { + sizeof(*consumer_addr)) != 0) { perror("Connect failed"); kill(0, SIGTERM); exit(1); diff --git a/tools/testing/selftests/net/arp_ndisc_evict_nocarrier.sh b/tools/testing/selftests/net/arp_ndisc_evict_nocarrier.sh index b5af08af8559..4a110bb01e53 100755 --- a/tools/testing/selftests/net/arp_ndisc_evict_nocarrier.sh +++ b/tools/testing/selftests/net/arp_ndisc_evict_nocarrier.sh @@ -18,14 +18,15 @@ readonly V4_ADDR1=10.0.10.2 readonly V6_ADDR0=2001:db8:91::1 readonly V6_ADDR1=2001:db8:91::2 nsid=100 +ret=0 cleanup_v6() { ip netns del me ip netns del peer - sysctl -w net.ipv4.conf.veth0.ndisc_evict_nocarrier=1 >/dev/null 2>&1 - sysctl -w net.ipv4.conf.all.ndisc_evict_nocarrier=1 >/dev/null 2>&1 + sysctl -w net.ipv6.conf.veth1.ndisc_evict_nocarrier=1 >/dev/null 2>&1 + sysctl -w net.ipv6.conf.all.ndisc_evict_nocarrier=1 >/dev/null 2>&1 } create_ns() @@ -61,7 +62,7 @@ setup_v6() { if [ $? -ne 0 ]; then cleanup_v6 echo "failed" - exit + exit 1 fi # Set veth2 down, which will put veth1 in NOCARRIER state @@ -88,7 +89,7 @@ setup_v4() { if [ $? -ne 0 ]; then cleanup_v4 echo "failed" - exit + exit 1 fi # Set veth1 down, which will put veth0 in NOCARRIER state @@ -115,6 +116,7 @@ run_arp_evict_nocarrier_enabled() { if [ $? -eq 0 ];then echo "failed" + ret=1 else echo "ok" fi @@ -134,6 +136,7 @@ run_arp_evict_nocarrier_disabled() { echo "ok" else echo "failed" + ret=1 fi cleanup_v4 @@ -164,6 +167,7 @@ run_ndisc_evict_nocarrier_enabled() { if [ $? -eq 0 ];then echo "failed" + ret=1 else echo "ok" fi @@ -182,6 +186,7 @@ run_ndisc_evict_nocarrier_disabled() { echo "ok" else echo "failed" + ret=1 fi cleanup_v6 @@ -198,6 +203,7 @@ run_ndisc_evict_nocarrier_disabled_all() { echo "ok" else echo "failed" + ret=1 fi cleanup_v6 @@ -218,3 +224,4 @@ if [ "$(id -u)" -ne 0 ];then fi run_all_tests +exit $ret diff --git a/tools/testing/selftests/net/bind_timewait.c b/tools/testing/selftests/net/bind_timewait.c new file mode 100644 index 000000000000..cb9fdf51ea59 --- /dev/null +++ b/tools/testing/selftests/net/bind_timewait.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright Amazon.com Inc. or its affiliates. */ + +#include <sys/socket.h> +#include <netinet/in.h> + +#include "../kselftest_harness.h" + +FIXTURE(bind_timewait) +{ + struct sockaddr_in addr; + socklen_t addrlen; +}; + +FIXTURE_VARIANT(bind_timewait) +{ + __u32 addr_const; +}; + +FIXTURE_VARIANT_ADD(bind_timewait, localhost) +{ + .addr_const = INADDR_LOOPBACK +}; + +FIXTURE_VARIANT_ADD(bind_timewait, addrany) +{ + .addr_const = INADDR_ANY +}; + +FIXTURE_SETUP(bind_timewait) +{ + self->addr.sin_family = AF_INET; + self->addr.sin_port = 0; + self->addr.sin_addr.s_addr = htonl(variant->addr_const); + self->addrlen = sizeof(self->addr); +} + +FIXTURE_TEARDOWN(bind_timewait) +{ +} + +void create_timewait_socket(struct __test_metadata *_metadata, + FIXTURE_DATA(bind_timewait) *self) +{ + int server_fd, client_fd, child_fd, ret; + struct sockaddr_in addr; + socklen_t addrlen; + + server_fd = socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GT(server_fd, 0); + + ret = bind(server_fd, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(ret, 0); + + ret = listen(server_fd, 1); + ASSERT_EQ(ret, 0); + + ret = getsockname(server_fd, (struct sockaddr *)&self->addr, &self->addrlen); + ASSERT_EQ(ret, 0); + + client_fd = socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GT(client_fd, 0); + + ret = connect(client_fd, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(ret, 0); + + addrlen = sizeof(addr); + child_fd = accept(server_fd, (struct sockaddr *)&addr, &addrlen); + ASSERT_GT(child_fd, 0); + + close(child_fd); + close(client_fd); + close(server_fd); +} + +TEST_F(bind_timewait, 1) +{ + int fd, ret; + + create_timewait_socket(_metadata, self); + + fd = socket(AF_INET, SOCK_STREAM, 0); + ASSERT_GT(fd, 0); + + ret = bind(fd, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(ret, -1); + ASSERT_EQ(errno, EADDRINUSE); + + close(fd); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/net/cmsg_ipv6.sh b/tools/testing/selftests/net/cmsg_ipv6.sh index 2d89cb0ad288..330d0b1ceced 100755 --- a/tools/testing/selftests/net/cmsg_ipv6.sh +++ b/tools/testing/selftests/net/cmsg_ipv6.sh @@ -6,7 +6,7 @@ ksft_skip=4 NS=ns IP6=2001:db8:1::1/64 TGT6=2001:db8:1::2 -TMPF=`mktemp` +TMPF=$(mktemp --suffix ".pcap") cleanup() { diff --git a/tools/testing/selftests/net/cmsg_sender.c b/tools/testing/selftests/net/cmsg_sender.c index 75dd83e39207..24b21b15ed3f 100644 --- a/tools/testing/selftests/net/cmsg_sender.c +++ b/tools/testing/selftests/net/cmsg_sender.c @@ -110,7 +110,7 @@ static void __attribute__((noreturn)) cs_usage(const char *bin) static void cs_parse_args(int argc, char *argv[]) { - char o; + int o; while ((o = getopt(argc, argv, "46sS:p:m:M:d:tf:F:c:C:l:L:H:")) != -1) { switch (o) { diff --git a/tools/testing/selftests/net/fib_rule_tests.sh b/tools/testing/selftests/net/fib_rule_tests.sh index c245476fa29d..63c3eaec8d30 100755 --- a/tools/testing/selftests/net/fib_rule_tests.sh +++ b/tools/testing/selftests/net/fib_rule_tests.sh @@ -10,8 +10,10 @@ ret=0 PAUSE_ON_FAIL=${PAUSE_ON_FAIL:=no} IP="ip -netns testns" +IP_PEER="ip -netns peerns" RTABLE=100 +RTABLE_PEER=101 GW_IP4=192.51.100.2 SRC_IP=192.51.100.3 GW_IP6=2001:db8:1::2 @@ -20,7 +22,9 @@ SRC_IP6=2001:db8:1::3 DEV_ADDR=192.51.100.1 DEV_ADDR6=2001:db8:1::1 DEV=dummy0 -TESTS="fib_rule6 fib_rule4" +TESTS="fib_rule6 fib_rule4 fib_rule6_connect fib_rule4_connect" + +SELFTEST_PATH="" log_test() { @@ -52,6 +56,31 @@ log_section() echo "######################################################################" } +check_nettest() +{ + if which nettest > /dev/null 2>&1; then + return 0 + fi + + # Add the selftest directory to PATH if not already done + if [ "${SELFTEST_PATH}" = "" ]; then + SELFTEST_PATH="$(dirname $0)" + PATH="${PATH}:${SELFTEST_PATH}" + + # Now retry with the new path + if which nettest > /dev/null 2>&1; then + return 0 + fi + + if [ "${ret}" -eq 0 ]; then + ret="${ksft_skip}" + fi + echo "nettest not found (try 'make -C ${SELFTEST_PATH} nettest')" + fi + + return 1 +} + setup() { set -e @@ -72,6 +101,39 @@ cleanup() ip netns del testns } +setup_peer() +{ + set -e + + ip netns add peerns + $IP_PEER link set dev lo up + + ip link add name veth0 netns testns type veth \ + peer name veth1 netns peerns + $IP link set dev veth0 up + $IP_PEER link set dev veth1 up + + $IP address add 192.0.2.10 peer 192.0.2.11/32 dev veth0 + $IP_PEER address add 192.0.2.11 peer 192.0.2.10/32 dev veth1 + + $IP address add 2001:db8::10 peer 2001:db8::11/128 dev veth0 nodad + $IP_PEER address add 2001:db8::11 peer 2001:db8::10/128 dev veth1 nodad + + $IP_PEER address add 198.51.100.11/32 dev lo + $IP route add table $RTABLE_PEER 198.51.100.11/32 via 192.0.2.11 + + $IP_PEER address add 2001:db8::1:11/128 dev lo + $IP route add table $RTABLE_PEER 2001:db8::1:11/128 via 2001:db8::11 + + set +e +} + +cleanup_peer() +{ + $IP link del dev veth0 + ip netns del peerns +} + fib_check_iproute_support() { ip rule help 2>&1 | grep -q $1 @@ -190,6 +252,37 @@ fib_rule6_test() fi } +# Verify that the IPV6_TCLASS option of UDPv6 and TCPv6 sockets is properly +# taken into account when connecting the socket and when sending packets. +fib_rule6_connect_test() +{ + local dsfield + + if ! check_nettest; then + echo "SKIP: Could not run test without nettest tool" + return + fi + + setup_peer + $IP -6 rule add dsfield 0x04 table $RTABLE_PEER + + # Combine the base DS Field value (0x04) with all possible ECN values + # (Not-ECT: 0, ECT(1): 1, ECT(0): 2, CE: 3). + # The ECN bits shouldn't influence the result of the test. + for dsfield in 0x04 0x05 0x06 0x07; do + nettest -q -6 -B -t 5 -N testns -O peerns -U -D \ + -Q "${dsfield}" -l 2001:db8::1:11 -r 2001:db8::1:11 + log_test $? 0 "rule6 dsfield udp connect (dsfield ${dsfield})" + + nettest -q -6 -B -t 5 -N testns -O peerns -Q "${dsfield}" \ + -l 2001:db8::1:11 -r 2001:db8::1:11 + log_test $? 0 "rule6 dsfield tcp connect (dsfield ${dsfield})" + done + + $IP -6 rule del dsfield 0x04 table $RTABLE_PEER + cleanup_peer +} + fib_rule4_del() { $IP rule del $1 @@ -296,6 +389,37 @@ fib_rule4_test() fi } +# Verify that the IP_TOS option of UDPv4 and TCPv4 sockets is properly taken +# into account when connecting the socket and when sending packets. +fib_rule4_connect_test() +{ + local dsfield + + if ! check_nettest; then + echo "SKIP: Could not run test without nettest tool" + return + fi + + setup_peer + $IP -4 rule add dsfield 0x04 table $RTABLE_PEER + + # Combine the base DS Field value (0x04) with all possible ECN values + # (Not-ECT: 0, ECT(1): 1, ECT(0): 2, CE: 3). + # The ECN bits shouldn't influence the result of the test. + for dsfield in 0x04 0x05 0x06 0x07; do + nettest -q -B -t 5 -N testns -O peerns -D -U -Q "${dsfield}" \ + -l 198.51.100.11 -r 198.51.100.11 + log_test $? 0 "rule4 dsfield udp connect (dsfield ${dsfield})" + + nettest -q -B -t 5 -N testns -O peerns -Q "${dsfield}" \ + -l 198.51.100.11 -r 198.51.100.11 + log_test $? 0 "rule4 dsfield tcp connect (dsfield ${dsfield})" + done + + $IP -4 rule del dsfield 0x04 table $RTABLE_PEER + cleanup_peer +} + run_fibrule_tests() { log_section "IPv4 fib rule" @@ -345,6 +469,8 @@ do case $t in fib_rule6_test|fib_rule6) fib_rule6_test;; fib_rule4_test|fib_rule4) fib_rule4_test;; + fib_rule6_connect_test|fib_rule6_connect) fib_rule6_connect_test;; + fib_rule4_connect_test|fib_rule4_connect) fib_rule4_connect_test;; help) echo "Test names: $TESTS"; exit 0;; diff --git a/tools/testing/selftests/net/forwarding/lib.sh b/tools/testing/selftests/net/forwarding/lib.sh index 1c4f866de7d7..3d8e4ebda1b6 100755 --- a/tools/testing/selftests/net/forwarding/lib.sh +++ b/tools/testing/selftests/net/forwarding/lib.sh @@ -914,14 +914,14 @@ sysctl_set() local value=$1; shift SYSCTL_ORIG[$key]=$(sysctl -n $key) - sysctl -qw $key=$value + sysctl -qw $key="$value" } sysctl_restore() { local key=$1; shift - sysctl -qw $key=${SYSCTL_ORIG["$key"]} + sysctl -qw $key="${SYSCTL_ORIG[$key]}" } forwarding_enable() diff --git a/tools/testing/selftests/net/l2_tos_ttl_inherit.sh b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh index dca1e6f777a8..f11756e7df2f 100755 --- a/tools/testing/selftests/net/l2_tos_ttl_inherit.sh +++ b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh @@ -12,19 +12,27 @@ # In addition this script also checks if forcing a specific field in the # outer header is working. +# Return 4 by default (Kselftest SKIP code) +ERR=4 + if [ "$(id -u)" != "0" ]; then echo "Please run as root." - exit 0 + exit $ERR fi if ! which tcpdump > /dev/null 2>&1; then echo "No tcpdump found. Required for this test." - exit 0 + exit $ERR fi expected_tos="0x00" expected_ttl="0" failed=false +readonly NS0=$(mktemp -u ns0-XXXXXXXX) +readonly NS1=$(mktemp -u ns1-XXXXXXXX) + +RUN_NS0="ip netns exec ${NS0}" + get_random_tos() { # Get a random hex tos value between 0x00 and 0xfc, a multiple of 4 echo "0x$(tr -dc '0-9a-f' < /dev/urandom | head -c 1)\ @@ -61,7 +69,6 @@ setup() { local vlan="$5" local test_tos="0x00" local test_ttl="0" - local ns="ip netns exec testing" # We don't want a test-tos of 0x00, # because this is the value that we get when no tos is set. @@ -94,14 +101,15 @@ setup() { printf "│%7s │%6s │%6s │%13s │%13s │%6s │" \ "$type" "$outer" "$inner" "$tos" "$ttl" "$vlan" - # Create 'testing' netns, veth pair and connect main ns with testing ns - ip netns add testing - ip link add type veth - ip link set veth1 netns testing - ip link set veth0 up - $ns ip link set veth1 up - ip addr flush dev veth0 - $ns ip addr flush dev veth1 + # Create netns NS0 and NS1 and connect them with a veth pair + ip netns add "${NS0}" + ip netns add "${NS1}" + ip link add name veth0 netns "${NS0}" type veth \ + peer name veth1 netns "${NS1}" + ip -netns "${NS0}" link set dev veth0 up + ip -netns "${NS1}" link set dev veth1 up + ip -netns "${NS0}" address flush dev veth0 + ip -netns "${NS1}" address flush dev veth1 local local_addr1="" local local_addr2="" @@ -127,51 +135,59 @@ setup() { if [ "$type" = "gre" ]; then type="gretap" fi - ip addr add 198.18.0.1/24 dev veth0 - $ns ip addr add 198.18.0.2/24 dev veth1 - ip link add name tep0 type $type $local_addr1 remote \ - 198.18.0.2 tos $test_tos ttl $test_ttl $vxlan $geneve - $ns ip link add name tep1 type $type $local_addr2 remote \ - 198.18.0.1 tos $test_tos ttl $test_ttl $vxlan $geneve + ip -netns "${NS0}" address add 198.18.0.1/24 dev veth0 + ip -netns "${NS1}" address add 198.18.0.2/24 dev veth1 + ip -netns "${NS0}" link add name tep0 type $type $local_addr1 \ + remote 198.18.0.2 tos $test_tos ttl $test_ttl \ + $vxlan $geneve + ip -netns "${NS1}" link add name tep1 type $type $local_addr2 \ + remote 198.18.0.1 tos $test_tos ttl $test_ttl \ + $vxlan $geneve elif [ "$outer" = "6" ]; then if [ "$type" = "gre" ]; then type="ip6gretap" fi - ip addr add fdd1:ced0:5d88:3fce::1/64 dev veth0 - $ns ip addr add fdd1:ced0:5d88:3fce::2/64 dev veth1 - ip link add name tep0 type $type $local_addr1 \ - remote fdd1:ced0:5d88:3fce::2 tos $test_tos ttl $test_ttl \ - $vxlan $geneve - $ns ip link add name tep1 type $type $local_addr2 \ - remote fdd1:ced0:5d88:3fce::1 tos $test_tos ttl $test_ttl \ - $vxlan $geneve + ip -netns "${NS0}" address add fdd1:ced0:5d88:3fce::1/64 \ + dev veth0 nodad + ip -netns "${NS1}" address add fdd1:ced0:5d88:3fce::2/64 \ + dev veth1 nodad + ip -netns "${NS0}" link add name tep0 type $type $local_addr1 \ + remote fdd1:ced0:5d88:3fce::2 tos $test_tos \ + ttl $test_ttl $vxlan $geneve + ip -netns "${NS1}" link add name tep1 type $type $local_addr2 \ + remote fdd1:ced0:5d88:3fce::1 tos $test_tos \ + ttl $test_ttl $vxlan $geneve fi # Bring L2-tunnel link up and create VLAN on top - ip link set tep0 up - $ns ip link set tep1 up - ip addr flush dev tep0 - $ns ip addr flush dev tep1 + ip -netns "${NS0}" link set tep0 up + ip -netns "${NS1}" link set tep1 up + ip -netns "${NS0}" address flush dev tep0 + ip -netns "${NS1}" address flush dev tep1 local parent if $vlan; then parent="vlan99-" - ip link add link tep0 name ${parent}0 type vlan id 99 - $ns ip link add link tep1 name ${parent}1 type vlan id 99 - ip link set ${parent}0 up - $ns ip link set ${parent}1 up - ip addr flush dev ${parent}0 - $ns ip addr flush dev ${parent}1 + ip -netns "${NS0}" link add link tep0 name ${parent}0 \ + type vlan id 99 + ip -netns "${NS1}" link add link tep1 name ${parent}1 \ + type vlan id 99 + ip -netns "${NS0}" link set dev ${parent}0 up + ip -netns "${NS1}" link set dev ${parent}1 up + ip -netns "${NS0}" address flush dev ${parent}0 + ip -netns "${NS1}" address flush dev ${parent}1 else parent="tep" fi # Assign inner IPv4/IPv6 addresses if [ "$inner" = "4" ] || [ "$inner" = "other" ]; then - ip addr add 198.19.0.1/24 brd + dev ${parent}0 - $ns ip addr add 198.19.0.2/24 brd + dev ${parent}1 + ip -netns "${NS0}" address add 198.19.0.1/24 brd + dev ${parent}0 + ip -netns "${NS1}" address add 198.19.0.2/24 brd + dev ${parent}1 elif [ "$inner" = "6" ]; then - ip addr add fdd4:96cf:4eae:443b::1/64 dev ${parent}0 - $ns ip addr add fdd4:96cf:4eae:443b::2/64 dev ${parent}1 + ip -netns "${NS0}" address add fdd4:96cf:4eae:443b::1/64 \ + dev ${parent}0 nodad + ip -netns "${NS1}" address add fdd4:96cf:4eae:443b::2/64 \ + dev ${parent}1 nodad fi } @@ -192,10 +208,10 @@ verify() { ping_dst="198.19.0.3" # Generates ARPs which are not IPv4/IPv6 fi if [ "$tos_ttl" = "inherit" ]; then - ping -i 0.1 $ping_dst -Q "$expected_tos" -t "$expected_ttl" \ - 2>/dev/null 1>&2 & ping_pid="$!" + ${RUN_NS0} ping -i 0.1 $ping_dst -Q "$expected_tos" \ + -t "$expected_ttl" 2>/dev/null 1>&2 & ping_pid="$!" else - ping -i 0.1 $ping_dst 2>/dev/null 1>&2 & ping_pid="$!" + ${RUN_NS0} ping -i 0.1 $ping_dst 2>/dev/null 1>&2 & ping_pid="$!" fi local tunnel_type_offset tunnel_type_proto req_proto_offset req_offset if [ "$type" = "gre" ]; then @@ -216,10 +232,12 @@ verify() { req_proto_offset="$((req_proto_offset + 4))" req_offset="$((req_offset + 4))" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip[$tunnel_type_offset] = $tunnel_type_proto and \ - ip[$req_proto_offset] = 0x01 and \ - ip[$req_offset] = 0x08 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x01 and \ + ip[$req_offset] = 0x08 2>/dev/null \ + | head -n 1)" elif [ "$inner" = "6" ]; then req_proto_offset="44" req_offset="78" @@ -231,10 +249,12 @@ verify() { req_proto_offset="$((req_proto_offset + 4))" req_offset="$((req_offset + 4))" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip[$tunnel_type_offset] = $tunnel_type_proto and \ - ip[$req_proto_offset] = 0x3a and \ - ip[$req_offset] = 0x80 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x3a and \ + ip[$req_offset] = 0x80 2>/dev/null \ + | head -n 1)" elif [ "$inner" = "other" ]; then req_proto_offset="36" req_offset="45" @@ -250,11 +270,13 @@ verify() { expected_tos="0x00" expected_ttl="64" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip[$tunnel_type_offset] = $tunnel_type_proto and \ - ip[$req_proto_offset] = 0x08 and \ - ip[$((req_proto_offset + 1))] = 0x06 and \ - ip[$req_offset] = 0x01 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x08 and \ + ip[$((req_proto_offset + 1))] = 0x06 and \ + ip[$req_offset] = 0x01 2>/dev/null \ + | head -n 1)" fi elif [ "$outer" = "6" ]; then if [ "$type" = "gre" ]; then @@ -273,10 +295,12 @@ verify() { req_proto_offset="$((req_proto_offset + 4))" req_offset="$((req_offset + 4))" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip6[$tunnel_type_offset] = $tunnel_type_proto and \ - ip6[$req_proto_offset] = 0x01 and \ - ip6[$req_offset] = 0x08 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x01 and \ + ip6[$req_offset] = 0x08 2>/dev/null \ + | head -n 1)" elif [ "$inner" = "6" ]; then local req_proto_offset="72" local req_offset="106" @@ -288,10 +312,12 @@ verify() { req_proto_offset="$((req_proto_offset + 4))" req_offset="$((req_offset + 4))" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip6[$tunnel_type_offset] = $tunnel_type_proto and \ - ip6[$req_proto_offset] = 0x3a and \ - ip6[$req_offset] = 0x80 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x3a and \ + ip6[$req_offset] = 0x80 2>/dev/null \ + | head -n 1)" elif [ "$inner" = "other" ]; then local req_proto_offset="64" local req_offset="73" @@ -307,15 +333,17 @@ verify() { expected_tos="0x00" expected_ttl="64" fi - out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ - ip6[$tunnel_type_offset] = $tunnel_type_proto and \ - ip6[$req_proto_offset] = 0x08 and \ - ip6[$((req_proto_offset + 1))] = 0x06 and \ - ip6[$req_offset] = 0x01 2>/dev/null | head -n 1)" + out="$(${RUN_NS0} tcpdump --immediate-mode -p -c 1 -v \ + -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x08 and \ + ip6[$((req_proto_offset + 1))] = 0x06 and \ + ip6[$req_offset] = 0x01 2>/dev/null \ + | head -n 1)" fi fi kill -9 $ping_pid - wait $ping_pid 2>/dev/null + wait $ping_pid 2>/dev/null || true result="FAIL" if [ "$outer" = "4" ]; then captured_ttl="$(get_field "ttl" "$out")" @@ -351,11 +379,35 @@ verify() { } cleanup() { - ip link del veth0 2>/dev/null - ip netns del testing 2>/dev/null - ip link del tep0 2>/dev/null + ip netns del "${NS0}" 2>/dev/null + ip netns del "${NS1}" 2>/dev/null } +exit_handler() { + # Don't exit immediately if one of the intermediate commands fails. + # We might be called at the end of the script, when the network + # namespaces have already been deleted. So cleanup() may fail, but we + # still need to run until 'exit $ERR' or the script won't return the + # correct error code. + set +e + + cleanup + + exit $ERR +} + +# Restore the default SIGINT handler (just in case) and exit. +# The exit handler will take care of cleaning everything up. +interrupted() { + trap - INT + + exit $ERR +} + +set -e +trap exit_handler EXIT +trap interrupted INT + printf "┌────────┬───────┬───────┬──────────────┬" printf "──────────────┬───────┬────────┐\n" for type in gre vxlan geneve; do @@ -385,6 +437,10 @@ done printf "└────────┴───────┴───────┴──────────────┴" printf "──────────────┴───────┴────────┘\n" +# All tests done. +# Set ERR appropriately: it will be returned by the exit handler. if $failed; then - exit 1 + ERR=1 +else + ERR=0 fi diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index d11d3d566608..079f8f46849d 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -498,6 +498,12 @@ kill_events_pids() kill_wait $evts_ns2_pid } +kill_tests_wait() +{ + kill -SIGUSR1 $(ip netns pids $ns2) $(ip netns pids $ns1) + wait +} + pm_nl_set_limits() { local ns=$1 @@ -1694,6 +1700,7 @@ chk_subflow_nr() local subflow_nr=$3 local cnt1 local cnt2 + local dump_stats if [ -n "${need_title}" ]; then printf "%03u %-36s %s" "${TEST_COUNT}" "${TEST_NAME}" "${msg}" @@ -1711,7 +1718,12 @@ chk_subflow_nr() echo "[ ok ]" fi - [ "${dump_stats}" = 1 ] && ( ss -N $ns1 -tOni ; ss -N $ns1 -tOni | grep token; ip -n $ns1 mptcp endpoint ) + if [ "${dump_stats}" = 1 ]; then + ss -N $ns1 -tOni + ss -N $ns1 -tOni | grep token + ip -n $ns1 mptcp endpoint + dump_stats + fi } chk_link_usage() @@ -3049,7 +3061,7 @@ endpoint_tests() pm_nl_set_limits $ns1 2 2 pm_nl_set_limits $ns2 2 2 pm_nl_add_endpoint $ns1 10.0.2.1 flags signal - run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow & + run_tests $ns1 $ns2 10.0.1.1 0 0 0 slow 2>/dev/null & wait_mpj $ns1 pm_nl_check_endpoint 1 "creation" \ @@ -3062,14 +3074,14 @@ endpoint_tests() pm_nl_add_endpoint $ns2 10.0.2.2 flags signal pm_nl_check_endpoint 0 "modif is allowed" \ $ns2 10.0.2.2 id 1 flags signal - wait + kill_tests_wait fi if reset "delete and re-add"; then pm_nl_set_limits $ns1 1 1 pm_nl_set_limits $ns2 1 1 pm_nl_add_endpoint $ns2 10.0.2.2 id 2 dev ns2eth2 flags subflow - run_tests $ns1 $ns2 10.0.1.1 4 0 0 slow & + run_tests $ns1 $ns2 10.0.1.1 4 0 0 speed_20 2>/dev/null & wait_mpj $ns2 pm_nl_del_endpoint $ns2 2 10.0.2.2 @@ -3079,7 +3091,7 @@ endpoint_tests() pm_nl_add_endpoint $ns2 10.0.2.2 dev ns2eth2 flags subflow wait_mpj $ns2 chk_subflow_nr "" "after re-add" 2 - wait + kill_tests_wait fi } diff --git a/tools/testing/selftests/net/mptcp/userspace_pm.sh b/tools/testing/selftests/net/mptcp/userspace_pm.sh index a29deb9fa024..ab2d581f28a1 100755 --- a/tools/testing/selftests/net/mptcp/userspace_pm.sh +++ b/tools/testing/selftests/net/mptcp/userspace_pm.sh @@ -752,6 +752,52 @@ test_subflows() "$server4_token" > /dev/null 2>&1 } +test_subflows_v4_v6_mix() +{ + # Attempt to add a listener at 10.0.2.1:<subflow-port> + ip netns exec "$ns1" ./pm_nl_ctl listen 10.0.2.1\ + $app6_port > /dev/null 2>&1 & + local listener_pid=$! + + # ADD_ADDR4 from server to client machine reusing the subflow port on + # the established v6 connection + :>"$client_evts" + ip netns exec "$ns1" ./pm_nl_ctl ann 10.0.2.1 token "$server6_token" id\ + $server_addr_id dev ns1eth2 > /dev/null 2>&1 + stdbuf -o0 -e0 printf "ADD_ADDR4 id:%d 10.0.2.1 (ns1) => ns2, reuse port\t\t" $server_addr_id + sleep 0.5 + verify_announce_event "$client_evts" "$ANNOUNCED" "$client6_token" "10.0.2.1"\ + "$server_addr_id" "$app6_port" + + # CREATE_SUBFLOW from client to server machine + :>"$client_evts" + ip netns exec "$ns2" ./pm_nl_ctl csf lip 10.0.2.2 lid 23 rip 10.0.2.1 rport\ + $app6_port token "$client6_token" > /dev/null 2>&1 + sleep 0.5 + verify_subflow_events "$client_evts" "$SUB_ESTABLISHED" "$client6_token"\ + "$AF_INET" "10.0.2.2" "10.0.2.1" "$app6_port" "23"\ + "$server_addr_id" "ns2" "ns1" + + # Delete the listener from the server ns, if one was created + kill_wait $listener_pid + + sport=$(sed --unbuffered -n 's/.*\(sport:\)\([[:digit:]]*\).*$/\2/p;q' "$client_evts") + + # DESTROY_SUBFLOW from client to server machine + :>"$client_evts" + ip netns exec "$ns2" ./pm_nl_ctl dsf lip 10.0.2.2 lport "$sport" rip 10.0.2.1 rport\ + $app6_port token "$client6_token" > /dev/null 2>&1 + sleep 0.5 + verify_subflow_events "$client_evts" "$SUB_CLOSED" "$client6_token" \ + "$AF_INET" "10.0.2.2" "10.0.2.1" "$app6_port" "23"\ + "$server_addr_id" "ns2" "ns1" + + # RM_ADDR from server to client machine + ip netns exec "$ns1" ./pm_nl_ctl rem id $server_addr_id token\ + "$server6_token" > /dev/null 2>&1 + sleep 0.5 +} + test_prio() { local count @@ -861,6 +907,7 @@ make_connection "v6" test_announce test_remove test_subflows +test_subflows_v4_v6_mix test_prio test_listener diff --git a/tools/testing/selftests/net/nettest.c b/tools/testing/selftests/net/nettest.c index 7900fa98eccb..ee9a72982705 100644 --- a/tools/testing/selftests/net/nettest.c +++ b/tools/testing/selftests/net/nettest.c @@ -87,6 +87,7 @@ struct sock_args { int use_setsockopt; int use_freebind; int use_cmsg; + uint8_t dsfield; const char *dev; const char *server_dev; int ifindex; @@ -580,6 +581,36 @@ static int set_reuseaddr(int sd) return rc; } +static int set_dsfield(int sd, int version, int dsfield) +{ + if (!dsfield) + return 0; + + switch (version) { + case AF_INET: + if (setsockopt(sd, SOL_IP, IP_TOS, &dsfield, + sizeof(dsfield)) < 0) { + log_err_errno("setsockopt(IP_TOS)"); + return -1; + } + break; + + case AF_INET6: + if (setsockopt(sd, SOL_IPV6, IPV6_TCLASS, &dsfield, + sizeof(dsfield)) < 0) { + log_err_errno("setsockopt(IPV6_TCLASS)"); + return -1; + } + break; + + default: + log_error("Invalid address family\n"); + return -1; + } + + return 0; +} + static int str_to_uint(const char *str, int min, int max, unsigned int *value) { int number; @@ -1317,6 +1348,9 @@ static int msock_init(struct sock_args *args, int server) (char *)&one, sizeof(one)) < 0) log_err_errno("Setting SO_BROADCAST error"); + if (set_dsfield(sd, AF_INET, args->dsfield) != 0) + goto out_err; + if (args->dev && bind_to_device(sd, args->dev) != 0) goto out_err; else if (args->use_setsockopt && @@ -1445,6 +1479,9 @@ static int lsock_init(struct sock_args *args) if (set_reuseport(sd) != 0) goto err; + if (set_dsfield(sd, args->version, args->dsfield) != 0) + goto err; + if (args->dev && bind_to_device(sd, args->dev) != 0) goto err; else if (args->use_setsockopt && @@ -1658,6 +1695,9 @@ static int connectsock(void *addr, socklen_t alen, struct sock_args *args) if (set_reuseport(sd) != 0) goto err; + if (set_dsfield(sd, args->version, args->dsfield) != 0) + goto err; + if (args->dev && bind_to_device(sd, args->dev) != 0) goto err; else if (args->use_setsockopt && @@ -1862,7 +1902,7 @@ static int ipc_parent(int cpid, int fd, struct sock_args *args) return client_status; } -#define GETOPT_STR "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SUCi6xL:0:1:2:3:Fbqf" +#define GETOPT_STR "sr:l:c:Q:p:t:g:P:DRn:M:X:m:d:I:BN:O:SUCi6xL:0:1:2:3:Fbqf" #define OPT_FORCE_BIND_KEY_IFINDEX 1001 #define OPT_NO_BIND_KEY_IFINDEX 1002 @@ -1893,6 +1933,8 @@ static void print_usage(char *prog) " -D|R datagram (D) / raw (R) socket (default stream)\n" " -l addr local address to bind to in server mode\n" " -c addr local address to bind to in client mode\n" + " -Q dsfield DS Field value of the socket (the IP_TOS or\n" + " IPV6_TCLASS socket option)\n" " -x configure XFRM policy on socket\n" "\n" " -d dev bind socket to given device name\n" @@ -1971,6 +2013,13 @@ int main(int argc, char *argv[]) args.has_local_ip = 1; args.client_local_addr_str = optarg; break; + case 'Q': + if (str_to_uint(optarg, 0, 255, &tmp) != 0) { + fprintf(stderr, "Invalid DS Field\n"); + return 1; + } + args.dsfield = tmp; + break; case 'p': if (str_to_uint(optarg, 1, 65535, &tmp) != 0) { fprintf(stderr, "Invalid port\n"); diff --git a/tools/testing/selftests/net/test_vxlan_vnifiltering.sh b/tools/testing/selftests/net/test_vxlan_vnifiltering.sh index 704997ffc244..8c3ac0a72545 100755 --- a/tools/testing/selftests/net/test_vxlan_vnifiltering.sh +++ b/tools/testing/selftests/net/test_vxlan_vnifiltering.sh @@ -293,19 +293,11 @@ setup-vm() { elif [[ -n $vtype && $vtype == "vnifilterg" ]]; then # Add per vni group config with 'bridge vni' api if [ -n "$group" ]; then - if [ "$family" == "v4" ]; then - if [ $mcast -eq 1 ]; then - bridge -netns hv-$hvid vni add dev $vxlandev vni $tid group $group - else - bridge -netns hv-$hvid vni add dev $vxlandev vni $tid remote $group - fi - else - if [ $mcast -eq 1 ]; then - bridge -netns hv-$hvid vni add dev $vxlandev vni $tid group6 $group - else - bridge -netns hv-$hvid vni add dev $vxlandev vni $tid remote6 $group - fi - fi + if [ $mcast -eq 1 ]; then + bridge -netns hv-$hvid vni add dev $vxlandev vni $tid group $group + else + bridge -netns hv-$hvid vni add dev $vxlandev vni $tid remote $group + fi fi fi done diff --git a/tools/testing/selftests/net/toeplitz.c b/tools/testing/selftests/net/toeplitz.c index 90026a27eac0..9ba03164d73a 100644 --- a/tools/testing/selftests/net/toeplitz.c +++ b/tools/testing/selftests/net/toeplitz.c @@ -215,7 +215,7 @@ static char *recv_frame(const struct ring_state *ring, char *frame) } /* A single TPACKET_V3 block can hold multiple frames */ -static void recv_block(struct ring_state *ring) +static bool recv_block(struct ring_state *ring) { struct tpacket_block_desc *block; char *frame; @@ -223,7 +223,7 @@ static void recv_block(struct ring_state *ring) block = (void *)(ring->mmap + ring->idx * ring_block_sz); if (!(block->hdr.bh1.block_status & TP_STATUS_USER)) - return; + return false; frame = (char *)block; frame += block->hdr.bh1.offset_to_first_pkt; @@ -235,6 +235,8 @@ static void recv_block(struct ring_state *ring) block->hdr.bh1.block_status = TP_STATUS_KERNEL; ring->idx = (ring->idx + 1) % ring_block_nr; + + return true; } /* simple test: sleep once unconditionally and then process all rings */ @@ -245,7 +247,7 @@ static void process_rings(void) usleep(1000 * cfg_timeout_msec); for (i = 0; i < num_cpus; i++) - recv_block(&rings[i]); + do {} while (recv_block(&rings[i])); fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n", frames_received - frames_nohash - frames_error, @@ -257,12 +259,12 @@ static char *setup_ring(int fd) struct tpacket_req3 req3 = {0}; void *ring; - req3.tp_retire_blk_tov = cfg_timeout_msec; + req3.tp_retire_blk_tov = cfg_timeout_msec / 8; req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH; req3.tp_frame_size = 2048; req3.tp_frame_nr = 1 << 10; - req3.tp_block_nr = 2; + req3.tp_block_nr = 16; req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr; req3.tp_block_size /= req3.tp_block_nr; diff --git a/tools/testing/selftests/net/udpgso_bench.sh b/tools/testing/selftests/net/udpgso_bench.sh index dc932fd65363..640bc43452fa 100755 --- a/tools/testing/selftests/net/udpgso_bench.sh +++ b/tools/testing/selftests/net/udpgso_bench.sh @@ -7,6 +7,7 @@ readonly GREEN='\033[0;92m' readonly YELLOW='\033[0;33m' readonly RED='\033[0;31m' readonly NC='\033[0m' # No Color +readonly TESTPORT=8000 readonly KSFT_PASS=0 readonly KSFT_FAIL=1 @@ -56,11 +57,26 @@ trap wake_children EXIT run_one() { local -r args=$@ + local nr_socks=0 + local i=0 + local -r timeout=10 + + ./udpgso_bench_rx -p "$TESTPORT" & + ./udpgso_bench_rx -p "$TESTPORT" -t & + + # Wait for the above test program to get ready to receive connections. + while [ "$i" -lt "$timeout" ]; do + nr_socks="$(ss -lnHi | grep -c "\*:${TESTPORT}")" + [ "$nr_socks" -eq 2 ] && break + i=$((i + 1)) + sleep 1 + done + if [ "$nr_socks" -ne 2 ]; then + echo "timed out while waiting for udpgso_bench_rx" + exit 1 + fi - ./udpgso_bench_rx & - ./udpgso_bench_rx -t & - - ./udpgso_bench_tx ${args} + ./udpgso_bench_tx -p "$TESTPORT" ${args} } run_in_netns() { diff --git a/tools/testing/selftests/net/udpgso_bench_rx.c b/tools/testing/selftests/net/udpgso_bench_rx.c index 6a193425c367..4058c7451e70 100644 --- a/tools/testing/selftests/net/udpgso_bench_rx.c +++ b/tools/testing/selftests/net/udpgso_bench_rx.c @@ -250,7 +250,7 @@ static int recv_msg(int fd, char *buf, int len, int *gso_size) static void do_flush_udp(int fd) { static char rbuf[ETH_MAX_MTU]; - int ret, len, gso_size, budget = 256; + int ret, len, gso_size = 0, budget = 256; len = cfg_read_all ? sizeof(rbuf) : 0; while (budget--) { @@ -336,6 +336,8 @@ static void parse_opts(int argc, char **argv) cfg_verify = true; cfg_read_all = true; break; + default: + exit(1); } } diff --git a/tools/testing/selftests/net/udpgso_bench_tx.c b/tools/testing/selftests/net/udpgso_bench_tx.c index f1fdaa270291..477392715a9a 100644 --- a/tools/testing/selftests/net/udpgso_bench_tx.c +++ b/tools/testing/selftests/net/udpgso_bench_tx.c @@ -62,6 +62,7 @@ static int cfg_payload_len = (1472 * 42); static int cfg_port = 8000; static int cfg_runtime_ms = -1; static bool cfg_poll; +static int cfg_poll_loop_timeout_ms = 2000; static bool cfg_segment; static bool cfg_sendmmsg; static bool cfg_tcp; @@ -235,16 +236,17 @@ static void flush_errqueue_recv(int fd) } } -static void flush_errqueue(int fd, const bool do_poll) +static void flush_errqueue(int fd, const bool do_poll, + unsigned long poll_timeout, const bool poll_err) { if (do_poll) { struct pollfd fds = {0}; int ret; fds.fd = fd; - ret = poll(&fds, 1, 500); + ret = poll(&fds, 1, poll_timeout); if (ret == 0) { - if (cfg_verbose) + if ((cfg_verbose) && (poll_err)) fprintf(stderr, "poll timeout\n"); } else if (ret < 0) { error(1, errno, "poll"); @@ -254,6 +256,20 @@ static void flush_errqueue(int fd, const bool do_poll) flush_errqueue_recv(fd); } +static void flush_errqueue_retry(int fd, unsigned long num_sends) +{ + unsigned long tnow, tstop; + bool first_try = true; + + tnow = gettimeofday_ms(); + tstop = tnow + cfg_poll_loop_timeout_ms; + do { + flush_errqueue(fd, true, tstop - tnow, first_try); + first_try = false; + tnow = gettimeofday_ms(); + } while ((stat_zcopies != num_sends) && (tnow < tstop)); +} + static int send_tcp(int fd, char *data) { int ret, done = 0, count = 0; @@ -413,7 +429,8 @@ static int send_udp_segment(int fd, char *data) static void usage(const char *filepath) { - error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]", + error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] " + "[-L secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]", filepath); } @@ -423,7 +440,7 @@ static void parse_opts(int argc, char **argv) int max_len, hdrlen; int c; - while ((c = getopt(argc, argv, "46acC:D:Hl:mM:p:s:PS:tTuvz")) != -1) { + while ((c = getopt(argc, argv, "46acC:D:Hl:L:mM:p:s:PS:tTuvz")) != -1) { switch (c) { case '4': if (cfg_family != PF_UNSPEC) @@ -452,6 +469,9 @@ static void parse_opts(int argc, char **argv) case 'l': cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000; break; + case 'L': + cfg_poll_loop_timeout_ms = strtoul(optarg, NULL, 10) * 1000; + break; case 'm': cfg_sendmmsg = true; break; @@ -490,6 +510,8 @@ static void parse_opts(int argc, char **argv) case 'z': cfg_zerocopy = true; break; + default: + exit(1); } } @@ -677,7 +699,7 @@ int main(int argc, char **argv) num_sends += send_udp(fd, buf[i]); num_msgs++; if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp) - flush_errqueue(fd, cfg_poll); + flush_errqueue(fd, cfg_poll, 500, true); if (cfg_msg_nr && num_msgs >= cfg_msg_nr) break; @@ -696,7 +718,7 @@ int main(int argc, char **argv) } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop)); if (cfg_zerocopy || cfg_tx_tstamp) - flush_errqueue(fd, true); + flush_errqueue_retry(fd, num_sends); if (close(fd)) error(1, errno, "close"); diff --git a/tools/testing/selftests/netfilter/nft_trans_stress.sh b/tools/testing/selftests/netfilter/nft_trans_stress.sh index a7f62ad4f661..2ffba45a78bf 100755 --- a/tools/testing/selftests/netfilter/nft_trans_stress.sh +++ b/tools/testing/selftests/netfilter/nft_trans_stress.sh @@ -10,12 +10,20 @@ ksft_skip=4 testns=testns-$(mktemp -u "XXXXXXXX") +tmp="" tables="foo bar baz quux" global_ret=0 eret=0 lret=0 +cleanup() { + ip netns pids "$testns" | xargs kill 2>/dev/null + ip netns del "$testns" + + rm -f "$tmp" +} + check_result() { local r=$1 @@ -43,6 +51,7 @@ if [ $? -ne 0 ];then exit $ksft_skip fi +trap cleanup EXIT tmp=$(mktemp) for table in $tables; do @@ -139,11 +148,4 @@ done check_result $lret "add/delete with nftrace enabled" -pkill -9 ping - -wait - -rm -f "$tmp" -ip netns del "$testns" - exit $global_ret diff --git a/tools/testing/selftests/netfilter/settings b/tools/testing/selftests/netfilter/settings new file mode 100644 index 000000000000..6091b45d226b --- /dev/null +++ b/tools/testing/selftests/netfilter/settings @@ -0,0 +1 @@ +timeout=120 diff --git a/tools/testing/selftests/proc/proc-empty-vm.c b/tools/testing/selftests/proc/proc-empty-vm.c index d95b1cb43d9d..7588428b8fcd 100644 --- a/tools/testing/selftests/proc/proc-empty-vm.c +++ b/tools/testing/selftests/proc/proc-empty-vm.c @@ -25,6 +25,7 @@ #undef NDEBUG #include <assert.h> #include <errno.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -41,7 +42,7 @@ * 1: vsyscall VMA is --xp vsyscall=xonly * 2: vsyscall VMA is r-xp vsyscall=emulate */ -static int g_vsyscall; +static volatile int g_vsyscall; static const char *g_proc_pid_maps_vsyscall; static const char *g_proc_pid_smaps_vsyscall; @@ -147,11 +148,12 @@ static void vsyscall(void) g_vsyscall = 0; /* gettimeofday(NULL, NULL); */ + uint64_t rax = 0xffffffffff600000; asm volatile ( - "call %P0" - : - : "i" (0xffffffffff600000), "D" (NULL), "S" (NULL) - : "rax", "rcx", "r11" + "call *%[rax]" + : [rax] "+a" (rax) + : "D" (NULL), "S" (NULL) + : "rcx", "r11" ); g_vsyscall = 1; diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c index 69551bfa215c..cacbd2a4aec9 100644 --- a/tools/testing/selftests/proc/proc-pid-vm.c +++ b/tools/testing/selftests/proc/proc-pid-vm.c @@ -257,11 +257,12 @@ static void vsyscall(void) g_vsyscall = 0; /* gettimeofday(NULL, NULL); */ + uint64_t rax = 0xffffffffff600000; asm volatile ( - "call %P0" - : - : "i" (0xffffffffff600000), "D" (NULL), "S" (NULL) - : "rax", "rcx", "r11" + "call *%[rax]" + : [rax] "+a" (rax) + : "D" (NULL), "S" (NULL) + : "rcx", "r11" ); g_vsyscall = 1; diff --git a/tools/testing/selftests/rseq/.gitignore b/tools/testing/selftests/rseq/.gitignore index 5910888ebfe1..16496de5f6ce 100644 --- a/tools/testing/selftests/rseq/.gitignore +++ b/tools/testing/selftests/rseq/.gitignore @@ -1,7 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-only basic_percpu_ops_test +basic_percpu_ops_mm_cid_test basic_test basic_rseq_op_test param_test param_test_benchmark param_test_compare_twice +param_test_mm_cid +param_test_mm_cid_benchmark +param_test_mm_cid_compare_twice diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile index 215e1067f037..82a52810a649 100644 --- a/tools/testing/selftests/rseq/Makefile +++ b/tools/testing/selftests/rseq/Makefile @@ -12,8 +12,9 @@ LDLIBS += -lpthread -ldl # still track changes to header files and depend on shared object. OVERRIDE_TARGETS = 1 -TEST_GEN_PROGS = basic_test basic_percpu_ops_test param_test \ - param_test_benchmark param_test_compare_twice +TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test param_test \ + param_test_benchmark param_test_compare_twice param_test_mm_cid \ + param_test_mm_cid_benchmark param_test_mm_cid_compare_twice TEST_GEN_PROGS_EXTENDED = librseq.so @@ -29,6 +30,9 @@ $(OUTPUT)/librseq.so: rseq.c rseq.h rseq-*.h $(OUTPUT)/%: %.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h $(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@ +$(OUTPUT)/basic_percpu_ops_mm_cid_test: basic_percpu_ops_test.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID_ID $< $(LDLIBS) -lrseq -o $@ + $(OUTPUT)/param_test_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ rseq.h rseq-*.h $(CC) $(CFLAGS) -DBENCHMARK $< $(LDLIBS) -lrseq -o $@ @@ -36,3 +40,15 @@ $(OUTPUT)/param_test_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ $(OUTPUT)/param_test_compare_twice: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ rseq.h rseq-*.h $(CC) $(CFLAGS) -DRSEQ_COMPARE_TWICE $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID -DBENCHMARK $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid_compare_twice: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID -DRSEQ_COMPARE_TWICE $< $(LDLIBS) -lrseq -o $@ diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c b/tools/testing/selftests/rseq/basic_percpu_ops_test.c index 517756afc2a4..887542961968 100644 --- a/tools/testing/selftests/rseq/basic_percpu_ops_test.c +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c @@ -12,6 +12,32 @@ #include "../kselftest.h" #include "rseq.h" +#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID +# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID +static +int get_current_cpu_id(void) +{ + return rseq_current_mm_cid(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_mm_cid_available(); +} +#else +# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID +static +int get_current_cpu_id(void) +{ + return rseq_cpu_start(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_current_cpu_raw() >= 0; +} +#endif + struct percpu_lock_entry { intptr_t v; } __attribute__((aligned(128))); @@ -51,9 +77,9 @@ int rseq_this_cpu_lock(struct percpu_lock *lock) for (;;) { int ret; - cpu = rseq_cpu_start(); - ret = rseq_cmpeqv_storev(&lock->c[cpu].v, - 0, 1, cpu); + cpu = get_current_cpu_id(); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -141,13 +167,14 @@ void this_cpu_list_push(struct percpu_list *list, intptr_t *targetptr, newval, expect; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; - ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -170,12 +197,13 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, long offset; int ret, cpu; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; - ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, + ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expectnot, offset, load, cpu); if (rseq_likely(!ret)) { if (_cpu) @@ -295,6 +323,10 @@ int main(int argc, char **argv) errno, strerror(errno)); goto error; } + if (!rseq_validate_cpu_id()) { + fprintf(stderr, "Error: cpu id getter unavailable\n"); + goto error; + } printf("spinlock\n"); test_percpu_spinlock(); printf("percpu_list\n"); diff --git a/tools/testing/selftests/rseq/basic_test.c b/tools/testing/selftests/rseq/basic_test.c index d8efbfb89193..295eea16466f 100644 --- a/tools/testing/selftests/rseq/basic_test.c +++ b/tools/testing/selftests/rseq/basic_test.c @@ -22,6 +22,8 @@ void test_cpu_pointer(void) CPU_ZERO(&test_affinity); for (i = 0; i < CPU_SETSIZE; i++) { if (CPU_ISSET(i, &affinity)) { + int node; + CPU_SET(i, &test_affinity); sched_setaffinity(0, sizeof(test_affinity), &test_affinity); @@ -29,6 +31,8 @@ void test_cpu_pointer(void) assert(rseq_current_cpu() == i); assert(rseq_current_cpu_raw() == i); assert(rseq_cpu_start() == i); + node = rseq_fallback_current_node(); + assert(rseq_current_node_id() == node); CPU_CLR(i, &test_affinity); } } diff --git a/tools/testing/selftests/rseq/compiler.h b/tools/testing/selftests/rseq/compiler.h index 876eb6a7f75b..f47092bddeba 100644 --- a/tools/testing/selftests/rseq/compiler.h +++ b/tools/testing/selftests/rseq/compiler.h @@ -27,4 +27,10 @@ */ #define rseq_after_asm_goto() asm volatile ("" : : : "memory") +/* Combine two tokens. */ +#define RSEQ__COMBINE_TOKENS(_tokena, _tokenb) \ + _tokena##_tokenb +#define RSEQ_COMBINE_TOKENS(_tokena, _tokenb) \ + RSEQ__COMBINE_TOKENS(_tokena, _tokenb) + #endif /* RSEQ_COMPILER_H_ */ diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c index ef29bc16f358..bf951a490bb4 100644 --- a/tools/testing/selftests/rseq/param_test.c +++ b/tools/testing/selftests/rseq/param_test.c @@ -16,6 +16,7 @@ #include <signal.h> #include <errno.h> #include <stddef.h> +#include <stdbool.h> static inline pid_t rseq_gettid(void) { @@ -36,13 +37,9 @@ static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep, opt_disable_rseq, opt_threads = 200, - opt_disable_mod = 0, opt_test = 's', opt_mb = 0; + opt_disable_mod = 0, opt_test = 's'; -#ifndef RSEQ_SKIP_FASTPATH static long long opt_reps = 5000; -#else -static long long opt_reps = 100; -#endif static __thread __attribute__((tls_model("initial-exec"))) unsigned int signals_delivered; @@ -268,6 +265,63 @@ unsigned int yield_mod_cnt, nr_abort; #include "rseq.h" +static enum rseq_mo opt_mo = RSEQ_MO_RELAXED; + +#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#define TEST_MEMBARRIER + +static int sys_membarrier(int cmd, int flags, int cpu_id) +{ + return syscall(__NR_membarrier, cmd, flags, cpu_id); +} +#endif + +#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID +# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID +static +int get_current_cpu_id(void) +{ + return rseq_current_mm_cid(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_mm_cid_available(); +} +# ifdef TEST_MEMBARRIER +/* + * Membarrier does not currently support targeting a mm_cid, so + * issue the barrier on all cpus. + */ +static +int rseq_membarrier_expedited(int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + 0, 0); +} +# endif /* TEST_MEMBARRIER */ +#else +# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID +static +int get_current_cpu_id(void) +{ + return rseq_cpu_start(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_current_cpu_raw() >= 0; +} +# ifdef TEST_MEMBARRIER +static +int rseq_membarrier_expedited(int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + MEMBARRIER_CMD_FLAG_CPU, cpu); +} +# endif /* TEST_MEMBARRIER */ +#endif + struct percpu_lock_entry { intptr_t v; } __attribute__((aligned(128))); @@ -355,8 +409,14 @@ static int rseq_this_cpu_lock(struct percpu_lock *lock) for (;;) { int ret; - cpu = rseq_cpu_start(); - ret = rseq_cmpeqv_storev(&lock->c[cpu].v, + cpu = get_current_cpu_id(); + if (cpu < 0) { + fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n", + getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu); + abort(); + } + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; @@ -473,8 +533,9 @@ void *test_percpu_inc_thread(void *arg) do { int cpu; - cpu = rseq_cpu_start(); - ret = rseq_addv(&data->c[cpu].count, 1, cpu); + cpu = get_current_cpu_id(); + ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &data->c[cpu].count, 1, cpu); } while (rseq_unlikely(ret)); #ifndef BENCHMARK if (i != 0 && !(i % (reps / 10))) @@ -543,13 +604,14 @@ void this_cpu_list_push(struct percpu_list *list, intptr_t *targetptr, newval, expect; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; - ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -575,13 +637,14 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, long offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; - ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, - offset, load, cpu); + ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expectnot, + offset, load, cpu); if (rseq_likely(!ret)) { node = head; break; @@ -719,7 +782,7 @@ bool this_cpu_buffer_push(struct percpu_buffer *buffer, intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) break; @@ -727,14 +790,9 @@ bool this_cpu_buffer_push(struct percpu_buffer *buffer, targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trystorev_storev_release( - targetptr_final, offset, targetptr_spec, - newval_spec, newval_final, cpu); - else - ret = rseq_cmpeqv_trystorev_storev(targetptr_final, - offset, targetptr_spec, newval_spec, - newval_final, cpu); + ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU, + targetptr_final, offset, targetptr_spec, + newval_spec, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -757,7 +815,7 @@ struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) { @@ -767,7 +825,8 @@ struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); newval = offset - 1; targetptr = (intptr_t *)&buffer->c[cpu].offset; - ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, + ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, offset, (intptr_t *)&buffer->c[cpu].array[offset - 1], (intptr_t)head, newval, cpu); if (rseq_likely(!ret)) @@ -924,7 +983,7 @@ bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) @@ -935,15 +994,11 @@ bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, copylen = sizeof(item); newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trymemcpy_storev_release( - targetptr_final, offset, - destptr, srcptr, copylen, - newval_final, cpu); - else - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, - newval_final, cpu); + ret = rseq_cmpeqv_trymemcpy_storev( + opt_mo, RSEQ_PERCPU, + targetptr_final, offset, + destptr, srcptr, copylen, + newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -968,7 +1023,7 @@ bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) @@ -979,8 +1034,8 @@ bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, copylen = sizeof(*item); newval_final = offset - 1; targetptr_final = &buffer->c[cpu].offset; - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, + ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr_final, offset, destptr, srcptr, copylen, newval_final, cpu); if (rseq_likely(!ret)) { result = true; @@ -1155,7 +1210,7 @@ static int set_signal_handler(void) } /* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */ -#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#ifdef TEST_MEMBARRIER struct test_membarrier_thread_args { int stop; intptr_t percpu_list_ptr; @@ -1182,9 +1237,10 @@ void *test_membarrier_worker_thread(void *arg) int ret; do { - int cpu = rseq_cpu_start(); + int cpu = get_current_cpu_id(); - ret = rseq_offset_deref_addv(&args->percpu_list_ptr, + ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &args->percpu_list_ptr, sizeof(struct percpu_list_entry) * cpu, 1, cpu); } while (rseq_unlikely(ret)); } @@ -1221,11 +1277,6 @@ void test_membarrier_free_percpu_list(struct percpu_list *list) free(list->c[i].head); } -static int sys_membarrier(int cmd, int flags, int cpu_id) -{ - return syscall(__NR_membarrier, cmd, flags, cpu_id); -} - /* * The manager thread swaps per-cpu lists that worker threads see, * and validates that there are no unexpected modifications. @@ -1264,8 +1315,7 @@ void *test_membarrier_manager_thread(void *arg) /* Make list_b "active". */ atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b); - if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, - MEMBARRIER_CMD_FLAG_CPU, cpu_a) && + if (rseq_membarrier_expedited(cpu_a) && errno != ENXIO /* missing CPU */) { perror("sys_membarrier"); abort(); @@ -1288,8 +1338,7 @@ void *test_membarrier_manager_thread(void *arg) /* Make list_a "active". */ atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a); - if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, - MEMBARRIER_CMD_FLAG_CPU, cpu_b) && + if (rseq_membarrier_expedited(cpu_b) && errno != ENXIO /* missing CPU*/) { perror("sys_membarrier"); abort(); @@ -1360,7 +1409,7 @@ void test_membarrier(void) abort(); } } -#else /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV */ +#else /* TEST_MEMBARRIER */ void test_membarrier(void) { fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. " @@ -1517,7 +1566,7 @@ int main(int argc, char **argv) verbose = 1; break; case 'M': - opt_mb = 1; + opt_mo = RSEQ_MO_RELEASE; break; default: show_usage(argc, argv); @@ -1537,6 +1586,10 @@ int main(int argc, char **argv) if (!opt_disable_rseq && rseq_register_current_thread()) goto error; + if (!opt_disable_rseq && !rseq_validate_cpu_id()) { + fprintf(stderr, "Error: cpu id getter unavailable\n"); + goto error; + } switch (opt_test) { case 's': printf_verbose("spinlock\n"); diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h index a8c44d9af71f..fb4ec8a75dd4 100644 --- a/tools/testing/selftests/rseq/rseq-abi.h +++ b/tools/testing/selftests/rseq/rseq-abi.h @@ -146,6 +146,28 @@ struct rseq_abi { * this thread. */ __u32 flags; + + /* + * Restartable sequences node_id field. Updated by the kernel. Read by + * user-space with single-copy atomicity semantics. This field should + * only be read by the thread which registered this data structure. + * Aligned on 32-bit. Contains the current NUMA node ID. + */ + __u32 node_id; + + /* + * Restartable sequences mm_cid field. Updated by the kernel. Read by + * user-space with single-copy atomicity semantics. This field should + * only be read by the thread which registered this data structure. + * Aligned on 32-bit. Contains the current thread's concurrency ID + * (allocated uniquely within a memory map). + */ + __u32 mm_cid; + + /* + * Flexible array member at end of structure, after last feature field. + */ + char end[]; } __attribute__((aligned(4 * sizeof(__u64)))); #endif /* _RSEQ_ABI_H */ diff --git a/tools/testing/selftests/rseq/rseq-arm-bits.h b/tools/testing/selftests/rseq/rseq-arm-bits.h new file mode 100644 index 000000000000..4f03cb395462 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-arm-bits.h @@ -0,0 +1,505 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-arm-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expectnot], r0\n\t" + "beq %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expectnot], r0\n\t" + "beq %l[error2]\n\t" +#endif + "str r0, %[load]\n\t" + "add r0, %[voffp]\n\t" + "ldr r0, [r0]\n\t" + /* final store */ + "str r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "Ir" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + "ldr r0, %[v]\n\t" + "add r0, %[count]\n\t" + /* final store */ + "str r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "Ir" (count) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "ldr r0, %[v2]\n\t" + "cmp %[expect2], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" + "ldr r0, %[v2]\n\t" + "cmp %[expect2], r0\n\t" + "bne %l[error3]\n\t" +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" +#endif + /* try store */ + "str %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "dmb\n\t" /* full mb provides store-release */ +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint32_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "str %[src], %[rseq_scratch0]\n\t" + "str %[dst], %[rseq_scratch1]\n\t" + "str %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne 7f\n\t" +#endif + /* try memcpy */ + "cmp %[len], #0\n\t" \ + "beq 333f\n\t" \ + "222:\n\t" \ + "ldrb %%r0, [%[src]]\n\t" \ + "strb %%r0, [%[dst]]\n\t" \ + "adds %[src], #1\n\t" \ + "adds %[dst], #1\n\t" \ + "subs %[len], #1\n\t" \ + "bne 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "dmb\n\t" /* full mb provides store-release */ +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t" + "b 8f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + abort, 1b, 2b, 4f) + RSEQ_ASM_DEFINE_CMPFAIL(5, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + error2) +#endif + "8:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h index 893a11eca9d5..8414fc3eac15 100644 --- a/tools/testing/selftests/rseq/rseq-arm.h +++ b/tools/testing/selftests/rseq/rseq-arm.h @@ -2,7 +2,7 @@ /* * rseq-arm.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ /* @@ -79,10 +79,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -147,681 +143,34 @@ do { \ teardown \ "b %l[" __rseq_str(cmpfail_label) "]\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expectnot], r0\n\t" - "beq %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expectnot], r0\n\t" - "beq %l[error2]\n\t" -#endif - "str r0, %[load]\n\t" - "add r0, %[voffp]\n\t" - "ldr r0, [r0]\n\t" - /* final store */ - "str r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "Ir" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - "ldr r0, %[v]\n\t" - "add r0, %[count]\n\t" - /* final store */ - "str r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [count] "Ir" (count) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* try store */ - "str %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-cpu-id indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* try store */ - "str %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "dmb\n\t" /* full mb provides store-release */ - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "ldr r0, %[v2]\n\t" - "cmp %[expect2], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" - "ldr r0, %[v2]\n\t" - "cmp %[expect2], r0\n\t" - "bne %l[error3]\n\t" -#endif - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("1st expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} +/* Per-mm-cid indexing. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "str %[src], %[rseq_scratch0]\n\t" - "str %[dst], %[rseq_scratch1]\n\t" - "str %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 7f\n\t" -#endif - /* try memcpy */ - "cmp %[len], #0\n\t" \ - "beq 333f\n\t" \ - "222:\n\t" \ - "ldrb %%r0, [%[src]]\n\t" \ - "strb %%r0, [%[dst]]\n\t" \ - "adds %[src], #1\n\t" \ - "adds %[dst], #1\n\t" \ - "subs %[len], #1\n\t" \ - "bne 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "str %[src], %[rseq_scratch0]\n\t" - "str %[dst], %[rseq_scratch1]\n\t" - "str %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 7f\n\t" -#endif - /* try memcpy */ - "cmp %[len], #0\n\t" \ - "beq 333f\n\t" \ - "222:\n\t" \ - "ldrb %%r0, [%[src]]\n\t" \ - "strb %%r0, [%[dst]]\n\t" \ - "adds %[src], #1\n\t" \ - "adds %[dst], #1\n\t" \ - "subs %[len], #1\n\t" \ - "bne 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "dmb\n\t" /* full mb provides store-release */ - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-arm64-bits.h b/tools/testing/selftests/rseq/rseq-arm64-bits.h new file mode 100644 index 000000000000..cc7226b1efe1 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-arm64-bits.h @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-arm64-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_STORE(load) + RSEQ_ASM_OP_R_LOAD_OFF(voffp) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expectnot] "r" (expectnot), + [load] "Qo" (*load), + [voffp] "r" (voffp) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_ADD(count) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expect] "r" (expect), + [v2] "Qo" (*v2), + [expect2] "r" (expect2), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "Qo" (*v), + [newv] "r" (newv), + [v2] "Qo" (*v2), + [newv2] "r" (newv2) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "Qo" (*v), + [newv] "r" (newv), + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h index cbe190a4d005..85b90977e7e6 100644 --- a/tools/testing/selftests/rseq/rseq-arm64.h +++ b/tools/testing/selftests/rseq/rseq-arm64.h @@ -2,7 +2,7 @@ /* * rseq-arm64.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> */ @@ -85,10 +85,6 @@ do { \ } \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define RSEQ_ASM_TMP_REG32 "w15" #define RSEQ_ASM_TMP_REG "x15" #define RSEQ_ASM_TMP_REG_2 "x14" @@ -204,492 +200,34 @@ do { \ " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ "333:\n" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "Qo" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_STORE(load) - RSEQ_ASM_OP_R_LOAD_OFF(voffp) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "Qo" (*v), - [expectnot] "r" (expectnot), - [load] "Qo" (*load), - [voffp] "r" (voffp) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-cpu-id indexing. */ -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_ADD(count) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "Qo" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [v2] "Qo" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [v2] "Qo" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "Qo" (*v), - [expect] "r" (expect), - [v2] "Qo" (*v2), - [expect2] "r" (expect2), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID + +/* Per-mm-cid indexing. */ + +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-bits-reset.h b/tools/testing/selftests/rseq/rseq-bits-reset.h new file mode 100644 index 000000000000..e8655089f9cb --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-bits-reset.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-bits-reset.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#undef RSEQ_TEMPLATE_IDENTIFIER +#undef RSEQ_TEMPLATE_CPU_ID_FIELD +#undef RSEQ_TEMPLATE_CPU_ID_OFFSET +#undef RSEQ_TEMPLATE_SUFFIX diff --git a/tools/testing/selftests/rseq/rseq-bits-template.h b/tools/testing/selftests/rseq/rseq-bits-template.h new file mode 100644 index 000000000000..65698d6a6cf9 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-bits-template.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-bits-template.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifdef RSEQ_TEMPLATE_CPU_ID +# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_CPU_ID_OFFSET +# define RSEQ_TEMPLATE_CPU_ID_FIELD cpu_id +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release_cpu_id +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed_cpu_id +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#elif defined(RSEQ_TEMPLATE_MM_CID) +# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_MM_CID_OFFSET +# define RSEQ_TEMPLATE_CPU_ID_FIELD mm_cid +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release_mm_cid +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed_mm_cid +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#elif defined (RSEQ_TEMPLATE_CPU_ID_NONE) +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +#endif + +#define RSEQ_TEMPLATE_IDENTIFIER(x) RSEQ_COMBINE_TOKENS(x, RSEQ_TEMPLATE_SUFFIX) + diff --git a/tools/testing/selftests/rseq/rseq-mips-bits.h b/tools/testing/selftests/rseq/rseq-mips-bits.h new file mode 100644 index 000000000000..6c48af4d0944 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-mips-bits.h @@ -0,0 +1,462 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * Author: Paul Burton <paul.burton@mips.com> + * (C) Copyright 2018 MIPS Tech LLC + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "beq $4, %[expectnot], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "beq $4, %[expectnot], %l[error2]\n\t" +#endif + LONG_S " $4, %[load]\n\t" + LONG_ADDI " $4, %[voffp]\n\t" + LONG_L " $4, 0($4)\n\t" + /* final store */ + LONG_S " $4, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "Ir" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + LONG_L " $4, %[v]\n\t" + LONG_ADDI " $4, %[count]\n\t" + /* final store */ + LONG_S " $4, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "Ir" (count) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + LONG_L " $4, %[v2]\n\t" + "bne $4, %[expect2], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" + LONG_L " $4, %[v2]\n\t" + "bne $4, %[expect2], %l[error3]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("1st expected value comparison failed"); +error3: + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" +#endif + /* try store */ + LONG_S " %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "sync\n\t" /* full sync provides store-release */ +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uintptr_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + LONG_S " %[src], %[rseq_scratch0]\n\t" + LONG_S " %[dst], %[rseq_scratch1]\n\t" + LONG_S " %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], 7f\n\t" +#endif + /* try memcpy */ + "beqz %[len], 333f\n\t" \ + "222:\n\t" \ + "lb $4, 0(%[src])\n\t" \ + "sb $4, 0(%[dst])\n\t" \ + LONG_ADDI " %[src], 1\n\t" \ + LONG_ADDI " %[dst], 1\n\t" \ + LONG_ADDI " %[len], -1\n\t" \ + "bnez %[len], 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "sync\n\t" /* full sync provides store-release */ +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t" + "b 8f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + abort, 1b, 2b, 4f) + RSEQ_ASM_DEFINE_CMPFAIL(5, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error2) +#endif + "8:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h index 878739fae2fd..50b950cf9585 100644 --- a/tools/testing/selftests/rseq/rseq-mips.h +++ b/tools/testing/selftests/rseq/rseq-mips.h @@ -2,9 +2,7 @@ /* * Author: Paul Burton <paul.burton@mips.com> * (C) Copyright 2018 MIPS Tech LLC - * - * Based on rseq-arm.h: - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ /* @@ -60,10 +58,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #if _MIPS_SZLONG == 64 # define LONG ".dword" # define LONG_LA "dla" @@ -154,624 +148,34 @@ do { \ teardown \ "b %l[" __rseq_str(cmpfail_label) "]\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "beq $4, %[expectnot], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "beq $4, %[expectnot], %l[error2]\n\t" -#endif - LONG_S " $4, %[load]\n\t" - LONG_ADDI " $4, %[voffp]\n\t" - LONG_L " $4, 0($4)\n\t" - /* final store */ - LONG_S " $4, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "Ir" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-cpu-id indexing. */ -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - LONG_L " $4, %[v]\n\t" - LONG_ADDI " $4, %[count]\n\t" - /* final store */ - LONG_S " $4, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [count] "Ir" (count) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-mm-cid indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "sync\n\t" /* full sync provides store-release */ - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - LONG_L " $4, %[v2]\n\t" - "bne $4, %[expect2], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" - LONG_L " $4, %[v2]\n\t" - "bne $4, %[expect2], %l[error3]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uintptr_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 7f\n\t" -#endif - /* try memcpy */ - "beqz %[len], 333f\n\t" \ - "222:\n\t" \ - "lb $4, 0(%[src])\n\t" \ - "sb $4, 0(%[dst])\n\t" \ - LONG_ADDI " %[src], 1\n\t" \ - LONG_ADDI " %[dst], 1\n\t" \ - LONG_ADDI " %[len], -1\n\t" \ - "bnez %[len], 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uintptr_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 7f\n\t" -#endif - /* try memcpy */ - "beqz %[len], 333f\n\t" \ - "222:\n\t" \ - "lb $4, 0(%[src])\n\t" \ - "sb $4, 0(%[dst])\n\t" \ - LONG_ADDI " %[src], 1\n\t" \ - LONG_ADDI " %[dst], 1\n\t" \ - LONG_ADDI " %[len], -1\n\t" \ - "bnez %[len], 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "sync\n\t" /* full sync provides store-release */ - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-ppc-bits.h b/tools/testing/selftests/rseq/rseq-ppc-bits.h new file mode 100644 index 000000000000..98e69eae1e62 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-ppc-bits.h @@ -0,0 +1,454 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-ppc-bits.h + * + * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v not equal to @expectnot */ + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v not equal to @expectnot */ + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) +#endif + /* load the value of @v */ + RSEQ_ASM_OP_R_LOAD(v) + /* store it in @load */ + RSEQ_ASM_OP_R_STORE(load) + /* dereference voffp(v) */ + RSEQ_ASM_OP_R_LOADX(voffp) + /* final store the value at voffp(v) */ + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "b" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + /* load the value of @v */ + RSEQ_ASM_OP_R_LOAD(v) + /* add @count to it */ + RSEQ_ASM_OP_R_ADD(count) + /* final store */ + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) + /* cmp @v2 equal to @expct2 */ + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + /* cmp @v2 equal to @expct2 */ + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* try store */ + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + /* for 'release' */ + "lwsync\n\t" +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* setup for mempcy */ + "mr %%r19, %[len]\n\t" + "mr %%r20, %[src]\n\t" + "mr %%r21, %[dst]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* try memcpy */ + RSEQ_ASM_OP_R_MEMCPY() + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + /* for 'release' */ + "lwsync\n\t" +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + /* teardown */ + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17", "r18", "r19", "r20", "r21" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h index bab8e0b9fb11..dc9190facee9 100644 --- a/tools/testing/selftests/rseq/rseq-ppc.h +++ b/tools/testing/selftests/rseq/rseq-ppc.h @@ -2,7 +2,7 @@ /* * rseq-ppc.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> */ @@ -36,10 +36,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - /* * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to * better handle single-stepping through the restartable critical sections. @@ -209,583 +205,34 @@ do { \ RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ __rseq_str(post_commit_label) ":\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) -#endif - /* load the value of @v */ - RSEQ_ASM_OP_R_LOAD(v) - /* store it in @load */ - RSEQ_ASM_OP_R_STORE(load) - /* dereference voffp(v) */ - RSEQ_ASM_OP_R_LOADX(voffp) - /* final store the value at voffp(v) */ - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "b" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - /* load the value of @v */ - RSEQ_ASM_OP_R_LOAD(v) - /* add @count to it */ - RSEQ_ASM_OP_R_ADD(count) - /* final store */ - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try store */ - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try store */ - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - /* for 'release' */ - "lwsync\n\t" - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) - /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) - /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) -#endif - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("1st expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* setup for mempcy */ - "mr %%r19, %[len]\n\t" - "mr %%r20, %[src]\n\t" - "mr %%r21, %[dst]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try memcpy */ - RSEQ_ASM_OP_R_MEMCPY() - RSEQ_INJECT_ASM(5) - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - /* teardown */ - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* setup for mempcy */ - "mr %%r19, %[len]\n\t" - "mr %%r20, %[src]\n\t" - "mr %%r21, %[dst]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try memcpy */ - RSEQ_ASM_OP_R_MEMCPY() - RSEQ_INJECT_ASM(5) - /* for 'release' */ - "lwsync\n\t" - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - /* teardown */ - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -#endif /* !RSEQ_SKIP_FASTPATH */ +/* Per-cpu-id indexing. */ + +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID + +/* Per-mm-cid indexing. */ + +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ + +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-riscv-bits.h b/tools/testing/selftests/rseq/rseq-riscv-bits.h new file mode 100644 index 000000000000..de31a0143139 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-riscv-bits.h @@ -0,0 +1,410 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_STORE(load) + RSEQ_ASM_OP_R_LOAD_OFF(voffp) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expectnot] "r" (expectnot), + [load] "m" (*load), + [voffp] "r" (voffp) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_ADD(count) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) + RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") + RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [v2] "m" (*v2), + [expect2] "r" (expect2), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +error3: + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV + +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") +#endif + RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [ptr] "r" (ptr), + [off] "er" (off), + [inc] "er" (inc) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "m" (*v), + [newv] "r" (newv), + [v2] "m" (*v2), + [newv2] "r" (newv2) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "m" (*v), + [newv] "r" (newv), + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, + RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-riscv.h b/tools/testing/selftests/rseq/rseq-riscv.h index 3a391c9bf468..17932a79e066 100644 --- a/tools/testing/selftests/rseq/rseq-riscv.h +++ b/tools/testing/selftests/rseq/rseq-riscv.h @@ -49,10 +49,6 @@ do { \ RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n" \ @@ -169,509 +165,34 @@ do { \ RSEQ_ASM_OP_R_ADD(inc) \ __rseq_str(post_commit_label) ":\n" -static inline __always_inline -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-cpu-id indexing. */ - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_STORE(load) - RSEQ_ASM_OP_R_LOAD_OFF(voffp) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expectnot] "r" (expectnot), - [load] "m" (*load), - [voffp] "r" (voffp) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __always_inline -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_ADD(count) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "m" (*v), - [newv] "r" (newv), - [v2] "m" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "m" (*v), - [newv] "r" (newv), - [v2] "m" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) - RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") - RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [v2] "m" (*v2), - [expect2] "r" (expect2), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "m" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, - RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __always_inline -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [expect] "r" (expect), - [v] "m" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, - RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-mm-cid indexing. */ -#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -/* - * pval = *(ptr+off) - * *pval += inc; - */ -static inline __always_inline -int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") -#endif - RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [ptr] "r" (ptr), - [off] "er" (off), - [inc] "er" (inc) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG_1 - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-s390-bits.h b/tools/testing/selftests/rseq/rseq-s390-bits.h new file mode 100644 index 000000000000..0cf17d9f170a --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-s390-bits.h @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " %%r1, %[v]\n\t" + LONG_CMP_R " %%r1, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " %%r1, %[v]\n\t" + LONG_CMP_R " %%r1, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + LONG_S " %%r1, %[load]\n\t" + LONG_ADD_R " %%r1, %[voffp]\n\t" + LONG_L " %%r1, 0(%%r1)\n\t" + /* final store */ + LONG_S " %%r1, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "r" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0", "r1" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + LONG_L " %%r0, %[v]\n\t" + LONG_ADD_R " %%r0, %[count]\n\t" + /* final store */ + LONG_S " %%r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + LONG_CMP " %[expect2], %[v2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" + LONG_CMP " %[expect2], %[v2]\n\t" + "jnz %l[error3]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +/* s390 is TSO. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + LONG_S " %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* s390 is TSO. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint64_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + LONG_S " %[src], %[rseq_scratch0]\n\t" + LONG_S " %[dst], %[rseq_scratch1]\n\t" + LONG_S " %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + LONG_CMP " %[expect], %[v]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + LONG_LT_R " %[len], %[len]\n\t" + "jz 333f\n\t" + "222:\n\t" + "ic %%r0,0(%[src])\n\t" + "stc %%r0,0(%[dst])\n\t" + LONG_ADDI " %[src], 1\n\t" + LONG_ADDI " %[dst], 1\n\t" + LONG_ADDI " %[len], -1\n\t" + "jnz 222b\n\t" + "333:\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h index 4e6dc5f0cb42..46c92598acc7 100644 --- a/tools/testing/selftests/rseq/rseq-s390.h +++ b/tools/testing/selftests/rseq/rseq-s390.h @@ -28,10 +28,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #ifdef __s390x__ #define LONG_L "lg" @@ -134,477 +130,34 @@ do { \ "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-cpu-id indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " %%r1, %[v]\n\t" - LONG_CMP_R " %%r1, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " %%r1, %[v]\n\t" - LONG_CMP_R " %%r1, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - LONG_S " %%r1, %[load]\n\t" - LONG_ADD_R " %%r1, %[voffp]\n\t" - LONG_L " %%r1, 0(%%r1)\n\t" - /* final store */ - LONG_S " %%r1, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "r" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0", "r1" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-mm-cid indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - LONG_L " %%r0, %[v]\n\t" - LONG_ADD_R " %%r0, %[count]\n\t" - /* final store */ - LONG_S " %%r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -/* s390 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - LONG_CMP " %[expect2], %[v2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" - LONG_CMP " %[expect2], %[v2]\n\t" - "jnz %l[error3]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("1st expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint64_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_CMP " %[expect], %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - LONG_LT_R " %[len], %[len]\n\t" - "jz 333f\n\t" - "222:\n\t" - "ic %%r0,0(%[src])\n\t" - "stc %%r0,0(%[dst])\n\t" - LONG_ADDI " %[src], 1\n\t" - LONG_ADDI " %[dst], 1\n\t" - LONG_ADDI " %[len], -1\n\t" - "jnz 222b\n\t" - "333:\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -/* s390 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, - newv, cpu); -} -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-skip.h b/tools/testing/selftests/rseq/rseq-skip.h deleted file mode 100644 index 7b53dac1fcdd..000000000000 --- a/tools/testing/selftests/rseq/rseq-skip.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * rseq-skip.h - * - * (C) Copyright 2017-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> - */ - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return -1; -} diff --git a/tools/testing/selftests/rseq/rseq-x86-bits.h b/tools/testing/selftests/rseq/rseq-x86-bits.h new file mode 100644 index 000000000000..8a9431eec467 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-x86-bits.h @@ -0,0 +1,993 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-x86-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#ifdef __x86_64__ + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movq %[v], %%rbx\n\t" + "cmpq %%rbx, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movq %[v], %%rbx\n\t" + "cmpq %%rbx, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + "movq %%rbx, %[load]\n\t" + "addq %[voffp], %%rbx\n\t" + "movq (%%rbx), %%rbx\n\t" + /* final store */ + "movq %%rbx, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "er" (voffp), + [load] "m" (*load) + : "memory", "cc", "rax", "rbx" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* final store */ + "addq %[count], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [count] "er" (count) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV + +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* get p+v */ + "movq %[ptr], %%rbx\n\t" + "addq %[off], %%rbx\n\t" + /* get pv */ + "movq (%%rbx), %%rcx\n\t" + /* *pv += inc */ + "addq %[inc], (%%rcx)\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [ptr] "m" (*ptr), + [off] "er" (off), + [inc] "er" (inc) + : "memory", "cc", "rax", "rbx", "rcx" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "cmpq %[v2], %[expect2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" + "cmpq %[v2], %[expect2]\n\t" + "jnz %l[error3]\n\t" +#endif + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + "movq %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint64_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "movq %[src], %[rseq_scratch0]\n\t" + "movq %[dst], %[rseq_scratch1]\n\t" + "movq %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + "cmpq %[v], %[expect]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + "test %[len], %[len]\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "movb (%[src]), %%al\n\t" \ + "movb %%al, (%[dst])\n\t" \ + "inc %[src]\n\t" \ + "inc %[dst]\n\t" \ + "dec %[len]\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#elif defined(__i386__) + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpl %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpl %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + "movl %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[v], %%ebx\n\t" + "cmpl %%ebx, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movl %[v], %%ebx\n\t" + "cmpl %%ebx, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + "movl %%ebx, %[load]\n\t" + "addl %[voffp], %%ebx\n\t" + "movl (%%ebx), %%ebx\n\t" + /* final store */ + "movl %%ebx, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "ir" (voffp), + [load] "m" (*load) + : "memory", "cc", "eax", "ebx" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* final store */ + "addl %[count], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [count] "ir" (count) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpl %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "cmpl %[expect2], %[v2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpl %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" + "cmpl %[expect2], %[v2]\n\t" + "jnz %l[error3]\n\t" +#endif + "movl %[newv], %%eax\n\t" + /* final store */ + "movl %%eax, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "m" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[expect], %%eax\n\t" + "cmpl %[v], %%eax\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movl %[expect], %%eax\n\t" + "cmpl %[v], %%eax\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + "movl %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "lock; addl $0,-128(%%esp)\n\t" +#endif + /* final store */ + "movl %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "m" (expect), + [newv] "r" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif + +} + +/* TODO: implement a faster memcpy. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint32_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "movl %[src], %[rseq_scratch0]\n\t" + "movl %[dst], %[rseq_scratch1]\n\t" + "movl %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[expect], %%eax\n\t" + "cmpl %%eax, %[v]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + "movl %[expect], %%eax\n\t" + "cmpl %%eax, %[v]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + "test %[len], %[len]\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "movb (%[src]), %%al\n\t" \ + "movb %%al, (%[dst])\n\t" \ + "inc %[src]\n\t" \ + "inc %[dst]\n\t" \ + "dec %[len]\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "lock; addl $0,-128(%%esp)\n\t" +#endif + "movl %[newv], %%eax\n\t" + /* final store */ + "movl %%eax, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expect] "m" (expect), + [newv] "m" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#endif + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h index bd01dc41ca13..fb65ef54b0fb 100644 --- a/tools/testing/selftests/rseq/rseq-x86.h +++ b/tools/testing/selftests/rseq/rseq-x86.h @@ -2,9 +2,13 @@ /* * rseq-x86.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ +#ifndef RSEQ_H +#error "Never use <rseq-x86.h> directly; include <rseq.h> instead." +#endif + #include <stdint.h> /* @@ -22,9 +26,10 @@ * address through a "r" input operand. */ -/* Offset of cpu_id and rseq_cs fields in struct rseq. */ +/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ #define RSEQ_CPU_ID_OFFSET 4 #define RSEQ_CS_OFFSET 8 +#define RSEQ_MM_CID_OFFSET 24 #ifdef __x86_64__ @@ -50,10 +55,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -112,525 +113,6 @@ do { \ "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "movq %[v], %%rbx\n\t" - "cmpq %%rbx, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "movq %[v], %%rbx\n\t" - "cmpq %%rbx, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - "movq %%rbx, %[load]\n\t" - "addq %[voffp], %%rbx\n\t" - "movq (%%rbx), %%rbx\n\t" - /* final store */ - "movq %%rbx, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "er" (voffp), - [load] "m" (*load) - : "memory", "cc", "rax", "rbx" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) -#endif - /* final store */ - "addq %[count], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [count] "er" (count) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} - -#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV - -/* - * pval = *(ptr+off) - * *pval += inc; - */ -static inline __attribute__((always_inline)) -int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) -#endif - /* get p+v */ - "movq %[ptr], %%rbx\n\t" - "addq %[off], %%rbx\n\t" - /* get pv */ - "movq (%%rbx), %%rcx\n\t" - /* *pv += inc */ - "addq %[inc], (%%rcx)\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [ptr] "m" (*ptr), - [off] "er" (off), - [inc] "er" (inc) - : "memory", "cc", "rax", "rbx", "rcx" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movq %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -/* x86-64 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "cmpq %[v2], %[expect2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" - "cmpq %[v2], %[expect2]\n\t" - "jnz %l[error3]\n\t" -#endif - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("1st expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint64_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movq %[src], %[rseq_scratch0]\n\t" - "movq %[dst], %[rseq_scratch1]\n\t" - "movq %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) - "cmpq %[v], %[expect]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -/* x86-64 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, - newv, cpu); -} - -#endif /* !RSEQ_SKIP_FASTPATH */ - #elif defined(__i386__) #define RSEQ_ASM_TP_SEGMENT %%gs @@ -657,10 +139,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - /* * Use eax as scratch register and take memory operands as input to * lessen register pressure. Especially needed when compiling in O0. @@ -721,645 +199,36 @@ do { \ "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - long voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[v], %%ebx\n\t" - "cmpl %%ebx, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "movl %[v], %%ebx\n\t" - "cmpl %%ebx, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - "movl %%ebx, %[load]\n\t" - "addl %[voffp], %%ebx\n\t" - "movl (%%ebx), %%ebx\n\t" - /* final store */ - "movl %%ebx, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "ir" (voffp), - [load] "m" (*load) - : "memory", "cc", "eax", "ebx" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) -#endif - /* final store */ - "addl %[count], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [count] "ir" (count) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movl %[newv2], %%eax\n\t" - "movl %%eax, %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* try store input */ - [v2] "m" (*v2), - [newv2] "m" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %[v], %%eax\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "movl %[expect], %%eax\n\t" - "cmpl %[v], %%eax\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movl %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "lock; addl $0,-128(%%esp)\n\t" - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif - -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "cmpl %[expect2], %[v2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" - "cmpl %[expect2], %[v2]\n\t" - "jnz %l[error3]\n\t" #endif - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "m" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("1st expected value comparison failed"); -error3: - rseq_after_asm_goto(); - rseq_bug("2nd expected value comparison failed"); -#endif -} -/* TODO: implement a faster memcpy. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; +/* Per-cpu-id indexing. */ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movl %[src], %[rseq_scratch0]\n\t" - "movl %[dst], %[rseq_scratch1]\n\t" - "movl %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "m" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID -/* TODO: implement a faster memcpy. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; +/* Per-mm-cid indexing. */ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movl %[src], %[rseq_scratch0]\n\t" - "movl %[dst], %[rseq_scratch1]\n\t" - "movl %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "lock; addl $0,-128(%%esp)\n\t" - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_offset] "r" (rseq_offset), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "m" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_after_asm_goto(); - return 0; -abort: - rseq_after_asm_goto(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_after_asm_goto(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_after_asm_goto(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_after_asm_goto(); - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID -#endif /* !RSEQ_SKIP_FASTPATH */ +/* APIs which are not based on cpu ids. */ -#endif +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index 4177f9507bbe..4e4aa006004c 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -28,6 +28,8 @@ #include <limits.h> #include <dlfcn.h> #include <stddef.h> +#include <sys/auxv.h> +#include <linux/auxvec.h> #include "../kselftest.h" #include "rseq.h" @@ -36,20 +38,38 @@ static const ptrdiff_t *libc_rseq_offset_p; static const unsigned int *libc_rseq_size_p; static const unsigned int *libc_rseq_flags_p; -/* Offset from the thread pointer to the rseq area. */ +/* Offset from the thread pointer to the rseq area. */ ptrdiff_t rseq_offset; -/* Size of the registered rseq area. 0 if the registration was - unsuccessful. */ +/* + * Size of the registered rseq area. 0 if the registration was + * unsuccessful. + */ unsigned int rseq_size = -1U; /* Flags used during rseq registration. */ unsigned int rseq_flags; +/* + * rseq feature size supported by the kernel. 0 if the registration was + * unsuccessful. + */ +unsigned int rseq_feature_size = -1U; + static int rseq_ownership; +static int rseq_reg_success; /* At least one rseq registration has succeded. */ + +/* Allocate a large area for the TLS. */ +#define RSEQ_THREAD_AREA_ALLOC_SIZE 1024 + +/* Original struct rseq feature size is 20 bytes. */ +#define ORIG_RSEQ_FEATURE_SIZE 20 + +/* Original struct rseq allocation size is 32 bytes. */ +#define ORIG_RSEQ_ALLOC_SIZE 32 static -__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = { +__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE))) = { .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, }; @@ -59,6 +79,11 @@ static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); } +static int sys_getcpu(unsigned *cpu, unsigned *node) +{ + return syscall(__NR_getcpu, cpu, node, NULL); +} + int rseq_available(void) { int rc; @@ -84,10 +109,16 @@ int rseq_register_current_thread(void) /* Treat libc's ownership as a successful registration. */ return 0; } - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG); - if (rc) + rc = sys_rseq(&__rseq_abi, rseq_size, 0, RSEQ_SIG); + if (rc) { + if (RSEQ_READ_ONCE(rseq_reg_success)) { + /* Incoherent success/failure within process. */ + abort(); + } return -1; + } assert(rseq_current_cpu_raw() >= 0); + RSEQ_WRITE_ONCE(rseq_reg_success, 1); return 0; } @@ -99,12 +130,28 @@ int rseq_unregister_current_thread(void) /* Treat libc's ownership as a successful unregistration. */ return 0; } - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); + rc = sys_rseq(&__rseq_abi, rseq_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); if (rc) return -1; return 0; } +static +unsigned int get_rseq_feature_size(void) +{ + unsigned long auxv_rseq_feature_size, auxv_rseq_align; + + auxv_rseq_align = getauxval(AT_RSEQ_ALIGN); + assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE); + + auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE); + assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE); + if (auxv_rseq_feature_size) + return auxv_rseq_feature_size; + else + return ORIG_RSEQ_FEATURE_SIZE; +} + static __attribute__((constructor)) void rseq_init(void) { @@ -117,14 +164,24 @@ void rseq_init(void) rseq_offset = *libc_rseq_offset_p; rseq_size = *libc_rseq_size_p; rseq_flags = *libc_rseq_flags_p; + rseq_feature_size = get_rseq_feature_size(); + if (rseq_feature_size > rseq_size) + rseq_feature_size = rseq_size; return; } - if (!rseq_available()) - return; rseq_ownership = 1; + if (!rseq_available()) { + rseq_size = 0; + rseq_feature_size = 0; + return; + } rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); - rseq_size = sizeof(struct rseq_abi); rseq_flags = 0; + rseq_feature_size = get_rseq_feature_size(); + if (rseq_feature_size == ORIG_RSEQ_FEATURE_SIZE) + rseq_size = ORIG_RSEQ_ALLOC_SIZE; + else + rseq_size = RSEQ_THREAD_AREA_ALLOC_SIZE; } static __attribute__((destructor)) @@ -134,6 +191,7 @@ void rseq_exit(void) return; rseq_offset = 0; rseq_size = -1U; + rseq_feature_size = -1U; rseq_ownership = 0; } @@ -148,3 +206,16 @@ int32_t rseq_fallback_current_cpu(void) } return cpu; } + +int32_t rseq_fallback_current_node(void) +{ + uint32_t cpu_id, node_id; + int ret; + + ret = sys_getcpu(&cpu_id, &node_id); + if (ret) { + perror("sys_getcpu()"); + return ret; + } + return (int32_t) node_id; +} diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h index 6f7513384bf5..d7364ea4d201 100644 --- a/tools/testing/selftests/rseq/rseq.h +++ b/tools/testing/selftests/rseq/rseq.h @@ -20,6 +20,15 @@ #include "rseq-abi.h" #include "compiler.h" +#ifndef rseq_sizeof_field +#define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) +#endif + +#ifndef rseq_offsetofend +#define rseq_offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) +#endif + /* * Empty code injection macros, override when testing. * It is important to consider that the ASM injection macros need to be @@ -47,14 +56,38 @@ #include "rseq-thread-pointer.h" -/* Offset from the thread pointer to the rseq area. */ +/* Offset from the thread pointer to the rseq area. */ extern ptrdiff_t rseq_offset; -/* Size of the registered rseq area. 0 if the registration was - unsuccessful. */ + +/* + * Size of the registered rseq area. 0 if the registration was + * unsuccessful. + */ extern unsigned int rseq_size; -/* Flags used during rseq registration. */ + +/* Flags used during rseq registration. */ extern unsigned int rseq_flags; +/* + * rseq feature size supported by the kernel. 0 if the registration was + * unsuccessful. + */ +extern unsigned int rseq_feature_size; + +enum rseq_mo { + RSEQ_MO_RELAXED = 0, + RSEQ_MO_CONSUME = 1, /* Unused */ + RSEQ_MO_ACQUIRE = 2, /* Unused */ + RSEQ_MO_RELEASE = 3, + RSEQ_MO_ACQ_REL = 4, /* Unused */ + RSEQ_MO_SEQ_CST = 5, /* Unused */ +}; + +enum rseq_percpu_mode { + RSEQ_PERCPU_CPU_ID = 0, + RSEQ_PERCPU_MM_CID = 1, +}; + static inline struct rseq_abi *rseq_get_abi(void) { return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); @@ -119,6 +152,11 @@ int rseq_unregister_current_thread(void); int32_t rseq_fallback_current_cpu(void); /* + * Restartable sequence fallback for reading the current node number. + */ +int32_t rseq_fallback_current_node(void); + +/* * Values returned can be either the current CPU number, -1 (rseq is * uninitialized), or -2 (rseq initialization has failed). */ @@ -153,6 +191,30 @@ static inline uint32_t rseq_current_cpu(void) return cpu; } +static inline bool rseq_node_id_available(void) +{ + return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id); +} + +/* + * Current NUMA node number. + */ +static inline uint32_t rseq_current_node_id(void) +{ + assert(rseq_node_id_available()); + return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); +} + +static inline bool rseq_mm_cid_available(void) +{ + return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid); +} + +static inline uint32_t rseq_current_mm_cid(void) +{ + return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); +} + static inline void rseq_clear_rseq_cs(void) { RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); @@ -174,4 +236,149 @@ static inline void rseq_prepare_unload(void) rseq_clear_rseq_cs(); } +static inline __attribute__((always_inline)) +int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t newv, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); + } + return -1; +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, + int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); + } + return -1; +} + +static inline __attribute__((always_inline)) +int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t count, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_addv_relaxed_cpu_id(v, count, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_addv_relaxed_mm_cid(v, count, cpu); + } + return -1; +} + +#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __attribute__((always_inline)) +int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *ptr, long off, intptr_t inc, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); + } + return -1; +} +#endif + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + switch (rseq_mo) { + case RSEQ_MO_RELAXED: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); + } + return -1; + case RSEQ_MO_RELEASE: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); + } + return -1; + default: + return -1; + } +} + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); + } + return -1; +} + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + switch (rseq_mo) { + case RSEQ_MO_RELAXED: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); + } + return -1; + case RSEQ_MO_RELEASE: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); + } + return -1; + default: + return -1; + } +} + #endif /* RSEQ_H_ */ diff --git a/tools/testing/selftests/rseq/run_param_test.sh b/tools/testing/selftests/rseq/run_param_test.sh index f51bc83c9e41..8d31426ab41f 100755 --- a/tools/testing/selftests/rseq/run_param_test.sh +++ b/tools/testing/selftests/rseq/run_param_test.sh @@ -42,6 +42,11 @@ function do_tests() ./param_test ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 echo "Running compare-twice test ${TEST_NAME[$i]}" ./param_test_compare_twice ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 + + echo "Running mm_cid test ${TEST_NAME[$i]}" + ./param_test_mm_cid ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 + echo "Running mm_cid compare-twice test ${TEST_NAME[$i]}" + ./param_test_mm_cid_compare_twice ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 let "i++" done } diff --git a/tools/testing/selftests/vm/hugetlb-madvise.c b/tools/testing/selftests/vm/hugetlb-madvise.c index a634f47d1e56..9a127a8fe176 100644 --- a/tools/testing/selftests/vm/hugetlb-madvise.c +++ b/tools/testing/selftests/vm/hugetlb-madvise.c @@ -17,7 +17,6 @@ #include <stdio.h> #include <unistd.h> #include <sys/mman.h> -#define __USE_GNU #include <fcntl.h> #define MIN_FREE_PAGES 20 diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c index 5b45e6986aea..47cab972807c 100644 --- a/tools/testing/selftests/x86/test_vsyscall.c +++ b/tools/testing/selftests/x86/test_vsyscall.c @@ -92,11 +92,8 @@ static void init_vdso(void) printf("[WARN]\tfailed to find time in vDSO\n"); vdso_getcpu = (getcpu_t)dlsym(vdso, "__vdso_getcpu"); - if (!vdso_getcpu) { - /* getcpu() was never wired up in the 32-bit vDSO. */ - printf("[%s]\tfailed to find getcpu in vDSO\n", - sizeof(long) == 8 ? "WARN" : "NOTE"); - } + if (!vdso_getcpu) + printf("[WARN]\tfailed to find getcpu in vDSO\n"); } static int init_vsys(void) |