summaryrefslogtreecommitdiff
path: root/scripts/atomic
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/atomic')
-rwxr-xr-xscripts/atomic/atomic-tbl.sh112
-rw-r--r--scripts/atomic/atomics.tbl2
-rwxr-xr-xscripts/atomic/fallbacks/acquire4
-rwxr-xr-xscripts/atomic/fallbacks/add_negative14
-rwxr-xr-xscripts/atomic/fallbacks/add_unless15
-rwxr-xr-xscripts/atomic/fallbacks/andnot6
-rw-r--r--scripts/atomic/fallbacks/cmpxchg3
-rwxr-xr-xscripts/atomic/fallbacks/dec6
-rwxr-xr-xscripts/atomic/fallbacks/dec_and_test14
-rwxr-xr-xscripts/atomic/fallbacks/dec_if_positive8
-rwxr-xr-xscripts/atomic/fallbacks/dec_unless_positive8
-rwxr-xr-xscripts/atomic/fallbacks/fence4
-rwxr-xr-xscripts/atomic/fallbacks/fetch_add_unless17
-rwxr-xr-xscripts/atomic/fallbacks/inc6
-rwxr-xr-xscripts/atomic/fallbacks/inc_and_test14
-rwxr-xr-xscripts/atomic/fallbacks/inc_not_zero13
-rwxr-xr-xscripts/atomic/fallbacks/inc_unless_negative8
-rwxr-xr-xscripts/atomic/fallbacks/read_acquire6
-rwxr-xr-xscripts/atomic/fallbacks/release4
-rwxr-xr-xscripts/atomic/fallbacks/set_release6
-rwxr-xr-xscripts/atomic/fallbacks/sub_and_test15
-rwxr-xr-xscripts/atomic/fallbacks/try_cmpxchg6
-rw-r--r--scripts/atomic/fallbacks/xchg3
-rwxr-xr-xscripts/atomic/gen-atomic-fallback.sh266
-rwxr-xr-xscripts/atomic/gen-atomic-instrumented.sh42
-rwxr-xr-xscripts/atomic/gen-atomic-long.sh38
-rw-r--r--scripts/atomic/kerneldoc/add13
-rw-r--r--scripts/atomic/kerneldoc/add_negative13
-rw-r--r--scripts/atomic/kerneldoc/add_unless18
-rw-r--r--scripts/atomic/kerneldoc/and13
-rw-r--r--scripts/atomic/kerneldoc/andnot13
-rw-r--r--scripts/atomic/kerneldoc/cmpxchg14
-rw-r--r--scripts/atomic/kerneldoc/dec12
-rw-r--r--scripts/atomic/kerneldoc/dec_and_test12
-rw-r--r--scripts/atomic/kerneldoc/dec_if_positive12
-rw-r--r--scripts/atomic/kerneldoc/dec_unless_positive12
-rw-r--r--scripts/atomic/kerneldoc/inc12
-rw-r--r--scripts/atomic/kerneldoc/inc_and_test12
-rw-r--r--scripts/atomic/kerneldoc/inc_not_zero12
-rw-r--r--scripts/atomic/kerneldoc/inc_unless_negative12
-rw-r--r--scripts/atomic/kerneldoc/or13
-rw-r--r--scripts/atomic/kerneldoc/read12
-rw-r--r--scripts/atomic/kerneldoc/set13
-rw-r--r--scripts/atomic/kerneldoc/sub13
-rw-r--r--scripts/atomic/kerneldoc/sub_and_test13
-rw-r--r--scripts/atomic/kerneldoc/try_cmpxchg15
-rw-r--r--scripts/atomic/kerneldoc/xchg13
-rw-r--r--scripts/atomic/kerneldoc/xor13
48 files changed, 629 insertions, 296 deletions
diff --git a/scripts/atomic/atomic-tbl.sh b/scripts/atomic/atomic-tbl.sh
index 81d5c32039dd..608ff39ebd8c 100755
--- a/scripts/atomic/atomic-tbl.sh
+++ b/scripts/atomic/atomic-tbl.sh
@@ -36,9 +36,16 @@ meta_has_relaxed()
meta_in "$1" "BFIR"
}
-#find_fallback_template(pfx, name, sfx, order)
-find_fallback_template()
+#meta_is_implicitly_relaxed(meta)
+meta_is_implicitly_relaxed()
{
+ meta_in "$1" "vls"
+}
+
+#find_template(tmpltype, pfx, name, sfx, order)
+find_template()
+{
+ local tmpltype="$1"; shift
local pfx="$1"; shift
local name="$1"; shift
local sfx="$1"; shift
@@ -52,8 +59,8 @@ find_fallback_template()
#
# Start at the most specific, and fall back to the most general. Once
# we find a specific fallback, don't bother looking for more.
- for base in "${pfx}${name}${sfx}${order}" "${name}"; do
- file="${ATOMICDIR}/fallbacks/${base}"
+ for base in "${pfx}${name}${sfx}${order}" "${pfx}${name}${sfx}" "${name}"; do
+ file="${ATOMICDIR}/${tmpltype}/${base}"
if [ -f "${file}" ]; then
printf "${file}"
@@ -62,6 +69,18 @@ find_fallback_template()
done
}
+#find_fallback_template(pfx, name, sfx, order)
+find_fallback_template()
+{
+ find_template "fallbacks" "$@"
+}
+
+#find_kerneldoc_template(pfx, name, sfx, order)
+find_kerneldoc_template()
+{
+ find_template "kerneldoc" "$@"
+}
+
#gen_ret_type(meta, int)
gen_ret_type() {
local meta="$1"; shift
@@ -142,6 +161,91 @@ gen_args()
done
}
+#gen_desc_return(meta)
+gen_desc_return()
+{
+ local meta="$1"; shift
+
+ case "${meta}" in
+ [v])
+ printf "Return: Nothing."
+ ;;
+ [Ff])
+ printf "Return: The original value of @v."
+ ;;
+ [R])
+ printf "Return: The updated value of @v."
+ ;;
+ [l])
+ printf "Return: The value of @v."
+ ;;
+ esac
+}
+
+#gen_template_kerneldoc(template, class, meta, pfx, name, sfx, order, atomic, int, args...)
+gen_template_kerneldoc()
+{
+ local template="$1"; shift
+ local class="$1"; shift
+ local meta="$1"; shift
+ local pfx="$1"; shift
+ local name="$1"; shift
+ local sfx="$1"; shift
+ local order="$1"; shift
+ local atomic="$1"; shift
+ local int="$1"; shift
+
+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
+
+ local ret="$(gen_ret_type "${meta}" "${int}")"
+ local retstmt="$(gen_ret_stmt "${meta}")"
+ local params="$(gen_params "${int}" "${atomic}" "$@")"
+ local args="$(gen_args "$@")"
+ local desc_order=""
+ local desc_instrumentation=""
+ local desc_return=""
+
+ if [ ! -z "${order}" ]; then
+ desc_order="${order##_}"
+ elif meta_is_implicitly_relaxed "${meta}"; then
+ desc_order="relaxed"
+ else
+ desc_order="full"
+ fi
+
+ if [ -z "${class}" ]; then
+ desc_noinstr="Unsafe to use in noinstr code; use raw_${atomicname}() there."
+ else
+ desc_noinstr="Safe to use in noinstr code; prefer ${atomicname}() elsewhere."
+ fi
+
+ desc_return="$(gen_desc_return "${meta}")"
+
+ . ${template}
+}
+
+#gen_kerneldoc(class, meta, pfx, name, sfx, order, atomic, int, args...)
+gen_kerneldoc()
+{
+ local class="$1"; shift
+ local meta="$1"; shift
+ local pfx="$1"; shift
+ local name="$1"; shift
+ local sfx="$1"; shift
+ local order="$1"; shift
+
+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
+
+ local tmpl="$(find_kerneldoc_template "${pfx}" "${name}" "${sfx}" "${order}")"
+ if [ -z "${tmpl}" ]; then
+ printf "/*\n"
+ printf " * No kerneldoc available for ${class}${atomicname}\n"
+ printf " */\n"
+ else
+ gen_template_kerneldoc "${tmpl}" "${class}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
+ fi
+}
+
#gen_proto_order_variants(meta, pfx, name, sfx, ...)
gen_proto_order_variants()
{
diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl
index 85ca8d9b5c27..903946cbf1b3 100644
--- a/scripts/atomic/atomics.tbl
+++ b/scripts/atomic/atomics.tbl
@@ -27,7 +27,7 @@ and vF i v
andnot vF i v
or vF i v
xor vF i v
-xchg I v i
+xchg I v i:new
cmpxchg I v i:old i:new
try_cmpxchg B v p:old i:new
sub_and_test b i v
diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire
index ef764085c79a..4da0cab3604e 100755
--- a/scripts/atomic/fallbacks/acquire
+++ b/scripts/atomic/fallbacks/acquire
@@ -1,9 +1,5 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}${name}${sfx}_acquire(${params})
-{
${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
__atomic_acquire_fence();
return ret;
-}
EOF
diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative
index e5980abf5904..1d3d4ab3a9d2 100755
--- a/scripts/atomic/fallbacks/add_negative
+++ b/scripts/atomic/fallbacks/add_negative
@@ -1,15 +1,3 @@
cat <<EOF
-/**
- * arch_${atomic}_add_negative${order} - Add and test if negative
- * @i: integer value to add
- * @v: pointer of type ${atomic}_t
- *
- * Atomically adds @i to @v and returns true if the result is negative,
- * or false when the result is greater than or equal to zero.
- */
-static __always_inline bool
-arch_${atomic}_add_negative${order}(${int} i, ${atomic}_t *v)
-{
- return arch_${atomic}_add_return${order}(i, v) < 0;
-}
+ return raw_${atomic}_add_return${order}(i, v) < 0;
EOF
diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless
index 9e5159c2ccfc..95ecb2b7405b 100755
--- a/scripts/atomic/fallbacks/add_unless
+++ b/scripts/atomic/fallbacks/add_unless
@@ -1,16 +1,3 @@
cat << EOF
-/**
- * arch_${atomic}_add_unless - add unless the number is already a given value
- * @v: pointer of type ${atomic}_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
- *
- * Atomically adds @a to @v, if @v was not already @u.
- * Returns true if the addition was done.
- */
-static __always_inline bool
-arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
-{
- return arch_${atomic}_fetch_add_unless(v, a, u) != u;
-}
+ return raw_${atomic}_fetch_add_unless(v, a, u) != u;
EOF
diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot
index 5a42f54a3595..66760457e67a 100755
--- a/scripts/atomic/fallbacks/andnot
+++ b/scripts/atomic/fallbacks/andnot
@@ -1,7 +1,3 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v)
-{
- ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v);
-}
+ ${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v);
EOF
diff --git a/scripts/atomic/fallbacks/cmpxchg b/scripts/atomic/fallbacks/cmpxchg
new file mode 100644
index 000000000000..1c8507f62e04
--- /dev/null
+++ b/scripts/atomic/fallbacks/cmpxchg
@@ -0,0 +1,3 @@
+cat <<EOF
+ return raw_cmpxchg${order}(&v->counter, old, new);
+EOF
diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec
index 8c144c818e9e..60d286d40300 100755
--- a/scripts/atomic/fallbacks/dec
+++ b/scripts/atomic/fallbacks/dec
@@ -1,7 +1,3 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v)
-{
- ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v);
-}
+ ${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v);
EOF
diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test
index 8549f359bd0e..3a0278e0ddd7 100755
--- a/scripts/atomic/fallbacks/dec_and_test
+++ b/scripts/atomic/fallbacks/dec_and_test
@@ -1,15 +1,3 @@
cat <<EOF
-/**
- * arch_${atomic}_dec_and_test - decrement and test
- * @v: pointer of type ${atomic}_t
- *
- * Atomically decrements @v by 1 and
- * returns true if the result is 0, or false for all other
- * cases.
- */
-static __always_inline bool
-arch_${atomic}_dec_and_test(${atomic}_t *v)
-{
- return arch_${atomic}_dec_return(v) == 0;
-}
+ return raw_${atomic}_dec_return(v) == 0;
EOF
diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive
index 86bdced3428d..f65c11b4b85b 100755
--- a/scripts/atomic/fallbacks/dec_if_positive
+++ b/scripts/atomic/fallbacks/dec_if_positive
@@ -1,15 +1,11 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_dec_if_positive(${atomic}_t *v)
-{
- ${int} dec, c = arch_${atomic}_read(v);
+ ${int} dec, c = raw_${atomic}_read(v);
do {
dec = c - 1;
if (unlikely(dec < 0))
break;
- } while (!arch_${atomic}_try_cmpxchg(v, &c, dec));
+ } while (!raw_${atomic}_try_cmpxchg(v, &c, dec));
return dec;
-}
EOF
diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive
index c531d5afecc4..d025361d7b85 100755
--- a/scripts/atomic/fallbacks/dec_unless_positive
+++ b/scripts/atomic/fallbacks/dec_unless_positive
@@ -1,14 +1,10 @@
cat <<EOF
-static __always_inline bool
-arch_${atomic}_dec_unless_positive(${atomic}_t *v)
-{
- ${int} c = arch_${atomic}_read(v);
+ ${int} c = raw_${atomic}_read(v);
do {
if (unlikely(c > 0))
return false;
- } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1));
+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1));
return true;
-}
EOF
diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence
index 07757d8e338e..40d5b397658f 100755
--- a/scripts/atomic/fallbacks/fence
+++ b/scripts/atomic/fallbacks/fence
@@ -1,11 +1,7 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}${name}${sfx}(${params})
-{
${ret} ret;
__atomic_pre_full_fence();
ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
__atomic_post_full_fence();
return ret;
-}
EOF
diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless
index 68ce13c8b9da..8db7e9e17fac 100755
--- a/scripts/atomic/fallbacks/fetch_add_unless
+++ b/scripts/atomic/fallbacks/fetch_add_unless
@@ -1,23 +1,10 @@
cat << EOF
-/**
- * arch_${atomic}_fetch_add_unless - add unless the number is already a given value
- * @v: pointer of type ${atomic}_t
- * @a: the amount to add to v...
- * @u: ...unless v is equal to u.
- *
- * Atomically adds @a to @v, so long as @v was not already @u.
- * Returns original value of @v
- */
-static __always_inline ${int}
-arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
-{
- ${int} c = arch_${atomic}_read(v);
+ ${int} c = raw_${atomic}_read(v);
do {
if (unlikely(c == u))
break;
- } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a));
+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c + a));
return c;
-}
EOF
diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc
index 3c2c3739169e..56c770f5919c 100755
--- a/scripts/atomic/fallbacks/inc
+++ b/scripts/atomic/fallbacks/inc
@@ -1,7 +1,3 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v)
-{
- ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v);
-}
+ ${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v);
EOF
diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test
index 0cf23fe1efb8..7d16a10f2257 100755
--- a/scripts/atomic/fallbacks/inc_and_test
+++ b/scripts/atomic/fallbacks/inc_and_test
@@ -1,15 +1,3 @@
cat <<EOF
-/**
- * arch_${atomic}_inc_and_test - increment and test
- * @v: pointer of type ${atomic}_t
- *
- * Atomically increments @v by 1
- * and returns true if the result is zero, or false for all
- * other cases.
- */
-static __always_inline bool
-arch_${atomic}_inc_and_test(${atomic}_t *v)
-{
- return arch_${atomic}_inc_return(v) == 0;
-}
+ return raw_${atomic}_inc_return(v) == 0;
EOF
diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero
index ed8a1f562667..1fcef1e55bc9 100755
--- a/scripts/atomic/fallbacks/inc_not_zero
+++ b/scripts/atomic/fallbacks/inc_not_zero
@@ -1,14 +1,3 @@
cat <<EOF
-/**
- * arch_${atomic}_inc_not_zero - increment unless the number is zero
- * @v: pointer of type ${atomic}_t
- *
- * Atomically increments @v by 1, if @v is non-zero.
- * Returns true if the increment was done.
- */
-static __always_inline bool
-arch_${atomic}_inc_not_zero(${atomic}_t *v)
-{
- return arch_${atomic}_add_unless(v, 1, 0);
-}
+ return raw_${atomic}_add_unless(v, 1, 0);
EOF
diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative
index 95d8ce48233f..7b4b09868842 100755
--- a/scripts/atomic/fallbacks/inc_unless_negative
+++ b/scripts/atomic/fallbacks/inc_unless_negative
@@ -1,14 +1,10 @@
cat <<EOF
-static __always_inline bool
-arch_${atomic}_inc_unless_negative(${atomic}_t *v)
-{
- ${int} c = arch_${atomic}_read(v);
+ ${int} c = raw_${atomic}_read(v);
do {
if (unlikely(c < 0))
return false;
- } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1));
+ } while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1));
return true;
-}
EOF
diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire
index a0ea1d26e6b2..e319862d2f1a 100755
--- a/scripts/atomic/fallbacks/read_acquire
+++ b/scripts/atomic/fallbacks/read_acquire
@@ -1,16 +1,12 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_read_acquire(const ${atomic}_t *v)
-{
${int} ret;
if (__native_word(${atomic}_t)) {
ret = smp_load_acquire(&(v)->counter);
} else {
- ret = arch_${atomic}_read(v);
+ ret = raw_${atomic}_read(v);
__atomic_acquire_fence();
}
return ret;
-}
EOF
diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release
index b46feb56d69c..1e6daf57b4ba 100755
--- a/scripts/atomic/fallbacks/release
+++ b/scripts/atomic/fallbacks/release
@@ -1,8 +1,4 @@
cat <<EOF
-static __always_inline ${ret}
-arch_${atomic}_${pfx}${name}${sfx}_release(${params})
-{
__atomic_release_fence();
${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args});
-}
EOF
diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release
index 05cdb7f42477..16a374ae6bb1 100755
--- a/scripts/atomic/fallbacks/set_release
+++ b/scripts/atomic/fallbacks/set_release
@@ -1,12 +1,8 @@
cat <<EOF
-static __always_inline void
-arch_${atomic}_set_release(${atomic}_t *v, ${int} i)
-{
if (__native_word(${atomic}_t)) {
smp_store_release(&(v)->counter, i);
} else {
__atomic_release_fence();
- arch_${atomic}_set(v, i);
+ raw_${atomic}_set(v, i);
}
-}
EOF
diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test
index 260f37341c88..d1f746fe0ca4 100755
--- a/scripts/atomic/fallbacks/sub_and_test
+++ b/scripts/atomic/fallbacks/sub_and_test
@@ -1,16 +1,3 @@
cat <<EOF
-/**
- * arch_${atomic}_sub_and_test - subtract value from variable and test result
- * @i: integer value to subtract
- * @v: pointer of type ${atomic}_t
- *
- * Atomically subtracts @i from @v and returns
- * true if the result is zero, or false for all
- * other cases.
- */
-static __always_inline bool
-arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
-{
- return arch_${atomic}_sub_return(i, v) == 0;
-}
+ return raw_${atomic}_sub_return(i, v) == 0;
EOF
diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg
index 890f850ede37..d4da82092baf 100755
--- a/scripts/atomic/fallbacks/try_cmpxchg
+++ b/scripts/atomic/fallbacks/try_cmpxchg
@@ -1,11 +1,7 @@
cat <<EOF
-static __always_inline bool
-arch_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new)
-{
${int} r, o = *old;
- r = arch_${atomic}_cmpxchg${order}(v, o, new);
+ r = raw_${atomic}_cmpxchg${order}(v, o, new);
if (unlikely(r != o))
*old = r;
return likely(r == o);
-}
EOF
diff --git a/scripts/atomic/fallbacks/xchg b/scripts/atomic/fallbacks/xchg
new file mode 100644
index 000000000000..e4def1e0d092
--- /dev/null
+++ b/scripts/atomic/fallbacks/xchg
@@ -0,0 +1,3 @@
+cat <<EOF
+ return raw_xchg${order}(&v->counter, new);
+EOF
diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh
index 6e853f0dad8d..c0c8a85d7c81 100755
--- a/scripts/atomic/gen-atomic-fallback.sh
+++ b/scripts/atomic/gen-atomic-fallback.sh
@@ -17,23 +17,16 @@ gen_template_fallback()
local atomic="$1"; shift
local int="$1"; shift
- local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}"
-
local ret="$(gen_ret_type "${meta}" "${int}")"
local retstmt="$(gen_ret_stmt "${meta}")"
local params="$(gen_params "${int}" "${atomic}" "$@")"
local args="$(gen_args "$@")"
- if [ ! -z "${template}" ]; then
- printf "#ifndef ${atomicname}\n"
- . ${template}
- printf "#define ${atomicname} ${atomicname}\n"
- printf "#endif\n\n"
- fi
+ . ${template}
}
-#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
-gen_proto_fallback()
+#gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_order_fallback()
{
local meta="$1"; shift
local pfx="$1"; shift
@@ -41,87 +34,124 @@ gen_proto_fallback()
local sfx="$1"; shift
local order="$1"; shift
- local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
+ local tmpl_order=${order#_}
+ local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}"
gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
}
-#gen_basic_fallbacks(basename)
-gen_basic_fallbacks()
-{
- local basename="$1"; shift
-cat << EOF
-#define ${basename}_acquire ${basename}
-#define ${basename}_release ${basename}
-#define ${basename}_relaxed ${basename}
-EOF
-}
-
-gen_proto_order_variant()
+#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_proto_fallback()
{
local meta="$1"; shift
local pfx="$1"; shift
local name="$1"; shift
local sfx="$1"; shift
local order="$1"; shift
- local atomic="$1"
- local basename="arch_${atomic}_${pfx}${name}${sfx}"
-
- printf "#define ${basename}${order} ${basename}${order}\n"
+ local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
+ gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
}
-#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
-gen_proto_order_variants()
+#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_proto_order_variant()
{
local meta="$1"; shift
local pfx="$1"; shift
local name="$1"; shift
local sfx="$1"; shift
- local atomic="$1"
+ local order="$1"; shift
+ local atomic="$1"; shift
+ local int="$1"; shift
- local basename="arch_${atomic}_${pfx}${name}${sfx}"
+ local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
+ local basename="${atomic}_${pfx}${name}${sfx}"
local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
- # If we don't have relaxed atomics, then we don't bother with ordering fallbacks
- # read_acquire and set_release need to be templated, though
- if ! meta_has_relaxed "${meta}"; then
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
+ local ret="$(gen_ret_type "${meta}" "${int}")"
+ local retstmt="$(gen_ret_stmt "${meta}")"
+ local params="$(gen_params "${int}" "${atomic}" "$@")"
+ local args="$(gen_args "$@")"
- if meta_has_acquire "${meta}"; then
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
- fi
+ gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+
+ printf "static __always_inline ${ret}\n"
+ printf "raw_${atomicname}(${params})\n"
+ printf "{\n"
+
+ # Where there is no possible fallback, this order variant is mandatory
+ # and must be provided by arch code. Add a comment to the header to
+ # make this obvious.
+ #
+ # Ideally we'd error on a missing definition, but arch code might
+ # define this order variant as a C function without a preprocessor
+ # symbol.
+ if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then
+ printf "\t${retstmt}arch_${atomicname}(${args});\n"
+ printf "}\n\n"
+ return
+ fi
- if meta_has_release "${meta}"; then
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
- fi
+ printf "#if defined(arch_${atomicname})\n"
+ printf "\t${retstmt}arch_${atomicname}(${args});\n"
- return
+ # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops
+ if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then
+ printf "#elif defined(arch_${basename}_relaxed)\n"
+ gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
fi
- printf "#ifndef ${basename}_relaxed\n"
+ # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops
+ if [ ! -z "${order}" ]; then
+ printf "#elif defined(arch_${basename})\n"
+ printf "\t${retstmt}arch_${basename}(${args});\n"
+ fi
+ printf "#else\n"
if [ ! -z "${template}" ]; then
- printf "#ifdef ${basename}\n"
+ gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+ else
+ printf "#error \"Unable to define raw_${atomicname}\"\n"
fi
- gen_basic_fallbacks "${basename}"
+ printf "#endif\n"
+ printf "}\n\n"
+}
- if [ ! -z "${template}" ]; then
- printf "#endif /* ${basename} */\n\n"
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
- gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
+
+#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
+gen_proto_order_variants()
+{
+ local meta="$1"; shift
+ local pfx="$1"; shift
+ local name="$1"; shift
+ local sfx="$1"; shift
+ local atomic="$1"
+
+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
+
+ if meta_has_acquire "${meta}"; then
+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
fi
- printf "#else /* ${basename}_relaxed */\n\n"
+ if meta_has_release "${meta}"; then
+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
+ fi
- gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
- gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
- gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
+ if meta_has_relaxed "${meta}"; then
+ gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
+ fi
+}
- printf "#endif /* ${basename}_relaxed */\n\n"
+#gen_basic_fallbacks(basename)
+gen_basic_fallbacks()
+{
+ local basename="$1"; shift
+cat << EOF
+#define raw_${basename}_acquire arch_${basename}
+#define raw_${basename}_release arch_${basename}
+#define raw_${basename}_relaxed arch_${basename}
+EOF
}
gen_order_fallbacks()
@@ -130,36 +160,65 @@ gen_order_fallbacks()
cat <<EOF
-#ifndef ${xchg}_acquire
-#define ${xchg}_acquire(...) \\
- __atomic_op_acquire(${xchg}, __VA_ARGS__)
+#define raw_${xchg}_relaxed arch_${xchg}_relaxed
+
+#ifdef arch_${xchg}_acquire
+#define raw_${xchg}_acquire arch_${xchg}_acquire
+#else
+#define raw_${xchg}_acquire(...) \\
+ __atomic_op_acquire(arch_${xchg}, __VA_ARGS__)
#endif
-#ifndef ${xchg}_release
-#define ${xchg}_release(...) \\
- __atomic_op_release(${xchg}, __VA_ARGS__)
+#ifdef arch_${xchg}_release
+#define raw_${xchg}_release arch_${xchg}_release
+#else
+#define raw_${xchg}_release(...) \\
+ __atomic_op_release(arch_${xchg}, __VA_ARGS__)
#endif
-#ifndef ${xchg}
-#define ${xchg}(...) \\
- __atomic_op_fence(${xchg}, __VA_ARGS__)
+#ifdef arch_${xchg}
+#define raw_${xchg} arch_${xchg}
+#else
+#define raw_${xchg}(...) \\
+ __atomic_op_fence(arch_${xchg}, __VA_ARGS__)
#endif
EOF
}
-gen_xchg_fallbacks()
+gen_xchg_order_fallback()
{
local xchg="$1"; shift
- printf "#ifndef ${xchg}_relaxed\n"
+ local order="$1"; shift
+ local forder="${order:-_fence}"
- gen_basic_fallbacks ${xchg}
+ printf "#if defined(arch_${xchg}${order})\n"
+ printf "#define raw_${xchg}${order} arch_${xchg}${order}\n"
- printf "#else /* ${xchg}_relaxed */\n"
+ if [ "${order}" != "_relaxed" ]; then
+ printf "#elif defined(arch_${xchg}_relaxed)\n"
+ printf "#define raw_${xchg}${order}(...) \\\\\n"
+ printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n"
+ fi
+
+ if [ ! -z "${order}" ]; then
+ printf "#elif defined(arch_${xchg})\n"
+ printf "#define raw_${xchg}${order} arch_${xchg}\n"
+ fi
- gen_order_fallbacks ${xchg}
+ printf "#else\n"
+ printf "extern void raw_${xchg}${order}_not_implemented(void);\n"
+ printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n"
+ printf "#endif\n\n"
+}
+
+gen_xchg_fallbacks()
+{
+ local xchg="$1"; shift
- printf "#endif /* ${xchg}_relaxed */\n\n"
+ for order in "" "_acquire" "_release" "_relaxed"; do
+ gen_xchg_order_fallback "${xchg}" "${order}"
+ done
}
gen_try_cmpxchg_fallback()
@@ -168,40 +227,61 @@ gen_try_cmpxchg_fallback()
local order="$1"; shift;
cat <<EOF
-#ifndef arch_try_${cmpxchg}${order}
-#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
+#define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\
({ \\
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
- ___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
+ ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\
if (unlikely(___r != ___o)) \\
*___op = ___r; \\
likely(___r == ___o); \\
})
-#endif /* arch_try_${cmpxchg}${order} */
-
EOF
}
-gen_try_cmpxchg_fallbacks()
+gen_try_cmpxchg_order_fallback()
{
- local cmpxchg="$1"; shift;
+ local cmpxchg="$1"; shift
+ local order="$1"; shift
+ local forder="${order:-_fence}"
- printf "#ifndef arch_try_${cmpxchg}_relaxed\n"
- printf "#ifdef arch_try_${cmpxchg}\n"
+ printf "#if defined(arch_try_${cmpxchg}${order})\n"
+ printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n"
+
+ if [ "${order}" != "_relaxed" ]; then
+ printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n"
+ printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n"
+ printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n"
+ fi
- gen_basic_fallbacks "arch_try_${cmpxchg}"
+ if [ ! -z "${order}" ]; then
+ printf "#elif defined(arch_try_${cmpxchg})\n"
+ printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n"
+ fi
- printf "#endif /* arch_try_${cmpxchg} */\n\n"
+ printf "#else\n"
+ gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
+ printf "#endif\n\n"
+}
+
+gen_try_cmpxchg_fallbacks()
+{
+ local cmpxchg="$1"; shift;
for order in "" "_acquire" "_release" "_relaxed"; do
- gen_try_cmpxchg_fallback "${cmpxchg}" "${order}"
+ gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}"
done
+}
- printf "#else /* arch_try_${cmpxchg}_relaxed */\n"
-
- gen_order_fallbacks "arch_try_${cmpxchg}"
+gen_cmpxchg_local_fallbacks()
+{
+ local cmpxchg="$1"; shift
- printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n"
+ printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
+ printf "#ifdef arch_try_${cmpxchg}\n"
+ printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n"
+ printf "#else\n"
+ gen_try_cmpxchg_fallback "${cmpxchg}" ""
+ printf "#endif\n\n"
}
cat << EOF
@@ -217,16 +297,20 @@ cat << EOF
EOF
-for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
+for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do
gen_xchg_fallbacks "${xchg}"
done
-for cmpxchg in "cmpxchg" "cmpxchg64"; do
+for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
gen_try_cmpxchg_fallbacks "${cmpxchg}"
done
-for cmpxchg in "cmpxchg_local" "cmpxchg64_local"; do
- gen_try_cmpxchg_fallback "${cmpxchg}" ""
+for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do
+ gen_cmpxchg_local_fallbacks "${cmpxchg}" ""
+done
+
+for cmpxchg in "sync_cmpxchg"; do
+ printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n"
done
grep '^[a-z]' "$1" | while read name meta args; do
diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh
index d9ffd74f73ca..8f8f8e3b20f9 100755
--- a/scripts/atomic/gen-atomic-instrumented.sh
+++ b/scripts/atomic/gen-atomic-instrumented.sh
@@ -68,12 +68,14 @@ gen_proto_order_variant()
local args="$(gen_args "$@")"
local retstmt="$(gen_ret_stmt "${meta}")"
+ gen_kerneldoc "" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+
cat <<EOF
static __always_inline ${ret}
${atomicname}(${params})
{
${checks}
- ${retstmt}arch_${atomicname}(${args});
+ ${retstmt}raw_${atomicname}(${args});
}
EOF
@@ -84,7 +86,6 @@ gen_xchg()
{
local xchg="$1"; shift
local order="$1"; shift
- local mult="$1"; shift
kcsan_barrier=""
if [ "${xchg%_local}" = "${xchg}" ]; then
@@ -104,9 +105,9 @@ cat <<EOF
EOF
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
cat <<EOF
- instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
- instrument_read_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\
- arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
+ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
+ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\
+ raw_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\
})
EOF
@@ -119,8 +120,8 @@ cat <<EOF
EOF
[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n"
cat <<EOF
- instrument_atomic_read_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
- arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
+ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\
+ raw_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\
})
EOF
@@ -134,15 +135,10 @@ cat << EOF
// DO NOT MODIFY THIS FILE DIRECTLY
/*
- * This file provides wrappers with KASAN instrumentation for atomic operations.
- * To use this functionality an arch's atomic.h file needs to define all
- * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
- * this file at the end. This file provides atomic_read() that forwards to
- * arch_atomic_read() for actual atomic operation.
- * Note: if an arch atomic operation is implemented by means of other atomic
- * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
- * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
- * double instrumentation.
+ * This file provoides atomic operations with explicit instrumentation (e.g.
+ * KASAN, KCSAN), which should be used unless it is necessary to avoid
+ * instrumentation. Where it is necessary to aovid instrumenation, the
+ * raw_atomic*() operations should be used.
*/
#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
#define _LINUX_ATOMIC_INSTRUMENTED_H
@@ -166,24 +162,18 @@ grep '^[a-z]' "$1" | while read name meta args; do
done
-for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
+for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
for order in "" "_acquire" "_release" "_relaxed"; do
- gen_xchg "${xchg}" "${order}" ""
+ gen_xchg "${xchg}" "${order}"
printf "\n"
done
done
-for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do
- gen_xchg "${xchg}" "" ""
+for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
+ gen_xchg "${xchg}" ""
printf "\n"
done
-gen_xchg "cmpxchg_double" "" "2 * "
-
-printf "\n\n"
-
-gen_xchg "cmpxchg_double_local" "" "2 * "
-
cat <<EOF
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
diff --git a/scripts/atomic/gen-atomic-long.sh b/scripts/atomic/gen-atomic-long.sh
index eda89cea6e1d..9826be3ba986 100755
--- a/scripts/atomic/gen-atomic-long.sh
+++ b/scripts/atomic/gen-atomic-long.sh
@@ -32,24 +32,34 @@ gen_args_cast()
done
}
-#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)
+#gen_proto_order_variant(meta, pfx, name, sfx, order, arg...)
gen_proto_order_variant()
{
local meta="$1"; shift
- local name="$1$2$3$4"; shift; shift; shift; shift
- local atomic="$1"; shift
- local int="$1"; shift
+ local pfx="$1"; shift
+ local name="$1"; shift
+ local sfx="$1"; shift
+ local order="$1"; shift
+
+ local atomicname="${pfx}${name}${sfx}${order}"
local ret="$(gen_ret_type "${meta}" "long")"
local params="$(gen_params "long" "atomic_long" "$@")"
- local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")"
+ local argscast_32="$(gen_args_cast "int" "atomic" "$@")"
+ local argscast_64="$(gen_args_cast "s64" "atomic64" "$@")"
local retstmt="$(gen_ret_stmt "${meta}")"
+ gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "atomic_long" "long" "$@"
+
cat <<EOF
static __always_inline ${ret}
-arch_atomic_long_${name}(${params})
+raw_atomic_long_${atomicname}(${params})
{
- ${retstmt}arch_${atomic}_${name}(${argscast});
+#ifdef CONFIG_64BIT
+ ${retstmt}raw_atomic64_${atomicname}(${argscast_64});
+#else
+ ${retstmt}raw_atomic_${atomicname}(${argscast_32});
+#endif
}
EOF
@@ -79,24 +89,12 @@ typedef atomic_t atomic_long_t;
#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
#endif
-#ifdef CONFIG_64BIT
-
-EOF
-
-grep '^[a-z]' "$1" | while read name meta args; do
- gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
-done
-
-cat <<EOF
-#else /* CONFIG_64BIT */
-
EOF
grep '^[a-z]' "$1" | while read name meta args; do
- gen_proto "${meta}" "${name}" "atomic" "int" ${args}
+ gen_proto "${meta}" "${name}" ${args}
done
cat <<EOF
-#endif /* CONFIG_64BIT */
#endif /* _LINUX_ATOMIC_LONG_H */
EOF
diff --git a/scripts/atomic/kerneldoc/add b/scripts/atomic/kerneldoc/add
new file mode 100644
index 000000000000..991f3dafceea
--- /dev/null
+++ b/scripts/atomic/kerneldoc/add
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic add with ${desc_order} ordering
+ * @i: ${int} value to add
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/add_negative b/scripts/atomic/kerneldoc/add_negative
new file mode 100644
index 000000000000..f4ca1f05d1d8
--- /dev/null
+++ b/scripts/atomic/kerneldoc/add_negative
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic add and test if negative with ${desc_order} ordering
+ * @i: ${int} value to add
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v + @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if the resulting value of @v is negative, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/add_unless b/scripts/atomic/kerneldoc/add_unless
new file mode 100644
index 000000000000..f828e5f6750c
--- /dev/null
+++ b/scripts/atomic/kerneldoc/add_unless
@@ -0,0 +1,18 @@
+if [ -z "${pfx}" ]; then
+ desc_return="Return: @true if @v was updated, @false otherwise."
+fi
+
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic add unless value with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ * @a: ${int} value to add
+ * @u: ${int} value to compare with
+ *
+ * If (@v != @u), atomically updates @v to (@v + @a) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/and b/scripts/atomic/kerneldoc/and
new file mode 100644
index 000000000000..a923574351fc
--- /dev/null
+++ b/scripts/atomic/kerneldoc/and
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic bitwise AND with ${desc_order} ordering
+ * @i: ${int} value
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v & @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/andnot b/scripts/atomic/kerneldoc/andnot
new file mode 100644
index 000000000000..64bb509f866b
--- /dev/null
+++ b/scripts/atomic/kerneldoc/andnot
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic bitwise AND NOT with ${desc_order} ordering
+ * @i: ${int} value
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v & ~@i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/cmpxchg b/scripts/atomic/kerneldoc/cmpxchg
new file mode 100644
index 000000000000..3bce328f50cf
--- /dev/null
+++ b/scripts/atomic/kerneldoc/cmpxchg
@@ -0,0 +1,14 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ * @old: ${int} value to compare with
+ * @new: ${int} value to assign
+ *
+ * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: The original value of @v.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/dec b/scripts/atomic/kerneldoc/dec
new file mode 100644
index 000000000000..bbeecbc4c20a
--- /dev/null
+++ b/scripts/atomic/kerneldoc/dec
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic decrement with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/dec_and_test b/scripts/atomic/kerneldoc/dec_and_test
new file mode 100644
index 000000000000..71bbd23ce4bc
--- /dev/null
+++ b/scripts/atomic/kerneldoc/dec_and_test
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic decrement and test if zero with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v - 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if the resulting value of @v is zero, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/dec_if_positive b/scripts/atomic/kerneldoc/dec_if_positive
new file mode 100644
index 000000000000..04f1aed3cf83
--- /dev/null
+++ b/scripts/atomic/kerneldoc/dec_if_positive
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic decrement if positive with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * If (@v > 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: The old value of (@v - 1), regardless of whether @v was updated.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/dec_unless_positive b/scripts/atomic/kerneldoc/dec_unless_positive
new file mode 100644
index 000000000000..ee73612f0354
--- /dev/null
+++ b/scripts/atomic/kerneldoc/dec_unless_positive
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic decrement unless positive with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * If (@v <= 0), atomically updates @v to (@v - 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if @v was updated, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/inc b/scripts/atomic/kerneldoc/inc
new file mode 100644
index 000000000000..9f14f1b3d2ef
--- /dev/null
+++ b/scripts/atomic/kerneldoc/inc
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic increment with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/inc_and_test b/scripts/atomic/kerneldoc/inc_and_test
new file mode 100644
index 000000000000..971694d59bbd
--- /dev/null
+++ b/scripts/atomic/kerneldoc/inc_and_test
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic increment and test if zero with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v + 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if the resulting value of @v is zero, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/inc_not_zero b/scripts/atomic/kerneldoc/inc_not_zero
new file mode 100644
index 000000000000..618be08e653e
--- /dev/null
+++ b/scripts/atomic/kerneldoc/inc_not_zero
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic increment unless zero with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * If (@v != 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if @v was updated, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/inc_unless_negative b/scripts/atomic/kerneldoc/inc_unless_negative
new file mode 100644
index 000000000000..597f23d4dc8d
--- /dev/null
+++ b/scripts/atomic/kerneldoc/inc_unless_negative
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic increment unless negative with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * If (@v >= 0), atomically updates @v to (@v + 1) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if @v was updated, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/or b/scripts/atomic/kerneldoc/or
new file mode 100644
index 000000000000..55b33de50416
--- /dev/null
+++ b/scripts/atomic/kerneldoc/or
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic bitwise OR with ${desc_order} ordering
+ * @i: ${int} value
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v | @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/read b/scripts/atomic/kerneldoc/read
new file mode 100644
index 000000000000..89fe6147c964
--- /dev/null
+++ b/scripts/atomic/kerneldoc/read
@@ -0,0 +1,12 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic load with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically loads the value of @v with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: The value loaded from @v.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/set b/scripts/atomic/kerneldoc/set
new file mode 100644
index 000000000000..e82cb9ebbc42
--- /dev/null
+++ b/scripts/atomic/kerneldoc/set
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic set with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ * @i: ${int} value to assign
+ *
+ * Atomically sets @v to @i with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: Nothing.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/sub b/scripts/atomic/kerneldoc/sub
new file mode 100644
index 000000000000..3ba642d04407
--- /dev/null
+++ b/scripts/atomic/kerneldoc/sub
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic subtract with ${desc_order} ordering
+ * @i: ${int} value to subtract
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/sub_and_test b/scripts/atomic/kerneldoc/sub_and_test
new file mode 100644
index 000000000000..d3760f7749d4
--- /dev/null
+++ b/scripts/atomic/kerneldoc/sub_and_test
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic subtract and test if zero with ${desc_order} ordering
+ * @i: ${int} value to add
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v - @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if the resulting value of @v is zero, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/try_cmpxchg b/scripts/atomic/kerneldoc/try_cmpxchg
new file mode 100644
index 000000000000..296553206c06
--- /dev/null
+++ b/scripts/atomic/kerneldoc/try_cmpxchg
@@ -0,0 +1,15 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ * @old: pointer to ${int} value to compare with
+ * @new: ${int} value to assign
+ *
+ * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering.
+ * Otherwise, updates @old to the current value of @v.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: @true if the exchange occured, @false otherwise.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/xchg b/scripts/atomic/kerneldoc/xchg
new file mode 100644
index 000000000000..75f04c085f25
--- /dev/null
+++ b/scripts/atomic/kerneldoc/xchg
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic exchange with ${desc_order} ordering
+ * @v: pointer to ${atomic}_t
+ * @new: ${int} value to assign
+ *
+ * Atomically updates @v to @new with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * Return: The original value of @v.
+ */
+EOF
diff --git a/scripts/atomic/kerneldoc/xor b/scripts/atomic/kerneldoc/xor
new file mode 100644
index 000000000000..8837270f2806
--- /dev/null
+++ b/scripts/atomic/kerneldoc/xor
@@ -0,0 +1,13 @@
+cat <<EOF
+/**
+ * ${class}${atomicname}() - atomic bitwise XOR with ${desc_order} ordering
+ * @i: ${int} value
+ * @v: pointer to ${atomic}_t
+ *
+ * Atomically updates @v to (@v ^ @i) with ${desc_order} ordering.
+ *
+ * ${desc_noinstr}
+ *
+ * ${desc_return}
+ */
+EOF