1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Support for Floating Point and Vector Instructions
*
*/
#ifndef __ASM_S390_FPU_INSN_H
#define __ASM_S390_FPU_INSN_H
#include <asm/fpu-insn-asm.h>
#ifndef __ASSEMBLY__
#include <linux/instrumented.h>
#include <asm/asm-extable.h>
asm(".include \"asm/fpu-insn-asm.h\"\n");
/*
* Various small helper functions, which can and should be used within
* kernel fpu code sections. Each function represents only one floating
* point or vector instruction (except for helper functions which require
* exception handling).
*
* This allows to use floating point and vector instructions like C
* functions, which has the advantage that all supporting code, like
* e.g. loops, can be written in easy to read C code.
*
* Each of the helper functions provides support for code instrumentation,
* like e.g. KASAN. Therefore instrumentation is also covered automatically
* when using these functions.
*
* In order to ensure that code generated with the helper functions stays
* within kernel fpu sections, which are guarded with kernel_fpu_begin()
* and kernel_fpu_end() calls, each function has a mandatory "memory"
* barrier.
*/
static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg)
{
instrument_read(reg, sizeof(*reg));
asm volatile("ld %[fpr],%[reg]\n"
:
: [fpr] "I" (fpr), [reg] "Q" (reg->ui)
: "memory");
}
static __always_inline void fpu_lfpc(unsigned int *fpc)
{
instrument_read(fpc, sizeof(*fpc));
asm volatile("lfpc %[fpc]"
:
: [fpc] "Q" (*fpc)
: "memory");
}
/**
* fpu_lfpc_safe - Load floating point control register safely.
* @fpc: new value for floating point control register
*
* Load floating point control register. This may lead to an exception,
* since a saved value may have been modified by user space (ptrace,
* signal return, kvm registers) to an invalid value. In such a case
* set the floating point control register to zero.
*/
static inline void fpu_lfpc_safe(unsigned int *fpc)
{
u32 tmp;
instrument_read(fpc, sizeof(*fpc));
asm volatile("\n"
"0: lfpc %[fpc]\n"
"1: nopr %%r7\n"
".pushsection .fixup, \"ax\"\n"
"2: lghi %[tmp],0\n"
" sfpc %[tmp]\n"
" jg 1b\n"
".popsection\n"
EX_TABLE(1b, 2b)
: [tmp] "=d" (tmp)
: [fpc] "Q" (*fpc)
: "memory");
}
static __always_inline void fpu_std(unsigned short fpr, freg_t *reg)
{
instrument_write(reg, sizeof(*reg));
asm volatile("std %[fpr],%[reg]\n"
: [reg] "=Q" (reg->ui)
: [fpr] "I" (fpr)
: "memory");
}
static __always_inline void fpu_sfpc(unsigned int fpc)
{
asm volatile("sfpc %[fpc]"
:
: [fpc] "d" (fpc)
: "memory");
}
static __always_inline void fpu_stfpc(unsigned int *fpc)
{
instrument_write(fpc, sizeof(*fpc));
asm volatile("stfpc %[fpc]"
: [fpc] "=Q" (*fpc)
:
: "memory");
}
#ifdef CONFIG_CC_IS_CLANG
#define fpu_vlm(_v1, _v3, _vxrs) \
({ \
unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
struct { \
__vector128 _v[(_v3) - (_v1) + 1]; \
} *_v = (void *)(_vxrs); \
\
instrument_read(_v, size); \
asm volatile("\n" \
" la 1,%[vxrs]\n" \
" VLM %[v1],%[v3],0,1\n" \
: \
: [vxrs] "R" (*_v), \
[v1] "I" (_v1), [v3] "I" (_v3) \
: "memory", "1"); \
(_v3) - (_v1) + 1; \
})
#else /* CONFIG_CC_IS_CLANG */
#define fpu_vlm(_v1, _v3, _vxrs) \
({ \
unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
struct { \
__vector128 _v[(_v3) - (_v1) + 1]; \
} *_v = (void *)(_vxrs); \
\
instrument_read(_v, size); \
asm volatile("VLM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \
: \
: [vxrs] "Q" (*_v), \
[v1] "I" (_v1), [v3] "I" (_v3) \
: "memory"); \
(_v3) - (_v1) + 1; \
})
#endif /* CONFIG_CC_IS_CLANG */
#ifdef CONFIG_CC_IS_CLANG
#define fpu_vstm(_v1, _v3, _vxrs) \
({ \
unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
struct { \
__vector128 _v[(_v3) - (_v1) + 1]; \
} *_v = (void *)(_vxrs); \
\
instrument_write(_v, size); \
asm volatile("\n" \
" la 1,%[vxrs]\n" \
" VSTM %[v1],%[v3],0,1\n" \
: [vxrs] "=R" (*_v) \
: [v1] "I" (_v1), [v3] "I" (_v3) \
: "memory", "1"); \
(_v3) - (_v1) + 1; \
})
#else /* CONFIG_CC_IS_CLANG */
#define fpu_vstm(_v1, _v3, _vxrs) \
({ \
unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
struct { \
__vector128 _v[(_v3) - (_v1) + 1]; \
} *_v = (void *)(_vxrs); \
\
instrument_write(_v, size); \
asm volatile("VSTM %[v1],%[v3],%O[vxrs],%R[vxrs]\n" \
: [vxrs] "=Q" (*_v) \
: [v1] "I" (_v1), [v3] "I" (_v3) \
: "memory"); \
(_v3) - (_v1) + 1; \
})
#endif /* CONFIG_CC_IS_CLANG */
#endif /* __ASSEMBLY__ */
#endif /* __ASM_S390_FPU_INSN_H */
|
