1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2019-2020 Arm Ltd.
#include <linux/compiler.h>
#include <linux/kasan-checks.h>
#include <linux/kernel.h>
#include <net/checksum.h>
static u64 accumulate(u64 sum, u64 data)
{
sum += data;
if (sum < data)
sum += 1;
return sum;
}
/*
* We over-read the buffer and this makes KASAN unhappy. Instead, disable
* instrumentation and call kasan explicitly.
*/
unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
{
unsigned int offset, shift, sum;
const u64 *ptr;
u64 data, sum64 = 0;
if (unlikely(len == 0))
return 0;
offset = (unsigned long)buff & 7;
/*
* This is to all intents and purposes safe, since rounding down cannot
* result in a different page or cache line being accessed, and @buff
* should absolutely not be pointing to anything read-sensitive. We do,
* however, have to be careful not to piss off KASAN, which means using
* unchecked reads to accommodate the head and tail, for which we'll
* compensate with an explicit check up-front.
*/
kasan_check_read(buff, len);
ptr = (u64 *)(buff - offset);
len = len + offset - 8;
/*
* Head: zero out any excess leading bytes. Shifting back by the same
* amount should be at least as fast as any other way of handling the
* odd/even alignment, and means we can ignore it until the very end.
*/
shift = offset * 8;
data = *ptr++;
data = (data >> shift) << shift;
/*
* Body: straightforward aligned loads from here on (the paired loads
* underlying the quadword type still only need dword alignment). The
* main loop strictly excludes the tail, so the second loop will always
* run at least once.
*/
while (unlikely(len > 64)) {
__uint128_t tmp1, tmp2, tmp3, tmp4;
tmp1 = *(__uint128_t *)ptr;
tmp2 = *(__uint128_t *)(ptr + 2);
tmp3 = *(__uint128_t *)(ptr + 4);
tmp4 = *(__uint128_t *)(ptr + 6);
len -= 64;
ptr += 8;
/* This is the "don't dump the carry flag into a GPR" idiom */
tmp1 += (tmp1 >> 64) | (tmp1 << 64);
tmp2 += (tmp2 >> 64) | (tmp2 << 64);
tmp3 += (tmp3 >> 64) | (tmp3 << 64);
tmp4 += (tmp4 >> 64) | (tmp4 << 64);
tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
tmp1 += (tmp1 >> 64) | (tmp1 << 64);
tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
tmp3 += (tmp3 >> 64) | (tmp3 << 64);
tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
tmp1 += (tmp1 >> 64) | (tmp1 << 64);
tmp1 = ((tmp1 >> 64) << 64) | sum64;
tmp1 += (tmp1 >> 64) | (tmp1 << 64);
sum64 = tmp1 >> 64;
}
while (len > 8) {
__uint128_t tmp;
sum64 = accumulate(sum64, data);
tmp = *(__uint128_t *)ptr;
len -= 16;
ptr += 2;
data = tmp >> 64;
sum64 = accumulate(sum64, tmp);
}
if (len > 0) {
sum64 = accumulate(sum64, data);
data = *ptr;
len -= 8;
}
/*
* Tail: zero any over-read bytes similarly to the head, again
* preserving odd/even alignment.
*/
shift = len * -8;
data = (data << shift) >> shift;
sum64 = accumulate(sum64, data);
/* Finally, folding */
sum64 += (sum64 >> 32) | (sum64 << 32);
sum = sum64 >> 32;
sum += (sum >> 16) | (sum << 16);
if (offset & 1)
return (u16)swab32(sum);
return sum >> 16;
}
__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
const struct in6_addr *daddr,
__u32 len, __u8 proto, __wsum csum)
{
__uint128_t src, dst;
u64 sum = (__force u64)csum;
src = *(const __uint128_t *)saddr->s6_addr;
dst = *(const __uint128_t *)daddr->s6_addr;
sum += (__force u32)htonl(len);
sum += (u32)proto << 24;
src += (src >> 64) | (src << 64);
dst += (dst >> 64) | (dst << 64);
sum = accumulate(sum, src >> 64);
sum = accumulate(sum, dst >> 64);
sum += ((sum >> 32) | (sum << 32));
return csum_fold((__force __wsum)(sum >> 32));
}
EXPORT_SYMBOL(csum_ipv6_magic);
|
