From e08d0b3d172311e2bb500865c0d0038533d0ff11 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 22 Sep 2023 03:42:16 +0000 Subject: inet: implement lockless IP_TOS Some reads of inet->tos are racy. Add needed READ_ONCE() annotations and convert IP_TOS option lockless. v2: missing changes in include/net/route.h (David Ahern) Signed-off-by: Eric Dumazet Reviewed-by: David Ahern Signed-off-by: David S. Miller --- net/dccp/ipv4.c | 2 +- net/ipv4/inet_diag.c | 2 +- net/ipv4/ip_output.c | 4 ++-- net/ipv4/ip_sockglue.c | 29 ++++++++++++----------------- net/ipv4/tcp_ipv4.c | 9 +++++---- net/mptcp/sockopt.c | 8 ++++---- net/sctp/protocol.c | 4 ++-- 7 files changed, 27 insertions(+), 31 deletions(-) (limited to 'net') diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c index 69453b936bd5..1b8cbfda6e5d 100644 --- a/net/dccp/ipv4.c +++ b/net/dccp/ipv4.c @@ -511,7 +511,7 @@ static int dccp_v4_send_response(const struct sock *sk, struct request_sock *req err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr, ireq->ir_rmt_addr, rcu_dereference(ireq->ireq_opt), - inet_sk(sk)->tos); + READ_ONCE(inet_sk(sk)->tos)); rcu_read_unlock(); err = net_xmit_eval(err); } diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c index 9f0bd518901a..f01aee832aab 100644 --- a/net/ipv4/inet_diag.c +++ b/net/ipv4/inet_diag.c @@ -134,7 +134,7 @@ int inet_diag_msg_attrs_fill(struct sock *sk, struct sk_buff *skb, * hence this needs to be included regardless of socket family. */ if (ext & (1 << (INET_DIAG_TOS - 1))) - if (nla_put_u8(skb, INET_DIAG_TOS, inet->tos) < 0) + if (nla_put_u8(skb, INET_DIAG_TOS, READ_ONCE(inet->tos)) < 0) goto errout; #if IS_ENABLED(CONFIG_IPV6) diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 9fc7be2c2033..89e62ed08dad 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -544,7 +544,7 @@ EXPORT_SYMBOL(__ip_queue_xmit); int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl) { - return __ip_queue_xmit(sk, skb, fl, inet_sk(sk)->tos); + return __ip_queue_xmit(sk, skb, fl, READ_ONCE(inet_sk(sk)->tos)); } EXPORT_SYMBOL(ip_queue_xmit); @@ -1438,7 +1438,7 @@ struct sk_buff *__ip_make_skb(struct sock *sk, iph = ip_hdr(skb); iph->version = 4; iph->ihl = 5; - iph->tos = (cork->tos != -1) ? cork->tos : inet->tos; + iph->tos = (cork->tos != -1) ? cork->tos : READ_ONCE(inet->tos); iph->frag_off = df; iph->ttl = ttl; iph->protocol = sk->sk_protocol; diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 6d874cc03c8b..50c008efbb6d 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -585,25 +585,20 @@ out: return err; } -void __ip_sock_set_tos(struct sock *sk, int val) +void ip_sock_set_tos(struct sock *sk, int val) { + u8 old_tos = READ_ONCE(inet_sk(sk)->tos); + if (sk->sk_type == SOCK_STREAM) { val &= ~INET_ECN_MASK; - val |= inet_sk(sk)->tos & INET_ECN_MASK; + val |= old_tos & INET_ECN_MASK; } - if (inet_sk(sk)->tos != val) { - inet_sk(sk)->tos = val; + if (old_tos != val) { + WRITE_ONCE(inet_sk(sk)->tos, val); WRITE_ONCE(sk->sk_priority, rt_tos2priority(val)); sk_dst_reset(sk); } } - -void ip_sock_set_tos(struct sock *sk, int val) -{ - lock_sock(sk); - __ip_sock_set_tos(sk, val); - release_sock(sk); -} EXPORT_SYMBOL(ip_sock_set_tos); void ip_sock_set_freebind(struct sock *sk) @@ -1050,6 +1045,9 @@ int do_ip_setsockopt(struct sock *sk, int level, int optname, return 0; case IP_MTU_DISCOVER: return ip_sock_set_mtu_discover(sk, val); + case IP_TOS: /* This sets both TOS and Precedence */ + ip_sock_set_tos(sk, val); + return 0; } err = 0; @@ -1104,9 +1102,6 @@ int do_ip_setsockopt(struct sock *sk, int level, int optname, } } break; - case IP_TOS: /* This sets both TOS and Precedence */ - __ip_sock_set_tos(sk, val); - break; case IP_UNICAST_IF: { struct net_device *dev = NULL; @@ -1593,6 +1588,9 @@ int do_ip_getsockopt(struct sock *sk, int level, int optname, case IP_MTU_DISCOVER: val = READ_ONCE(inet->pmtudisc); goto copyval; + case IP_TOS: + val = READ_ONCE(inet->tos); + goto copyval; } if (needs_rtnl) @@ -1629,9 +1627,6 @@ int do_ip_getsockopt(struct sock *sk, int level, int optname, return -EFAULT; return 0; } - case IP_TOS: - val = inet->tos; - break; case IP_MTU: { struct dst_entry *dst; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 95e972be0c05..a441740616d7 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1024,10 +1024,11 @@ static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst, if (skb) { __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr); - tos = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ? - (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | - (inet_sk(sk)->tos & INET_ECN_MASK) : - inet_sk(sk)->tos; + tos = READ_ONCE(inet_sk(sk)->tos); + + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) + tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | + (tos & INET_ECN_MASK); if (!INET_ECN_is_capable(tos) && tcp_bpf_ca_needs_ecn((struct sock *)req)) diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index f3485a6b35e7..18ce624bfde2 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -734,11 +734,11 @@ static int mptcp_setsockopt_v4_set_tos(struct mptcp_sock *msk, int optname, lock_sock(sk); sockopt_seq_inc(msk); - val = inet_sk(sk)->tos; + val = READ_ONCE(inet_sk(sk)->tos); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); - __ip_sock_set_tos(ssk, val); + ip_sock_set_tos(ssk, val); } release_sock(sk); @@ -1343,7 +1343,7 @@ static int mptcp_getsockopt_v4(struct mptcp_sock *msk, int optname, switch (optname) { case IP_TOS: - return mptcp_put_int_option(msk, optval, optlen, inet_sk(sk)->tos); + return mptcp_put_int_option(msk, optval, optlen, READ_ONCE(inet_sk(sk)->tos)); } return -EOPNOTSUPP; @@ -1411,7 +1411,7 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk) ssk->sk_bound_dev_if = sk->sk_bound_dev_if; ssk->sk_incoming_cpu = sk->sk_incoming_cpu; ssk->sk_ipv6only = sk->sk_ipv6only; - __ip_sock_set_tos(ssk, inet_sk(sk)->tos); + ip_sock_set_tos(ssk, inet_sk(sk)->tos); if (sk->sk_userlocks & tx_rx_locks) { ssk->sk_userlocks |= sk->sk_userlocks & tx_rx_locks; diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index 2185f44198de..94c6dd53cd62 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -426,7 +426,7 @@ static void sctp_v4_get_dst(struct sctp_transport *t, union sctp_addr *saddr, struct dst_entry *dst = NULL; union sctp_addr *daddr = &t->ipaddr; union sctp_addr dst_saddr; - __u8 tos = inet_sk(sk)->tos; + u8 tos = READ_ONCE(inet_sk(sk)->tos); if (t->dscp & SCTP_DSCP_SET_MASK) tos = t->dscp & SCTP_DSCP_VAL_MASK; @@ -1057,7 +1057,7 @@ static inline int sctp_v4_xmit(struct sk_buff *skb, struct sctp_transport *t) struct flowi4 *fl4 = &t->fl.u.ip4; struct sock *sk = skb->sk; struct inet_sock *inet = inet_sk(sk); - __u8 dscp = inet->tos; + __u8 dscp = READ_ONCE(inet->tos); __be16 df = 0; pr_debug("%s: skb:%p, len:%d, src:%pI4, dst:%pI4\n", __func__, skb, -- cgit v1.2.3