b73c3d0e4f
For a connected socket we can precompute the flow hash for setting
in skb->hash on output. This is a performance advantage over
calculating the skb->hash for every packet on the connection. The
computation is done using the common hash algorithm to be consistent
with computations done for packets of the connection in other states
where thers is no socket (e.g. time-wait, syn-recv, syn-cookies).
This patch adds sk_txhash to the sock structure. inet_set_txhash and
ip6_set_txhash functions are added which are called from points in
TCP and UDP where socket moves to established state.
skb_set_hash_from_sk is a function which sets skb->hash from the
sock txhash value. This is called in UDP and TCP transmit path when
transmitting within the context of a socket.
Tested: ran super_netperf with 200 TCP_RR streams over a vxlan
interface (in this case skb_get_hash called on every TX packet to
create a UDP source port).
Before fix:
95.02% CPU utilization
154/256/505 90/95/99% latencies
1.13042e+06 tps
Time in functions:
0.28% skb_flow_dissect
0.21% __skb_get_hash
After fix:
94.95% CPU utilization
156/254/485 90/95/99% latencies
1.15447e+06
Neither __skb_get_hash nor skb_flow_dissect appear in perf
Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
124 lines
3.0 KiB
C
124 lines
3.0 KiB
C
/*
|
|
* common UDP/RAW code
|
|
* Linux INET implementation
|
|
*
|
|
* Authors:
|
|
* Hideaki YOSHIFUJI <yoshfuji@linux-ipv6.org>
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/module.h>
|
|
#include <linux/ip.h>
|
|
#include <linux/in.h>
|
|
#include <net/ip.h>
|
|
#include <net/sock.h>
|
|
#include <net/route.h>
|
|
#include <net/tcp_states.h>
|
|
|
|
int ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
|
|
{
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
struct sockaddr_in *usin = (struct sockaddr_in *) uaddr;
|
|
struct flowi4 *fl4;
|
|
struct rtable *rt;
|
|
__be32 saddr;
|
|
int oif;
|
|
int err;
|
|
|
|
|
|
if (addr_len < sizeof(*usin))
|
|
return -EINVAL;
|
|
|
|
if (usin->sin_family != AF_INET)
|
|
return -EAFNOSUPPORT;
|
|
|
|
sk_dst_reset(sk);
|
|
|
|
lock_sock(sk);
|
|
|
|
oif = sk->sk_bound_dev_if;
|
|
saddr = inet->inet_saddr;
|
|
if (ipv4_is_multicast(usin->sin_addr.s_addr)) {
|
|
if (!oif)
|
|
oif = inet->mc_index;
|
|
if (!saddr)
|
|
saddr = inet->mc_addr;
|
|
}
|
|
fl4 = &inet->cork.fl.u.ip4;
|
|
rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr,
|
|
RT_CONN_FLAGS(sk), oif,
|
|
sk->sk_protocol,
|
|
inet->inet_sport, usin->sin_port, sk);
|
|
if (IS_ERR(rt)) {
|
|
err = PTR_ERR(rt);
|
|
if (err == -ENETUNREACH)
|
|
IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
|
|
goto out;
|
|
}
|
|
|
|
if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) {
|
|
ip_rt_put(rt);
|
|
err = -EACCES;
|
|
goto out;
|
|
}
|
|
if (!inet->inet_saddr)
|
|
inet->inet_saddr = fl4->saddr; /* Update source address */
|
|
if (!inet->inet_rcv_saddr) {
|
|
inet->inet_rcv_saddr = fl4->saddr;
|
|
if (sk->sk_prot->rehash)
|
|
sk->sk_prot->rehash(sk);
|
|
}
|
|
inet->inet_daddr = fl4->daddr;
|
|
inet->inet_dport = usin->sin_port;
|
|
sk->sk_state = TCP_ESTABLISHED;
|
|
inet_set_txhash(sk);
|
|
inet->inet_id = jiffies;
|
|
|
|
sk_dst_set(sk, &rt->dst);
|
|
err = 0;
|
|
out:
|
|
release_sock(sk);
|
|
return err;
|
|
}
|
|
EXPORT_SYMBOL(ip4_datagram_connect);
|
|
|
|
/* Because UDP xmit path can manipulate sk_dst_cache without holding
|
|
* socket lock, we need to use sk_dst_set() here,
|
|
* even if we own the socket lock.
|
|
*/
|
|
void ip4_datagram_release_cb(struct sock *sk)
|
|
{
|
|
const struct inet_sock *inet = inet_sk(sk);
|
|
const struct ip_options_rcu *inet_opt;
|
|
__be32 daddr = inet->inet_daddr;
|
|
struct dst_entry *dst;
|
|
struct flowi4 fl4;
|
|
struct rtable *rt;
|
|
|
|
rcu_read_lock();
|
|
|
|
dst = __sk_dst_get(sk);
|
|
if (!dst || !dst->obsolete || dst->ops->check(dst, 0)) {
|
|
rcu_read_unlock();
|
|
return;
|
|
}
|
|
inet_opt = rcu_dereference(inet->inet_opt);
|
|
if (inet_opt && inet_opt->opt.srr)
|
|
daddr = inet_opt->opt.faddr;
|
|
rt = ip_route_output_ports(sock_net(sk), &fl4, sk, daddr,
|
|
inet->inet_saddr, inet->inet_dport,
|
|
inet->inet_sport, sk->sk_protocol,
|
|
RT_CONN_FLAGS(sk), sk->sk_bound_dev_if);
|
|
|
|
dst = !IS_ERR(rt) ? &rt->dst : NULL;
|
|
sk_dst_set(sk, dst);
|
|
|
|
rcu_read_unlock();
|
|
}
|
|
EXPORT_SYMBOL_GPL(ip4_datagram_release_cb);
|