6d7b857d54
Replace the per network namespace shared atomic "mem" accounting variable, in the fragmentation code, with a lib/percpu_counter. Getting percpu_counter to scale to the fragmentation code usage requires some tweaks. At first view, percpu_counter looks superfast, but it does not scale on multi-CPU/NUMA machines, because the default batch size is too small, for frag code usage. Thus, I have adjusted the batch size by using __percpu_counter_add() directly, instead of percpu_counter_sub() and percpu_counter_add(). The batch size is increased to 130.000, based on the largest 64K fragment memory usage. This does introduce some imprecise memory accounting, but its does not need to be strict for this use-case. It is also essential, that the percpu_counter, does not share cacheline with other writers, to make this scale. Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
120 lines
3.3 KiB
C
120 lines
3.3 KiB
C
#ifndef __NET_FRAG_H__
|
|
#define __NET_FRAG_H__
|
|
|
|
#include <linux/percpu_counter.h>
|
|
|
|
struct netns_frags {
|
|
int nqueues;
|
|
struct list_head lru_list;
|
|
|
|
/* The percpu_counter "mem" need to be cacheline aligned.
|
|
* mem.count must not share cacheline with other writers
|
|
*/
|
|
struct percpu_counter mem ____cacheline_aligned_in_smp;
|
|
|
|
/* sysctls */
|
|
int timeout;
|
|
int high_thresh;
|
|
int low_thresh;
|
|
};
|
|
|
|
struct inet_frag_queue {
|
|
spinlock_t lock;
|
|
struct timer_list timer; /* when will this queue expire? */
|
|
struct list_head lru_list; /* lru list member */
|
|
struct hlist_node list;
|
|
atomic_t refcnt;
|
|
struct sk_buff *fragments; /* list of received fragments */
|
|
struct sk_buff *fragments_tail;
|
|
ktime_t stamp;
|
|
int len; /* total length of orig datagram */
|
|
int meat;
|
|
__u8 last_in; /* first/last segment arrived? */
|
|
|
|
#define INET_FRAG_COMPLETE 4
|
|
#define INET_FRAG_FIRST_IN 2
|
|
#define INET_FRAG_LAST_IN 1
|
|
|
|
u16 max_size;
|
|
|
|
struct netns_frags *net;
|
|
};
|
|
|
|
#define INETFRAGS_HASHSZ 64
|
|
|
|
struct inet_frags {
|
|
struct hlist_head hash[INETFRAGS_HASHSZ];
|
|
/* This rwlock is a global lock (seperate per IPv4, IPv6 and
|
|
* netfilter). Important to keep this on a seperate cacheline.
|
|
*/
|
|
rwlock_t lock ____cacheline_aligned_in_smp;
|
|
int secret_interval;
|
|
struct timer_list secret_timer;
|
|
u32 rnd;
|
|
int qsize;
|
|
|
|
unsigned int (*hashfn)(struct inet_frag_queue *);
|
|
bool (*match)(struct inet_frag_queue *q, void *arg);
|
|
void (*constructor)(struct inet_frag_queue *q,
|
|
void *arg);
|
|
void (*destructor)(struct inet_frag_queue *);
|
|
void (*skb_free)(struct sk_buff *);
|
|
void (*frag_expire)(unsigned long data);
|
|
};
|
|
|
|
void inet_frags_init(struct inet_frags *);
|
|
void inet_frags_fini(struct inet_frags *);
|
|
|
|
void inet_frags_init_net(struct netns_frags *nf);
|
|
void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f);
|
|
|
|
void inet_frag_kill(struct inet_frag_queue *q, struct inet_frags *f);
|
|
void inet_frag_destroy(struct inet_frag_queue *q,
|
|
struct inet_frags *f, int *work);
|
|
int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force);
|
|
struct inet_frag_queue *inet_frag_find(struct netns_frags *nf,
|
|
struct inet_frags *f, void *key, unsigned int hash)
|
|
__releases(&f->lock);
|
|
|
|
static inline void inet_frag_put(struct inet_frag_queue *q, struct inet_frags *f)
|
|
{
|
|
if (atomic_dec_and_test(&q->refcnt))
|
|
inet_frag_destroy(q, f, NULL);
|
|
}
|
|
|
|
/* Memory Tracking Functions. */
|
|
|
|
/* The default percpu_counter batch size is not big enough to scale to
|
|
* fragmentation mem acct sizes.
|
|
* The mem size of a 64K fragment is approx:
|
|
* (44 fragments * 2944 truesize) + frag_queue struct(200) = 129736 bytes
|
|
*/
|
|
static unsigned int frag_percpu_counter_batch = 130000;
|
|
|
|
static inline int frag_mem_limit(struct netns_frags *nf)
|
|
{
|
|
return percpu_counter_read(&nf->mem);
|
|
}
|
|
|
|
static inline void sub_frag_mem_limit(struct inet_frag_queue *q, int i)
|
|
{
|
|
__percpu_counter_add(&q->net->mem, -i, frag_percpu_counter_batch);
|
|
}
|
|
|
|
static inline void add_frag_mem_limit(struct inet_frag_queue *q, int i)
|
|
{
|
|
__percpu_counter_add(&q->net->mem, i, frag_percpu_counter_batch);
|
|
}
|
|
|
|
static inline void init_frag_mem_limit(struct netns_frags *nf)
|
|
{
|
|
percpu_counter_init(&nf->mem, 0);
|
|
}
|
|
|
|
static inline int sum_frag_mem_limit(struct netns_frags *nf)
|
|
{
|
|
return percpu_counter_sum_positive(&nf->mem);
|
|
}
|
|
|
|
#endif
|