bde6f5f59c
Aviod TLB flush IPIs during C3 states by voluntary leave_mm() before entering C3. The performance impact of TLB flush on C3 should not be significant with respect to C3 wakeup latency. Also, CPUs tend to flush TLB in hardware while in C3 anyways. On a 8 logical CPU system, running make -j2, the number of tlbflush IPIs goes down from 40 per second to ~ 0. Total number of interrupts during the run of this workload was ~1200 per second, which makes it ~3% savings in wakeups. There was no measurable performance or power impact however. [ akpm@linux-foundation.org: symbol export fixes. ] Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
85 lines
1.9 KiB
C
85 lines
1.9 KiB
C
#ifndef __I386_SCHED_H
|
|
#define __I386_SCHED_H
|
|
|
|
#include <asm/desc.h>
|
|
#include <asm/atomic.h>
|
|
#include <asm/pgalloc.h>
|
|
#include <asm/tlbflush.h>
|
|
#include <asm/paravirt.h>
|
|
#ifndef CONFIG_PARAVIRT
|
|
#include <asm-generic/mm_hooks.h>
|
|
|
|
static inline void paravirt_activate_mm(struct mm_struct *prev,
|
|
struct mm_struct *next)
|
|
{
|
|
}
|
|
#endif /* !CONFIG_PARAVIRT */
|
|
|
|
|
|
/*
|
|
* Used for LDT copy/destruction.
|
|
*/
|
|
int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
|
|
void destroy_context(struct mm_struct *mm);
|
|
|
|
|
|
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
|
|
{
|
|
#ifdef CONFIG_SMP
|
|
unsigned cpu = smp_processor_id();
|
|
if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
|
|
per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_LAZY;
|
|
#endif
|
|
}
|
|
|
|
static inline void switch_mm(struct mm_struct *prev,
|
|
struct mm_struct *next,
|
|
struct task_struct *tsk)
|
|
{
|
|
int cpu = smp_processor_id();
|
|
|
|
if (likely(prev != next)) {
|
|
/* stop flush ipis for the previous mm */
|
|
cpu_clear(cpu, prev->cpu_vm_mask);
|
|
#ifdef CONFIG_SMP
|
|
per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
|
|
per_cpu(cpu_tlbstate, cpu).active_mm = next;
|
|
#endif
|
|
cpu_set(cpu, next->cpu_vm_mask);
|
|
|
|
/* Re-load page tables */
|
|
load_cr3(next->pgd);
|
|
|
|
/*
|
|
* load the LDT, if the LDT is different:
|
|
*/
|
|
if (unlikely(prev->context.ldt != next->context.ldt))
|
|
load_LDT_nolock(&next->context);
|
|
}
|
|
#ifdef CONFIG_SMP
|
|
else {
|
|
per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
|
|
BUG_ON(per_cpu(cpu_tlbstate, cpu).active_mm != next);
|
|
|
|
if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
|
|
/* We were in lazy tlb mode and leave_mm disabled
|
|
* tlb flush IPI delivery. We must reload %cr3.
|
|
*/
|
|
load_cr3(next->pgd);
|
|
load_LDT_nolock(&next->context);
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
#define deactivate_mm(tsk, mm) \
|
|
asm("movl %0,%%gs": :"r" (0));
|
|
|
|
#define activate_mm(prev, next) \
|
|
do { \
|
|
paravirt_activate_mm(prev, next); \
|
|
switch_mm((prev),(next),NULL); \
|
|
} while(0);
|
|
|
|
#endif
|