da181a8b39
Add the three bare TLB accessor functions to paravirt-ops. Most amusingly, flush_tlb is redefined on SMP, so I can't call the paravirt op flush_tlb. Instead, I chose to indicate the actual flush type, kernel (global) vs. user (non-global). Global in this sense means using the global bit in the page table entry, which makes TLB entries persistent across CR3 reloads, not global as in the SMP sense of invoking remote shootdowns, so the term is confusingly overloaded. AK: folded in fix from Zach for PAE compilation Signed-off-by: Zachary Amsden <zach@vmware.com> Signed-off-by: Chris Wright <chrisw@sous-sol.org> Signed-off-by: Andi Kleen <ak@suse.de> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Jeremy Fitzhardinge <jeremy@goop.org> Signed-off-by: Andrew Morton <akpm@osdl.org>
155 lines
3.9 KiB
C
155 lines
3.9 KiB
C
#ifndef _I386_TLBFLUSH_H
|
|
#define _I386_TLBFLUSH_H
|
|
|
|
#include <linux/mm.h>
|
|
#include <asm/processor.h>
|
|
|
|
#ifdef CONFIG_PARAVIRT
|
|
#include <asm/paravirt.h>
|
|
#else
|
|
#define __flush_tlb() __native_flush_tlb()
|
|
#define __flush_tlb_global() __native_flush_tlb_global()
|
|
#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
|
|
#endif
|
|
|
|
#define __native_flush_tlb() \
|
|
do { \
|
|
unsigned int tmpreg; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
"movl %%cr3, %0; \n" \
|
|
"movl %0, %%cr3; # flush TLB \n" \
|
|
: "=r" (tmpreg) \
|
|
:: "memory"); \
|
|
} while (0)
|
|
|
|
/*
|
|
* Global pages have to be flushed a bit differently. Not a real
|
|
* performance problem because this does not happen often.
|
|
*/
|
|
#define __native_flush_tlb_global() \
|
|
do { \
|
|
unsigned int tmpreg, cr4, cr4_orig; \
|
|
\
|
|
__asm__ __volatile__( \
|
|
"movl %%cr4, %2; # turn off PGE \n" \
|
|
"movl %2, %1; \n" \
|
|
"andl %3, %1; \n" \
|
|
"movl %1, %%cr4; \n" \
|
|
"movl %%cr3, %0; \n" \
|
|
"movl %0, %%cr3; # flush TLB \n" \
|
|
"movl %2, %%cr4; # turn PGE back on \n" \
|
|
: "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \
|
|
: "i" (~X86_CR4_PGE) \
|
|
: "memory"); \
|
|
} while (0)
|
|
|
|
#define __native_flush_tlb_single(addr) \
|
|
__asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory")
|
|
|
|
# define __flush_tlb_all() \
|
|
do { \
|
|
if (cpu_has_pge) \
|
|
__flush_tlb_global(); \
|
|
else \
|
|
__flush_tlb(); \
|
|
} while (0)
|
|
|
|
#define cpu_has_invlpg (boot_cpu_data.x86 > 3)
|
|
|
|
#ifdef CONFIG_X86_INVLPG
|
|
# define __flush_tlb_one(addr) __flush_tlb_single(addr)
|
|
#else
|
|
# define __flush_tlb_one(addr) \
|
|
do { \
|
|
if (cpu_has_invlpg) \
|
|
__flush_tlb_single(addr); \
|
|
else \
|
|
__flush_tlb(); \
|
|
} while (0)
|
|
#endif
|
|
|
|
/*
|
|
* TLB flushing:
|
|
*
|
|
* - flush_tlb() flushes the current mm struct TLBs
|
|
* - flush_tlb_all() flushes all processes TLBs
|
|
* - flush_tlb_mm(mm) flushes the specified mm context TLB's
|
|
* - flush_tlb_page(vma, vmaddr) flushes one page
|
|
* - flush_tlb_range(vma, start, end) flushes a range of pages
|
|
* - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
|
|
* - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
|
|
*
|
|
* ..but the i386 has somewhat limited tlb flushing capabilities,
|
|
* and page-granular flushes are available only on i486 and up.
|
|
*/
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
#define flush_tlb() __flush_tlb()
|
|
#define flush_tlb_all() __flush_tlb_all()
|
|
#define local_flush_tlb() __flush_tlb()
|
|
|
|
static inline void flush_tlb_mm(struct mm_struct *mm)
|
|
{
|
|
if (mm == current->active_mm)
|
|
__flush_tlb();
|
|
}
|
|
|
|
static inline void flush_tlb_page(struct vm_area_struct *vma,
|
|
unsigned long addr)
|
|
{
|
|
if (vma->vm_mm == current->active_mm)
|
|
__flush_tlb_one(addr);
|
|
}
|
|
|
|
static inline void flush_tlb_range(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
if (vma->vm_mm == current->active_mm)
|
|
__flush_tlb();
|
|
}
|
|
|
|
#else
|
|
|
|
#include <asm/smp.h>
|
|
|
|
#define local_flush_tlb() \
|
|
__flush_tlb()
|
|
|
|
extern void flush_tlb_all(void);
|
|
extern void flush_tlb_current_task(void);
|
|
extern void flush_tlb_mm(struct mm_struct *);
|
|
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
|
|
|
|
#define flush_tlb() flush_tlb_current_task()
|
|
|
|
static inline void flush_tlb_range(struct vm_area_struct * vma, unsigned long start, unsigned long end)
|
|
{
|
|
flush_tlb_mm(vma->vm_mm);
|
|
}
|
|
|
|
#define TLBSTATE_OK 1
|
|
#define TLBSTATE_LAZY 2
|
|
|
|
struct tlb_state
|
|
{
|
|
struct mm_struct *active_mm;
|
|
int state;
|
|
char __cacheline_padding[L1_CACHE_BYTES-8];
|
|
};
|
|
DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
|
|
|
|
|
|
#endif
|
|
|
|
#define flush_tlb_kernel_range(start, end) flush_tlb_all()
|
|
|
|
static inline void flush_tlb_pgtables(struct mm_struct *mm,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
/* i386 does not keep any page table caches in TLB */
|
|
}
|
|
|
|
#endif /* _I386_TLBFLUSH_H */
|