81 lines
2.2 KiB
Diff
81 lines
2.2 KiB
Diff
From e168a466bde2761439bfae6ba78d7321a9a3af58 Mon Sep 17 00:00:00 2001
|
|
From: Fedora Kernel Team <kernel-team@fedoraproject.org>
|
|
Date: Sun, 25 Aug 2019 09:04:31 +0000
|
|
Subject: [PATCH] riscv: optimize tlb flush path
|
|
|
|
|
|
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
|
|
index 4d9bbe8..d669d87 100644
|
|
--- a/arch/riscv/include/asm/tlbflush.h
|
|
+++ b/arch/riscv/include/asm/tlbflush.h
|
|
@@ -8,6 +8,7 @@
|
|
#define _ASM_RISCV_TLBFLUSH_H
|
|
|
|
#include <linux/mm_types.h>
|
|
+#include <linux/sched.h>
|
|
#include <asm/smp.h>
|
|
|
|
/*
|
|
@@ -42,20 +43,44 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
|
|
|
|
#include <asm/sbi.h>
|
|
|
|
-static inline void remote_sfence_vma(struct cpumask *cmask, unsigned long start,
|
|
- unsigned long size)
|
|
+static void __riscv_flush_tlb(struct cpumask *cmask, unsigned long start,
|
|
+ unsigned long size)
|
|
{
|
|
struct cpumask hmask;
|
|
+ unsigned int hartid;
|
|
+ unsigned int cpuid;
|
|
|
|
cpumask_clear(&hmask);
|
|
+
|
|
+ if (!cmask) {
|
|
+ riscv_cpuid_to_hartid_mask(cpu_online_mask, &hmask);
|
|
+ goto issue_sfence;
|
|
+ }
|
|
+
|
|
+ cpuid = get_cpu();
|
|
+ if (cpumask_test_cpu(cpuid, cmask)) {
|
|
+ /* Save trap cost by issuing a local tlb flush here */
|
|
+ if ((start == 0 && size == -1) || (size > PAGE_SIZE))
|
|
+ local_flush_tlb_all();
|
|
+ else if (size == PAGE_SIZE)
|
|
+ local_flush_tlb_page(start);
|
|
+ }
|
|
+ if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids)
|
|
+ goto done;
|
|
+
|
|
riscv_cpuid_to_hartid_mask(cmask, &hmask);
|
|
+ hartid = cpuid_to_hartid_map(cpuid);
|
|
+ cpumask_clear_cpu(hartid, &hmask);
|
|
+
|
|
+issue_sfence:
|
|
sbi_remote_sfence_vma(hmask.bits, start, size);
|
|
+done:
|
|
+ put_cpu();
|
|
}
|
|
|
|
-#define flush_tlb_all() sbi_remote_sfence_vma(NULL, 0, -1)
|
|
-
|
|
+#define flush_tlb_all() __riscv_flush_tlb(NULL, 0, -1)
|
|
#define flush_tlb_range(vma, start, end) \
|
|
- remote_sfence_vma(mm_cpumask((vma)->vm_mm), start, (end) - (start))
|
|
+ __riscv_flush_tlb(mm_cpumask((vma)->vm_mm), start, (end) - (start))
|
|
|
|
static inline void flush_tlb_page(struct vm_area_struct *vma,
|
|
unsigned long addr)
|
|
@@ -64,7 +89,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
|
|
}
|
|
|
|
#define flush_tlb_mm(mm) \
|
|
- remote_sfence_vma(mm_cpumask(mm), 0, -1)
|
|
+ __riscv_flush_tlb(mm_cpumask(mm), 0, -1)
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
--
|
|
2.23.0
|
|
|