summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
authorNicholas Piggin <npiggin@gmail.com>2020-12-17 16:47:30 +0300
committerMichael Ellerman <mpe@ellerman.id.au>2021-02-08 17:09:45 +0300
commit032b7f08932c9b212952d6d585e45b2941b3e8be (patch)
treeb177ec1a30b3623584c124d6957b580afe663ee6 /arch
parent9393544842d6c85ebfc387c43a5059f8171d598f (diff)
downloadlinux-032b7f08932c9b212952d6d585e45b2941b3e8be.tar.xz
powerpc/64s/radix: serialize_against_pte_lookup IPIs trim mm_cpumask
serialize_against_pte_lookup() performs IPIs to all CPUs in mm_cpumask. Take this opportunity to try trim the CPU out of mm_cpumask. This can reduce the cost of future serialize_against_pte_lookup() and/or the cost of future TLB flushes. Reported-by: kernel test robot <lkp@intel.com> Signed-off-by: Nicholas Piggin <npiggin@gmail.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/20201217134731.488135-7-npiggin@gmail.com
Diffstat (limited to 'arch')
-rw-r--r--arch/powerpc/mm/book3s64/pgtable.c13
-rw-r--r--arch/powerpc/mm/book3s64/radix_tlb.c20
2 files changed, 23 insertions, 10 deletions
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index 5b3a3bae21aa..78c492e86752 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -79,10 +79,17 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
}
-static void do_nothing(void *unused)
-{
+void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush);
+static void do_serialize(void *arg)
+{
+ /* We've taken the IPI, so try to trim the mask while here */
+ if (radix_enabled()) {
+ struct mm_struct *mm = arg;
+ exit_lazy_flush_tlb(mm, false);
+ }
}
+
/*
* Serialize against find_current_mm_pte which does lock-less
* lookup in page tables with local interrupts disabled. For huge pages
@@ -96,7 +103,7 @@ static void do_nothing(void *unused)
void serialize_against_pte_lookup(struct mm_struct *mm)
{
smp_mb();
- smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
+ smp_call_function_many(mm_cpumask(mm), do_serialize, mm, 1);
}
/*
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 1ef5d4e1a50a..d7f1a6bd08ef 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -639,7 +639,11 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm)
return false;
}
-static void exit_lazy_flush_tlb(struct mm_struct *mm)
+/*
+ * If always_flush is true, then flush even if this CPU can't be removed
+ * from mm_cpumask.
+ */
+void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush)
{
unsigned long pid = mm->context.id;
int cpu = smp_processor_id();
@@ -652,7 +656,7 @@ static void exit_lazy_flush_tlb(struct mm_struct *mm)
* done with interrupts off.
*/
if (current->mm == mm)
- goto out_flush;
+ goto out;
if (current->active_mm == mm) {
WARN_ON_ONCE(current->mm != NULL);
@@ -674,17 +678,19 @@ static void exit_lazy_flush_tlb(struct mm_struct *mm)
if (cpumask_test_cpu(cpu, mm_cpumask(mm))) {
atomic_dec(&mm->context.active_cpus);
cpumask_clear_cpu(cpu, mm_cpumask(mm));
+ always_flush = true;
}
-out_flush:
- _tlbiel_pid(pid, RIC_FLUSH_ALL);
+out:
+ if (always_flush)
+ _tlbiel_pid(pid, RIC_FLUSH_ALL);
}
#ifdef CONFIG_SMP
static void do_exit_flush_lazy_tlb(void *arg)
{
struct mm_struct *mm = arg;
- exit_lazy_flush_tlb(mm);
+ exit_lazy_flush_tlb(mm, true);
}
static void exit_flush_lazy_tlbs(struct mm_struct *mm)
@@ -746,7 +752,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
* to trim.
*/
if (tick_and_test_trim_clock()) {
- exit_lazy_flush_tlb(mm);
+ exit_lazy_flush_tlb(mm, true);
return FLUSH_TYPE_NONE;
}
}
@@ -792,7 +798,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm)
if (current->mm == mm)
return FLUSH_TYPE_LOCAL;
if (cpumask_test_cpu(cpu, mm_cpumask(mm)))
- exit_lazy_flush_tlb(mm);
+ exit_lazy_flush_tlb(mm, true);
return FLUSH_TYPE_NONE;
}