Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
- include/asm-generic/tlb.h | 6 ++++++
- mm/hugetlb.c | 19 +++++++++++++++++++
- 2 files changed, 25 insertions(+)
+ arch/arm/include/asm/tlb.h | 8 ++++++++
+ arch/ia64/include/asm/tlb.h | 10 ++++++++++
+ arch/s390/include/asm/tlb.h | 14 ++++++++++++++
+ arch/sh/include/asm/tlb.h | 9 +++++++++
+ arch/um/include/asm/tlb.h | 12 ++++++++++++
+ include/asm-generic/tlb.h | 2 ++
+ mm/hugetlb.c | 19 +++++++++++++++++++
+ mm/memory.c | 10 ++++++++++
+ 8 files changed, 84 insertions(+)
---- a/include/asm-generic/tlb.h
-+++ b/include/asm-generic/tlb.h
-@@ -196,6 +196,12 @@ static inline void tlb_remove_check_page
- #define tlb_end_vma __tlb_end_vma
- #endif
+--- a/arch/arm/include/asm/tlb.h
++++ b/arch/arm/include/asm/tlb.h
+@@ -280,6 +280,14 @@ tlb_remove_pmd_tlb_entry(struct mmu_gath
+ tlb_add_flush(tlb, addr);
+ }
+
++static inline void
++tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size)
++{
++ tlb_add_flush(tlb, address);
++ tlb_add_flush(tlb, address + size - PMD_SIZE);
++}
++
+ #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
+ #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
+ #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
+--- a/arch/ia64/include/asm/tlb.h
++++ b/arch/ia64/include/asm/tlb.h
+@@ -269,6 +269,16 @@ __tlb_remove_tlb_entry (struct mmu_gathe
+ tlb->end_addr = address + PAGE_SIZE;
+ }
+
++static inline void
++tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size)
++{
++ if (tlb->start_addr > address)
++ tlb->start_addr = address;
++ if (tlb->end_addr < address + size)
++ tlb->end_addr = address + size;
++}
++
+ #define tlb_migrate_finish(mm) platform_tlb_migrate_finish(mm)
+
+ #define tlb_start_vma(tlb, vma) do { } while (0)
+--- a/arch/s390/include/asm/tlb.h
++++ b/arch/s390/include/asm/tlb.h
+@@ -116,6 +116,20 @@ static inline void tlb_remove_page_size(
+ return tlb_remove_page(tlb, page);
+ }
+static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
+ unsigned long address, unsigned long size)
+{
-+ __tlb_adjust_range(tlb, address, size);
++ /*
++ * the range might exceed the original range that was provided to
++ * tlb_gather_mmu(), so we need to update it despite the fact it is
++ * usually not updated.
++ */
++ if (tlb->start > address)
++ tlb->start = address;
++ if (tlb->end < address + size)
++ tlb->end = address + size;
+}
+
- #ifndef __tlb_remove_tlb_entry
- #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
- #endif
+ /*
+ * pte_free_tlb frees a pte table and clears the CRSTE for the
+ * page table from the tlb.
+--- a/arch/sh/include/asm/tlb.h
++++ b/arch/sh/include/asm/tlb.h
+@@ -127,6 +127,15 @@ static inline void tlb_remove_page_size(
+ return tlb_remove_page(tlb, page);
+ }
+
++static inline tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size)
++{
++ if (tlb->start > address)
++ tlb->start = address;
++ if (tlb->end < address + size)
++ tlb->end = address + size;
++}
++
+ #define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
+ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
+ unsigned int page_size)
+--- a/arch/um/include/asm/tlb.h
++++ b/arch/um/include/asm/tlb.h
+@@ -130,6 +130,18 @@ static inline void tlb_remove_page_size(
+ return tlb_remove_page(tlb, page);
+ }
+
++static inline void
++tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size)
++{
++ tlb->need_flush = 1;
++
++ if (tlb->start > address)
++ tlb->start = address;
++ if (tlb->end < address + size)
++ tlb->end = address + size;
++}
++
+ /**
+ * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
+ *
+--- a/include/asm-generic/tlb.h
++++ b/include/asm-generic/tlb.h
+@@ -117,6 +117,8 @@ void arch_tlb_gather_mmu(struct mmu_gath
+ void tlb_flush_mmu(struct mmu_gather *tlb);
+ void arch_tlb_finish_mmu(struct mmu_gather *tlb,
+ unsigned long start, unsigned long end, bool force);
++void tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size);
+ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
+ int page_size);
+
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3386,6 +3386,7 @@ void __unmap_hugepage_range(struct mmu_g
}
void __unmap_hugepage_range_final(struct mmu_gather *tlb,
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -335,6 +335,16 @@ bool __tlb_remove_page_size(struct mmu_g
+ return false;
+ }
+
++void tlb_flush_pmd_range(struct mmu_gather *tlb, unsigned long address,
++ unsigned long size)
++{
++ if (tlb->page_size != 0 && tlb->page_size != PMD_SIZE)
++ tlb_flush_mmu(tlb);
++
++ tlb->page_size = PMD_SIZE;
++ tlb->start = min(tlb->start, address);
++ tlb->end = max(tlb->end, address + size);
++}
+ #endif /* HAVE_GENERIC_MMU_GATHER */
+
+ #ifdef CONFIG_HAVE_RCU_TABLE_FREE