--- /dev/null
+From: Nick Piggin <npiggin@suse.de>
+Subject: mm: /dev/zero optimisation
+References: bnc#430738
+Patch-mainline: no (could be submit)
+
+Patch for removal of ZERO_PAGE from main VM paths also removed the
+/dev/zero optimisation to map directly from ZERO_PAGE when doing
+mmap() and also the interesting read(2) "hack" where the MMU was
+used to make zero-filling the target buffer zero-copy.
+
+Some benchmarks have run into issues with this. Customers sometimes
+use these benchmarks to qualify and test systems, so even if the
+benchmarks themselves are "stupid", it saves some trouble to retain
+this optimisation for them. Also, while I don't think it was established
+that there is a "real" workload where this helps, but it can't be proven
+that one does not exist.
+
+Signed-off-by: Nick Piggin <npiggin@suse.de>
+---
+ drivers/char/mem.c | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
+ include/linux/mm.h | 2 +
+ mm/memory.c | 90 ++++++++++++++++++++++++++++++++++++++++++++-
+ 3 files changed, 195 insertions(+), 2 deletions(-)
+
+--- a/drivers/char/mem.c
++++ b/drivers/char/mem.c
+@@ -702,6 +702,100 @@ static ssize_t splice_write_null(struct
+ return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null);
+ }
+
++#if 1 //ndef CONFIG_XEN
++/*
++ * For fun, we are using the MMU for this.
++ */
++static inline size_t read_zero_pagealigned(char __user * buf, size_t size)
++{
++ struct mm_struct *mm;
++ struct vm_area_struct * vma;
++ unsigned long addr=(unsigned long)buf;
++
++ mm = current->mm;
++ /* Oops, this was forgotten before. -ben */
++ down_read(&mm->mmap_sem);
++
++ /* For private mappings, just map in zero pages. */
++ for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
++ unsigned long count;
++
++ if (vma->vm_start > addr || (vma->vm_flags & VM_WRITE) == 0)
++ goto out_up;
++ if (vma->vm_flags & (VM_SHARED | VM_HUGETLB))
++ break;
++ count = vma->vm_end - addr;
++ if (count > size)
++ count = size;
++
++ zap_page_range(vma, addr, count, NULL);
++ if (zeromap_page_range(vma, addr, count, PAGE_COPY))
++ break;
++
++ size -= count;
++ buf += count;
++ addr += count;
++ if (size == 0)
++ goto out_up;
++ }
++
++ up_read(&mm->mmap_sem);
++
++ /* The shared case is hard. Let's do the conventional zeroing. */
++ do {
++ unsigned long unwritten = clear_user(buf, PAGE_SIZE);
++ if (unwritten)
++ return size + unwritten - PAGE_SIZE;
++ cond_resched();
++ buf += PAGE_SIZE;
++ size -= PAGE_SIZE;
++ } while (size);
++
++ return size;
++out_up:
++ up_read(&mm->mmap_sem);
++ return size;
++}
++
++static ssize_t read_zero(struct file * file, char __user * buf,
++ size_t count, loff_t *ppos)
++{
++ unsigned long left, unwritten, written = 0;
++
++ if (!count)
++ return 0;
++
++ if (!access_ok(VERIFY_WRITE, buf, count))
++ return -EFAULT;
++
++ left = count;
++
++ /* do we want to be clever? Arbitrary cut-off */
++ if (count >= PAGE_SIZE*4) {
++ unsigned long partial;
++
++ /* How much left of the page? */
++ partial = (PAGE_SIZE-1) & -(unsigned long) buf;
++ unwritten = clear_user(buf, partial);
++ written = partial - unwritten;
++ if (unwritten)
++ goto out;
++ left -= partial;
++ buf += partial;
++ unwritten = read_zero_pagealigned(buf, left & PAGE_MASK);
++ written += (left & PAGE_MASK) - unwritten;
++ if (unwritten)
++ goto out;
++ buf += left & PAGE_MASK;
++ left &= ~PAGE_MASK;
++ }
++ unwritten = clear_user(buf, left);
++ written += left - unwritten;
++out:
++ return written ? written : -EFAULT;
++}
++
++#else /* CONFIG_XEN */
+ static ssize_t read_zero(struct file * file, char __user * buf,
+ size_t count, loff_t *ppos)
+ {
+@@ -730,15 +824,24 @@ static ssize_t read_zero(struct file * f
+ }
+ return written ? written : -EFAULT;
+ }
++#endif /* CONFIG_XEN */
+
+ static int mmap_zero(struct file * file, struct vm_area_struct * vma)
+ {
++ int err = 0;
++
+ #ifndef CONFIG_MMU
+ return -ENOSYS;
+ #endif
++
+ if (vma->vm_flags & VM_SHARED)
+ return shmem_zero_setup(vma);
+- return 0;
++#if 1 //ndef CONFIG_XEN
++ err = zeromap_page_range(vma, vma->vm_start,
++ vma->vm_end - vma->vm_start, vma->vm_page_prot);
++ BUG_ON(err == -EEXIST);
++#endif
++ return err;
+ }
+
+ static ssize_t write_full(struct file * file, const char __user * buf,
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -791,6 +791,8 @@ void free_pgd_range(struct mmu_gather *t
+ unsigned long end, unsigned long floor, unsigned long ceiling);
+ int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
+ struct vm_area_struct *vma);
++int zeromap_page_range(struct vm_area_struct *vma, unsigned long from,
++ unsigned long size, pgprot_t prot);
+ void unmap_mapping_range(struct address_space *mapping,
+ loff_t const holebegin, loff_t const holelen, int even_cows);
+ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -1268,6 +1268,95 @@ int get_user_pages(struct task_struct *t
+ }
+ EXPORT_SYMBOL(get_user_pages);
+
++static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
++ unsigned long addr, unsigned long end, pgprot_t prot)
++{
++ pte_t *pte;
++ spinlock_t *ptl;
++ int err = 0;
++
++ pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
++ if (!pte)
++ return -EAGAIN;
++ arch_enter_lazy_mmu_mode();
++ do {
++ struct page *page = ZERO_PAGE(addr);
++ pte_t zero_pte = pte_wrprotect(mk_pte(page, prot));
++
++ if (unlikely(!pte_none(*pte))) {
++ err = -EEXIST;
++ pte++;
++ break;
++ }
++ page_cache_get(page);
++ page_add_file_rmap(page);
++ inc_mm_counter(mm, file_rss);
++ set_pte_at(mm, addr, pte, zero_pte);
++ } while (pte++, addr += PAGE_SIZE, addr != end);
++ arch_leave_lazy_mmu_mode();
++ pte_unmap_unlock(pte - 1, ptl);
++ return err;
++}
++
++static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud,
++ unsigned long addr, unsigned long end, pgprot_t prot)
++{
++ pmd_t *pmd;
++ unsigned long next;
++ int err;
++
++ pmd = pmd_alloc(mm, pud, addr);
++ if (!pmd)
++ return -EAGAIN;
++ do {
++ next = pmd_addr_end(addr, end);
++ err = zeromap_pte_range(mm, pmd, addr, next, prot);
++ if (err)
++ break;
++ } while (pmd++, addr = next, addr != end);
++ return err;
++}
++
++static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd,
++ unsigned long addr, unsigned long end, pgprot_t prot)
++{
++ pud_t *pud;
++ unsigned long next;
++ int err;
++
++ pud = pud_alloc(mm, pgd, addr);
++ if (!pud)
++ return -EAGAIN;
++ do {
++ next = pud_addr_end(addr, end);
++ err = zeromap_pmd_range(mm, pud, addr, next, prot);
++ if (err)
++ break;
++ } while (pud++, addr = next, addr != end);
++ return err;
++}
++
++int zeromap_page_range(struct vm_area_struct *vma,
++ unsigned long addr, unsigned long size, pgprot_t prot)
++{
++ pgd_t *pgd;
++ unsigned long next;
++ unsigned long end = addr + size;
++ struct mm_struct *mm = vma->vm_mm;
++ int err;
++
++ BUG_ON(addr >= end);
++ pgd = pgd_offset(mm, addr);
++ flush_cache_range(vma, addr, end);
++ do {
++ next = pgd_addr_end(addr, end);
++ err = zeromap_pud_range(mm, pgd, addr, next, prot);
++ if (err)
++ break;
++ } while (pgd++, addr = next, addr != end);
++ return err;
++}
++
+ pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
+ spinlock_t **ptl)
+ {
+@@ -1899,7 +1988,6 @@ gotten:
+
+ if (unlikely(anon_vma_prepare(vma)))
+ goto oom;
+- VM_BUG_ON(old_page == ZERO_PAGE(0));
+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
+ if (!new_page)
+ goto oom;
projects / people / pmueller / ipfire-2.x.git / blobdiff