From: Christoph Lameter <clameter@sgi.com>
The zeroing of a page of a arbitrary order in page_alloc.c and in hugetlb.c
may benefit from a clear_page that is capable of zeroing multiple pages at
once. The following patch adds a function "clear_pages" that is capable of
clearing multiple continuous pages at once.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/arch/sparc64/Kconfig | 4 ++++
25-akpm/arch/sparc64/lib/clear_page.S | 11 +++++++----
25-akpm/arch/x86_64/Kconfig | 4 ++++
25-akpm/arch/x86_64/kernel/x8664_ksyms.c | 2 +-
25-akpm/arch/x86_64/lib/clear_page.S | 29 ++++++++++++++++++-----------
25-akpm/include/asm-i386/mmx.h | 2 +-
25-akpm/include/asm-i386/page.h | 5 +++--
25-akpm/include/asm-ia64/page.h | 3 ++-
25-akpm/include/asm-sparc64/page.h | 6 ++++--
25-akpm/include/asm-x86_64/page.h | 3 ++-
25-akpm/include/linux/gfp.h | 1 +
25-akpm/mm/hugetlb.c | 4 +---
25-akpm/mm/page_alloc.c | 11 ++++++++++-
arch/i386/Kconfig | 0
arch/i386/lib/mmx.c | 0
arch/ia64/Kconfig | 0
arch/ia64/kernel/ia64_ksyms.c | 0
arch/ia64/lib/clear_page.S | 0
18 files changed, 58 insertions(+), 27 deletions(-)
diff -puN arch/i386/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/i386/Kconfig
diff -puN arch/i386/lib/mmx.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/i386/lib/mmx.c
diff -puN arch/ia64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/Kconfig
diff -puN arch/ia64/kernel/ia64_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/kernel/ia64_ksyms.c
diff -puN arch/ia64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/ia64/lib/clear_page.S
diff -puN arch/sparc64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/sparc64/Kconfig
--- 25/arch/sparc64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/arch/sparc64/Kconfig 2005-04-04 01:54:48.000000000 -0700
@@ -43,6 +43,10 @@ config SPARC64_PAGE_SIZE_4MB
endchoice
+config CLEAR_PAGES
+ bool
+ default y
+
source "init/Kconfig"
config SYSVIPC_COMPAT
diff -puN arch/sparc64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/sparc64/lib/clear_page.S
--- 25/arch/sparc64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/arch/sparc64/lib/clear_page.S 2005-04-04 01:54:48.000000000 -0700
@@ -28,9 +28,12 @@
.text
.globl _clear_page
-_clear_page: /* %o0=dest */
+_clear_page: /* %o0=dest, %o1=order */
+ sethi %hi(PAGE_SIZE/64), %o2
+ clr %o4
+ or %o2, %lo(PAGE_SIZE/64), %o2
ba,pt %xcc, clear_page_common
- clr %o4
+ sllx %o2, %o1, %o1
/* This thing is pretty important, it shows up
* on the profiles via do_anonymous_page().
@@ -69,16 +72,16 @@ clear_user_page: /* %o0=dest, %o1=vaddr
flush %g6
wrpr %o4, 0x0, %pstate
+ sethi %hi(PAGE_SIZE/64), %o1
mov 1, %o4
+ or %o1, %lo(PAGE_SIZE/64), %o1
clear_page_common:
VISEntryHalf
membar #StoreLoad | #StoreStore | #LoadStore
fzero %f0
- sethi %hi(PAGE_SIZE/64), %o1
mov %o0, %g1 ! remember vaddr for tlbflush
fzero %f2
- or %o1, %lo(PAGE_SIZE/64), %o1
faddd %f0, %f2, %f4
fmuld %f0, %f2, %f6
faddd %f0, %f2, %f8
diff -puN arch/x86_64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/Kconfig
--- 25/arch/x86_64/Kconfig~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/arch/x86_64/Kconfig 2005-04-04 01:54:48.000000000 -0700
@@ -61,6 +61,10 @@ config GENERIC_IOMAP
bool
default y
+config CLEAR_PAGES
+ bool
+ default y
+
source "init/Kconfig"
diff -puN arch/x86_64/kernel/x8664_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/kernel/x8664_ksyms.c
--- 25/arch/x86_64/kernel/x8664_ksyms.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/arch/x86_64/kernel/x8664_ksyms.c 2005-04-04 01:54:48.000000000 -0700
@@ -107,7 +107,7 @@ EXPORT_SYMBOL(pci_mem_start);
#endif
EXPORT_SYMBOL(copy_page);
-EXPORT_SYMBOL(clear_page);
+EXPORT_SYMBOL(clear_pages);
EXPORT_SYMBOL(cpu_pda);
#ifdef CONFIG_SMP
diff -puN arch/x86_64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher arch/x86_64/lib/clear_page.S
--- 25/arch/x86_64/lib/clear_page.S~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/arch/x86_64/lib/clear_page.S 2005-04-04 01:54:48.000000000 -0700
@@ -1,12 +1,16 @@
/*
* Zero a page.
* rdi page
+ * rsi order
*/
- .globl clear_page
+ .globl clear_pages
.p2align 4
-clear_page:
+clear_pages:
+ movl $4096/64,%eax
+ movl %esi, %ecx
+ shll %cl, %eax
+ movl %eax, %ecx
xorl %eax,%eax
- movl $4096/64,%ecx
.p2align 4
.Lloop:
decl %ecx
@@ -23,7 +27,7 @@ clear_page:
jnz .Lloop
nop
ret
-clear_page_end:
+clear_pages_end:
/* C stepping K8 run faster using the string instructions.
It is also a lot simpler. Use this when possible */
@@ -32,19 +36,22 @@ clear_page_end:
.section .altinstructions,"a"
.align 8
- .quad clear_page
- .quad clear_page_c
+ .quad clear_pages
+ .quad clear_pages_c
.byte X86_FEATURE_K8_C
- .byte clear_page_end-clear_page
- .byte clear_page_c_end-clear_page_c
+ .byte clear_pages_end-clear_pages
+ .byte clear_pages_c_end-clear_pages_c
.previous
.section .altinstr_replacement,"ax"
-clear_page_c:
- movl $4096/8,%ecx
+clear_pages_c:
+ movl $4096/8,%eax
+ movl %esi, %ecx
+ shll %cl, %eax
+ movl %eax, %ecx
xorl %eax,%eax
rep
stosq
ret
-clear_page_c_end:
+clear_pages_c_end:
.previous
diff -puN include/asm-i386/mmx.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-i386/mmx.h
--- 25/include/asm-i386/mmx.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/asm-i386/mmx.h 2005-04-04 01:54:48.000000000 -0700
@@ -8,7 +8,7 @@
#include <linux/types.h>
extern void *_mmx_memcpy(void *to, const void *from, size_t size);
-extern void mmx_clear_page(void *page);
+extern void mmx_clear_page(void *page, int order);
extern void mmx_copy_page(void *to, void *from);
#endif
diff -puN include/asm-i386/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-i386/page.h
--- 25/include/asm-i386/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/asm-i386/page.h 2005-04-04 01:54:48.000000000 -0700
@@ -18,7 +18,7 @@
#include <asm/mmx.h>
-#define clear_page(page) mmx_clear_page((void *)(page))
+#define clear_pages(page, order) mmx_clear_page((void *)(page),order)
#define copy_page(to,from) mmx_copy_page(to,from)
#else
@@ -28,11 +28,12 @@
* Maybe the K6-III ?
*/
-#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
+#define clear_pages(page, order) memset((void *)(page), 0, PAGE_SIZE << (order))
#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE)
#endif
+#define clear_page(page) clear_pages(page, 0)
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
diff -puN include/asm-ia64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-ia64/page.h
--- 25/include/asm-ia64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/asm-ia64/page.h 2005-04-04 01:54:48.000000000 -0700
@@ -56,8 +56,9 @@
# ifdef __KERNEL__
# define STRICT_MM_TYPECHECKS
-extern void clear_page (void *page);
+extern void clear_pages (void *page, int order);
extern void copy_page (void *to, void *from);
+#define clear_page(__page) clear_pages(__page, 0)
/*
* clear_user_page() and copy_user_page() can't be inline functions because
diff -puN include/asm-sparc64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-sparc64/page.h
--- 25/include/asm-sparc64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/asm-sparc64/page.h 2005-04-04 01:54:48.000000000 -0700
@@ -25,8 +25,10 @@
#ifndef __ASSEMBLY__
-extern void _clear_page(void *page);
-#define clear_page(X) _clear_page((void *)(X))
+extern void _clear_page(void *page, int order);
+#define clear_page(X) _clear_page((void *)(X), 0)
+#define clear_pages _clear_page
+
struct page;
extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page);
#define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE)
diff -puN include/asm-x86_64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher include/asm-x86_64/page.h
--- 25/include/asm-x86_64/page.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/asm-x86_64/page.h 2005-04-04 01:54:48.000000000 -0700
@@ -32,8 +32,9 @@
#ifdef __KERNEL__
#ifndef __ASSEMBLY__
-void clear_page(void *);
+void clear_pages(void *, int);
void copy_page(void *, void *);
+#define clear_page(__page) clear_pages(__page, 0)
#define clear_user_page(page, vaddr, pg) clear_page(page)
#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
diff -puN include/linux/gfp.h~add-a-clear_pages-function-to-clear-pages-of-higher include/linux/gfp.h
--- 25/include/linux/gfp.h~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/include/linux/gfp.h 2005-04-04 01:54:48.000000000 -0700
@@ -131,4 +131,5 @@ extern void FASTCALL(free_cold_page(stru
void page_alloc_init(void);
+void prep_zero_page(struct page *, int order, unsigned int __nocast gfp_flags);
#endif /* __LINUX_GFP_H */
diff -puN mm/hugetlb.c~add-a-clear_pages-function-to-clear-pages-of-higher mm/hugetlb.c
--- 25/mm/hugetlb.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/mm/hugetlb.c 2005-04-04 01:54:48.000000000 -0700
@@ -78,7 +78,6 @@ void free_huge_page(struct page *page)
struct page *alloc_huge_page(void)
{
struct page *page;
- int i;
spin_lock(&hugetlb_lock);
page = dequeue_huge_page();
@@ -89,8 +88,7 @@ struct page *alloc_huge_page(void)
spin_unlock(&hugetlb_lock);
set_page_count(page, 1);
page[1].mapping = (void *)free_huge_page;
- for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
- clear_highpage(&page[i]);
+ prep_zero_page(page, HUGETLB_PAGE_ORDER, GFP_HIGHUSER);
return page;
}
diff -puN mm/page_alloc.c~add-a-clear_pages-function-to-clear-pages-of-higher mm/page_alloc.c
--- 25/mm/page_alloc.c~add-a-clear_pages-function-to-clear-pages-of-higher 2005-04-04 01:54:48.000000000 -0700
+++ 25-akpm/mm/page_alloc.c 2005-04-04 01:54:48.000000000 -0700
@@ -633,11 +633,20 @@ void fastcall free_cold_page(struct page
free_hot_cold_page(page, 1);
}
-static inline void prep_zero_page(struct page *page, int order, unsigned int __nocast gfp_flags)
+void prep_zero_page(struct page *page, int order,
+ unsigned int __nocast gfp_flags)
{
int i;
BUG_ON((gfp_flags & (__GFP_WAIT | __GFP_HIGHMEM)) == __GFP_HIGHMEM);
+
+#ifdef CONFIG_CLEAR_PAGES
+ if (!PageHighMem(page)) {
+ clear_pages(page_address(page), order);
+ return;
+ }
+#endif
+
for(i = 0; i < (1 << order); i++)
clear_highpage(page + i);
}
_