diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/arch/ia64/mm/contig.c zero-slab/arch/ia64/mm/contig.c --- linux-pristine/arch/ia64/mm/contig.c 2004-10-11 14:57:16.000000000 -0700 +++ zero-slab/arch/ia64/mm/contig.c 2004-10-13 17:39:26.000000000 -0700 @@ -60,7 +60,6 @@ printk("%d reserved pages\n", reserved); printk("%d pages shared\n", shared); printk("%d pages swap cached\n", cached); - printk("%ld pages in page table cache\n", pgtable_cache_size); } /* physical address where the bootmem map is located */ diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/arch/ia64/mm/discontig.c zero-slab/arch/ia64/mm/discontig.c --- linux-pristine/arch/ia64/mm/discontig.c 2004-10-11 14:57:16.000000000 -0700 +++ zero-slab/arch/ia64/mm/discontig.c 2004-10-13 17:39:26.000000000 -0700 @@ -540,7 +540,6 @@ printk("%d reserved pages\n", total_reserved); printk("%d pages shared\n", total_shared); printk("%d pages swap cached\n", total_cached); - printk("Total of %ld pages in page table cache\n", pgtable_cache_size); printk("%d free buffer pages\n", nr_free_buffer_pages()); } diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/arch/ia64/mm/init.c zero-slab/arch/ia64/mm/init.c --- linux-pristine/arch/ia64/mm/init.c 2004-10-11 14:57:16.000000000 -0700 +++ zero-slab/arch/ia64/mm/init.c 2004-10-14 08:14:50.000000000 -0700 @@ -56,26 +56,6 @@ EXPORT_SYMBOL(zero_page_memmap_ptr); void -check_pgt_cache (void) -{ - int low, high; - - low = pgt_cache_water[0]; - high = pgt_cache_water[1]; - - preempt_disable(); - if (pgtable_cache_size > (u64) high) { - do { - if (pgd_quicklist) - free_page((unsigned long)pgd_alloc_one_fast(NULL)); - if (pmd_quicklist) - free_page((unsigned long)pmd_alloc_one_fast(NULL, 0)); - } while (pgtable_cache_size > (u64) low); - } - preempt_enable(); -} - -void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte) { unsigned long addr; @@ -254,7 +234,7 @@ return page; } -static void +static int __init setup_gate (void) { struct page *page; @@ -272,8 +252,11 @@ put_kernel_page(page, GATE_ADDR + PERCPU_PAGE_SIZE, PAGE_GATE); #endif ia64_patch_gate(); + return 0; } +core_initcall(setup_gate); + void __devinit ia64_mmu_init (void *my_cpu_data) { @@ -584,7 +567,6 @@ if (!fsyscall_table[i] || nolwsys) fsyscall_table[i] = sys_call_table[i] | 1; } - setup_gate(); #ifdef CONFIG_IA32_SUPPORT ia32_mem_init(); diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/arch/ppc64/mm/init.c zero-slab/arch/ppc64/mm/init.c --- linux-pristine/arch/ppc64/mm/init.c 2004-10-11 14:57:17.000000000 -0700 +++ zero-slab/arch/ppc64/mm/init.c 2004-10-13 17:59:16.000000000 -0700 @@ -886,21 +886,3 @@ return virt_addr; } -kmem_cache_t *zero_cache; - -static void zero_ctor(void *pte, kmem_cache_t *cache, unsigned long flags) -{ - memset(pte, 0, PAGE_SIZE); -} - -void pgtable_cache_init(void) -{ - zero_cache = kmem_cache_create("zero", - PAGE_SIZE, - 0, - SLAB_HWCACHE_ALIGN | SLAB_MUST_HWCACHE_ALIGN, - zero_ctor, - NULL); - if (!zero_cache) - panic("pgtable_cache_init(): could not create zero_cache!\n"); -} diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/include/asm-ia64/pgalloc.h zero-slab/include/asm-ia64/pgalloc.h --- linux-pristine/include/asm-ia64/pgalloc.h 2004-08-13 22:36:12.000000000 -0700 +++ zero-slab/include/asm-ia64/pgalloc.h 2004-10-13 17:57:38.000000000 -0700 @@ -17,65 +17,26 @@ #include #include +#include #include #include #include #include +#include -/* - * Very stupidly, we used to get new pgd's and pmd's, init their contents - * to point to the NULL versions of the next level page table, later on - * completely re-init them the same way, then free them up. This wasted - * a lot of work and caused unnecessary memory traffic. How broken... - * We fix this by caching them. - */ -#define pgd_quicklist (local_cpu_data->pgd_quick) -#define pmd_quicklist (local_cpu_data->pmd_quick) -#define pgtable_cache_size (local_cpu_data->pgtable_cache_sz) - -static inline pgd_t* -pgd_alloc_one_fast (struct mm_struct *mm) -{ - unsigned long *ret = NULL; - - preempt_disable(); - - ret = pgd_quicklist; - if (likely(ret != NULL)) { - pgd_quicklist = (unsigned long *)(*ret); - ret[0] = 0; - --pgtable_cache_size; - } else - ret = NULL; - - preempt_enable(); - - return (pgd_t *) ret; -} +extern kmem_cache_t *zero_page_cachep; static inline pgd_t* pgd_alloc (struct mm_struct *mm) { - /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */ - pgd_t *pgd = pgd_alloc_one_fast(mm); - - if (unlikely(pgd == NULL)) { - pgd = (pgd_t *)__get_free_page(GFP_KERNEL); - if (likely(pgd != NULL)) - clear_page(pgd); - } - return pgd; + return kmem_cache_alloc(zero_page_cachep, GFP_KERNEL); } static inline void pgd_free (pgd_t *pgd) { - preempt_disable(); - *(unsigned long *)pgd = (unsigned long) pgd_quicklist; - pgd_quicklist = (unsigned long *) pgd; - ++pgtable_cache_size; - preempt_enable(); + kmem_cache_free(zero_page_cachep, pgd); } static inline void @@ -86,92 +47,62 @@ static inline pmd_t* -pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr) -{ - unsigned long *ret = NULL; - - preempt_disable(); - - ret = (unsigned long *)pmd_quicklist; - if (likely(ret != NULL)) { - pmd_quicklist = (unsigned long *)(*ret); - ret[0] = 0; - --pgtable_cache_size; - } - - preempt_enable(); - - return (pmd_t *)ret; -} - -static inline pmd_t* pmd_alloc_one (struct mm_struct *mm, unsigned long addr) { - pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); - - if (likely(pmd != NULL)) - clear_page(pmd); - return pmd; + return kmem_cache_alloc(zero_page_cachep, GFP_KERNEL|__GFP_REPEAT); } static inline void pmd_free (pmd_t *pmd) { - preempt_disable(); - *(unsigned long *)pmd = (unsigned long) pmd_quicklist; - pmd_quicklist = (unsigned long *) pmd; - ++pgtable_cache_size; - preempt_enable(); + kmem_cache_free(zero_page_cachep, pmd); } #define __pmd_free_tlb(tlb, pmd) pmd_free(pmd) static inline void -pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte) -{ - pmd_val(*pmd_entry) = page_to_phys(pte); -} - -static inline void pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte) { pmd_val(*pmd_entry) = __pa(pte); } -static inline struct page * -pte_alloc_one (struct mm_struct *mm, unsigned long addr) +static inline void +pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte) { - struct page *pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0); - - if (likely(pte != NULL)) - clear_page(page_address(pte)); - return pte; + pmd_val(*pmd_entry) = page_to_phys(pte); } + static inline pte_t * pte_alloc_one_kernel (struct mm_struct *mm, unsigned long addr) { - pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); - - if (likely(pte != NULL)) - clear_page(pte); - return pte; + return kmem_cache_alloc(zero_page_cachep, GFP_KERNEL|__GFP_REPEAT); } static inline void -pte_free (struct page *pte) +pte_free_kernel (pte_t *pte) { - __free_page(pte); + kmem_cache_free(zero_page_cachep, pte); } -static inline void -pte_free_kernel (pte_t *pte) +static inline struct page * +pte_alloc_one (struct mm_struct *mm, unsigned long addr) { - free_page((unsigned long) pte); + pte_t *pte = pte_alloc_one_kernel(mm, addr); + + if (pte) + return virt_to_page(pte); + + return NULL; } -#define __pte_free_tlb(tlb, pte) tlb_remove_page((tlb), (pte)) +static inline void +pte_free (struct page *pte) +{ + pte_free_kernel(page_address(pte)); +} -extern void check_pgt_cache (void); +#define __pte_free_tlb(tlb, pte) pte_free(pte) +#define check_pgt_cache() do { } while (0) #endif /* _ASM_IA64_PGALLOC_H */ diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/include/asm-ppc64/pgalloc.h zero-slab/include/asm-ppc64/pgalloc.h --- linux-pristine/include/asm-ppc64/pgalloc.h 2004-10-11 14:57:20.000000000 -0700 +++ zero-slab/include/asm-ppc64/pgalloc.h 2004-10-13 18:04:26.000000000 -0700 @@ -9,7 +9,7 @@ #include #include -extern kmem_cache_t *zero_cache; +extern kmem_cache_t *zero_page_cachep; /* * This program is free software; you can redistribute it and/or @@ -21,13 +21,13 @@ static inline pgd_t * pgd_alloc(struct mm_struct *mm) { - return kmem_cache_alloc(zero_cache, GFP_KERNEL); + return kmem_cache_alloc(zero_page_cachep, GFP_KERNEL); } static inline void pgd_free(pgd_t *pgd) { - kmem_cache_free(zero_cache, pgd); + kmem_cache_free(zero_page_cachep, pgd); } #define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD) @@ -35,13 +35,13 @@ static inline pmd_t * pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(zero_cache, GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(zero_page_cachep, GFP_KERNEL|__GFP_REPEAT); } static inline void pmd_free(pmd_t *pmd) { - kmem_cache_free(zero_cache, pmd); + kmem_cache_free(zero_page_cachep, pmd); } #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, pte) @@ -52,7 +52,7 @@ pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; - pte = kmem_cache_alloc(zero_cache, GFP_KERNEL|__GFP_REPEAT); + pte = kmem_cache_alloc(zero_page_cachep, GFP_KERNEL|__GFP_REPEAT); if (pte) { struct page *ptepage = virt_to_page(pte); ptepage->mapping = (void *) mm; @@ -65,7 +65,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address) { pte_t *pte; - pte = kmem_cache_alloc(zero_cache, GFP_KERNEL|__GFP_REPEAT); + pte = kmem_cache_alloc(zero_page_cachep, GFP_KERNEL|__GFP_REPEAT); if (pte) { struct page *ptepage = virt_to_page(pte); ptepage->mapping = (void *) mm; @@ -78,13 +78,13 @@ static inline void pte_free_kernel(pte_t *pte) { virt_to_page(pte)->mapping = NULL; - kmem_cache_free(zero_cache, pte); + kmem_cache_free(zero_page_cachep, pte); } static inline void pte_free(struct page *ptepage) { ptepage->mapping = NULL; - kmem_cache_free(zero_cache, page_address(ptepage)); + kmem_cache_free(zero_page_cachep, page_address(ptepage)); } struct pte_freelist_batch diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/include/asm-ppc64/pgtable.h zero-slab/include/asm-ppc64/pgtable.h --- linux-pristine/include/asm-ppc64/pgtable.h 2004-10-11 14:57:20.000000000 -0700 +++ zero-slab/include/asm-ppc64/pgtable.h 2004-10-13 18:02:53.000000000 -0700 @@ -494,7 +494,10 @@ #define io_remap_page_range remap_page_range -void pgtable_cache_init(void); +/* + * No page table caches to initialise + */ +#define pgtable_cache_init() do { } while (0) extern void hpte_init_native(void); extern void hpte_init_lpar(void); diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/include/linux/slab.h zero-slab/include/linux/slab.h --- linux-pristine/include/linux/slab.h 2004-10-11 14:57:20.000000000 -0700 +++ zero-slab/include/linux/slab.h 2004-10-13 17:49:29.000000000 -0700 @@ -115,6 +115,7 @@ extern kmem_cache_t *signal_cachep; extern kmem_cache_t *sighand_cachep; extern kmem_cache_t *bio_cachep; +extern kmem_cache_t *zero_page_cachep; extern atomic_t slab_reclaim_pages; diff -urN -X /usr/people/mkp/bin/dontdiff linux-pristine/mm/slab.c zero-slab/mm/slab.c --- linux-pristine/mm/slab.c 2004-10-11 14:57:20.000000000 -0700 +++ zero-slab/mm/slab.c 2004-10-13 17:49:57.000000000 -0700 @@ -716,6 +716,13 @@ static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 }; +kmem_cache_t *zero_page_cachep; + +static void zero_page_ctor(void *pte, kmem_cache_t *cache, unsigned long flags) +{ + memset(pte, 0, PAGE_SIZE); +} + /* Initialisation. * Called after the gfp() functions have been enabled, and before smp_init(). */ @@ -837,6 +844,16 @@ /* The reap timers are started later, with a module init call: * That part of the kernel is not yet operational. */ + + /* General purpose cache of zeroed pages */ + zero_page_cachep = kmem_cache_create("zero_page_cache", + PAGE_SIZE, 0, + SLAB_HWCACHE_ALIGN | + SLAB_MUST_HWCACHE_ALIGN, + zero_page_ctor, + NULL); + if (!zero_page_cachep) + panic("could not create zero_page_cache!\n"); } static int __init cpucache_init(void)