Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * SPARC64 Huge TLB page support. |
| 3 | * |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 4 | * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | */ |
| 6 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | #include <linux/init.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | #include <linux/fs.h> |
| 9 | #include <linux/mm.h> |
| 10 | #include <linux/hugetlb.h> |
| 11 | #include <linux/pagemap.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/sysctl.h> |
| 13 | |
| 14 | #include <asm/mman.h> |
| 15 | #include <asm/pgalloc.h> |
| 16 | #include <asm/tlb.h> |
| 17 | #include <asm/tlbflush.h> |
| 18 | #include <asm/cacheflush.h> |
| 19 | #include <asm/mmu_context.h> |
| 20 | |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 21 | /* Slightly simplified from the non-hugepage variant because by |
| 22 | * definition we don't have to worry about any page coloring stuff |
| 23 | */ |
| 24 | #define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) |
| 25 | #define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) |
| 26 | |
| 27 | static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, |
| 28 | unsigned long addr, |
| 29 | unsigned long len, |
| 30 | unsigned long pgoff, |
| 31 | unsigned long flags) |
| 32 | { |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 33 | unsigned long task_size = TASK_SIZE; |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 34 | struct vm_unmapped_area_info info; |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 35 | |
| 36 | if (test_thread_flag(TIF_32BIT)) |
| 37 | task_size = STACK_TOP32; |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 38 | |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 39 | info.flags = 0; |
| 40 | info.length = len; |
| 41 | info.low_limit = TASK_UNMAPPED_BASE; |
| 42 | info.high_limit = min(task_size, VA_EXCLUDE_START); |
| 43 | info.align_mask = PAGE_MASK & ~HPAGE_MASK; |
| 44 | info.align_offset = 0; |
| 45 | addr = vm_unmapped_area(&info); |
| 46 | |
| 47 | if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { |
| 48 | VM_BUG_ON(addr != -ENOMEM); |
| 49 | info.low_limit = VA_EXCLUDE_END; |
| 50 | info.high_limit = task_size; |
| 51 | addr = vm_unmapped_area(&info); |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 52 | } |
| 53 | |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 54 | return addr; |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 55 | } |
| 56 | |
| 57 | static unsigned long |
| 58 | hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, |
| 59 | const unsigned long len, |
| 60 | const unsigned long pgoff, |
| 61 | const unsigned long flags) |
| 62 | { |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 63 | struct mm_struct *mm = current->mm; |
| 64 | unsigned long addr = addr0; |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 65 | struct vm_unmapped_area_info info; |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 66 | |
| 67 | /* This should only ever run for 32-bit processes. */ |
| 68 | BUG_ON(!test_thread_flag(TIF_32BIT)); |
| 69 | |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 70 | info.flags = VM_UNMAPPED_AREA_TOPDOWN; |
| 71 | info.length = len; |
| 72 | info.low_limit = PAGE_SIZE; |
| 73 | info.high_limit = mm->mmap_base; |
| 74 | info.align_mask = PAGE_MASK & ~HPAGE_MASK; |
| 75 | info.align_offset = 0; |
| 76 | addr = vm_unmapped_area(&info); |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 77 | |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 78 | /* |
| 79 | * A failed mmap() very likely causes application failure, |
| 80 | * so fall back to the bottom-up function here. This scenario |
| 81 | * can happen with large stack limits and large mmap() |
| 82 | * allocations. |
| 83 | */ |
Michel Lespinasse | 2aea28b | 2012-12-11 16:02:25 -0800 | [diff] [blame] | 84 | if (addr & ~PAGE_MASK) { |
| 85 | VM_BUG_ON(addr != -ENOMEM); |
| 86 | info.flags = 0; |
| 87 | info.low_limit = TASK_UNMAPPED_BASE; |
| 88 | info.high_limit = STACK_TOP32; |
| 89 | addr = vm_unmapped_area(&info); |
| 90 | } |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 91 | |
| 92 | return addr; |
| 93 | } |
| 94 | |
| 95 | unsigned long |
| 96 | hugetlb_get_unmapped_area(struct file *file, unsigned long addr, |
| 97 | unsigned long len, unsigned long pgoff, unsigned long flags) |
| 98 | { |
| 99 | struct mm_struct *mm = current->mm; |
| 100 | struct vm_area_struct *vma; |
| 101 | unsigned long task_size = TASK_SIZE; |
| 102 | |
| 103 | if (test_thread_flag(TIF_32BIT)) |
| 104 | task_size = STACK_TOP32; |
| 105 | |
| 106 | if (len & ~HPAGE_MASK) |
| 107 | return -EINVAL; |
| 108 | if (len > task_size) |
| 109 | return -ENOMEM; |
| 110 | |
Benjamin Herrenschmidt | ac35ee4 | 2007-05-06 14:50:10 -0700 | [diff] [blame] | 111 | if (flags & MAP_FIXED) { |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 112 | if (prepare_hugepage_range(file, addr, len)) |
Benjamin Herrenschmidt | ac35ee4 | 2007-05-06 14:50:10 -0700 | [diff] [blame] | 113 | return -EINVAL; |
| 114 | return addr; |
| 115 | } |
| 116 | |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 117 | if (addr) { |
| 118 | addr = ALIGN(addr, HPAGE_SIZE); |
| 119 | vma = find_vma(mm, addr); |
| 120 | if (task_size - len >= addr && |
| 121 | (!vma || addr + len <= vma->vm_start)) |
| 122 | return addr; |
| 123 | } |
| 124 | if (mm->get_unmapped_area == arch_get_unmapped_area) |
| 125 | return hugetlb_get_unmapped_area_bottomup(file, addr, len, |
| 126 | pgoff, flags); |
| 127 | else |
| 128 | return hugetlb_get_unmapped_area_topdown(file, addr, len, |
| 129 | pgoff, flags); |
| 130 | } |
| 131 | |
Andi Kleen | a551643 | 2008-07-23 21:27:41 -0700 | [diff] [blame] | 132 | pte_t *huge_pte_alloc(struct mm_struct *mm, |
| 133 | unsigned long addr, unsigned long sz) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 134 | { |
| 135 | pgd_t *pgd; |
| 136 | pud_t *pud; |
| 137 | pmd_t *pmd; |
| 138 | pte_t *pte = NULL; |
| 139 | |
David S. Miller | 9df1dab | 2006-03-31 00:36:25 -0800 | [diff] [blame] | 140 | /* We must align the address, because our caller will run |
| 141 | * set_huge_pte_at() on whatever we return, which writes out |
| 142 | * all of the sub-ptes for the hugepage range. So we have |
| 143 | * to give it the first such sub-pte. |
| 144 | */ |
| 145 | addr &= HPAGE_MASK; |
| 146 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 147 | pgd = pgd_offset(mm, addr); |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 148 | pud = pud_alloc(mm, pgd, addr); |
| 149 | if (pud) { |
| 150 | pmd = pmd_alloc(mm, pud, addr); |
| 151 | if (pmd) |
Andrea Arcangeli | 8ac1f83 | 2011-01-13 15:46:43 -0800 | [diff] [blame] | 152 | pte = pte_alloc_map(mm, NULL, pmd, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | } |
| 154 | return pte; |
| 155 | } |
| 156 | |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 157 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 158 | { |
| 159 | pgd_t *pgd; |
| 160 | pud_t *pud; |
| 161 | pmd_t *pmd; |
| 162 | pte_t *pte = NULL; |
| 163 | |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 164 | addr &= HPAGE_MASK; |
| 165 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | pgd = pgd_offset(mm, addr); |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 167 | if (!pgd_none(*pgd)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | pud = pud_offset(pgd, addr); |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 169 | if (!pud_none(*pud)) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | pmd = pmd_offset(pud, addr); |
David S. Miller | f6b83f0 | 2006-03-20 01:17:17 -0800 | [diff] [blame] | 171 | if (!pmd_none(*pmd)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 172 | pte = pte_offset_map(pmd, addr); |
| 173 | } |
| 174 | } |
| 175 | return pte; |
| 176 | } |
| 177 | |
Chen, Kenneth W | 39dde65 | 2006-12-06 20:32:03 -0800 | [diff] [blame] | 178 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) |
| 179 | { |
| 180 | return 0; |
| 181 | } |
| 182 | |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 183 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
| 184 | pte_t *ptep, pte_t entry) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 185 | { |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 186 | int i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 187 | |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 188 | if (!pte_present(*ptep) && pte_present(entry)) |
| 189 | mm->context.huge_pte_count++; |
| 190 | |
David S. Miller | bb8236f | 2007-03-12 22:55:39 -0700 | [diff] [blame] | 191 | addr &= HPAGE_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 193 | set_pte_at(mm, addr, ptep, entry); |
| 194 | ptep++; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 195 | addr += PAGE_SIZE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | pte_val(entry) += PAGE_SIZE; |
| 197 | } |
| 198 | } |
| 199 | |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 200 | pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
| 201 | pte_t *ptep) |
| 202 | { |
| 203 | pte_t entry; |
| 204 | int i; |
| 205 | |
| 206 | entry = *ptep; |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 207 | if (pte_present(entry)) |
| 208 | mm->context.huge_pte_count--; |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 209 | |
David S. Miller | bb8236f | 2007-03-12 22:55:39 -0700 | [diff] [blame] | 210 | addr &= HPAGE_MASK; |
| 211 | |
David Gibson | 63551ae | 2005-06-21 17:14:44 -0700 | [diff] [blame] | 212 | for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { |
| 213 | pte_clear(mm, addr, ptep); |
| 214 | addr += PAGE_SIZE; |
| 215 | ptep++; |
| 216 | } |
| 217 | |
| 218 | return entry; |
| 219 | } |
| 220 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 221 | struct page *follow_huge_addr(struct mm_struct *mm, |
| 222 | unsigned long address, int write) |
| 223 | { |
| 224 | return ERR_PTR(-EINVAL); |
| 225 | } |
| 226 | |
| 227 | int pmd_huge(pmd_t pmd) |
| 228 | { |
| 229 | return 0; |
| 230 | } |
| 231 | |
Andi Kleen | ceb8687 | 2008-07-23 21:27:50 -0700 | [diff] [blame] | 232 | int pud_huge(pud_t pud) |
| 233 | { |
| 234 | return 0; |
| 235 | } |
| 236 | |
Naoya Horiguchi | 83467ef | 2013-09-11 14:22:11 -0700 | [diff] [blame^] | 237 | int pmd_huge_support(void) |
| 238 | { |
| 239 | return 0; |
| 240 | } |
| 241 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 242 | struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, |
| 243 | pmd_t *pmd, int write) |
| 244 | { |
| 245 | return NULL; |
| 246 | } |