source: mainline/kernel/arch/mips64/src/mm/tlb.c@ 9d58539

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 9d58539 was 9d58539, checked in by Prutkov Alex <prutkov.alex@…>, 14 years ago

Fixed unix permissions for all files
  • Property mode set to 100644
File size: 11.8 KB
Line 
1/*
2 * Copyright (c) 2003-2004 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup mips64mm
30 * @{
31 */
32/** @file
33 */
34
35#include <arch/mm/tlb.h>
36#include <mm/asid.h>
37#include <mm/tlb.h>
38#include <mm/page.h>
39#include <mm/as.h>
40#include <arch/cp0.h>
41#include <panic.h>
42#include <arch.h>
43#include <synch/mutex.h>
44#include <print.h>
45#include <debug.h>
46#include <align.h>
47#include <interrupt.h>
48#include <symtab.h>
49
50/** Initialize TLB.
51 *
52 * Invalidate all entries and mark wired entries.
53 *
54 */
55void tlb_arch_init(void)
56{
57 cp0_pagemask_write(TLB_PAGE_MASK_16K);
58 cp0_entry_hi_write(0);
59 cp0_entry_lo0_write(0);
60 cp0_entry_lo1_write(0);
61
62 /* Clear and initialize TLB. */
63
64 for (unsigned int i = 0; i < TLB_ENTRY_COUNT; i++) {
65 cp0_index_write(i);
66 tlbwi();
67 }
68
69 /*
70 * The kernel is going to make use of some wired
71 * entries (e.g. mapping kernel stacks in kseg3).
72 */
73 cp0_wired_write(TLB_WIRED);
74}
75
76/** Try to find PTE for faulting address.
77 *
78 * @param badvaddr Faulting virtual address.
79 * @param access Access mode that caused the fault.
80 * @param istate Pointer to interrupted state.
81 * @param pfrc Pointer to variable where as_page_fault()
82 * return code will be stored.
83 *
84 * @return PTE on success, NULL otherwise.
85 *
86 */
87static pte_t *find_mapping_and_check(uintptr_t badvaddr, int access,
88 istate_t *istate, int *pfrc)
89{
90 entry_hi_t hi;
91 hi.value = cp0_entry_hi_read();
92
93 /*
94 * Handler cannot succeed if the ASIDs don't match.
95 */
96 if (hi.asid != AS->asid) {
97 printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
98 return NULL;
99 }
100
101 /*
102 * Check if the mapping exists in page tables.
103 */
104 pte_t *pte = page_mapping_find(AS, badvaddr, true);
105 if ((pte) && (pte->p) && ((pte->w) || (access != PF_ACCESS_WRITE))) {
106 /*
107 * Mapping found in page tables.
108 * Immediately succeed.
109 */
110 return pte;
111 } else {
112 int rc;
113
114 /*
115 * Mapping not found in page tables.
116 * Resort to higher-level page fault handler.
117 */
118 switch (rc = as_page_fault(badvaddr, access, istate)) {
119 case AS_PF_OK:
120 /*
121 * The higher-level page fault handler succeeded,
122 * The mapping ought to be in place.
123 */
124 pte = page_mapping_find(AS, badvaddr, true);
125 ASSERT(pte);
126 ASSERT(pte->p);
127 ASSERT((pte->w) || (access != PF_ACCESS_WRITE));
128 return pte;
129 case AS_PF_DEFER:
130 *pfrc = AS_PF_DEFER;
131 return NULL;
132 case AS_PF_FAULT:
133 *pfrc = AS_PF_FAULT;
134 return NULL;
135 default:
136 panic("Unexpected return code (%d).", rc);
137 }
138 }
139}
140
141void tlb_prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d,
142 bool c, uintptr_t addr)
143{
144 lo->value = 0;
145 lo->g = g;
146 lo->v = v;
147 lo->d = d;
148 lo->c = c ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
149 lo->pfn = ADDR2PFN(addr);
150}
151
152void tlb_prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
153{
154 hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
155 hi->asid = asid;
156}
157
158static void tlb_refill_fail(istate_t *istate)
159{
160 uintptr_t va = cp0_badvaddr_read();
161
162 fault_if_from_uspace(istate, "TLB Refill Exception on %p.",
163 (void *) va);
164 panic_memtrap(istate, PF_ACCESS_UNKNOWN, va, "TLB Refill Exception.");
165}
166
167static void tlb_invalid_fail(istate_t *istate)
168{
169 uintptr_t va = cp0_badvaddr_read();
170
171 fault_if_from_uspace(istate, "TLB Invalid Exception on %p.",
172 (void *) va);
173 panic_memtrap(istate, PF_ACCESS_UNKNOWN, va, "TLB Invalid Exception.");
174}
175
176static void tlb_modified_fail(istate_t *istate)
177{
178 uintptr_t va = cp0_badvaddr_read();
179
180 fault_if_from_uspace(istate, "TLB Modified Exception on %p.",
181 (void *) va);
182 panic_memtrap(istate, PF_ACCESS_WRITE, va, "TLB Modified Exception.");
183}
184
185/** Process TLB Refill Exception.
186 *
187 * @param istate Interrupted register context.
188 *
189 */
190void tlb_refill(istate_t *istate)
191{
192 uintptr_t badvaddr = cp0_badvaddr_read();
193
194 mutex_lock(&AS->lock);
195 asid_t asid = AS->asid;
196 mutex_unlock(&AS->lock);
197
198 int pfrc;
199 pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ,
200 istate, &pfrc);
201 if (!pte) {
202 switch (pfrc) {
203 case AS_PF_FAULT:
204 goto fail;
205 break;
206 case AS_PF_DEFER:
207 /*
208 * The page fault came during copy_from_uspace()
209 * or copy_to_uspace().
210 */
211 return;
212 default:
213 panic("Unexpected pfrc (%d).", pfrc);
214 }
215 }
216
217 /*
218 * Record access to PTE.
219 */
220 pte->a = 1;
221
222 entry_lo_t lo;
223 entry_hi_t hi;
224
225 tlb_prepare_entry_hi(&hi, asid, badvaddr);
226 tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->c,
227 pte->frame);
228
229 /*
230 * New entry is to be inserted into TLB
231 */
232 cp0_entry_hi_write(hi.value);
233
234 if ((badvaddr / PAGE_SIZE) % 2 == 0) {
235 cp0_entry_lo0_write(lo.value);
236 cp0_entry_lo1_write(0);
237 } else {
238 cp0_entry_lo0_write(0);
239 cp0_entry_lo1_write(lo.value);
240 }
241
242 cp0_pagemask_write(TLB_PAGE_MASK_16K);
243 tlbwr();
244
245 return;
246
247fail:
248 tlb_refill_fail(istate);
249}
250
251/** Process TLB Invalid Exception.
252 *
253 * @param istate Interrupted register context.
254 *
255 */
256void tlb_invalid(istate_t *istate)
257{
258 uintptr_t badvaddr = cp0_badvaddr_read();
259
260 /*
261 * Locate the faulting entry in TLB.
262 */
263 entry_hi_t hi;
264 hi.value = cp0_entry_hi_read();
265
266 tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
267 cp0_entry_hi_write(hi.value);
268 tlbp();
269
270 tlb_index_t index;
271 index.value = cp0_index_read();
272
273 /*
274 * Fail if the entry is not in TLB.
275 */
276 if (index.p) {
277 printf("TLB entry not found.\n");
278 goto fail;
279 }
280
281 int pfrc;
282 pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ,
283 istate, &pfrc);
284 if (!pte) {
285 switch (pfrc) {
286 case AS_PF_FAULT:
287 goto fail;
288 break;
289 case AS_PF_DEFER:
290 /*
291 * The page fault came during copy_from_uspace()
292 * or copy_to_uspace().
293 */
294 return;
295 default:
296 panic("Unexpected pfrc (%d).", pfrc);
297 }
298 }
299
300 /*
301 * Read the faulting TLB entry.
302 */
303 tlbr();
304
305 /*
306 * Record access to PTE.
307 */
308 pte->a = 1;
309
310 entry_lo_t lo;
311 tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->c,
312 pte->frame);
313
314 /*
315 * The entry is to be updated in TLB.
316 */
317 if ((badvaddr / PAGE_SIZE) % 2 == 0)
318 cp0_entry_lo0_write(lo.value);
319 else
320 cp0_entry_lo1_write(lo.value);
321
322 cp0_pagemask_write(TLB_PAGE_MASK_16K);
323 tlbwi();
324
325 return;
326
327fail:
328 tlb_invalid_fail(istate);
329}
330
331/** Process TLB Modified Exception.
332 *
333 * @param istate Interrupted register context.
334 *
335 */
336void tlb_modified(istate_t *istate)
337{
338 uintptr_t badvaddr = cp0_badvaddr_read();
339
340 /*
341 * Locate the faulting entry in TLB.
342 */
343 entry_hi_t hi;
344 hi.value = cp0_entry_hi_read();
345
346 tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
347 cp0_entry_hi_write(hi.value);
348 tlbp();
349
350 tlb_index_t index;
351 index.value = cp0_index_read();
352
353 /*
354 * Fail if the entry is not in TLB.
355 */
356 if (index.p) {
357 printf("TLB entry not found.\n");
358 goto fail;
359 }
360
361 int pfrc;
362 pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE,
363 istate, &pfrc);
364 if (!pte) {
365 switch (pfrc) {
366 case AS_PF_FAULT:
367 goto fail;
368 break;
369 case AS_PF_DEFER:
370 /*
371 * The page fault came during copy_from_uspace()
372 * or copy_to_uspace().
373 */
374 return;
375 default:
376 panic("Unexpected pfrc (%d).", pfrc);
377 }
378 }
379
380 /*
381 * Read the faulting TLB entry.
382 */
383 tlbr();
384
385 /*
386 * Record access and write to PTE.
387 */
388 pte->a = 1;
389 pte->d = 1;
390
391 entry_lo_t lo;
392 tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->c,
393 pte->frame);
394
395 /*
396 * The entry is to be updated in TLB.
397 */
398 if ((badvaddr / PAGE_SIZE) % 2 == 0)
399 cp0_entry_lo0_write(lo.value);
400 else
401 cp0_entry_lo1_write(lo.value);
402
403 cp0_pagemask_write(TLB_PAGE_MASK_16K);
404 tlbwi();
405
406 return;
407
408fail:
409 tlb_modified_fail(istate);
410}
411
412/** Print contents of TLB. */
413void tlb_print(void)
414{
415 entry_hi_t hi_save;
416 hi_save.value = cp0_entry_hi_read();
417
418 printf("[nr] [asid] [vpn2] [mask] [gvdc] [pfn ]\n");
419
420 for (unsigned int i = 0; i < TLB_ENTRY_COUNT; i++) {
421 cp0_index_write(i);
422 tlbr();
423
424 page_mask_t mask;
425 mask.value = cp0_pagemask_read();
426
427 entry_hi_t hi;
428 hi.value = cp0_entry_hi_read();
429
430 entry_lo_t lo0;
431 lo0.value = cp0_entry_lo0_read();
432
433 entry_lo_t lo1;
434 lo1.value = cp0_entry_lo1_read();
435
436 printf("%-4u %-6u %#6x %#6x %1u%1u%1u%1u %#6x\n",
437 i, hi.asid, hi.vpn2, mask.mask,
438 lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn);
439 printf(" %1u%1u%1u%1u %#6x\n",
440 lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
441 }
442
443 cp0_entry_hi_write(hi_save.value);
444}
445
446/** Invalidate all not wired TLB entries. */
447void tlb_invalidate_all(void)
448{
449 entry_hi_t hi_save;
450 hi_save.value = cp0_entry_hi_read();
451 ipl_t ipl = interrupts_disable();
452
453 for (unsigned int i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
454 cp0_index_write(i);
455 tlbr();
456
457 entry_lo_t lo0;
458 lo0.value = cp0_entry_lo0_read();
459
460 entry_lo_t lo1;
461 lo1.value = cp0_entry_lo1_read();
462
463 lo0.v = 0;
464 lo1.v = 0;
465
466 cp0_entry_lo0_write(lo0.value);
467 cp0_entry_lo1_write(lo1.value);
468
469 tlbwi();
470 }
471
472 interrupts_restore(ipl);
473 cp0_entry_hi_write(hi_save.value);
474}
475
476/** Invalidate all TLB entries belonging to specified address space.
477 *
478 * @param asid Address space identifier.
479 *
480 */
481void tlb_invalidate_asid(asid_t asid)
482{
483 ASSERT(asid != ASID_INVALID);
484
485 entry_hi_t hi_save;
486 hi_save.value = cp0_entry_hi_read();
487 ipl_t ipl = interrupts_disable();
488
489 for (unsigned int i = 0; i < TLB_ENTRY_COUNT; i++) {
490 cp0_index_write(i);
491 tlbr();
492
493 entry_hi_t hi;
494 hi.value = cp0_entry_hi_read();
495
496 if (hi.asid == asid) {
497 entry_lo_t lo0;
498 lo0.value = cp0_entry_lo0_read();
499
500 entry_lo_t lo1;
501 lo1.value = cp0_entry_lo1_read();
502
503 lo0.v = 0;
504 lo1.v = 0;
505
506 cp0_entry_lo0_write(lo0.value);
507 cp0_entry_lo1_write(lo1.value);
508
509 tlbwi();
510 }
511 }
512
513 interrupts_restore(ipl);
514 cp0_entry_hi_write(hi_save.value);
515}
516
517/** Invalidate TLB entries for specified page range belonging to specified
518 * address space.
519 *
520 * @param asid Address space identifier.
521 * @param page First page whose TLB entry is to be invalidated.
522 * @param cnt Number of entries to invalidate.
523 *
524 */
525void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
526{
527 if (asid == ASID_INVALID)
528 return;
529
530 entry_hi_t hi_save;
531 hi_save.value = cp0_entry_hi_read();
532 ipl_t ipl = interrupts_disable();
533
534 for (unsigned int i = 0; i < cnt + 1; i += 2) {
535 entry_hi_t hi;
536 hi.value = 0;
537 tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
538 cp0_entry_hi_write(hi.value);
539
540 tlbp();
541
542 tlb_index_t index;
543 index.value = cp0_index_read();
544
545 if (!index.p) {
546 /*
547 * Entry was found, index register contains valid
548 * index.
549 */
550 tlbr();
551
552 entry_lo_t lo0;
553 lo0.value = cp0_entry_lo0_read();
554
555 entry_lo_t lo1;
556 lo1.value = cp0_entry_lo1_read();
557
558 lo0.v = 0;
559 lo1.v = 0;
560
561 cp0_entry_lo0_write(lo0.value);
562 cp0_entry_lo1_write(lo1.value);
563
564 tlbwi();
565 }
566 }
567
568 interrupts_restore(ipl);
569 cp0_entry_hi_write(hi_save.value);
570}
571
572/** @}
573 */
Note: See TracBrowser for help on using the repository browser.