00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00035 #include <arch/mm/tlb.h>
00036 #include <mm/asid.h>
00037 #include <mm/tlb.h>
00038 #include <mm/page.h>
00039 #include <mm/as.h>
00040 #include <arch/cp0.h>
00041 #include <panic.h>
00042 #include <arch.h>
00043 #include <symtab.h>
00044 #include <synch/spinlock.h>
00045 #include <print.h>
00046 #include <debug.h>
00047 #include <align.h>
00048 #include <interrupt.h>
00049
00050 static void tlb_refill_fail(istate_t *istate);
00051 static void tlb_invalid_fail(istate_t *istate);
00052 static void tlb_modified_fail(istate_t *istate);
00053
00054 static pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc);
00055
00056 static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn);
00057 static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr);
00058
00064 void tlb_arch_init(void)
00065 {
00066 int i;
00067
00068 cp0_pagemask_write(TLB_PAGE_MASK_16K);
00069 cp0_entry_hi_write(0);
00070 cp0_entry_lo0_write(0);
00071 cp0_entry_lo1_write(0);
00072
00073
00074
00075 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
00076 cp0_index_write(i);
00077 tlbwi();
00078 }
00079
00080
00081
00082
00083
00084
00085 cp0_wired_write(TLB_WIRED);
00086 }
00087
00094 void tlb_refill(istate_t *istate)
00095 {
00096 entry_lo_t lo;
00097 entry_hi_t hi;
00098 asid_t asid;
00099 __address badvaddr;
00100 pte_t *pte;
00101 int pfrc;
00102
00103 badvaddr = cp0_badvaddr_read();
00104
00105 spinlock_lock(&AS->lock);
00106 asid = AS->asid;
00107 spinlock_unlock(&AS->lock);
00108
00109 page_table_lock(AS, true);
00110
00111 pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
00112 if (!pte) {
00113 switch (pfrc) {
00114 case AS_PF_FAULT:
00115 goto fail;
00116 break;
00117 case AS_PF_DEFER:
00118
00119
00120
00121
00122 page_table_unlock(AS, true);
00123 return;
00124 default:
00125 panic("unexpected pfrc (%d)\n", pfrc);
00126 }
00127 }
00128
00129
00130
00131
00132 pte->a = 1;
00133
00134 prepare_entry_hi(&hi, asid, badvaddr);
00135 prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
00136
00137
00138
00139
00140 cp0_entry_hi_write(hi.value);
00141 if ((badvaddr/PAGE_SIZE) % 2 == 0) {
00142 cp0_entry_lo0_write(lo.value);
00143 cp0_entry_lo1_write(0);
00144 }
00145 else {
00146 cp0_entry_lo0_write(0);
00147 cp0_entry_lo1_write(lo.value);
00148 }
00149 cp0_pagemask_write(TLB_PAGE_MASK_16K);
00150 tlbwr();
00151
00152 page_table_unlock(AS, true);
00153 return;
00154
00155 fail:
00156 page_table_unlock(AS, true);
00157 tlb_refill_fail(istate);
00158 }
00159
00166 void tlb_invalid(istate_t *istate)
00167 {
00168 tlb_index_t index;
00169 __address badvaddr;
00170 entry_lo_t lo;
00171 entry_hi_t hi;
00172 pte_t *pte;
00173 int pfrc;
00174
00175 badvaddr = cp0_badvaddr_read();
00176
00177
00178
00179
00180 hi.value = cp0_entry_hi_read();
00181 prepare_entry_hi(&hi, hi.asid, badvaddr);
00182 cp0_entry_hi_write(hi.value);
00183 tlbp();
00184 index.value = cp0_index_read();
00185
00186 page_table_lock(AS, true);
00187
00188
00189
00190
00191 if (index.p) {
00192 printf("TLB entry not found.\n");
00193 goto fail;
00194 }
00195
00196 pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
00197 if (!pte) {
00198 switch (pfrc) {
00199 case AS_PF_FAULT:
00200 goto fail;
00201 break;
00202 case AS_PF_DEFER:
00203
00204
00205
00206
00207 page_table_unlock(AS, true);
00208 return;
00209 default:
00210 panic("unexpected pfrc (%d)\n", pfrc);
00211 }
00212 }
00213
00214
00215
00216
00217 tlbr();
00218
00219
00220
00221
00222 pte->a = 1;
00223
00224 prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
00225
00226
00227
00228
00229 if ((badvaddr/PAGE_SIZE) % 2 == 0)
00230 cp0_entry_lo0_write(lo.value);
00231 else
00232 cp0_entry_lo1_write(lo.value);
00233 cp0_pagemask_write(TLB_PAGE_MASK_16K);
00234 tlbwi();
00235
00236 page_table_unlock(AS, true);
00237 return;
00238
00239 fail:
00240 page_table_unlock(AS, true);
00241 tlb_invalid_fail(istate);
00242 }
00243
00250 void tlb_modified(istate_t *istate)
00251 {
00252 tlb_index_t index;
00253 __address badvaddr;
00254 entry_lo_t lo;
00255 entry_hi_t hi;
00256 pte_t *pte;
00257 int pfrc;
00258
00259 badvaddr = cp0_badvaddr_read();
00260
00261
00262
00263
00264 hi.value = cp0_entry_hi_read();
00265 prepare_entry_hi(&hi, hi.asid, badvaddr);
00266 cp0_entry_hi_write(hi.value);
00267 tlbp();
00268 index.value = cp0_index_read();
00269
00270 page_table_lock(AS, true);
00271
00272
00273
00274
00275 if (index.p) {
00276 printf("TLB entry not found.\n");
00277 goto fail;
00278 }
00279
00280 pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
00281 if (!pte) {
00282 switch (pfrc) {
00283 case AS_PF_FAULT:
00284 goto fail;
00285 break;
00286 case AS_PF_DEFER:
00287
00288
00289
00290
00291 page_table_unlock(AS, true);
00292 return;
00293 default:
00294 panic("unexpected pfrc (%d)\n", pfrc);
00295 }
00296 }
00297
00298
00299
00300
00301 if (!pte->w)
00302 goto fail;
00303
00304
00305
00306
00307 tlbr();
00308
00309
00310
00311
00312 pte->a = 1;
00313 pte->d = 1;
00314
00315 prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
00316
00317
00318
00319
00320 if ((badvaddr/PAGE_SIZE) % 2 == 0)
00321 cp0_entry_lo0_write(lo.value);
00322 else
00323 cp0_entry_lo1_write(lo.value);
00324 cp0_pagemask_write(TLB_PAGE_MASK_16K);
00325 tlbwi();
00326
00327 page_table_unlock(AS, true);
00328 return;
00329
00330 fail:
00331 page_table_unlock(AS, true);
00332 tlb_modified_fail(istate);
00333 }
00334
00335 void tlb_refill_fail(istate_t *istate)
00336 {
00337 char *symbol = "";
00338 char *sym2 = "";
00339
00340 char *s = get_symtab_entry(istate->epc);
00341 if (s)
00342 symbol = s;
00343 s = get_symtab_entry(istate->ra);
00344 if (s)
00345 sym2 = s;
00346
00347 fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());
00348 panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
00349 }
00350
00351
00352 void tlb_invalid_fail(istate_t *istate)
00353 {
00354 char *symbol = "";
00355
00356 char *s = get_symtab_entry(istate->epc);
00357 if (s)
00358 symbol = s;
00359 fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());
00360 panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
00361 }
00362
00363 void tlb_modified_fail(istate_t *istate)
00364 {
00365 char *symbol = "";
00366
00367 char *s = get_symtab_entry(istate->epc);
00368 if (s)
00369 symbol = s;
00370 fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());
00371 panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
00372 }
00373
00386 pte_t *find_mapping_and_check(__address badvaddr, int access, istate_t *istate, int *pfrc)
00387 {
00388 entry_hi_t hi;
00389 pte_t *pte;
00390
00391 hi.value = cp0_entry_hi_read();
00392
00393
00394
00395
00396 if (hi.asid != AS->asid) {
00397 printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
00398 return NULL;
00399 }
00400
00401
00402
00403
00404 pte = page_mapping_find(AS, badvaddr);
00405 if (pte && pte->p) {
00406
00407
00408
00409
00410 return pte;
00411 } else {
00412 int rc;
00413
00414
00415
00416
00417
00418 page_table_unlock(AS, true);
00419 switch (rc = as_page_fault(badvaddr, access, istate)) {
00420 case AS_PF_OK:
00421
00422
00423
00424
00425 page_table_lock(AS, true);
00426 pte = page_mapping_find(AS, badvaddr);
00427 ASSERT(pte && pte->p);
00428 return pte;
00429 break;
00430 case AS_PF_DEFER:
00431 page_table_lock(AS, true);
00432 *pfrc = AS_PF_DEFER;
00433 return NULL;
00434 break;
00435 case AS_PF_FAULT:
00436 page_table_lock(AS, true);
00437 printf("Page fault.\n");
00438 *pfrc = AS_PF_FAULT;
00439 return NULL;
00440 break;
00441 default:
00442 panic("unexpected rc (%d)\n", rc);
00443 }
00444
00445 }
00446 }
00447
00448 void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, __address pfn)
00449 {
00450 lo->value = 0;
00451 lo->g = g;
00452 lo->v = v;
00453 lo->d = d;
00454 lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
00455 lo->pfn = pfn;
00456 }
00457
00458 void prepare_entry_hi(entry_hi_t *hi, asid_t asid, __address addr)
00459 {
00460 hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
00461 hi->asid = asid;
00462 }
00463
00465 void tlb_print(void)
00466 {
00467 page_mask_t mask;
00468 entry_lo_t lo0, lo1;
00469 entry_hi_t hi, hi_save;
00470 int i;
00471
00472 hi_save.value = cp0_entry_hi_read();
00473
00474 printf("TLB:\n");
00475 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
00476 cp0_index_write(i);
00477 tlbr();
00478
00479 mask.value = cp0_pagemask_read();
00480 hi.value = cp0_entry_hi_read();
00481 lo0.value = cp0_entry_lo0_read();
00482 lo1.value = cp0_entry_lo1_read();
00483
00484 printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%hhd, pfn[0]=%d\n"
00485 "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%hhd, pfn[1]=%d\n",
00486 i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
00487 lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
00488 }
00489
00490 cp0_entry_hi_write(hi_save.value);
00491 }
00492
00494 void tlb_invalidate_all(void)
00495 {
00496 ipl_t ipl;
00497 entry_lo_t lo0, lo1;
00498 entry_hi_t hi_save;
00499 int i;
00500
00501 hi_save.value = cp0_entry_hi_read();
00502 ipl = interrupts_disable();
00503
00504 for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
00505 cp0_index_write(i);
00506 tlbr();
00507
00508 lo0.value = cp0_entry_lo0_read();
00509 lo1.value = cp0_entry_lo1_read();
00510
00511 lo0.v = 0;
00512 lo1.v = 0;
00513
00514 cp0_entry_lo0_write(lo0.value);
00515 cp0_entry_lo1_write(lo1.value);
00516
00517 tlbwi();
00518 }
00519
00520 interrupts_restore(ipl);
00521 cp0_entry_hi_write(hi_save.value);
00522 }
00523
00528 void tlb_invalidate_asid(asid_t asid)
00529 {
00530 ipl_t ipl;
00531 entry_lo_t lo0, lo1;
00532 entry_hi_t hi, hi_save;
00533 int i;
00534
00535 ASSERT(asid != ASID_INVALID);
00536
00537 hi_save.value = cp0_entry_hi_read();
00538 ipl = interrupts_disable();
00539
00540 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
00541 cp0_index_write(i);
00542 tlbr();
00543
00544 hi.value = cp0_entry_hi_read();
00545
00546 if (hi.asid == asid) {
00547 lo0.value = cp0_entry_lo0_read();
00548 lo1.value = cp0_entry_lo1_read();
00549
00550 lo0.v = 0;
00551 lo1.v = 0;
00552
00553 cp0_entry_lo0_write(lo0.value);
00554 cp0_entry_lo1_write(lo1.value);
00555
00556 tlbwi();
00557 }
00558 }
00559
00560 interrupts_restore(ipl);
00561 cp0_entry_hi_write(hi_save.value);
00562 }
00563
00570 void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
00571 {
00572 int i;
00573 ipl_t ipl;
00574 entry_lo_t lo0, lo1;
00575 entry_hi_t hi, hi_save;
00576 tlb_index_t index;
00577
00578 ASSERT(asid != ASID_INVALID);
00579
00580 hi_save.value = cp0_entry_hi_read();
00581 ipl = interrupts_disable();
00582
00583 for (i = 0; i < cnt+1; i+=2) {
00584 hi.value = 0;
00585 prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
00586 cp0_entry_hi_write(hi.value);
00587
00588 tlbp();
00589 index.value = cp0_index_read();
00590
00591 if (!index.p) {
00592
00593 tlbr();
00594
00595 lo0.value = cp0_entry_lo0_read();
00596 lo1.value = cp0_entry_lo1_read();
00597
00598 lo0.v = 0;
00599 lo1.v = 0;
00600
00601 cp0_entry_lo0_write(lo0.value);
00602 cp0_entry_lo1_write(lo1.value);
00603
00604 tlbwi();
00605 }
00606 }
00607
00608 interrupts_restore(ipl);
00609 cp0_entry_hi_write(hi_save.value);
00610 }
00611