00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00035
00036
00037
00038
00039 #include <mm/tlb.h>
00040 #include <mm/asid.h>
00041 #include <mm/page.h>
00042 #include <mm/as.h>
00043 #include <arch/mm/tlb.h>
00044 #include <arch/mm/page.h>
00045 #include <arch/mm/vhpt.h>
00046 #include <arch/barrier.h>
00047 #include <arch/interrupt.h>
00048 #include <arch/pal/pal.h>
00049 #include <arch/asm.h>
00050 #include <typedefs.h>
00051 #include <panic.h>
00052 #include <print.h>
00053 #include <arch.h>
00054 #include <interrupt.h>
00055
00057 void tlb_invalidate_all(void)
00058 {
00059 ipl_t ipl;
00060 __address adr;
00061 __u32 count1, count2, stride1, stride2;
00062
00063 int i,j;
00064
00065 adr = PAL_PTCE_INFO_BASE();
00066 count1 = PAL_PTCE_INFO_COUNT1();
00067 count2 = PAL_PTCE_INFO_COUNT2();
00068 stride1 = PAL_PTCE_INFO_STRIDE1();
00069 stride2 = PAL_PTCE_INFO_STRIDE2();
00070
00071 ipl = interrupts_disable();
00072
00073 for(i = 0; i < count1; i++) {
00074 for(j = 0; j < count2; j++) {
00075 __asm__ volatile (
00076 "ptc.e %0 ;;"
00077 :
00078 : "r" (adr)
00079 );
00080 adr += stride2;
00081 }
00082 adr += stride1;
00083 }
00084
00085 interrupts_restore(ipl);
00086
00087 srlz_d();
00088 srlz_i();
00089 #ifdef CONFIG_VHPT
00090 vhpt_invalidate_all();
00091 #endif
00092 }
00093
00098 void tlb_invalidate_asid(asid_t asid)
00099 {
00100 tlb_invalidate_all();
00101 }
00102
00103
00104 void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
00105 {
00106 region_register rr;
00107 bool restore_rr = false;
00108 int b = 0;
00109 int c = cnt;
00110
00111 __address va;
00112 va = page;
00113
00114 rr.word = rr_read(VA2VRN(va));
00115 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00116
00117
00118
00119
00120 region_register rr0;
00121
00122 rr0 = rr;
00123 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00124 rr_write(VA2VRN(va), rr0.word);
00125 srlz_d();
00126 srlz_i();
00127 }
00128
00129 while(c >>= 1)
00130 b++;
00131 b >>= 1;
00132 __u64 ps;
00133
00134 switch (b) {
00135 case 0:
00136 ps = PAGE_WIDTH;
00137 break;
00138 case 1:
00139
00140 ps = PAGE_WIDTH+2;
00141 va &= ~((1<<ps)-1);
00142 break;
00143 case 2:
00144
00145 ps = PAGE_WIDTH+4;
00146 va &= ~((1<<ps)-1);
00147 break;
00148 case 3:
00149
00150 ps = PAGE_WIDTH+6;
00151 va &= ~((1<<ps)-1);
00152 break;
00153 case 4:
00154
00155 ps = PAGE_WIDTH+8;
00156 va &= ~((1<<ps)-1);
00157 break;
00158 case 5:
00159
00160 ps = PAGE_WIDTH+10;
00161 va &= ~((1<<ps)-1);
00162 break;
00163 case 6:
00164
00165 ps = PAGE_WIDTH+12;
00166 va &= ~((1<<ps)-1);
00167 break;
00168 case 7:
00169 case 8:
00170
00171 ps = PAGE_WIDTH+14;
00172 va &= ~((1<<ps)-1);
00173 break;
00174 default:
00175
00176 ps=PAGE_WIDTH+18;
00177 va&=~((1<<ps)-1);
00178 break;
00179 }
00180
00181 for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
00182 __asm__ volatile (
00183 "ptc.l %0,%1;;"
00184 :
00185 : "r" (va), "r" (ps<<2)
00186 );
00187 }
00188 srlz_d();
00189 srlz_i();
00190
00191 if (restore_rr) {
00192 rr_write(VA2VRN(va), rr.word);
00193 srlz_d();
00194 srlz_i();
00195 }
00196 }
00197
00204 void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
00205 {
00206 tc_mapping_insert(va, asid, entry, true);
00207 }
00208
00215 void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
00216 {
00217 tc_mapping_insert(va, asid, entry, false);
00218 }
00219
00227 void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
00228 {
00229 region_register rr;
00230 bool restore_rr = false;
00231
00232 rr.word = rr_read(VA2VRN(va));
00233 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00234
00235
00236
00237
00238 region_register rr0;
00239
00240 rr0 = rr;
00241 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00242 rr_write(VA2VRN(va), rr0.word);
00243 srlz_d();
00244 srlz_i();
00245 }
00246
00247 __asm__ volatile (
00248 "mov r8=psr;;\n"
00249 "rsm %0;;\n"
00250 "srlz.d;;\n"
00251 "srlz.i;;\n"
00252 "mov cr.ifa=%1\n"
00253 "mov cr.itir=%2;;\n"
00254 "cmp.eq p6,p7 = %4,r0;;\n"
00255 "(p6) itc.i %3;;\n"
00256 "(p7) itc.d %3;;\n"
00257 "mov psr.l=r8;;\n"
00258 "srlz.d;;\n"
00259 :
00260 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
00261 : "p6", "p7", "r8"
00262 );
00263
00264 if (restore_rr) {
00265 rr_write(VA2VRN(va), rr.word);
00266 srlz_d();
00267 srlz_i();
00268 }
00269 }
00270
00278 void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
00279 {
00280 tr_mapping_insert(va, asid, entry, false, tr);
00281 }
00282
00290 void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
00291 {
00292 tr_mapping_insert(va, asid, entry, true, tr);
00293 }
00294
00303 void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
00304 {
00305 region_register rr;
00306 bool restore_rr = false;
00307
00308 rr.word = rr_read(VA2VRN(va));
00309 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00310
00311
00312
00313
00314 region_register rr0;
00315
00316 rr0 = rr;
00317 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00318 rr_write(VA2VRN(va), rr0.word);
00319 srlz_d();
00320 srlz_i();
00321 }
00322
00323 __asm__ volatile (
00324 "mov r8=psr;;\n"
00325 "rsm %0;;\n"
00326 "srlz.d;;\n"
00327 "srlz.i;;\n"
00328 "mov cr.ifa=%1\n"
00329 "mov cr.itir=%2;;\n"
00330 "cmp.eq p6,p7=%5,r0;;\n"
00331 "(p6) itr.i itr[%4]=%3;;\n"
00332 "(p7) itr.d dtr[%4]=%3;;\n"
00333 "mov psr.l=r8;;\n"
00334 "srlz.d;;\n"
00335 :
00336 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
00337 : "p6", "p7", "r8"
00338 );
00339
00340 if (restore_rr) {
00341 rr_write(VA2VRN(va), rr.word);
00342 srlz_d();
00343 srlz_i();
00344 }
00345 }
00346
00354 void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
00355 {
00356 tlb_entry_t entry;
00357
00358 entry.word[0] = 0;
00359 entry.word[1] = 0;
00360
00361 entry.p = true;
00362 entry.ma = MA_WRITEBACK;
00363 entry.a = true;
00364 entry.d = true;
00365 entry.pl = PL_KERNEL;
00366 entry.ar = AR_READ | AR_WRITE;
00367 entry.ppn = frame >> PPN_SHIFT;
00368 entry.ps = PAGE_WIDTH;
00369
00370 if (dtr)
00371 dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
00372 else
00373 dtc_mapping_insert(page, ASID_KERNEL, entry);
00374 }
00375
00383 void dtr_purge(__address page, count_t width)
00384 {
00385 __asm__ volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));
00386 }
00387
00388
00393 void dtc_pte_copy(pte_t *t)
00394 {
00395 tlb_entry_t entry;
00396
00397 entry.word[0] = 0;
00398 entry.word[1] = 0;
00399
00400 entry.p = t->p;
00401 entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
00402 entry.a = t->a;
00403 entry.d = t->d;
00404 entry.pl = t->k ? PL_KERNEL : PL_USER;
00405 entry.ar = t->w ? AR_WRITE : AR_READ;
00406 entry.ppn = t->frame >> PPN_SHIFT;
00407 entry.ps = PAGE_WIDTH;
00408
00409 dtc_mapping_insert(t->page, t->as->asid, entry);
00410 #ifdef CONFIG_VHPT
00411 vhpt_mapping_insert(t->page, t->as->asid, entry);
00412 #endif
00413 }
00414
00419 void itc_pte_copy(pte_t *t)
00420 {
00421 tlb_entry_t entry;
00422
00423 entry.word[0] = 0;
00424 entry.word[1] = 0;
00425
00426 ASSERT(t->x);
00427
00428 entry.p = t->p;
00429 entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
00430 entry.a = t->a;
00431 entry.pl = t->k ? PL_KERNEL : PL_USER;
00432 entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
00433 entry.ppn = t->frame >> PPN_SHIFT;
00434 entry.ps = PAGE_WIDTH;
00435
00436 itc_mapping_insert(t->page, t->as->asid, entry);
00437 #ifdef CONFIG_VHPT
00438 vhpt_mapping_insert(t->page, t->as->asid, entry);
00439 #endif
00440 }
00441
00447 void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
00448 {
00449 region_register rr;
00450 rid_t rid;
00451 __address va;
00452 pte_t *t;
00453
00454 va = istate->cr_ifa;
00455 rr.word = rr_read(VA2VRN(va));
00456 rid = rr.map.rid;
00457
00458 page_table_lock(AS, true);
00459 t = page_mapping_find(AS, va);
00460 if (t) {
00461
00462
00463
00464
00465 itc_pte_copy(t);
00466 page_table_unlock(AS, true);
00467 } else {
00468
00469
00470
00471 page_table_unlock(AS, true);
00472 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
00473 fault_if_from_uspace(istate,"Page fault at %p",va);
00474 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00475 }
00476 }
00477 }
00478
00484 void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
00485 {
00486 region_register rr;
00487 rid_t rid;
00488 __address va;
00489 pte_t *t;
00490
00491 va = istate->cr_ifa;
00492 rr.word = rr_read(VA2VRN(va));
00493 rid = rr.map.rid;
00494 if (RID2ASID(rid) == ASID_KERNEL) {
00495 if (VA2VRN(va) == VRN_KERNEL) {
00496
00497
00498
00499
00500 dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
00501 return;
00502 }
00503 }
00504
00505 page_table_lock(AS, true);
00506 t = page_mapping_find(AS, va);
00507 if (t) {
00508
00509
00510
00511
00512 dtc_pte_copy(t);
00513 page_table_unlock(AS, true);
00514 } else {
00515
00516
00517
00518 page_table_unlock(AS, true);
00519 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00520 fault_if_from_uspace(istate,"Page fault at %p",va);
00521 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00522 }
00523 }
00524 }
00525
00533 void data_nested_tlb_fault(__u64 vector, istate_t *istate)
00534 {
00535 panic("%s\n", __FUNCTION__);
00536 }
00537
00543 void data_dirty_bit_fault(__u64 vector, istate_t *istate)
00544 {
00545 region_register rr;
00546 rid_t rid;
00547 __address va;
00548 pte_t *t;
00549
00550 va = istate->cr_ifa;
00551 rr.word = rr_read(VA2VRN(va));
00552 rid = rr.map.rid;
00553
00554 page_table_lock(AS, true);
00555 t = page_mapping_find(AS, va);
00556 ASSERT(t && t->p);
00557 if (t && t->p && t->w) {
00558
00559
00560
00561
00562 t->d = true;
00563 dtc_pte_copy(t);
00564 } else {
00565 if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
00566 fault_if_from_uspace(istate,"Page fault at %p",va);
00567 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00568 t->d = true;
00569 dtc_pte_copy(t);
00570 }
00571 }
00572 page_table_unlock(AS, true);
00573 }
00574
00580 void instruction_access_bit_fault(__u64 vector, istate_t *istate)
00581 {
00582 region_register rr;
00583 rid_t rid;
00584 __address va;
00585 pte_t *t;
00586
00587 va = istate->cr_ifa;
00588 rr.word = rr_read(VA2VRN(va));
00589 rid = rr.map.rid;
00590
00591 page_table_lock(AS, true);
00592 t = page_mapping_find(AS, va);
00593 ASSERT(t && t->p);
00594 if (t && t->p && t->x) {
00595
00596
00597
00598
00599 t->a = true;
00600 itc_pte_copy(t);
00601 } else {
00602 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
00603 fault_if_from_uspace(istate,"Page fault at %p",va);
00604 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00605 t->a = true;
00606 itc_pte_copy(t);
00607 }
00608 }
00609 page_table_unlock(AS, true);
00610 }
00611
00617 void data_access_bit_fault(__u64 vector, istate_t *istate)
00618 {
00619 region_register rr;
00620 rid_t rid;
00621 __address va;
00622 pte_t *t;
00623
00624 va = istate->cr_ifa;
00625 rr.word = rr_read(VA2VRN(va));
00626 rid = rr.map.rid;
00627
00628 page_table_lock(AS, true);
00629 t = page_mapping_find(AS, va);
00630 ASSERT(t && t->p);
00631 if (t && t->p) {
00632
00633
00634
00635
00636 t->a = true;
00637 dtc_pte_copy(t);
00638 } else {
00639 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00640 fault_if_from_uspace(istate,"Page fault at %p",va);
00641 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00642 t->a = true;
00643 itc_pte_copy(t);
00644 }
00645 }
00646 page_table_unlock(AS, true);
00647 }
00648
00654 void page_not_present(__u64 vector, istate_t *istate)
00655 {
00656 region_register rr;
00657 rid_t rid;
00658 __address va;
00659 pte_t *t;
00660
00661 va = istate->cr_ifa;
00662 rr.word = rr_read(VA2VRN(va));
00663 rid = rr.map.rid;
00664
00665 page_table_lock(AS, true);
00666 t = page_mapping_find(AS, va);
00667 ASSERT(t);
00668
00669 if (t->p) {
00670
00671
00672
00673
00674 if (t->x)
00675 itc_pte_copy(t);
00676 else
00677 dtc_pte_copy(t);
00678 page_table_unlock(AS, true);
00679 } else {
00680 page_table_unlock(AS, true);
00681 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00682 fault_if_from_uspace(istate,"Page fault at %p",va);
00683 panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rid);
00684 }
00685 }
00686 }
00687