tlb.c

Go to the documentation of this file.
00001 /*
00002  * Copyright (C) 2006 Jakub Jermar
00003  * All rights reserved.
00004  *
00005  * Redistribution and use in source and binary forms, with or without
00006  * modification, are permitted provided that the following conditions
00007  * are met:
00008  *
00009  * - Redistributions of source code must retain the above copyright
00010  *   notice, this list of conditions and the following disclaimer.
00011  * - Redistributions in binary form must reproduce the above copyright
00012  *   notice, this list of conditions and the following disclaimer in the
00013  *   documentation and/or other materials provided with the distribution.
00014  * - The name of the author may not be used to endorse or promote products
00015  *   derived from this software without specific prior written permission.
00016  *
00017  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
00018  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00019  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
00020  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
00021  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
00022  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
00023  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
00024  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
00025  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
00026  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00027  */
00028 
00035 /*
00036  * TLB management.
00037  */
00038 
00039 #include <mm/tlb.h>
00040 #include <mm/asid.h>
00041 #include <mm/page.h>
00042 #include <mm/as.h>
00043 #include <arch/mm/tlb.h>
00044 #include <arch/mm/page.h>
00045 #include <arch/mm/vhpt.h>
00046 #include <arch/barrier.h>
00047 #include <arch/interrupt.h>
00048 #include <arch/pal/pal.h>
00049 #include <arch/asm.h>
00050 #include <typedefs.h>
00051 #include <panic.h>
00052 #include <print.h>
00053 #include <arch.h>
00054 #include <interrupt.h>
00055 
00057 void tlb_invalidate_all(void)
00058 {
00059                 ipl_t ipl;
00060                 __address adr;
00061                 __u32 count1, count2, stride1, stride2;
00062                 
00063                 int i,j;
00064                 
00065                 adr = PAL_PTCE_INFO_BASE();
00066                 count1 = PAL_PTCE_INFO_COUNT1();
00067                 count2 = PAL_PTCE_INFO_COUNT2();
00068                 stride1 = PAL_PTCE_INFO_STRIDE1();
00069                 stride2 = PAL_PTCE_INFO_STRIDE2();
00070                 
00071                 ipl = interrupts_disable();
00072 
00073                 for(i = 0; i < count1; i++) {
00074                         for(j = 0; j < count2; j++) {
00075                                 __asm__ volatile (
00076                                         "ptc.e %0 ;;"
00077                                         :
00078                                         : "r" (adr)
00079                                 );
00080                                 adr += stride2;
00081                         }
00082                         adr += stride1;
00083                 }
00084 
00085                 interrupts_restore(ipl);
00086 
00087                 srlz_d();
00088                 srlz_i();
00089 #ifdef CONFIG_VHPT
00090                 vhpt_invalidate_all();
00091 #endif  
00092 }
00093 
00098 void tlb_invalidate_asid(asid_t asid)
00099 {
00100         tlb_invalidate_all();
00101 }
00102 
00103 
00104 void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
00105 {
00106         region_register rr;
00107         bool restore_rr = false;
00108         int b = 0;
00109         int c = cnt;
00110 
00111         __address va;
00112         va = page;
00113 
00114         rr.word = rr_read(VA2VRN(va));
00115         if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00116                 /*
00117                  * The selected region register does not contain required RID.
00118                  * Save the old content of the register and replace the RID.
00119                  */
00120                 region_register rr0;
00121 
00122                 rr0 = rr;
00123                 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00124                 rr_write(VA2VRN(va), rr0.word);
00125                 srlz_d();
00126                 srlz_i();
00127         }
00128         
00129         while(c >>= 1)
00130                 b++;
00131         b >>= 1;
00132         __u64 ps;
00133         
00134         switch (b) {
00135                 case 0: /*cnt 1-3*/
00136                         ps = PAGE_WIDTH;
00137                         break;
00138                 case 1: /*cnt 4-15*/
00139                         /*cnt=((cnt-1)/4)+1;*/
00140                         ps = PAGE_WIDTH+2;
00141                         va &= ~((1<<ps)-1);
00142                         break;
00143                 case 2: /*cnt 16-63*/
00144                         /*cnt=((cnt-1)/16)+1;*/
00145                         ps = PAGE_WIDTH+4;
00146                         va &= ~((1<<ps)-1);
00147                         break;
00148                 case 3: /*cnt 64-255*/
00149                         /*cnt=((cnt-1)/64)+1;*/
00150                         ps = PAGE_WIDTH+6;
00151                         va &= ~((1<<ps)-1);
00152                         break;
00153                 case 4: /*cnt 256-1023*/
00154                         /*cnt=((cnt-1)/256)+1;*/
00155                         ps = PAGE_WIDTH+8;
00156                         va &= ~((1<<ps)-1);
00157                         break;
00158                 case 5: /*cnt 1024-4095*/
00159                         /*cnt=((cnt-1)/1024)+1;*/
00160                         ps = PAGE_WIDTH+10;
00161                         va &= ~((1<<ps)-1);
00162                         break;
00163                 case 6: /*cnt 4096-16383*/
00164                         /*cnt=((cnt-1)/4096)+1;*/
00165                         ps = PAGE_WIDTH+12;
00166                         va &= ~((1<<ps)-1);
00167                         break;
00168                 case 7: /*cnt 16384-65535*/
00169                 case 8: /*cnt 65536-(256K-1)*/
00170                         /*cnt=((cnt-1)/16384)+1;*/
00171                         ps = PAGE_WIDTH+14;
00172                         va &= ~((1<<ps)-1);
00173                         break;
00174                 default:
00175                         /*cnt=((cnt-1)/(16384*16))+1;*/
00176                         ps=PAGE_WIDTH+18;
00177                         va&=~((1<<ps)-1);
00178                         break;
00179         }
00180         /*cnt+=(page!=va);*/
00181         for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
00182                 __asm__ volatile (
00183                         "ptc.l %0,%1;;"
00184                         :
00185                         : "r" (va), "r" (ps<<2)
00186                 );
00187         }
00188         srlz_d();
00189         srlz_i();
00190         
00191         if (restore_rr) {
00192                 rr_write(VA2VRN(va), rr.word);
00193                 srlz_d();
00194                 srlz_i();
00195         }
00196 }
00197 
00204 void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
00205 {
00206         tc_mapping_insert(va, asid, entry, true);
00207 }
00208 
00215 void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
00216 {
00217         tc_mapping_insert(va, asid, entry, false);
00218 }
00219 
00227 void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
00228 {
00229         region_register rr;
00230         bool restore_rr = false;
00231 
00232         rr.word = rr_read(VA2VRN(va));
00233         if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00234                 /*
00235                  * The selected region register does not contain required RID.
00236                  * Save the old content of the register and replace the RID.
00237                  */
00238                 region_register rr0;
00239 
00240                 rr0 = rr;
00241                 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00242                 rr_write(VA2VRN(va), rr0.word);
00243                 srlz_d();
00244                 srlz_i();
00245         }
00246         
00247         __asm__ volatile (
00248                 "mov r8=psr;;\n"
00249                 "rsm %0;;\n"                    /* PSR_IC_MASK */
00250                 "srlz.d;;\n"
00251                 "srlz.i;;\n"
00252                 "mov cr.ifa=%1\n"               /* va */
00253                 "mov cr.itir=%2;;\n"            /* entry.word[1] */
00254                 "cmp.eq p6,p7 = %4,r0;;\n"      /* decide between itc and dtc */ 
00255                 "(p6) itc.i %3;;\n"
00256                 "(p7) itc.d %3;;\n"
00257                 "mov psr.l=r8;;\n"
00258                 "srlz.d;;\n"
00259                 :
00260                 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
00261                 : "p6", "p7", "r8"
00262         );
00263         
00264         if (restore_rr) {
00265                 rr_write(VA2VRN(va), rr.word);
00266                 srlz_d();
00267                 srlz_i();
00268         }
00269 }
00270 
00278 void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
00279 {
00280         tr_mapping_insert(va, asid, entry, false, tr);
00281 }
00282 
00290 void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
00291 {
00292         tr_mapping_insert(va, asid, entry, true, tr);
00293 }
00294 
00303 void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
00304 {
00305         region_register rr;
00306         bool restore_rr = false;
00307 
00308         rr.word = rr_read(VA2VRN(va));
00309         if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
00310                 /*
00311                  * The selected region register does not contain required RID.
00312                  * Save the old content of the register and replace the RID.
00313                  */
00314                 region_register rr0;
00315 
00316                 rr0 = rr;
00317                 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
00318                 rr_write(VA2VRN(va), rr0.word);
00319                 srlz_d();
00320                 srlz_i();
00321         }
00322 
00323         __asm__ volatile (
00324                 "mov r8=psr;;\n"
00325                 "rsm %0;;\n"                    /* PSR_IC_MASK */
00326                 "srlz.d;;\n"
00327                 "srlz.i;;\n"
00328                 "mov cr.ifa=%1\n"               /* va */                 
00329                 "mov cr.itir=%2;;\n"            /* entry.word[1] */ 
00330                 "cmp.eq p6,p7=%5,r0;;\n"        /* decide between itr and dtr */
00331                 "(p6) itr.i itr[%4]=%3;;\n"
00332                 "(p7) itr.d dtr[%4]=%3;;\n"
00333                 "mov psr.l=r8;;\n"
00334                 "srlz.d;;\n"
00335                 :
00336                 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
00337                 : "p6", "p7", "r8"
00338         );
00339         
00340         if (restore_rr) {
00341                 rr_write(VA2VRN(va), rr.word);
00342                 srlz_d();
00343                 srlz_i();
00344         }
00345 }
00346 
00354 void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
00355 {
00356         tlb_entry_t entry;
00357         
00358         entry.word[0] = 0;
00359         entry.word[1] = 0;
00360         
00361         entry.p = true;                 /* present */
00362         entry.ma = MA_WRITEBACK;
00363         entry.a = true;                 /* already accessed */
00364         entry.d = true;                 /* already dirty */
00365         entry.pl = PL_KERNEL;
00366         entry.ar = AR_READ | AR_WRITE;
00367         entry.ppn = frame >> PPN_SHIFT;
00368         entry.ps = PAGE_WIDTH;
00369         
00370         if (dtr)
00371                 dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
00372         else
00373                 dtc_mapping_insert(page, ASID_KERNEL, entry);
00374 }
00375 
00383 void dtr_purge(__address page, count_t width)
00384 {
00385         __asm__ volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));
00386 }
00387 
00388 
00393 void dtc_pte_copy(pte_t *t)
00394 {
00395         tlb_entry_t entry;
00396 
00397         entry.word[0] = 0;
00398         entry.word[1] = 0;
00399         
00400         entry.p = t->p;
00401         entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
00402         entry.a = t->a;
00403         entry.d = t->d;
00404         entry.pl = t->k ? PL_KERNEL : PL_USER;
00405         entry.ar = t->w ? AR_WRITE : AR_READ;
00406         entry.ppn = t->frame >> PPN_SHIFT;
00407         entry.ps = PAGE_WIDTH;
00408         
00409         dtc_mapping_insert(t->page, t->as->asid, entry);
00410 #ifdef CONFIG_VHPT
00411         vhpt_mapping_insert(t->page, t->as->asid, entry);
00412 #endif  
00413 }
00414 
00419 void itc_pte_copy(pte_t *t)
00420 {
00421         tlb_entry_t entry;
00422 
00423         entry.word[0] = 0;
00424         entry.word[1] = 0;
00425         
00426         ASSERT(t->x);
00427         
00428         entry.p = t->p;
00429         entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
00430         entry.a = t->a;
00431         entry.pl = t->k ? PL_KERNEL : PL_USER;
00432         entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
00433         entry.ppn = t->frame >> PPN_SHIFT;
00434         entry.ps = PAGE_WIDTH;
00435         
00436         itc_mapping_insert(t->page, t->as->asid, entry);
00437 #ifdef CONFIG_VHPT
00438         vhpt_mapping_insert(t->page, t->as->asid, entry);
00439 #endif  
00440 }
00441 
00447 void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
00448 {
00449         region_register rr;
00450         rid_t rid;
00451         __address va;
00452         pte_t *t;
00453         
00454         va = istate->cr_ifa;    /* faulting address */
00455         rr.word = rr_read(VA2VRN(va));
00456         rid = rr.map.rid;
00457 
00458         page_table_lock(AS, true);
00459         t = page_mapping_find(AS, va);
00460         if (t) {
00461                 /*
00462                  * The mapping was found in software page hash table.
00463                  * Insert it into data translation cache.
00464                  */
00465                 itc_pte_copy(t);
00466                 page_table_unlock(AS, true);
00467         } else {
00468                 /*
00469                  * Forward the page fault to address space page fault handler.
00470                  */
00471                 page_table_unlock(AS, true);
00472                 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
00473                         fault_if_from_uspace(istate,"Page fault at %p",va);
00474                         panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00475                 }
00476         }
00477 }
00478 
00484 void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
00485 {
00486         region_register rr;
00487         rid_t rid;
00488         __address va;
00489         pte_t *t;
00490         
00491         va = istate->cr_ifa;    /* faulting address */
00492         rr.word = rr_read(VA2VRN(va));
00493         rid = rr.map.rid;
00494         if (RID2ASID(rid) == ASID_KERNEL) {
00495                 if (VA2VRN(va) == VRN_KERNEL) {
00496                         /*
00497                          * Provide KA2PA(identity) mapping for faulting piece of
00498                          * kernel address space.
00499                          */
00500                         dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
00501                         return;
00502                 }
00503         }
00504 
00505         page_table_lock(AS, true);
00506         t = page_mapping_find(AS, va);
00507         if (t) {
00508                 /*
00509                  * The mapping was found in software page hash table.
00510                  * Insert it into data translation cache.
00511                  */
00512                 dtc_pte_copy(t);
00513                 page_table_unlock(AS, true);
00514         } else {
00515                 /*
00516                  * Forward the page fault to address space page fault handler.
00517                  */
00518                 page_table_unlock(AS, true);
00519                 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00520                         fault_if_from_uspace(istate,"Page fault at %p",va);
00521                         panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00522                 }
00523         }
00524 }
00525 
00533 void data_nested_tlb_fault(__u64 vector, istate_t *istate)
00534 {
00535         panic("%s\n", __FUNCTION__);
00536 }
00537 
00543 void data_dirty_bit_fault(__u64 vector, istate_t *istate)
00544 {
00545         region_register rr;
00546         rid_t rid;
00547         __address va;
00548         pte_t *t;
00549         
00550         va = istate->cr_ifa;    /* faulting address */
00551         rr.word = rr_read(VA2VRN(va));
00552         rid = rr.map.rid;
00553 
00554         page_table_lock(AS, true);
00555         t = page_mapping_find(AS, va);
00556         ASSERT(t && t->p);
00557         if (t && t->p && t->w) {
00558                 /*
00559                  * Update the Dirty bit in page tables and reinsert
00560                  * the mapping into DTC.
00561                  */
00562                 t->d = true;
00563                 dtc_pte_copy(t);
00564         } else {
00565                 if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
00566                         fault_if_from_uspace(istate,"Page fault at %p",va);
00567                         panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00568                         t->d = true;
00569                         dtc_pte_copy(t);
00570                 }
00571         }
00572         page_table_unlock(AS, true);
00573 }
00574 
00580 void instruction_access_bit_fault(__u64 vector, istate_t *istate)
00581 {
00582         region_register rr;
00583         rid_t rid;
00584         __address va;
00585         pte_t *t;       
00586 
00587         va = istate->cr_ifa;    /* faulting address */
00588         rr.word = rr_read(VA2VRN(va));
00589         rid = rr.map.rid;
00590 
00591         page_table_lock(AS, true);
00592         t = page_mapping_find(AS, va);
00593         ASSERT(t && t->p);
00594         if (t && t->p && t->x) {
00595                 /*
00596                  * Update the Accessed bit in page tables and reinsert
00597                  * the mapping into ITC.
00598                  */
00599                 t->a = true;
00600                 itc_pte_copy(t);
00601         } else {
00602                 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
00603                         fault_if_from_uspace(istate,"Page fault at %p",va);
00604                         panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00605                         t->a = true;
00606                         itc_pte_copy(t);
00607                 }
00608         }
00609         page_table_unlock(AS, true);
00610 }
00611 
00617 void data_access_bit_fault(__u64 vector, istate_t *istate)
00618 {
00619         region_register rr;
00620         rid_t rid;
00621         __address va;
00622         pte_t *t;
00623 
00624         va = istate->cr_ifa;    /* faulting address */
00625         rr.word = rr_read(VA2VRN(va));
00626         rid = rr.map.rid;
00627 
00628         page_table_lock(AS, true);
00629         t = page_mapping_find(AS, va);
00630         ASSERT(t && t->p);
00631         if (t && t->p) {
00632                 /*
00633                  * Update the Accessed bit in page tables and reinsert
00634                  * the mapping into DTC.
00635                  */
00636                 t->a = true;
00637                 dtc_pte_copy(t);
00638         } else {
00639                 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00640                         fault_if_from_uspace(istate,"Page fault at %p",va);
00641                         panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
00642                         t->a = true;
00643                         itc_pte_copy(t);
00644                 }
00645         }
00646         page_table_unlock(AS, true);
00647 }
00648 
00654 void page_not_present(__u64 vector, istate_t *istate)
00655 {
00656         region_register rr;
00657         rid_t rid;
00658         __address va;
00659         pte_t *t;
00660         
00661         va = istate->cr_ifa;    /* faulting address */
00662         rr.word = rr_read(VA2VRN(va));
00663         rid = rr.map.rid;
00664 
00665         page_table_lock(AS, true);
00666         t = page_mapping_find(AS, va);
00667         ASSERT(t);
00668         
00669         if (t->p) {
00670                 /*
00671                  * If the Present bit is set in page hash table, just copy it
00672                  * and update ITC/DTC.
00673                  */
00674                 if (t->x)
00675                         itc_pte_copy(t);
00676                 else
00677                         dtc_pte_copy(t);
00678                 page_table_unlock(AS, true);
00679         } else {
00680                 page_table_unlock(AS, true);
00681                 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
00682                         fault_if_from_uspace(istate,"Page fault at %p",va);
00683                         panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rid);
00684                 }
00685         }
00686 }
00687 

Generated on Sun Jun 18 16:51:21 2006 for HelenOS Kernel (ia64) by  doxygen 1.4.6