source: mainline/arch/ia64/src/mm/tlb.c@ 208259c

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 208259c was 208259c, checked in by Jakub Jermar <jakub@…>, 19 years ago

On ia64, purge DTR entry before overwriting it with new contents.
  • Property mode set to 100644
File size: 16.5 KB
Line 
1/*
2 * Copyright (C) 2006 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * TLB management.
31 */
32
33#include <mm/tlb.h>
34#include <mm/asid.h>
35#include <mm/page.h>
36#include <mm/as.h>
37#include <arch/mm/tlb.h>
38#include <arch/mm/page.h>
39#include <arch/mm/vhpt.h>
40#include <arch/barrier.h>
41#include <arch/interrupt.h>
42#include <arch/pal/pal.h>
43#include <arch/asm.h>
44#include <typedefs.h>
45#include <panic.h>
46#include <print.h>
47#include <arch.h>
48#include <interrupt.h>
49
50/** Invalidate all TLB entries. */
51void tlb_invalidate_all(void)
52{
53 ipl_t ipl;
54 __address adr;
55 __u32 count1, count2, stride1, stride2;
56
57 int i,j;
58
59 adr = PAL_PTCE_INFO_BASE();
60 count1 = PAL_PTCE_INFO_COUNT1();
61 count2 = PAL_PTCE_INFO_COUNT2();
62 stride1 = PAL_PTCE_INFO_STRIDE1();
63 stride2 = PAL_PTCE_INFO_STRIDE2();
64
65 ipl = interrupts_disable();
66
67 for(i = 0; i < count1; i++) {
68 for(j = 0; j < count2; j++) {
69 __asm__ volatile (
70 "ptc.e %0 ;;"
71 :
72 : "r" (adr)
73 );
74 adr += stride2;
75 }
76 adr += stride1;
77 }
78
79 interrupts_restore(ipl);
80
81 srlz_d();
82 srlz_i();
83#ifdef CONFIG_VHPT
84 vhpt_invalidate_all();
85#endif
86}
87
88/** Invalidate entries belonging to an address space.
89 *
90 * @param asid Address space identifier.
91 */
92void tlb_invalidate_asid(asid_t asid)
93{
94 tlb_invalidate_all();
95}
96
97
98void tlb_invalidate_pages(asid_t asid, __address page, count_t cnt)
99{
100 region_register rr;
101 bool restore_rr = false;
102 int b = 0;
103 int c = cnt;
104
105 __address va;
106 va = page;
107
108 rr.word = rr_read(VA2VRN(va));
109 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
110 /*
111 * The selected region register does not contain required RID.
112 * Save the old content of the register and replace the RID.
113 */
114 region_register rr0;
115
116 rr0 = rr;
117 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
118 rr_write(VA2VRN(va), rr0.word);
119 srlz_d();
120 srlz_i();
121 }
122
123 while(c >>= 1)
124 b++;
125 b >>= 1;
126 __u64 ps;
127
128 switch (b) {
129 case 0: /*cnt 1-3*/
130 ps = PAGE_WIDTH;
131 break;
132 case 1: /*cnt 4-15*/
133 /*cnt=((cnt-1)/4)+1;*/
134 ps = PAGE_WIDTH+2;
135 va &= ~((1<<ps)-1);
136 break;
137 case 2: /*cnt 16-63*/
138 /*cnt=((cnt-1)/16)+1;*/
139 ps = PAGE_WIDTH+4;
140 va &= ~((1<<ps)-1);
141 break;
142 case 3: /*cnt 64-255*/
143 /*cnt=((cnt-1)/64)+1;*/
144 ps = PAGE_WIDTH+6;
145 va &= ~((1<<ps)-1);
146 break;
147 case 4: /*cnt 256-1023*/
148 /*cnt=((cnt-1)/256)+1;*/
149 ps = PAGE_WIDTH+8;
150 va &= ~((1<<ps)-1);
151 break;
152 case 5: /*cnt 1024-4095*/
153 /*cnt=((cnt-1)/1024)+1;*/
154 ps = PAGE_WIDTH+10;
155 va &= ~((1<<ps)-1);
156 break;
157 case 6: /*cnt 4096-16383*/
158 /*cnt=((cnt-1)/4096)+1;*/
159 ps = PAGE_WIDTH+12;
160 va &= ~((1<<ps)-1);
161 break;
162 case 7: /*cnt 16384-65535*/
163 case 8: /*cnt 65536-(256K-1)*/
164 /*cnt=((cnt-1)/16384)+1;*/
165 ps = PAGE_WIDTH+14;
166 va &= ~((1<<ps)-1);
167 break;
168 default:
169 /*cnt=((cnt-1)/(16384*16))+1;*/
170 ps=PAGE_WIDTH+18;
171 va&=~((1<<ps)-1);
172 break;
173 }
174 /*cnt+=(page!=va);*/
175 for(; va<(page+cnt*(PAGE_SIZE)); va += (1<<ps)) {
176 __asm__ volatile (
177 "ptc.l %0,%1;;"
178 :
179 : "r" (va), "r" (ps<<2)
180 );
181 }
182 srlz_d();
183 srlz_i();
184
185 if (restore_rr) {
186 rr_write(VA2VRN(va), rr.word);
187 srlz_d();
188 srlz_i();
189 }
190}
191
192/** Insert data into data translation cache.
193 *
194 * @param va Virtual page address.
195 * @param asid Address space identifier.
196 * @param entry The rest of TLB entry as required by TLB insertion format.
197 */
198void dtc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
199{
200 tc_mapping_insert(va, asid, entry, true);
201}
202
203/** Insert data into instruction translation cache.
204 *
205 * @param va Virtual page address.
206 * @param asid Address space identifier.
207 * @param entry The rest of TLB entry as required by TLB insertion format.
208 */
209void itc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry)
210{
211 tc_mapping_insert(va, asid, entry, false);
212}
213
214/** Insert data into instruction or data translation cache.
215 *
216 * @param va Virtual page address.
217 * @param asid Address space identifier.
218 * @param entry The rest of TLB entry as required by TLB insertion format.
219 * @param dtc If true, insert into data translation cache, use instruction translation cache otherwise.
220 */
221void tc_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtc)
222{
223 region_register rr;
224 bool restore_rr = false;
225
226 rr.word = rr_read(VA2VRN(va));
227 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
228 /*
229 * The selected region register does not contain required RID.
230 * Save the old content of the register and replace the RID.
231 */
232 region_register rr0;
233
234 rr0 = rr;
235 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
236 rr_write(VA2VRN(va), rr0.word);
237 srlz_d();
238 srlz_i();
239 }
240
241 __asm__ volatile (
242 "mov r8=psr;;\n"
243 "rsm %0;;\n" /* PSR_IC_MASK */
244 "srlz.d;;\n"
245 "srlz.i;;\n"
246 "mov cr.ifa=%1\n" /* va */
247 "mov cr.itir=%2;;\n" /* entry.word[1] */
248 "cmp.eq p6,p7 = %4,r0;;\n" /* decide between itc and dtc */
249 "(p6) itc.i %3;;\n"
250 "(p7) itc.d %3;;\n"
251 "mov psr.l=r8;;\n"
252 "srlz.d;;\n"
253 :
254 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (dtc)
255 : "p6", "p7", "r8"
256 );
257
258 if (restore_rr) {
259 rr_write(VA2VRN(va), rr.word);
260 srlz_d();
261 srlz_i();
262 }
263}
264
265/** Insert data into instruction translation register.
266 *
267 * @param va Virtual page address.
268 * @param asid Address space identifier.
269 * @param entry The rest of TLB entry as required by TLB insertion format.
270 * @param tr Translation register.
271 */
272void itr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
273{
274 tr_mapping_insert(va, asid, entry, false, tr);
275}
276
277/** Insert data into data translation register.
278 *
279 * @param va Virtual page address.
280 * @param asid Address space identifier.
281 * @param entry The rest of TLB entry as required by TLB insertion format.
282 * @param tr Translation register.
283 */
284void dtr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, index_t tr)
285{
286 tr_mapping_insert(va, asid, entry, true, tr);
287}
288
289/** Insert data into instruction or data translation register.
290 *
291 * @param va Virtual page address.
292 * @param asid Address space identifier.
293 * @param entry The rest of TLB entry as required by TLB insertion format.
294 * @param dtc If true, insert into data translation register, use instruction translation register otherwise.
295 * @param tr Translation register.
296 */
297void tr_mapping_insert(__address va, asid_t asid, tlb_entry_t entry, bool dtr, index_t tr)
298{
299 region_register rr;
300 bool restore_rr = false;
301
302 rr.word = rr_read(VA2VRN(va));
303 if ((restore_rr = (rr.map.rid != ASID2RID(asid, VA2VRN(va))))) {
304 /*
305 * The selected region register does not contain required RID.
306 * Save the old content of the register and replace the RID.
307 */
308 region_register rr0;
309
310 rr0 = rr;
311 rr0.map.rid = ASID2RID(asid, VA2VRN(va));
312 rr_write(VA2VRN(va), rr0.word);
313 srlz_d();
314 srlz_i();
315 }
316
317 __asm__ volatile (
318 "mov r8=psr;;\n"
319 "rsm %0;;\n" /* PSR_IC_MASK */
320 "srlz.d;;\n"
321 "srlz.i;;\n"
322 "mov cr.ifa=%1\n" /* va */
323 "mov cr.itir=%2;;\n" /* entry.word[1] */
324 "cmp.eq p6,p7=%5,r0;;\n" /* decide between itr and dtr */
325 "(p6) itr.i itr[%4]=%3;;\n"
326 "(p7) itr.d dtr[%4]=%3;;\n"
327 "mov psr.l=r8;;\n"
328 "srlz.d;;\n"
329 :
330 : "i" (PSR_IC_MASK), "r" (va), "r" (entry.word[1]), "r" (entry.word[0]), "r" (tr), "r" (dtr)
331 : "p6", "p7", "r8"
332 );
333
334 if (restore_rr) {
335 rr_write(VA2VRN(va), rr.word);
336 srlz_d();
337 srlz_i();
338 }
339}
340
341/** Insert data into DTLB.
342 *
343 * @param page Virtual page address including VRN bits.
344 * @param frame Physical frame address.
345 * @param dtr If true, insert into data translation register, use data translation cache otherwise.
346 * @param tr Translation register if dtr is true, ignored otherwise.
347 */
348void dtlb_kernel_mapping_insert(__address page, __address frame, bool dtr, index_t tr)
349{
350 tlb_entry_t entry;
351
352 entry.word[0] = 0;
353 entry.word[1] = 0;
354
355 entry.p = true; /* present */
356 entry.ma = MA_WRITEBACK;
357 entry.a = true; /* already accessed */
358 entry.d = true; /* already dirty */
359 entry.pl = PL_KERNEL;
360 entry.ar = AR_READ | AR_WRITE;
361 entry.ppn = frame >> PPN_SHIFT;
362 entry.ps = PAGE_WIDTH;
363
364 if (dtr)
365 dtr_mapping_insert(page, ASID_KERNEL, entry, tr);
366 else
367 dtc_mapping_insert(page, ASID_KERNEL, entry);
368}
369
370/** Purge kernel entries from DTR.
371 *
372 * Purge DTR entries used by the kernel.
373 *
374 * @param page Virtual page address including VRN bits.
375 * @param width Width of the purge in bits.
376 */
377void dtr_purge(__address page, count_t width)
378{
379 __asm__ volatile ("ptr.d %0, %1\n" : : "r" (page), "r" (width<<2));
380}
381
382
383/** Copy content of PTE into data translation cache.
384 *
385 * @param t PTE.
386 */
387void dtc_pte_copy(pte_t *t)
388{
389 tlb_entry_t entry;
390
391 entry.word[0] = 0;
392 entry.word[1] = 0;
393
394 entry.p = t->p;
395 entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
396 entry.a = t->a;
397 entry.d = t->d;
398 entry.pl = t->k ? PL_KERNEL : PL_USER;
399 entry.ar = t->w ? AR_WRITE : AR_READ;
400 entry.ppn = t->frame >> PPN_SHIFT;
401 entry.ps = PAGE_WIDTH;
402
403 dtc_mapping_insert(t->page, t->as->asid, entry);
404#ifdef CONFIG_VHPT
405 vhpt_mapping_insert(t->page, t->as->asid, entry);
406#endif
407}
408
409/** Copy content of PTE into instruction translation cache.
410 *
411 * @param t PTE.
412 */
413void itc_pte_copy(pte_t *t)
414{
415 tlb_entry_t entry;
416
417 entry.word[0] = 0;
418 entry.word[1] = 0;
419
420 ASSERT(t->x);
421
422 entry.p = t->p;
423 entry.ma = t->c ? MA_WRITEBACK : MA_UNCACHEABLE;
424 entry.a = t->a;
425 entry.pl = t->k ? PL_KERNEL : PL_USER;
426 entry.ar = t->x ? (AR_EXECUTE | AR_READ) : AR_READ;
427 entry.ppn = t->frame >> PPN_SHIFT;
428 entry.ps = PAGE_WIDTH;
429
430 itc_mapping_insert(t->page, t->as->asid, entry);
431#ifdef CONFIG_VHPT
432 vhpt_mapping_insert(t->page, t->as->asid, entry);
433#endif
434}
435
436/** Instruction TLB fault handler for faults with VHPT turned off.
437 *
438 * @param vector Interruption vector.
439 * @param istate Structure with saved interruption state.
440 */
441void alternate_instruction_tlb_fault(__u64 vector, istate_t *istate)
442{
443 region_register rr;
444 rid_t rid;
445 __address va;
446 pte_t *t;
447
448 va = istate->cr_ifa; /* faulting address */
449 rr.word = rr_read(VA2VRN(va));
450 rid = rr.map.rid;
451
452 page_table_lock(AS, true);
453 t = page_mapping_find(AS, va);
454 if (t) {
455 /*
456 * The mapping was found in software page hash table.
457 * Insert it into data translation cache.
458 */
459 itc_pte_copy(t);
460 page_table_unlock(AS, true);
461 } else {
462 /*
463 * Forward the page fault to address space page fault handler.
464 */
465 page_table_unlock(AS, true);
466 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
467 fault_if_from_uspace(istate,"Page fault at %P",va);
468 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
469 }
470 }
471}
472
473/** Data TLB fault handler for faults with VHPT turned off.
474 *
475 * @param vector Interruption vector.
476 * @param istate Structure with saved interruption state.
477 */
478void alternate_data_tlb_fault(__u64 vector, istate_t *istate)
479{
480 region_register rr;
481 rid_t rid;
482 __address va;
483 pte_t *t;
484
485 va = istate->cr_ifa; /* faulting address */
486 rr.word = rr_read(VA2VRN(va));
487 rid = rr.map.rid;
488 if (RID2ASID(rid) == ASID_KERNEL) {
489 if (VA2VRN(va) == VRN_KERNEL) {
490 /*
491 * Provide KA2PA(identity) mapping for faulting piece of
492 * kernel address space.
493 */
494 dtlb_kernel_mapping_insert(va, KA2PA(va), false, 0);
495 return;
496 }
497 }
498
499 page_table_lock(AS, true);
500 t = page_mapping_find(AS, va);
501 if (t) {
502 /*
503 * The mapping was found in software page hash table.
504 * Insert it into data translation cache.
505 */
506 dtc_pte_copy(t);
507 page_table_unlock(AS, true);
508 } else {
509 /*
510 * Forward the page fault to address space page fault handler.
511 */
512 page_table_unlock(AS, true);
513 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
514 fault_if_from_uspace(istate,"Page fault at %P",va);
515 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
516 }
517 }
518}
519
520/** Data nested TLB fault handler.
521 *
522 * This fault should not occur.
523 *
524 * @param vector Interruption vector.
525 * @param istate Structure with saved interruption state.
526 */
527void data_nested_tlb_fault(__u64 vector, istate_t *istate)
528{
529 panic("%s\n", __FUNCTION__);
530}
531
532/** Data Dirty bit fault handler.
533 *
534 * @param vector Interruption vector.
535 * @param istate Structure with saved interruption state.
536 */
537void data_dirty_bit_fault(__u64 vector, istate_t *istate)
538{
539 region_register rr;
540 rid_t rid;
541 __address va;
542 pte_t *t;
543
544 va = istate->cr_ifa; /* faulting address */
545 rr.word = rr_read(VA2VRN(va));
546 rid = rr.map.rid;
547
548 page_table_lock(AS, true);
549 t = page_mapping_find(AS, va);
550 ASSERT(t && t->p);
551 if (t && t->p && t->w) {
552 /*
553 * Update the Dirty bit in page tables and reinsert
554 * the mapping into DTC.
555 */
556 t->d = true;
557 dtc_pte_copy(t);
558 } else {
559 if (as_page_fault(va, PF_ACCESS_WRITE, istate) == AS_PF_FAULT) {
560 fault_if_from_uspace(istate,"Page fault at %P",va);
561 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
562 t->d = true;
563 dtc_pte_copy(t);
564 }
565 }
566 page_table_unlock(AS, true);
567}
568
569/** Instruction access bit fault handler.
570 *
571 * @param vector Interruption vector.
572 * @param istate Structure with saved interruption state.
573 */
574void instruction_access_bit_fault(__u64 vector, istate_t *istate)
575{
576 region_register rr;
577 rid_t rid;
578 __address va;
579 pte_t *t;
580
581 va = istate->cr_ifa; /* faulting address */
582 rr.word = rr_read(VA2VRN(va));
583 rid = rr.map.rid;
584
585 page_table_lock(AS, true);
586 t = page_mapping_find(AS, va);
587 ASSERT(t && t->p);
588 if (t && t->p && t->x) {
589 /*
590 * Update the Accessed bit in page tables and reinsert
591 * the mapping into ITC.
592 */
593 t->a = true;
594 itc_pte_copy(t);
595 } else {
596 if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) {
597 fault_if_from_uspace(istate,"Page fault at %P",va);
598 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
599 t->a = true;
600 itc_pte_copy(t);
601 }
602 }
603 page_table_unlock(AS, true);
604}
605
606/** Data access bit fault handler.
607 *
608 * @param vector Interruption vector.
609 * @param istate Structure with saved interruption state.
610 */
611void data_access_bit_fault(__u64 vector, istate_t *istate)
612{
613 region_register rr;
614 rid_t rid;
615 __address va;
616 pte_t *t;
617
618 va = istate->cr_ifa; /* faulting address */
619 rr.word = rr_read(VA2VRN(va));
620 rid = rr.map.rid;
621
622 page_table_lock(AS, true);
623 t = page_mapping_find(AS, va);
624 ASSERT(t && t->p);
625 if (t && t->p) {
626 /*
627 * Update the Accessed bit in page tables and reinsert
628 * the mapping into DTC.
629 */
630 t->a = true;
631 dtc_pte_copy(t);
632 } else {
633 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
634 fault_if_from_uspace(istate,"Page fault at %P",va);
635 panic("%s: va=%p, rid=%d, iip=%p\n", __FUNCTION__, va, rid, istate->cr_iip);
636 t->a = true;
637 itc_pte_copy(t);
638 }
639 }
640 page_table_unlock(AS, true);
641}
642
643/** Page not present fault handler.
644 *
645 * @param vector Interruption vector.
646 * @param istate Structure with saved interruption state.
647 */
648void page_not_present(__u64 vector, istate_t *istate)
649{
650 region_register rr;
651 rid_t rid;
652 __address va;
653 pte_t *t;
654
655 va = istate->cr_ifa; /* faulting address */
656 rr.word = rr_read(VA2VRN(va));
657 rid = rr.map.rid;
658
659 page_table_lock(AS, true);
660 t = page_mapping_find(AS, va);
661 ASSERT(t);
662
663 if (t->p) {
664 /*
665 * If the Present bit is set in page hash table, just copy it
666 * and update ITC/DTC.
667 */
668 if (t->x)
669 itc_pte_copy(t);
670 else
671 dtc_pte_copy(t);
672 page_table_unlock(AS, true);
673 } else {
674 page_table_unlock(AS, true);
675 if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) {
676 fault_if_from_uspace(istate,"Page fault at %P",va);
677 panic("%s: va=%p, rid=%d\n", __FUNCTION__, va, rid);
678 }
679 }
680}
Note: See TracBrowser for help on using the repository browser.