source: mainline/kernel/arch/sparc64/src/mm/sun4u/tlb.c@ d99c1d2

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since d99c1d2 was d99c1d2, checked in by Martin Decky <martin@…>, 15 years ago

use [u]int{8|16|32|64}_t type definitions as detected by the autotool
replace direct usage of arch/types.h with typedefs.h
  • Property mode set to 100644
File size: 15.8 KB
Line 
1/*
2 * Copyright (c) 2005 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup sparc64mm
30 * @{
31 */
32/** @file
33 */
34
35#include <arch/mm/tlb.h>
36#include <mm/tlb.h>
37#include <mm/as.h>
38#include <mm/asid.h>
39#include <arch/mm/frame.h>
40#include <arch/mm/page.h>
41#include <arch/mm/mmu.h>
42#include <arch/interrupt.h>
43#include <interrupt.h>
44#include <arch.h>
45#include <print.h>
46#include <typedefs.h>
47#include <config.h>
48#include <arch/trap/trap.h>
49#include <arch/trap/exception.h>
50#include <panic.h>
51#include <arch/asm.h>
52#include <genarch/mm/page_ht.h>
53
54#ifdef CONFIG_TSB
55#include <arch/mm/tsb.h>
56#endif
57
58static void dtlb_pte_copy(pte_t *, size_t, bool);
59static void itlb_pte_copy(pte_t *, size_t);
60static void do_fast_instruction_access_mmu_miss_fault(istate_t *, const char *);
61static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t,
62 const char *);
63static void do_fast_data_access_protection_fault(istate_t *,
64 tlb_tag_access_reg_t, const char *);
65
66const char *context_encoding[] = {
67 "Primary",
68 "Secondary",
69 "Nucleus",
70 "Reserved"
71};
72
73void tlb_arch_init(void)
74{
75 /*
76 * Invalidate all non-locked DTLB and ITLB entries.
77 */
78 tlb_invalidate_all();
79
80 /*
81 * Clear both SFSRs.
82 */
83 dtlb_sfsr_write(0);
84 itlb_sfsr_write(0);
85}
86
87/** Insert privileged mapping into DMMU TLB.
88 *
89 * @param page Virtual page address.
90 * @param frame Physical frame address.
91 * @param pagesize Page size.
92 * @param locked True for permanent mappings, false otherwise.
93 * @param cacheable True if the mapping is cacheable, false otherwise.
94 */
95void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
96 bool locked, bool cacheable)
97{
98 tlb_tag_access_reg_t tag;
99 tlb_data_t data;
100 page_address_t pg;
101 frame_address_t fr;
102
103 pg.address = page;
104 fr.address = frame;
105
106 tag.context = ASID_KERNEL;
107 tag.vpn = pg.vpn;
108
109 dtlb_tag_access_write(tag.value);
110
111 data.value = 0;
112 data.v = true;
113 data.size = pagesize;
114 data.pfn = fr.pfn;
115 data.l = locked;
116 data.cp = cacheable;
117#ifdef CONFIG_VIRT_IDX_DCACHE
118 data.cv = cacheable;
119#endif /* CONFIG_VIRT_IDX_DCACHE */
120 data.p = true;
121 data.w = true;
122 data.g = false;
123
124 dtlb_data_in_write(data.value);
125}
126
127/** Copy PTE to TLB.
128 *
129 * @param t Page Table Entry to be copied.
130 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
131 * @param ro If true, the entry will be created read-only, regardless
132 * of its w field.
133 */
134void dtlb_pte_copy(pte_t *t, size_t index, bool ro)
135{
136 tlb_tag_access_reg_t tag;
137 tlb_data_t data;
138 page_address_t pg;
139 frame_address_t fr;
140
141 pg.address = t->page + (index << MMU_PAGE_WIDTH);
142 fr.address = t->frame + (index << MMU_PAGE_WIDTH);
143
144 tag.value = 0;
145 tag.context = t->as->asid;
146 tag.vpn = pg.vpn;
147
148 dtlb_tag_access_write(tag.value);
149
150 data.value = 0;
151 data.v = true;
152 data.size = PAGESIZE_8K;
153 data.pfn = fr.pfn;
154 data.l = false;
155 data.cp = t->c;
156#ifdef CONFIG_VIRT_IDX_DCACHE
157 data.cv = t->c;
158#endif /* CONFIG_VIRT_IDX_DCACHE */
159 data.p = t->k; /* p like privileged */
160 data.w = ro ? false : t->w;
161 data.g = t->g;
162
163 dtlb_data_in_write(data.value);
164}
165
166/** Copy PTE to ITLB.
167 *
168 * @param t Page Table Entry to be copied.
169 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
170 */
171void itlb_pte_copy(pte_t *t, size_t index)
172{
173 tlb_tag_access_reg_t tag;
174 tlb_data_t data;
175 page_address_t pg;
176 frame_address_t fr;
177
178 pg.address = t->page + (index << MMU_PAGE_WIDTH);
179 fr.address = t->frame + (index << MMU_PAGE_WIDTH);
180
181 tag.value = 0;
182 tag.context = t->as->asid;
183 tag.vpn = pg.vpn;
184
185 itlb_tag_access_write(tag.value);
186
187 data.value = 0;
188 data.v = true;
189 data.size = PAGESIZE_8K;
190 data.pfn = fr.pfn;
191 data.l = false;
192 data.cp = t->c;
193 data.p = t->k; /* p like privileged */
194 data.w = false;
195 data.g = t->g;
196
197 itlb_data_in_write(data.value);
198}
199
200/** ITLB miss handler. */
201void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate)
202{
203 uintptr_t page_16k = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
204 size_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
205 pte_t *t;
206
207 page_table_lock(AS, true);
208 t = page_mapping_find(AS, page_16k);
209 if (t && PTE_EXECUTABLE(t)) {
210 /*
211 * The mapping was found in the software page hash table.
212 * Insert it into ITLB.
213 */
214 t->a = true;
215 itlb_pte_copy(t, index);
216#ifdef CONFIG_TSB
217 itsb_pte_copy(t, index);
218#endif
219 page_table_unlock(AS, true);
220 } else {
221 /*
222 * Forward the page fault to the address space page fault
223 * handler.
224 */
225 page_table_unlock(AS, true);
226 if (as_page_fault(page_16k, PF_ACCESS_EXEC, istate) ==
227 AS_PF_FAULT) {
228 do_fast_instruction_access_mmu_miss_fault(istate,
229 __func__);
230 }
231 }
232}
233
234/** DTLB miss handler.
235 *
236 * Note that some faults (e.g. kernel faults) were already resolved by the
237 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
238 *
239 * @param tag Content of the TLB Tag Access register as it existed
240 * when the trap happened. This is to prevent confusion
241 * created by clobbered Tag Access register during a nested
242 * DTLB miss.
243 * @param istate Interrupted state saved on the stack.
244 */
245void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
246{
247 uintptr_t page_8k;
248 uintptr_t page_16k;
249 size_t index;
250 pte_t *t;
251
252 page_8k = (uint64_t) tag.vpn << MMU_PAGE_WIDTH;
253 page_16k = ALIGN_DOWN(page_8k, PAGE_SIZE);
254 index = tag.vpn % MMU_PAGES_PER_PAGE;
255
256 if (tag.context == ASID_KERNEL) {
257 if (!tag.vpn) {
258 /* NULL access in kernel */
259 do_fast_data_access_mmu_miss_fault(istate, tag,
260 __func__);
261 } else if (page_8k >= end_of_identity) {
262 /*
263 * The kernel is accessing the I/O space.
264 * We still do identity mapping for I/O,
265 * but without caching.
266 */
267 dtlb_insert_mapping(page_8k, KA2PA(page_8k),
268 PAGESIZE_8K, false, false);
269 return;
270 }
271 do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected "
272 "kernel page fault.");
273 }
274
275 page_table_lock(AS, true);
276 t = page_mapping_find(AS, page_16k);
277 if (t) {
278 /*
279 * The mapping was found in the software page hash table.
280 * Insert it into DTLB.
281 */
282 t->a = true;
283 dtlb_pte_copy(t, index, true);
284#ifdef CONFIG_TSB
285 dtsb_pte_copy(t, index, true);
286#endif
287 page_table_unlock(AS, true);
288 } else {
289 /*
290 * Forward the page fault to the address space page fault
291 * handler.
292 */
293 page_table_unlock(AS, true);
294 if (as_page_fault(page_16k, PF_ACCESS_READ, istate) ==
295 AS_PF_FAULT) {
296 do_fast_data_access_mmu_miss_fault(istate, tag,
297 __func__);
298 }
299 }
300}
301
302/** DTLB protection fault handler.
303 *
304 * @param tag Content of the TLB Tag Access register as it existed
305 * when the trap happened. This is to prevent confusion
306 * created by clobbered Tag Access register during a nested
307 * DTLB miss.
308 * @param istate Interrupted state saved on the stack.
309 */
310void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
311{
312 uintptr_t page_16k;
313 size_t index;
314 pte_t *t;
315
316 page_16k = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
317 index = tag.vpn % MMU_PAGES_PER_PAGE; /* 16K-page emulation */
318
319 page_table_lock(AS, true);
320 t = page_mapping_find(AS, page_16k);
321 if (t && PTE_WRITABLE(t)) {
322 /*
323 * The mapping was found in the software page hash table and is
324 * writable. Demap the old mapping and insert an updated mapping
325 * into DTLB.
326 */
327 t->a = true;
328 t->d = true;
329 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
330 page_16k + index * MMU_PAGE_SIZE);
331 dtlb_pte_copy(t, index, false);
332#ifdef CONFIG_TSB
333 dtsb_pte_copy(t, index, false);
334#endif
335 page_table_unlock(AS, true);
336 } else {
337 /*
338 * Forward the page fault to the address space page fault
339 * handler.
340 */
341 page_table_unlock(AS, true);
342 if (as_page_fault(page_16k, PF_ACCESS_WRITE, istate) ==
343 AS_PF_FAULT) {
344 do_fast_data_access_protection_fault(istate, tag,
345 __func__);
346 }
347 }
348}
349
350/** Print TLB entry (for debugging purposes).
351 *
352 * The diag field has been left out in order to make this function more generic
353 * (there is no diag field in US3 architeture).
354 *
355 * @param i TLB entry number
356 * @param t TLB entry tag
357 * @param d TLB entry data
358 */
359static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d)
360{
361 printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, "
362 "ie=%d, soft2=%#x, pfn=%#x, soft=%#x, l=%d, "
363 "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn,
364 t.context, d.v, d.size, d.nfo, d.ie, d.soft2,
365 d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
366}
367
368#if defined (US)
369
370/** Print contents of both TLBs. */
371void tlb_print(void)
372{
373 int i;
374 tlb_data_t d;
375 tlb_tag_read_reg_t t;
376
377 printf("I-TLB contents:\n");
378 for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
379 d.value = itlb_data_access_read(i);
380 t.value = itlb_tag_read_read(i);
381 print_tlb_entry(i, t, d);
382 }
383
384 printf("D-TLB contents:\n");
385 for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
386 d.value = dtlb_data_access_read(i);
387 t.value = dtlb_tag_read_read(i);
388 print_tlb_entry(i, t, d);
389 }
390}
391
392#elif defined (US3)
393
394/** Print contents of all TLBs. */
395void tlb_print(void)
396{
397 int i;
398 tlb_data_t d;
399 tlb_tag_read_reg_t t;
400
401 printf("TLB_ISMALL contents:\n");
402 for (i = 0; i < tlb_ismall_size(); i++) {
403 d.value = dtlb_data_access_read(TLB_ISMALL, i);
404 t.value = dtlb_tag_read_read(TLB_ISMALL, i);
405 print_tlb_entry(i, t, d);
406 }
407
408 printf("TLB_IBIG contents:\n");
409 for (i = 0; i < tlb_ibig_size(); i++) {
410 d.value = dtlb_data_access_read(TLB_IBIG, i);
411 t.value = dtlb_tag_read_read(TLB_IBIG, i);
412 print_tlb_entry(i, t, d);
413 }
414
415 printf("TLB_DSMALL contents:\n");
416 for (i = 0; i < tlb_dsmall_size(); i++) {
417 d.value = dtlb_data_access_read(TLB_DSMALL, i);
418 t.value = dtlb_tag_read_read(TLB_DSMALL, i);
419 print_tlb_entry(i, t, d);
420 }
421
422 printf("TLB_DBIG_1 contents:\n");
423 for (i = 0; i < tlb_dbig_size(); i++) {
424 d.value = dtlb_data_access_read(TLB_DBIG_0, i);
425 t.value = dtlb_tag_read_read(TLB_DBIG_0, i);
426 print_tlb_entry(i, t, d);
427 }
428
429 printf("TLB_DBIG_2 contents:\n");
430 for (i = 0; i < tlb_dbig_size(); i++) {
431 d.value = dtlb_data_access_read(TLB_DBIG_1, i);
432 t.value = dtlb_tag_read_read(TLB_DBIG_1, i);
433 print_tlb_entry(i, t, d);
434 }
435}
436
437#endif
438
439void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
440 const char *str)
441{
442 fault_if_from_uspace(istate, "%s.", str);
443 dump_istate(istate);
444 panic("%s.", str);
445}
446
447void do_fast_data_access_mmu_miss_fault(istate_t *istate,
448 tlb_tag_access_reg_t tag, const char *str)
449{
450 uintptr_t va;
451
452 va = tag.vpn << MMU_PAGE_WIDTH;
453 if (tag.context) {
454 fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d).", str, va,
455 tag.context);
456 }
457 dump_istate(istate);
458 printf("Faulting page: %p, ASID=%d.\n", va, tag.context);
459 panic("%s.", str);
460}
461
462void do_fast_data_access_protection_fault(istate_t *istate,
463 tlb_tag_access_reg_t tag, const char *str)
464{
465 uintptr_t va;
466
467 va = tag.vpn << MMU_PAGE_WIDTH;
468
469 if (tag.context) {
470 fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d).", str, va,
471 tag.context);
472 }
473 printf("Faulting page: %p, ASID=%d\n", va, tag.context);
474 dump_istate(istate);
475 panic("%s.", str);
476}
477
478void describe_dmmu_fault(void)
479{
480 tlb_sfsr_reg_t sfsr;
481 uintptr_t sfar;
482
483 sfsr.value = dtlb_sfsr_read();
484 sfar = dtlb_sfar_read();
485
486#if defined (US)
487 printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
488 "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
489 sfsr.ow, sfsr.fv);
490#elif defined (US3)
491 printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
492 "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
493 sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
494#endif
495
496 printf("DTLB SFAR: address=%p\n", sfar);
497
498 dtlb_sfsr_write(0);
499}
500
501void dump_sfsr_and_sfar(void)
502{
503 tlb_sfsr_reg_t sfsr;
504 uintptr_t sfar;
505
506 sfsr.value = dtlb_sfsr_read();
507 sfar = dtlb_sfar_read();
508
509#if defined (US)
510 printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
511 "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
512 sfsr.ow, sfsr.fv);
513#elif defined (US3)
514 printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
515 "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
516 sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
517#endif
518
519 printf("DTLB SFAR: address=%p\n", sfar);
520
521 dtlb_sfsr_write(0);
522}
523
524#if defined (US)
525/** Invalidate all unlocked ITLB and DTLB entries. */
526void tlb_invalidate_all(void)
527{
528 int i;
529
530 /*
531 * Walk all ITLB and DTLB entries and remove all unlocked mappings.
532 *
533 * The kernel doesn't use global mappings so any locked global mappings
534 * found must have been created by someone else. Their only purpose now
535 * is to collide with proper mappings. Invalidate immediately. It should
536 * be safe to invalidate them as late as now.
537 */
538
539 tlb_data_t d;
540 tlb_tag_read_reg_t t;
541
542 for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
543 d.value = itlb_data_access_read(i);
544 if (!d.l || d.g) {
545 t.value = itlb_tag_read_read(i);
546 d.v = false;
547 itlb_tag_access_write(t.value);
548 itlb_data_access_write(i, d.value);
549 }
550 }
551
552 for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
553 d.value = dtlb_data_access_read(i);
554 if (!d.l || d.g) {
555 t.value = dtlb_tag_read_read(i);
556 d.v = false;
557 dtlb_tag_access_write(t.value);
558 dtlb_data_access_write(i, d.value);
559 }
560 }
561
562}
563
564#elif defined (US3)
565
566/** Invalidate all unlocked ITLB and DTLB entries. */
567void tlb_invalidate_all(void)
568{
569 itlb_demap(TLB_DEMAP_ALL, 0, 0);
570 dtlb_demap(TLB_DEMAP_ALL, 0, 0);
571}
572
573#endif
574
575/** Invalidate all ITLB and DTLB entries that belong to specified ASID
576 * (Context).
577 *
578 * @param asid Address Space ID.
579 */
580void tlb_invalidate_asid(asid_t asid)
581{
582 tlb_context_reg_t pc_save, ctx;
583
584 /* switch to nucleus because we are mapped by the primary context */
585 nucleus_enter();
586
587 ctx.v = pc_save.v = mmu_primary_context_read();
588 ctx.context = asid;
589 mmu_primary_context_write(ctx.v);
590
591 itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
592 dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
593
594 mmu_primary_context_write(pc_save.v);
595
596 nucleus_leave();
597}
598
599/** Invalidate all ITLB and DTLB entries for specified page range in specified
600 * address space.
601 *
602 * @param asid Address Space ID.
603 * @param page First page which to sweep out from ITLB and DTLB.
604 * @param cnt Number of ITLB and DTLB entries to invalidate.
605 */
606void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
607{
608 unsigned int i;
609 tlb_context_reg_t pc_save, ctx;
610
611 /* switch to nucleus because we are mapped by the primary context */
612 nucleus_enter();
613
614 ctx.v = pc_save.v = mmu_primary_context_read();
615 ctx.context = asid;
616 mmu_primary_context_write(ctx.v);
617
618 for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
619 itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
620 page + i * MMU_PAGE_SIZE);
621 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
622 page + i * MMU_PAGE_SIZE);
623 }
624
625 mmu_primary_context_write(pc_save.v);
626
627 nucleus_leave();
628}
629
630/** @}
631 */
Note: See TracBrowser for help on using the repository browser.