source: mainline/kernel/arch/sparc64/src/mm/sun4u/tlb.c@ 1f5714e

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 1f5714e was 1f5714e, checked in by Jakub Jermar <jakub@…>, 14 years ago

sparc64: Kernel non-identity needs to be resolved by walking page tables.
  • Property mode set to 100644
File size: 15.4 KB
Line 
1/*
2 * Copyright (c) 2005 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup sparc64mm
30 * @{
31 */
32/** @file
33 */
34
35#include <arch/mm/tlb.h>
36#include <mm/tlb.h>
37#include <mm/as.h>
38#include <mm/asid.h>
39#include <arch/mm/frame.h>
40#include <arch/mm/page.h>
41#include <arch/mm/mmu.h>
42#include <arch/interrupt.h>
43#include <interrupt.h>
44#include <arch.h>
45#include <print.h>
46#include <typedefs.h>
47#include <config.h>
48#include <arch/trap/trap.h>
49#include <arch/trap/exception.h>
50#include <panic.h>
51#include <arch/asm.h>
52#include <genarch/mm/page_ht.h>
53
54#ifdef CONFIG_TSB
55#include <arch/mm/tsb.h>
56#endif
57
58static void dtlb_pte_copy(pte_t *, size_t, bool);
59static void itlb_pte_copy(pte_t *, size_t);
60static void do_fast_instruction_access_mmu_miss_fault(istate_t *, uintptr_t,
61 const char *);
62static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t,
63 const char *);
64static void do_fast_data_access_protection_fault(istate_t *,
65 tlb_tag_access_reg_t, const char *);
66
67const char *context_encoding[] = {
68 "Primary",
69 "Secondary",
70 "Nucleus",
71 "Reserved"
72};
73
74void tlb_arch_init(void)
75{
76 /*
77 * Invalidate all non-locked DTLB and ITLB entries.
78 */
79 tlb_invalidate_all();
80
81 /*
82 * Clear both SFSRs.
83 */
84 dtlb_sfsr_write(0);
85 itlb_sfsr_write(0);
86}
87
88/** Insert privileged mapping into DMMU TLB.
89 *
90 * @param page Virtual page address.
91 * @param frame Physical frame address.
92 * @param pagesize Page size.
93 * @param locked True for permanent mappings, false otherwise.
94 * @param cacheable True if the mapping is cacheable, false otherwise.
95 */
96void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize,
97 bool locked, bool cacheable)
98{
99 tlb_tag_access_reg_t tag;
100 tlb_data_t data;
101 page_address_t pg;
102 frame_address_t fr;
103
104 pg.address = page;
105 fr.address = frame;
106
107 tag.context = ASID_KERNEL;
108 tag.vpn = pg.vpn;
109
110 dtlb_tag_access_write(tag.value);
111
112 data.value = 0;
113 data.v = true;
114 data.size = pagesize;
115 data.pfn = fr.pfn;
116 data.l = locked;
117 data.cp = cacheable;
118#ifdef CONFIG_VIRT_IDX_DCACHE
119 data.cv = cacheable;
120#endif /* CONFIG_VIRT_IDX_DCACHE */
121 data.p = true;
122 data.w = true;
123 data.g = false;
124
125 dtlb_data_in_write(data.value);
126}
127
128/** Copy PTE to TLB.
129 *
130 * @param t Page Table Entry to be copied.
131 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
132 * @param ro If true, the entry will be created read-only, regardless
133 * of its w field.
134 */
135void dtlb_pte_copy(pte_t *t, size_t index, bool ro)
136{
137 tlb_tag_access_reg_t tag;
138 tlb_data_t data;
139 page_address_t pg;
140 frame_address_t fr;
141
142 pg.address = t->page + (index << MMU_PAGE_WIDTH);
143 fr.address = t->frame + (index << MMU_PAGE_WIDTH);
144
145 tag.value = 0;
146 tag.context = t->as->asid;
147 tag.vpn = pg.vpn;
148
149 dtlb_tag_access_write(tag.value);
150
151 data.value = 0;
152 data.v = true;
153 data.size = PAGESIZE_8K;
154 data.pfn = fr.pfn;
155 data.l = false;
156 data.cp = t->c;
157#ifdef CONFIG_VIRT_IDX_DCACHE
158 data.cv = t->c;
159#endif /* CONFIG_VIRT_IDX_DCACHE */
160 data.p = t->k; /* p like privileged */
161 data.w = ro ? false : t->w;
162 data.g = t->g;
163
164 dtlb_data_in_write(data.value);
165}
166
167/** Copy PTE to ITLB.
168 *
169 * @param t Page Table Entry to be copied.
170 * @param index Zero if lower 8K-subpage, one if higher 8K-subpage.
171 */
172void itlb_pte_copy(pte_t *t, size_t index)
173{
174 tlb_tag_access_reg_t tag;
175 tlb_data_t data;
176 page_address_t pg;
177 frame_address_t fr;
178
179 pg.address = t->page + (index << MMU_PAGE_WIDTH);
180 fr.address = t->frame + (index << MMU_PAGE_WIDTH);
181
182 tag.value = 0;
183 tag.context = t->as->asid;
184 tag.vpn = pg.vpn;
185
186 itlb_tag_access_write(tag.value);
187
188 data.value = 0;
189 data.v = true;
190 data.size = PAGESIZE_8K;
191 data.pfn = fr.pfn;
192 data.l = false;
193 data.cp = t->c;
194 data.p = t->k; /* p like privileged */
195 data.w = false;
196 data.g = t->g;
197
198 itlb_data_in_write(data.value);
199}
200
201/** ITLB miss handler. */
202void fast_instruction_access_mmu_miss(sysarg_t unused, istate_t *istate)
203{
204 uintptr_t page_16k = ALIGN_DOWN(istate->tpc, PAGE_SIZE);
205 size_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE;
206 pte_t *t;
207
208 t = page_mapping_find(AS, page_16k, true);
209 if (t && PTE_EXECUTABLE(t)) {
210 /*
211 * The mapping was found in the software page hash table.
212 * Insert it into ITLB.
213 */
214 t->a = true;
215 itlb_pte_copy(t, index);
216#ifdef CONFIG_TSB
217 itsb_pte_copy(t, index);
218#endif
219 } else {
220 /*
221 * Forward the page fault to the address space page fault
222 * handler.
223 */
224 if (as_page_fault(page_16k, PF_ACCESS_EXEC, istate) ==
225 AS_PF_FAULT) {
226 do_fast_instruction_access_mmu_miss_fault(istate,
227 istate->tpc, __func__);
228 }
229 }
230}
231
232/** DTLB miss handler.
233 *
234 * Note that some faults (e.g. kernel faults) were already resolved by the
235 * low-level, assembly language part of the fast_data_access_mmu_miss handler.
236 *
237 * @param tag Content of the TLB Tag Access register as it existed
238 * when the trap happened. This is to prevent confusion
239 * created by clobbered Tag Access register during a nested
240 * DTLB miss.
241 * @param istate Interrupted state saved on the stack.
242 */
243void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate)
244{
245 uintptr_t page_8k;
246 uintptr_t page_16k;
247 size_t index;
248 pte_t *t;
249
250 page_8k = (uint64_t) tag.vpn << MMU_PAGE_WIDTH;
251 page_16k = ALIGN_DOWN(page_8k, PAGE_SIZE);
252 index = tag.vpn % MMU_PAGES_PER_PAGE;
253
254 if (tag.context == ASID_KERNEL) {
255 if (!tag.vpn) {
256 /* NULL access in kernel */
257 do_fast_data_access_mmu_miss_fault(istate, tag,
258 "Dereferencing NULL pointer.");
259 } else if (page_8k >= end_of_identity) {
260 /* Kernel non-identity, fall through. */
261 } else {
262 do_fast_data_access_mmu_miss_fault(istate, tag,
263 "Unexpected kernel page fault.");
264 }
265 }
266
267 t = page_mapping_find(AS, page_16k, true);
268 if (t) {
269 /*
270 * The mapping was found in the software page hash table.
271 * Insert it into DTLB.
272 */
273 t->a = true;
274 dtlb_pte_copy(t, index, true);
275#ifdef CONFIG_TSB
276 dtsb_pte_copy(t, index, true);
277#endif
278 } else {
279 /*
280 * Forward the page fault to the address space page fault
281 * handler.
282 */
283 if (as_page_fault(page_16k, PF_ACCESS_READ, istate) ==
284 AS_PF_FAULT) {
285 do_fast_data_access_mmu_miss_fault(istate, tag,
286 __func__);
287 }
288 }
289}
290
291/** DTLB protection fault handler.
292 *
293 * @param tag Content of the TLB Tag Access register as it existed
294 * when the trap happened. This is to prevent confusion
295 * created by clobbered Tag Access register during a nested
296 * DTLB miss.
297 * @param istate Interrupted state saved on the stack.
298 */
299void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate)
300{
301 uintptr_t page_16k;
302 size_t index;
303 pte_t *t;
304
305 page_16k = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE);
306 index = tag.vpn % MMU_PAGES_PER_PAGE; /* 16K-page emulation */
307
308 t = page_mapping_find(AS, page_16k, true);
309 if (t && PTE_WRITABLE(t)) {
310 /*
311 * The mapping was found in the software page hash table and is
312 * writable. Demap the old mapping and insert an updated mapping
313 * into DTLB.
314 */
315 t->a = true;
316 t->d = true;
317 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY,
318 page_16k + index * MMU_PAGE_SIZE);
319 dtlb_pte_copy(t, index, false);
320#ifdef CONFIG_TSB
321 dtsb_pte_copy(t, index, false);
322#endif
323 } else {
324 /*
325 * Forward the page fault to the address space page fault
326 * handler.
327 */
328 if (as_page_fault(page_16k, PF_ACCESS_WRITE, istate) ==
329 AS_PF_FAULT) {
330 do_fast_data_access_protection_fault(istate, tag,
331 __func__);
332 }
333 }
334}
335
336/** Print TLB entry (for debugging purposes).
337 *
338 * The diag field has been left out in order to make this function more generic
339 * (there is no diag field in US3 architeture).
340 *
341 * @param i TLB entry number
342 * @param t TLB entry tag
343 * @param d TLB entry data
344 */
345static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d)
346{
347 printf("%u: vpn=%#" PRIx64 ", context=%u, v=%u, size=%u, nfo=%u, "
348 "ie=%u, soft2=%#x, pfn=%#x, soft=%#x, l=%u, "
349 "cp=%u, cv=%u, e=%u, p=%u, w=%u, g=%u\n", i, (uint64_t) t.vpn,
350 t.context, d.v, d.size, d.nfo, d.ie, d.soft2,
351 d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g);
352}
353
354#if defined (US)
355
356/** Print contents of both TLBs. */
357void tlb_print(void)
358{
359 int i;
360 tlb_data_t d;
361 tlb_tag_read_reg_t t;
362
363 printf("I-TLB contents:\n");
364 for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
365 d.value = itlb_data_access_read(i);
366 t.value = itlb_tag_read_read(i);
367 print_tlb_entry(i, t, d);
368 }
369
370 printf("D-TLB contents:\n");
371 for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
372 d.value = dtlb_data_access_read(i);
373 t.value = dtlb_tag_read_read(i);
374 print_tlb_entry(i, t, d);
375 }
376}
377
378#elif defined (US3)
379
380/** Print contents of all TLBs. */
381void tlb_print(void)
382{
383 int i;
384 tlb_data_t d;
385 tlb_tag_read_reg_t t;
386
387 printf("TLB_ISMALL contents:\n");
388 for (i = 0; i < tlb_ismall_size(); i++) {
389 d.value = dtlb_data_access_read(TLB_ISMALL, i);
390 t.value = dtlb_tag_read_read(TLB_ISMALL, i);
391 print_tlb_entry(i, t, d);
392 }
393
394 printf("TLB_IBIG contents:\n");
395 for (i = 0; i < tlb_ibig_size(); i++) {
396 d.value = dtlb_data_access_read(TLB_IBIG, i);
397 t.value = dtlb_tag_read_read(TLB_IBIG, i);
398 print_tlb_entry(i, t, d);
399 }
400
401 printf("TLB_DSMALL contents:\n");
402 for (i = 0; i < tlb_dsmall_size(); i++) {
403 d.value = dtlb_data_access_read(TLB_DSMALL, i);
404 t.value = dtlb_tag_read_read(TLB_DSMALL, i);
405 print_tlb_entry(i, t, d);
406 }
407
408 printf("TLB_DBIG_1 contents:\n");
409 for (i = 0; i < tlb_dbig_size(); i++) {
410 d.value = dtlb_data_access_read(TLB_DBIG_0, i);
411 t.value = dtlb_tag_read_read(TLB_DBIG_0, i);
412 print_tlb_entry(i, t, d);
413 }
414
415 printf("TLB_DBIG_2 contents:\n");
416 for (i = 0; i < tlb_dbig_size(); i++) {
417 d.value = dtlb_data_access_read(TLB_DBIG_1, i);
418 t.value = dtlb_tag_read_read(TLB_DBIG_1, i);
419 print_tlb_entry(i, t, d);
420 }
421}
422
423#endif
424
425void do_fast_instruction_access_mmu_miss_fault(istate_t *istate,
426 uintptr_t va, const char *str)
427{
428 fault_if_from_uspace(istate, "%s, address=%p.", str, (void *) va);
429 panic_memtrap(istate, PF_ACCESS_EXEC, va, str);
430}
431
432void do_fast_data_access_mmu_miss_fault(istate_t *istate,
433 tlb_tag_access_reg_t tag, const char *str)
434{
435 uintptr_t va;
436
437 va = tag.vpn << MMU_PAGE_WIDTH;
438 fault_if_from_uspace(istate, "%s, page=%p (asid=%u).", str,
439 (void *) va, tag.context);
440 panic_memtrap(istate, PF_ACCESS_UNKNOWN, va, str);
441}
442
443void do_fast_data_access_protection_fault(istate_t *istate,
444 tlb_tag_access_reg_t tag, const char *str)
445{
446 uintptr_t va;
447
448 va = tag.vpn << MMU_PAGE_WIDTH;
449 fault_if_from_uspace(istate, "%s, page=%p (asid=%u).", str,
450 (void *) va, tag.context);
451 panic_memtrap(istate, PF_ACCESS_WRITE, va, str);
452}
453
454void describe_dmmu_fault(void)
455{
456 tlb_sfsr_reg_t sfsr;
457 uintptr_t sfar;
458
459 sfsr.value = dtlb_sfsr_read();
460 sfar = dtlb_sfar_read();
461
462#if defined (US)
463 printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
464 "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
465 sfsr.ow, sfsr.fv);
466#elif defined (US3)
467 printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
468 "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
469 sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
470#endif
471
472 printf("DTLB SFAR: address=%p\n", (void *) sfar);
473
474 dtlb_sfsr_write(0);
475}
476
477void dump_sfsr_and_sfar(void)
478{
479 tlb_sfsr_reg_t sfsr;
480 uintptr_t sfar;
481
482 sfsr.value = dtlb_sfsr_read();
483 sfar = dtlb_sfar_read();
484
485#if defined (US)
486 printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, "
487 "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w,
488 sfsr.ow, sfsr.fv);
489#elif defined (US3)
490 printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, "
491 "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft,
492 sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv);
493#endif
494
495 printf("DTLB SFAR: address=%p\n", (void *) sfar);
496
497 dtlb_sfsr_write(0);
498}
499
500#if defined (US)
501/** Invalidate all unlocked ITLB and DTLB entries. */
502void tlb_invalidate_all(void)
503{
504 int i;
505
506 /*
507 * Walk all ITLB and DTLB entries and remove all unlocked mappings.
508 *
509 * The kernel doesn't use global mappings so any locked global mappings
510 * found must have been created by someone else. Their only purpose now
511 * is to collide with proper mappings. Invalidate immediately. It should
512 * be safe to invalidate them as late as now.
513 */
514
515 tlb_data_t d;
516 tlb_tag_read_reg_t t;
517
518 for (i = 0; i < ITLB_ENTRY_COUNT; i++) {
519 d.value = itlb_data_access_read(i);
520 if (!d.l || d.g) {
521 t.value = itlb_tag_read_read(i);
522 d.v = false;
523 itlb_tag_access_write(t.value);
524 itlb_data_access_write(i, d.value);
525 }
526 }
527
528 for (i = 0; i < DTLB_ENTRY_COUNT; i++) {
529 d.value = dtlb_data_access_read(i);
530 if (!d.l || d.g) {
531 t.value = dtlb_tag_read_read(i);
532 d.v = false;
533 dtlb_tag_access_write(t.value);
534 dtlb_data_access_write(i, d.value);
535 }
536 }
537
538}
539
540#elif defined (US3)
541
542/** Invalidate all unlocked ITLB and DTLB entries. */
543void tlb_invalidate_all(void)
544{
545 itlb_demap(TLB_DEMAP_ALL, 0, 0);
546 dtlb_demap(TLB_DEMAP_ALL, 0, 0);
547}
548
549#endif
550
551/** Invalidate all ITLB and DTLB entries that belong to specified ASID
552 * (Context).
553 *
554 * @param asid Address Space ID.
555 */
556void tlb_invalidate_asid(asid_t asid)
557{
558 tlb_context_reg_t pc_save, ctx;
559
560 /* switch to nucleus because we are mapped by the primary context */
561 nucleus_enter();
562
563 ctx.v = pc_save.v = mmu_primary_context_read();
564 ctx.context = asid;
565 mmu_primary_context_write(ctx.v);
566
567 itlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
568 dtlb_demap(TLB_DEMAP_CONTEXT, TLB_DEMAP_PRIMARY, 0);
569
570 mmu_primary_context_write(pc_save.v);
571
572 nucleus_leave();
573}
574
575/** Invalidate all ITLB and DTLB entries for specified page range in specified
576 * address space.
577 *
578 * @param asid Address Space ID.
579 * @param page First page which to sweep out from ITLB and DTLB.
580 * @param cnt Number of ITLB and DTLB entries to invalidate.
581 */
582void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
583{
584 unsigned int i;
585 tlb_context_reg_t pc_save, ctx;
586
587 /* switch to nucleus because we are mapped by the primary context */
588 nucleus_enter();
589
590 ctx.v = pc_save.v = mmu_primary_context_read();
591 ctx.context = asid;
592 mmu_primary_context_write(ctx.v);
593
594 for (i = 0; i < cnt * MMU_PAGES_PER_PAGE; i++) {
595 itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
596 page + i * MMU_PAGE_SIZE);
597 dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY,
598 page + i * MMU_PAGE_SIZE);
599 }
600
601 mmu_primary_context_write(pc_save.v);
602
603 nucleus_leave();
604}
605
606/** @}
607 */
Note: See TracBrowser for help on using the repository browser.