Index: kernel/arch/sparc64/include/mm/sun4u/tlb.h =================================================================== --- kernel/arch/sparc64/include/mm/sun4u/tlb.h (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) +++ kernel/arch/sparc64/include/mm/sun4u/tlb.h (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -0,0 +1,692 @@ +/* + * Copyright (c) 2005 Jakub Jermar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup sparc64mm + * @{ + */ +/** @file + */ + +#ifndef KERN_sparc64_TLB_sun4u_H_ +#define KERN_sparc64_TLB_sun4u_H_ + +#if defined (US) +#define ITLB_ENTRY_COUNT 64 +#define DTLB_ENTRY_COUNT 64 +#define DTLB_MAX_LOCKED_ENTRIES DTLB_ENTRY_COUNT +#endif + +/** TLB_DSMALL is the only of the three DMMUs that can hold locked entries. */ +#if defined (US3) +#define DTLB_MAX_LOCKED_ENTRIES 16 +#endif + +#define MEM_CONTEXT_KERNEL 0 +#define MEM_CONTEXT_TEMP 1 + +/** Page sizes. */ +#define PAGESIZE_8K 0 +#define PAGESIZE_64K 1 +#define PAGESIZE_512K 2 +#define PAGESIZE_4M 3 + +/** Bit width of the TLB-locked portion of kernel address space. */ +#define KERNEL_PAGE_WIDTH 22 /* 4M */ + +/* TLB Demap Operation types. */ +#define TLB_DEMAP_PAGE 0 +#define TLB_DEMAP_CONTEXT 1 +#if defined (US3) +#define TLB_DEMAP_ALL 2 +#endif + +#define TLB_DEMAP_TYPE_SHIFT 6 + +/* TLB Demap Operation Context register encodings. */ +#define TLB_DEMAP_PRIMARY 0 +#define TLB_DEMAP_SECONDARY 1 +#define TLB_DEMAP_NUCLEUS 2 + +/* There are more TLBs in one MMU in US3, their codes are defined here. */ +#if defined (US3) +/* D-MMU: one small (16-entry) TLB and two big (512-entry) TLBs */ +#define TLB_DSMALL 0 +#define TLB_DBIG_0 2 +#define TLB_DBIG_1 3 + +/* I-MMU: one small (16-entry) TLB and one big TLB */ +#define TLB_ISMALL 0 +#define TLB_IBIG 2 +#endif + +#define TLB_DEMAP_CONTEXT_SHIFT 4 + +/* TLB Tag Access shifts */ +#define TLB_TAG_ACCESS_CONTEXT_SHIFT 0 +#define TLB_TAG_ACCESS_CONTEXT_MASK ((1 << 13) - 1) +#define TLB_TAG_ACCESS_VPN_SHIFT 13 + +#ifndef __ASM__ + +#include +#include +#include +#include +#include +#include +#include +#include + +union tlb_context_reg { + uint64_t v; + struct { + unsigned long : 51; + unsigned context : 13; /**< Context/ASID. */ + } __attribute__ ((packed)); +}; +typedef union tlb_context_reg tlb_context_reg_t; + +/** I-/D-TLB Data In/Access Register type. */ +typedef tte_data_t tlb_data_t; + +/** I-/D-TLB Data Access Address in Alternate Space. */ + +#if defined (US) + +union tlb_data_access_addr { + uint64_t value; + struct { + uint64_t : 55; + unsigned tlb_entry : 6; + unsigned : 3; + } __attribute__ ((packed)); +}; +typedef union tlb_data_access_addr dtlb_data_access_addr_t; +typedef union tlb_data_access_addr dtlb_tag_read_addr_t; +typedef union tlb_data_access_addr itlb_data_access_addr_t; +typedef union tlb_data_access_addr itlb_tag_read_addr_t; + +#elif defined (US3) + +/* + * In US3, I-MMU and D-MMU have different formats of the data + * access register virtual address. In the corresponding + * structures the member variable for the entry number is + * called "local_tlb_entry" - it contrasts with the "tlb_entry" + * for the US data access register VA structure. The rationale + * behind this is to prevent careless mistakes in the code + * caused by setting only the entry number and not the TLB + * number in the US3 code (when taking the code from US). + */ + +union dtlb_data_access_addr { + uint64_t value; + struct { + uint64_t : 45; + unsigned : 1; + unsigned tlb_number : 2; + unsigned : 4; + unsigned local_tlb_entry : 9; + unsigned : 3; + } __attribute__ ((packed)); +}; +typedef union dtlb_data_access_addr dtlb_data_access_addr_t; +typedef union dtlb_data_access_addr dtlb_tag_read_addr_t; + +union itlb_data_access_addr { + uint64_t value; + struct { + uint64_t : 45; + unsigned : 1; + unsigned tlb_number : 2; + unsigned : 6; + unsigned local_tlb_entry : 7; + unsigned : 3; + } __attribute__ ((packed)); +}; +typedef union itlb_data_access_addr itlb_data_access_addr_t; +typedef union itlb_data_access_addr itlb_tag_read_addr_t; + +#endif + +/** I-/D-TLB Tag Read Register. */ +union tlb_tag_read_reg { + uint64_t value; + struct { + uint64_t vpn : 51; /**< Virtual Address bits 63:13. */ + unsigned context : 13; /**< Context identifier. */ + } __attribute__ ((packed)); +}; +typedef union tlb_tag_read_reg tlb_tag_read_reg_t; +typedef union tlb_tag_read_reg tlb_tag_access_reg_t; + + +/** TLB Demap Operation Address. */ +union tlb_demap_addr { + uint64_t value; + struct { + uint64_t vpn: 51; /**< Virtual Address bits 63:13. */ +#if defined (US) + unsigned : 6; /**< Ignored. */ + unsigned type : 1; /**< The type of demap operation. */ +#elif defined (US3) + unsigned : 5; /**< Ignored. */ + unsigned type: 2; /**< The type of demap operation. */ +#endif + unsigned context : 2; /**< Context register selection. */ + unsigned : 4; /**< Zero. */ + } __attribute__ ((packed)); +}; +typedef union tlb_demap_addr tlb_demap_addr_t; + +/** TLB Synchronous Fault Status Register. */ +union tlb_sfsr_reg { + uint64_t value; + struct { +#if defined (US) + unsigned long : 40; /**< Implementation dependent. */ + unsigned asi : 8; /**< ASI. */ + unsigned : 2; + unsigned ft : 7; /**< Fault type. */ +#elif defined (US3) + unsigned long : 39; /**< Implementation dependent. */ + unsigned nf : 1; /**< Non-faulting load. */ + unsigned asi : 8; /**< ASI. */ + unsigned tm : 1; /**< I-TLB miss. */ + unsigned : 3; /**< Reserved. */ + unsigned ft : 5; /**< Fault type. */ +#endif + unsigned e : 1; /**< Side-effect bit. */ + unsigned ct : 2; /**< Context Register selection. */ + unsigned pr : 1; /**< Privilege bit. */ + unsigned w : 1; /**< Write bit. */ + unsigned ow : 1; /**< Overwrite bit. */ + unsigned fv : 1; /**< Fault Valid bit. */ + } __attribute__ ((packed)); +}; +typedef union tlb_sfsr_reg tlb_sfsr_reg_t; + +#if defined (US3) + +/* + * Functions for determining the number of entries in TLBs. They either return + * a constant value or a value based on the CPU autodetection. + */ + +/** + * Determine the number of entries in the DMMU's small TLB. + */ +static inline uint16_t tlb_dsmall_size(void) +{ + return 16; +} + +/** + * Determine the number of entries in each DMMU's big TLB. + */ +static inline uint16_t tlb_dbig_size(void) +{ + return 512; +} + +/** + * Determine the number of entries in the IMMU's small TLB. + */ +static inline uint16_t tlb_ismall_size(void) +{ + return 16; +} + +/** + * Determine the number of entries in the IMMU's big TLB. + */ +static inline uint16_t tlb_ibig_size(void) +{ + if (((ver_reg_t) ver_read()).impl == IMPL_ULTRASPARCIV_PLUS) + return 512; + else + return 128; +} + +#endif + +/** Read MMU Primary Context Register. + * + * @return Current value of Primary Context Register. + */ +static inline uint64_t mmu_primary_context_read(void) +{ + return asi_u64_read(ASI_DMMU, VA_PRIMARY_CONTEXT_REG); +} + +/** Write MMU Primary Context Register. + * + * @param v New value of Primary Context Register. + */ +static inline void mmu_primary_context_write(uint64_t v) +{ + asi_u64_write(ASI_DMMU, VA_PRIMARY_CONTEXT_REG, v); + flush_pipeline(); +} + +/** Read MMU Secondary Context Register. + * + * @return Current value of Secondary Context Register. + */ +static inline uint64_t mmu_secondary_context_read(void) +{ + return asi_u64_read(ASI_DMMU, VA_SECONDARY_CONTEXT_REG); +} + +/** Write MMU Primary Context Register. + * + * @param v New value of Primary Context Register. + */ +static inline void mmu_secondary_context_write(uint64_t v) +{ + asi_u64_write(ASI_DMMU, VA_SECONDARY_CONTEXT_REG, v); + flush_pipeline(); +} + +#if defined (US) + +/** Read IMMU TLB Data Access Register. + * + * @param entry TLB Entry index. + * + * @return Current value of specified IMMU TLB Data Access + * Register. + */ +static inline uint64_t itlb_data_access_read(size_t entry) +{ + itlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_entry = entry; + return asi_u64_read(ASI_ITLB_DATA_ACCESS_REG, reg.value); +} + +/** Write IMMU TLB Data Access Register. + * + * @param entry TLB Entry index. + * @param value Value to be written. + */ +static inline void itlb_data_access_write(size_t entry, uint64_t value) +{ + itlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_entry = entry; + asi_u64_write(ASI_ITLB_DATA_ACCESS_REG, reg.value, value); + flush_pipeline(); +} + +/** Read DMMU TLB Data Access Register. + * + * @param entry TLB Entry index. + * + * @return Current value of specified DMMU TLB Data Access + * Register. + */ +static inline uint64_t dtlb_data_access_read(size_t entry) +{ + dtlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_entry = entry; + return asi_u64_read(ASI_DTLB_DATA_ACCESS_REG, reg.value); +} + +/** Write DMMU TLB Data Access Register. + * + * @param entry TLB Entry index. + * @param value Value to be written. + */ +static inline void dtlb_data_access_write(size_t entry, uint64_t value) +{ + dtlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_entry = entry; + asi_u64_write(ASI_DTLB_DATA_ACCESS_REG, reg.value, value); + membar(); +} + +/** Read IMMU TLB Tag Read Register. + * + * @param entry TLB Entry index. + * + * @return Current value of specified IMMU TLB Tag Read Register. + */ +static inline uint64_t itlb_tag_read_read(size_t entry) +{ + itlb_tag_read_addr_t tag; + + tag.value = 0; + tag.tlb_entry = entry; + return asi_u64_read(ASI_ITLB_TAG_READ_REG, tag.value); +} + +/** Read DMMU TLB Tag Read Register. + * + * @param entry TLB Entry index. + * + * @return Current value of specified DMMU TLB Tag Read Register. + */ +static inline uint64_t dtlb_tag_read_read(size_t entry) +{ + dtlb_tag_read_addr_t tag; + + tag.value = 0; + tag.tlb_entry = entry; + return asi_u64_read(ASI_DTLB_TAG_READ_REG, tag.value); +} + +#elif defined (US3) + + +/** Read IMMU TLB Data Access Register. + * + * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) + * @param entry TLB Entry index. + * + * @return Current value of specified IMMU TLB Data Access + * Register. + */ +static inline uint64_t itlb_data_access_read(int tlb, size_t entry) +{ + itlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_number = tlb; + reg.local_tlb_entry = entry; + return asi_u64_read(ASI_ITLB_DATA_ACCESS_REG, reg.value); +} + +/** Write IMMU TLB Data Access Register. + * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) + * @param entry TLB Entry index. + * @param value Value to be written. + */ +static inline void itlb_data_access_write(int tlb, size_t entry, + uint64_t value) +{ + itlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_number = tlb; + reg.local_tlb_entry = entry; + asi_u64_write(ASI_ITLB_DATA_ACCESS_REG, reg.value, value); + flush_pipeline(); +} + +/** Read DMMU TLB Data Access Register. + * + * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG, TLB_DBIG) + * @param entry TLB Entry index. + * + * @return Current value of specified DMMU TLB Data Access + * Register. + */ +static inline uint64_t dtlb_data_access_read(int tlb, size_t entry) +{ + dtlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_number = tlb; + reg.local_tlb_entry = entry; + return asi_u64_read(ASI_DTLB_DATA_ACCESS_REG, reg.value); +} + +/** Write DMMU TLB Data Access Register. + * + * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1) + * @param entry TLB Entry index. + * @param value Value to be written. + */ +static inline void dtlb_data_access_write(int tlb, size_t entry, + uint64_t value) +{ + dtlb_data_access_addr_t reg; + + reg.value = 0; + reg.tlb_number = tlb; + reg.local_tlb_entry = entry; + asi_u64_write(ASI_DTLB_DATA_ACCESS_REG, reg.value, value); + membar(); +} + +/** Read IMMU TLB Tag Read Register. + * + * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) + * @param entry TLB Entry index. + * + * @return Current value of specified IMMU TLB Tag Read Register. + */ +static inline uint64_t itlb_tag_read_read(int tlb, size_t entry) +{ + itlb_tag_read_addr_t tag; + + tag.value = 0; + tag.tlb_number = tlb; + tag.local_tlb_entry = entry; + return asi_u64_read(ASI_ITLB_TAG_READ_REG, tag.value); +} + +/** Read DMMU TLB Tag Read Register. + * + * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1) + * @param entry TLB Entry index. + * + * @return Current value of specified DMMU TLB Tag Read Register. + */ +static inline uint64_t dtlb_tag_read_read(int tlb, size_t entry) +{ + dtlb_tag_read_addr_t tag; + + tag.value = 0; + tag.tlb_number = tlb; + tag.local_tlb_entry = entry; + return asi_u64_read(ASI_DTLB_TAG_READ_REG, tag.value); +} + +#endif + + +/** Write IMMU TLB Tag Access Register. + * + * @param v Value to be written. + */ +static inline void itlb_tag_access_write(uint64_t v) +{ + asi_u64_write(ASI_IMMU, VA_IMMU_TAG_ACCESS, v); + flush_pipeline(); +} + +/** Read IMMU TLB Tag Access Register. + * + * @return Current value of IMMU TLB Tag Access Register. + */ +static inline uint64_t itlb_tag_access_read(void) +{ + return asi_u64_read(ASI_IMMU, VA_IMMU_TAG_ACCESS); +} + +/** Write DMMU TLB Tag Access Register. + * + * @param v Value to be written. + */ +static inline void dtlb_tag_access_write(uint64_t v) +{ + asi_u64_write(ASI_DMMU, VA_DMMU_TAG_ACCESS, v); + membar(); +} + +/** Read DMMU TLB Tag Access Register. + * + * @return Current value of DMMU TLB Tag Access Register. + */ +static inline uint64_t dtlb_tag_access_read(void) +{ + return asi_u64_read(ASI_DMMU, VA_DMMU_TAG_ACCESS); +} + + +/** Write IMMU TLB Data in Register. + * + * @param v Value to be written. + */ +static inline void itlb_data_in_write(uint64_t v) +{ + asi_u64_write(ASI_ITLB_DATA_IN_REG, 0, v); + flush_pipeline(); +} + +/** Write DMMU TLB Data in Register. + * + * @param v Value to be written. + */ +static inline void dtlb_data_in_write(uint64_t v) +{ + asi_u64_write(ASI_DTLB_DATA_IN_REG, 0, v); + membar(); +} + +/** Read ITLB Synchronous Fault Status Register. + * + * @return Current content of I-SFSR register. + */ +static inline uint64_t itlb_sfsr_read(void) +{ + return asi_u64_read(ASI_IMMU, VA_IMMU_SFSR); +} + +/** Write ITLB Synchronous Fault Status Register. + * + * @param v New value of I-SFSR register. + */ +static inline void itlb_sfsr_write(uint64_t v) +{ + asi_u64_write(ASI_IMMU, VA_IMMU_SFSR, v); + flush_pipeline(); +} + +/** Read DTLB Synchronous Fault Status Register. + * + * @return Current content of D-SFSR register. + */ +static inline uint64_t dtlb_sfsr_read(void) +{ + return asi_u64_read(ASI_DMMU, VA_DMMU_SFSR); +} + +/** Write DTLB Synchronous Fault Status Register. + * + * @param v New value of D-SFSR register. + */ +static inline void dtlb_sfsr_write(uint64_t v) +{ + asi_u64_write(ASI_DMMU, VA_DMMU_SFSR, v); + membar(); +} + +/** Read DTLB Synchronous Fault Address Register. + * + * @return Current content of D-SFAR register. + */ +static inline uint64_t dtlb_sfar_read(void) +{ + return asi_u64_read(ASI_DMMU, VA_DMMU_SFAR); +} + +/** Perform IMMU TLB Demap Operation. + * + * @param type Selects between context and page demap (and entire MMU + * demap on US3). + * @param context_encoding Specifies which Context register has Context ID for + * demap. + * @param page Address which is on the page to be demapped. + */ +static inline void itlb_demap(int type, int context_encoding, uintptr_t page) +{ + tlb_demap_addr_t da; + page_address_t pg; + + da.value = 0; + pg.address = page; + + da.type = type; + da.context = context_encoding; + da.vpn = pg.vpn; + + /* da.value is the address within the ASI */ + asi_u64_write(ASI_IMMU_DEMAP, da.value, 0); + + flush_pipeline(); +} + +/** Perform DMMU TLB Demap Operation. + * + * @param type Selects between context and page demap (and entire MMU + * demap on US3). + * @param context_encoding Specifies which Context register has Context ID for + * demap. + * @param page Address which is on the page to be demapped. + */ +static inline void dtlb_demap(int type, int context_encoding, uintptr_t page) +{ + tlb_demap_addr_t da; + page_address_t pg; + + da.value = 0; + pg.address = page; + + da.type = type; + da.context = context_encoding; + da.vpn = pg.vpn; + + /* da.value is the address within the ASI */ + asi_u64_write(ASI_DMMU_DEMAP, da.value, 0); + + membar(); +} + +extern void fast_instruction_access_mmu_miss(unative_t, istate_t *); +extern void fast_data_access_mmu_miss(tlb_tag_access_reg_t, istate_t *); +extern void fast_data_access_protection(tlb_tag_access_reg_t , istate_t *); + +extern void dtlb_insert_mapping(uintptr_t, uintptr_t, int, bool, bool); + +extern void dump_sfsr_and_sfar(void); + +#endif /* !def __ASM__ */ + +#endif + +/** @} + */ Index: kernel/arch/sparc64/include/mm/sun4v/frame.h =================================================================== --- kernel/arch/sparc64/include/mm/sun4v/frame.h (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/include/mm/sun4v/frame.h (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -44,10 +44,5 @@ #define MMU_FRAME_SIZE (1 << MMU_FRAME_WIDTH) -/* - * Page size exported to the generic memory management subsystems. - * This page size is not directly supported by the MMU, but we can emulate - * each 16K page with a pair of adjacent 8K pages. - */ -#define FRAME_WIDTH 14 /* 16K */ +#define FRAME_WIDTH 13 #define FRAME_SIZE (1 << FRAME_WIDTH) Index: kernel/arch/sparc64/include/mm/tlb.h =================================================================== --- kernel/arch/sparc64/include/mm/tlb.h (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/include/mm/tlb.h (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -36,654 +36,9 @@ #define KERN_sparc64_TLB_H_ -#if defined (US) -#define ITLB_ENTRY_COUNT 64 -#define DTLB_ENTRY_COUNT 64 -#define DTLB_MAX_LOCKED_ENTRIES DTLB_ENTRY_COUNT +#if defined (SUN4U) +#include +#elif defined (SUN4V) +#include #endif - -/** TLB_DSMALL is the only of the three DMMUs that can hold locked entries. */ -#if defined (US3) -#define DTLB_MAX_LOCKED_ENTRIES 16 -#endif - -#define MEM_CONTEXT_KERNEL 0 -#define MEM_CONTEXT_TEMP 1 - -/** Page sizes. */ -#define PAGESIZE_8K 0 -#define PAGESIZE_64K 1 -#define PAGESIZE_512K 2 -#define PAGESIZE_4M 3 - -/** Bit width of the TLB-locked portion of kernel address space. */ -#define KERNEL_PAGE_WIDTH 22 /* 4M */ - -/* TLB Demap Operation types. */ -#define TLB_DEMAP_PAGE 0 -#define TLB_DEMAP_CONTEXT 1 -#if defined (US3) -#define TLB_DEMAP_ALL 2 -#endif - -#define TLB_DEMAP_TYPE_SHIFT 6 - -/* TLB Demap Operation Context register encodings. */ -#define TLB_DEMAP_PRIMARY 0 -#define TLB_DEMAP_SECONDARY 1 -#define TLB_DEMAP_NUCLEUS 2 - -/* There are more TLBs in one MMU in US3, their codes are defined here. */ -#if defined (US3) -/* D-MMU: one small (16-entry) TLB and two big (512-entry) TLBs */ -#define TLB_DSMALL 0 -#define TLB_DBIG_0 2 -#define TLB_DBIG_1 3 - -/* I-MMU: one small (16-entry) TLB and one big TLB */ -#define TLB_ISMALL 0 -#define TLB_IBIG 2 -#endif - -#define TLB_DEMAP_CONTEXT_SHIFT 4 - -/* TLB Tag Access shifts */ -#define TLB_TAG_ACCESS_CONTEXT_SHIFT 0 -#define TLB_TAG_ACCESS_CONTEXT_MASK ((1 << 13) - 1) -#define TLB_TAG_ACCESS_VPN_SHIFT 13 - -#ifndef __ASM__ - -#include -#include -#include -#include -#include -#include -#include -#include - -union tlb_context_reg { - uint64_t v; - struct { - unsigned long : 51; - unsigned context : 13; /**< Context/ASID. */ - } __attribute__ ((packed)); -}; -typedef union tlb_context_reg tlb_context_reg_t; - -/** I-/D-TLB Data In/Access Register type. */ -typedef tte_data_t tlb_data_t; - -/** I-/D-TLB Data Access Address in Alternate Space. */ - -#if defined (US) - -union tlb_data_access_addr { - uint64_t value; - struct { - uint64_t : 55; - unsigned tlb_entry : 6; - unsigned : 3; - } __attribute__ ((packed)); -}; -typedef union tlb_data_access_addr dtlb_data_access_addr_t; -typedef union tlb_data_access_addr dtlb_tag_read_addr_t; -typedef union tlb_data_access_addr itlb_data_access_addr_t; -typedef union tlb_data_access_addr itlb_tag_read_addr_t; - -#elif defined (US3) - -/* - * In US3, I-MMU and D-MMU have different formats of the data - * access register virtual address. In the corresponding - * structures the member variable for the entry number is - * called "local_tlb_entry" - it contrasts with the "tlb_entry" - * for the US data access register VA structure. The rationale - * behind this is to prevent careless mistakes in the code - * caused by setting only the entry number and not the TLB - * number in the US3 code (when taking the code from US). - */ - -union dtlb_data_access_addr { - uint64_t value; - struct { - uint64_t : 45; - unsigned : 1; - unsigned tlb_number : 2; - unsigned : 4; - unsigned local_tlb_entry : 9; - unsigned : 3; - } __attribute__ ((packed)); -}; -typedef union dtlb_data_access_addr dtlb_data_access_addr_t; -typedef union dtlb_data_access_addr dtlb_tag_read_addr_t; - -union itlb_data_access_addr { - uint64_t value; - struct { - uint64_t : 45; - unsigned : 1; - unsigned tlb_number : 2; - unsigned : 6; - unsigned local_tlb_entry : 7; - unsigned : 3; - } __attribute__ ((packed)); -}; -typedef union itlb_data_access_addr itlb_data_access_addr_t; -typedef union itlb_data_access_addr itlb_tag_read_addr_t; - -#endif - -/** I-/D-TLB Tag Read Register. */ -union tlb_tag_read_reg { - uint64_t value; - struct { - uint64_t vpn : 51; /**< Virtual Address bits 63:13. */ - unsigned context : 13; /**< Context identifier. */ - } __attribute__ ((packed)); -}; -typedef union tlb_tag_read_reg tlb_tag_read_reg_t; -typedef union tlb_tag_read_reg tlb_tag_access_reg_t; - - -/** TLB Demap Operation Address. */ -union tlb_demap_addr { - uint64_t value; - struct { - uint64_t vpn: 51; /**< Virtual Address bits 63:13. */ -#if defined (US) - unsigned : 6; /**< Ignored. */ - unsigned type : 1; /**< The type of demap operation. */ -#elif defined (US3) - unsigned : 5; /**< Ignored. */ - unsigned type: 2; /**< The type of demap operation. */ -#endif - unsigned context : 2; /**< Context register selection. */ - unsigned : 4; /**< Zero. */ - } __attribute__ ((packed)); -}; -typedef union tlb_demap_addr tlb_demap_addr_t; - -/** TLB Synchronous Fault Status Register. */ -union tlb_sfsr_reg { - uint64_t value; - struct { -#if defined (US) - unsigned long : 40; /**< Implementation dependent. */ - unsigned asi : 8; /**< ASI. */ - unsigned : 2; - unsigned ft : 7; /**< Fault type. */ -#elif defined (US3) - unsigned long : 39; /**< Implementation dependent. */ - unsigned nf : 1; /**< Non-faulting load. */ - unsigned asi : 8; /**< ASI. */ - unsigned tm : 1; /**< I-TLB miss. */ - unsigned : 3; /**< Reserved. */ - unsigned ft : 5; /**< Fault type. */ -#endif - unsigned e : 1; /**< Side-effect bit. */ - unsigned ct : 2; /**< Context Register selection. */ - unsigned pr : 1; /**< Privilege bit. */ - unsigned w : 1; /**< Write bit. */ - unsigned ow : 1; /**< Overwrite bit. */ - unsigned fv : 1; /**< Fault Valid bit. */ - } __attribute__ ((packed)); -}; -typedef union tlb_sfsr_reg tlb_sfsr_reg_t; - -#if defined (US3) - -/* - * Functions for determining the number of entries in TLBs. They either return - * a constant value or a value based on the CPU autodetection. - */ - -/** - * Determine the number of entries in the DMMU's small TLB. - */ -static inline uint16_t tlb_dsmall_size(void) -{ - return 16; -} - -/** - * Determine the number of entries in each DMMU's big TLB. - */ -static inline uint16_t tlb_dbig_size(void) -{ - return 512; -} - -/** - * Determine the number of entries in the IMMU's small TLB. - */ -static inline uint16_t tlb_ismall_size(void) -{ - return 16; -} - -/** - * Determine the number of entries in the IMMU's big TLB. - */ -static inline uint16_t tlb_ibig_size(void) -{ - if (((ver_reg_t) ver_read()).impl == IMPL_ULTRASPARCIV_PLUS) - return 512; - else - return 128; -} - -#endif - -/** Read MMU Primary Context Register. - * - * @return Current value of Primary Context Register. - */ -static inline uint64_t mmu_primary_context_read(void) -{ - return asi_u64_read(ASI_DMMU, VA_PRIMARY_CONTEXT_REG); -} - -/** Write MMU Primary Context Register. - * - * @param v New value of Primary Context Register. - */ -static inline void mmu_primary_context_write(uint64_t v) -{ - asi_u64_write(ASI_DMMU, VA_PRIMARY_CONTEXT_REG, v); - flush_pipeline(); -} - -/** Read MMU Secondary Context Register. - * - * @return Current value of Secondary Context Register. - */ -static inline uint64_t mmu_secondary_context_read(void) -{ - return asi_u64_read(ASI_DMMU, VA_SECONDARY_CONTEXT_REG); -} - -/** Write MMU Primary Context Register. - * - * @param v New value of Primary Context Register. - */ -static inline void mmu_secondary_context_write(uint64_t v) -{ - asi_u64_write(ASI_DMMU, VA_SECONDARY_CONTEXT_REG, v); - flush_pipeline(); -} - -#if defined (US) - -/** Read IMMU TLB Data Access Register. - * - * @param entry TLB Entry index. - * - * @return Current value of specified IMMU TLB Data Access - * Register. - */ -static inline uint64_t itlb_data_access_read(size_t entry) -{ - itlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_entry = entry; - return asi_u64_read(ASI_ITLB_DATA_ACCESS_REG, reg.value); -} - -/** Write IMMU TLB Data Access Register. - * - * @param entry TLB Entry index. - * @param value Value to be written. - */ -static inline void itlb_data_access_write(size_t entry, uint64_t value) -{ - itlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_entry = entry; - asi_u64_write(ASI_ITLB_DATA_ACCESS_REG, reg.value, value); - flush_pipeline(); -} - -/** Read DMMU TLB Data Access Register. - * - * @param entry TLB Entry index. - * - * @return Current value of specified DMMU TLB Data Access - * Register. - */ -static inline uint64_t dtlb_data_access_read(size_t entry) -{ - dtlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_entry = entry; - return asi_u64_read(ASI_DTLB_DATA_ACCESS_REG, reg.value); -} - -/** Write DMMU TLB Data Access Register. - * - * @param entry TLB Entry index. - * @param value Value to be written. - */ -static inline void dtlb_data_access_write(size_t entry, uint64_t value) -{ - dtlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_entry = entry; - asi_u64_write(ASI_DTLB_DATA_ACCESS_REG, reg.value, value); - membar(); -} - -/** Read IMMU TLB Tag Read Register. - * - * @param entry TLB Entry index. - * - * @return Current value of specified IMMU TLB Tag Read Register. - */ -static inline uint64_t itlb_tag_read_read(size_t entry) -{ - itlb_tag_read_addr_t tag; - - tag.value = 0; - tag.tlb_entry = entry; - return asi_u64_read(ASI_ITLB_TAG_READ_REG, tag.value); -} - -/** Read DMMU TLB Tag Read Register. - * - * @param entry TLB Entry index. - * - * @return Current value of specified DMMU TLB Tag Read Register. - */ -static inline uint64_t dtlb_tag_read_read(size_t entry) -{ - dtlb_tag_read_addr_t tag; - - tag.value = 0; - tag.tlb_entry = entry; - return asi_u64_read(ASI_DTLB_TAG_READ_REG, tag.value); -} - -#elif defined (US3) - - -/** Read IMMU TLB Data Access Register. - * - * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) - * @param entry TLB Entry index. - * - * @return Current value of specified IMMU TLB Data Access - * Register. - */ -static inline uint64_t itlb_data_access_read(int tlb, size_t entry) -{ - itlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_number = tlb; - reg.local_tlb_entry = entry; - return asi_u64_read(ASI_ITLB_DATA_ACCESS_REG, reg.value); -} - -/** Write IMMU TLB Data Access Register. - * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) - * @param entry TLB Entry index. - * @param value Value to be written. - */ -static inline void itlb_data_access_write(int tlb, size_t entry, - uint64_t value) -{ - itlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_number = tlb; - reg.local_tlb_entry = entry; - asi_u64_write(ASI_ITLB_DATA_ACCESS_REG, reg.value, value); - flush_pipeline(); -} - -/** Read DMMU TLB Data Access Register. - * - * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG, TLB_DBIG) - * @param entry TLB Entry index. - * - * @return Current value of specified DMMU TLB Data Access - * Register. - */ -static inline uint64_t dtlb_data_access_read(int tlb, size_t entry) -{ - dtlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_number = tlb; - reg.local_tlb_entry = entry; - return asi_u64_read(ASI_DTLB_DATA_ACCESS_REG, reg.value); -} - -/** Write DMMU TLB Data Access Register. - * - * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1) - * @param entry TLB Entry index. - * @param value Value to be written. - */ -static inline void dtlb_data_access_write(int tlb, size_t entry, - uint64_t value) -{ - dtlb_data_access_addr_t reg; - - reg.value = 0; - reg.tlb_number = tlb; - reg.local_tlb_entry = entry; - asi_u64_write(ASI_DTLB_DATA_ACCESS_REG, reg.value, value); - membar(); -} - -/** Read IMMU TLB Tag Read Register. - * - * @param tlb TLB number (one of TLB_ISMALL or TLB_IBIG) - * @param entry TLB Entry index. - * - * @return Current value of specified IMMU TLB Tag Read Register. - */ -static inline uint64_t itlb_tag_read_read(int tlb, size_t entry) -{ - itlb_tag_read_addr_t tag; - - tag.value = 0; - tag.tlb_number = tlb; - tag.local_tlb_entry = entry; - return asi_u64_read(ASI_ITLB_TAG_READ_REG, tag.value); -} - -/** Read DMMU TLB Tag Read Register. - * - * @param tlb TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1) - * @param entry TLB Entry index. - * - * @return Current value of specified DMMU TLB Tag Read Register. - */ -static inline uint64_t dtlb_tag_read_read(int tlb, size_t entry) -{ - dtlb_tag_read_addr_t tag; - - tag.value = 0; - tag.tlb_number = tlb; - tag.local_tlb_entry = entry; - return asi_u64_read(ASI_DTLB_TAG_READ_REG, tag.value); -} - -#endif - - -/** Write IMMU TLB Tag Access Register. - * - * @param v Value to be written. - */ -static inline void itlb_tag_access_write(uint64_t v) -{ - asi_u64_write(ASI_IMMU, VA_IMMU_TAG_ACCESS, v); - flush_pipeline(); -} - -/** Read IMMU TLB Tag Access Register. - * - * @return Current value of IMMU TLB Tag Access Register. - */ -static inline uint64_t itlb_tag_access_read(void) -{ - return asi_u64_read(ASI_IMMU, VA_IMMU_TAG_ACCESS); -} - -/** Write DMMU TLB Tag Access Register. - * - * @param v Value to be written. - */ -static inline void dtlb_tag_access_write(uint64_t v) -{ - asi_u64_write(ASI_DMMU, VA_DMMU_TAG_ACCESS, v); - membar(); -} - -/** Read DMMU TLB Tag Access Register. - * - * @return Current value of DMMU TLB Tag Access Register. - */ -static inline uint64_t dtlb_tag_access_read(void) -{ - return asi_u64_read(ASI_DMMU, VA_DMMU_TAG_ACCESS); -} - - -/** Write IMMU TLB Data in Register. - * - * @param v Value to be written. - */ -static inline void itlb_data_in_write(uint64_t v) -{ - asi_u64_write(ASI_ITLB_DATA_IN_REG, 0, v); - flush_pipeline(); -} - -/** Write DMMU TLB Data in Register. - * - * @param v Value to be written. - */ -static inline void dtlb_data_in_write(uint64_t v) -{ - asi_u64_write(ASI_DTLB_DATA_IN_REG, 0, v); - membar(); -} - -/** Read ITLB Synchronous Fault Status Register. - * - * @return Current content of I-SFSR register. - */ -static inline uint64_t itlb_sfsr_read(void) -{ - return asi_u64_read(ASI_IMMU, VA_IMMU_SFSR); -} - -/** Write ITLB Synchronous Fault Status Register. - * - * @param v New value of I-SFSR register. - */ -static inline void itlb_sfsr_write(uint64_t v) -{ - asi_u64_write(ASI_IMMU, VA_IMMU_SFSR, v); - flush_pipeline(); -} - -/** Read DTLB Synchronous Fault Status Register. - * - * @return Current content of D-SFSR register. - */ -static inline uint64_t dtlb_sfsr_read(void) -{ - return asi_u64_read(ASI_DMMU, VA_DMMU_SFSR); -} - -/** Write DTLB Synchronous Fault Status Register. - * - * @param v New value of D-SFSR register. - */ -static inline void dtlb_sfsr_write(uint64_t v) -{ - asi_u64_write(ASI_DMMU, VA_DMMU_SFSR, v); - membar(); -} - -/** Read DTLB Synchronous Fault Address Register. - * - * @return Current content of D-SFAR register. - */ -static inline uint64_t dtlb_sfar_read(void) -{ - return asi_u64_read(ASI_DMMU, VA_DMMU_SFAR); -} - -/** Perform IMMU TLB Demap Operation. - * - * @param type Selects between context and page demap (and entire MMU - * demap on US3). - * @param context_encoding Specifies which Context register has Context ID for - * demap. - * @param page Address which is on the page to be demapped. - */ -static inline void itlb_demap(int type, int context_encoding, uintptr_t page) -{ - tlb_demap_addr_t da; - page_address_t pg; - - da.value = 0; - pg.address = page; - - da.type = type; - da.context = context_encoding; - da.vpn = pg.vpn; - - /* da.value is the address within the ASI */ - asi_u64_write(ASI_IMMU_DEMAP, da.value, 0); - - flush_pipeline(); -} - -/** Perform DMMU TLB Demap Operation. - * - * @param type Selects between context and page demap (and entire MMU - * demap on US3). - * @param context_encoding Specifies which Context register has Context ID for - * demap. - * @param page Address which is on the page to be demapped. - */ -static inline void dtlb_demap(int type, int context_encoding, uintptr_t page) -{ - tlb_demap_addr_t da; - page_address_t pg; - - da.value = 0; - pg.address = page; - - da.type = type; - da.context = context_encoding; - da.vpn = pg.vpn; - - /* da.value is the address within the ASI */ - asi_u64_write(ASI_DMMU_DEMAP, da.value, 0); - - membar(); -} - -extern void fast_instruction_access_mmu_miss(unative_t, istate_t *); -extern void fast_data_access_mmu_miss(tlb_tag_access_reg_t, istate_t *); -extern void fast_data_access_protection(tlb_tag_access_reg_t , istate_t *); - -extern void dtlb_insert_mapping(uintptr_t, uintptr_t, int, bool, bool); - -extern void dump_sfsr_and_sfar(void); - -#endif /* !def __ASM__ */ #endif Index: kernel/arch/sparc64/src/mm/sun4v/as.c =================================================================== --- kernel/arch/sparc64/src/mm/sun4v/as.c (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/src/mm/sun4v/as.c (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -61,22 +61,22 @@ { #ifdef CONFIG_TSB - /* - * The order must be calculated with respect to the emulated - * 16K page size. - */ - int order = fnzb32(((ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * - sizeof(tsb_entry_t)) >> FRAME_WIDTH); + int order = fnzb32( + (TSB_ENTRY_COUNT * sizeof(tsb_entry_t)) >> FRAME_WIDTH); - uintptr_t tsb = (uintptr_t) frame_alloc(order, flags | FRAME_KA); + uintptr_t tsb = (uintptr_t) frame_alloc(order, flags); if (!tsb) return -1; - as->arch.itsb = (tsb_entry_t *) tsb; - as->arch.dtsb = (tsb_entry_t *) (tsb + ITSB_ENTRY_COUNT * - sizeof(tsb_entry_t)); + as->arch.tsb_description.page_size = PAGESIZE_8K; + as->arch.tsb_description.associativity = 1; + as->arch.tsb_description.num_ttes = TSB_ENTRY_COUNT; + as->arch.tsb_description.pgsize_mask = 1 << PAGESIZE_8K; + as->arch.tsb_description.tsb_base = tsb; + as->arch.tsb_description.reserved = 0; + as->arch.tsb_description.context = 0; - memsetb(as->arch.itsb, - (ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * sizeof(tsb_entry_t), 0); + memsetb((void *) PA2KA(as->arch.tsb_description.tsb_base), + TSB_ENTRY_COUNT * sizeof(tsb_entry_t), 0); #endif return 0; @@ -86,11 +86,6 @@ { #ifdef CONFIG_TSB - /* - * The count must be calculated with respect to the emualted 16K page - * size. - */ - size_t cnt = ((ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * - sizeof(tsb_entry_t)) >> FRAME_WIDTH; - frame_free(KA2PA((uintptr_t) as->arch.itsb)); + count_t cnt = (TSB_ENTRY_COUNT * sizeof(tsb_entry_t)) >> FRAME_WIDTH; + frame_free((uintptr_t) as->arch.tsb_description.tsb_base); return cnt; #else @@ -116,74 +111,5 @@ void as_install_arch(as_t *as) { -#if 0 - tlb_context_reg_t ctx; - - /* - * Note that we don't and may not lock the address space. That's ok - * since we only read members that are currently read-only. - * - * Moreover, the as->asid is protected by asidlock, which is being held. - */ - - /* - * Write ASID to secondary context register. The primary context - * register has to be set from TL>0 so it will be filled from the - * secondary context register from the TL=1 code just before switch to - * userspace. - */ - ctx.v = 0; - ctx.context = as->asid; - mmu_secondary_context_write(ctx.v); - -#ifdef CONFIG_TSB - uintptr_t base = ALIGN_DOWN(config.base, 1 << KERNEL_PAGE_WIDTH); - - ASSERT(as->arch.itsb && as->arch.dtsb); - - uintptr_t tsb = (uintptr_t) as->arch.itsb; - - if (!overlaps(tsb, 8 * MMU_PAGE_SIZE, base, 1 << KERNEL_PAGE_WIDTH)) { - /* - * TSBs were allocated from memory not covered - * by the locked 4M kernel DTLB entry. We need - * to map both TSBs explicitly. - */ - dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_NUCLEUS, tsb); - dtlb_insert_mapping(tsb, KA2PA(tsb), PAGESIZE_64K, true, true); - } - - /* - * Setup TSB Base registers. - */ - tsb_base_reg_t tsb_base; - - tsb_base.value = 0; - tsb_base.size = TSB_SIZE; - tsb_base.split = 0; - - tsb_base.base = ((uintptr_t) as->arch.itsb) >> MMU_PAGE_WIDTH; - itsb_base_write(tsb_base.value); - tsb_base.base = ((uintptr_t) as->arch.dtsb) >> MMU_PAGE_WIDTH; - dtsb_base_write(tsb_base.value); - -#if defined (US3) - /* - * Clear the extension registers. - * In HelenOS, primary and secondary context registers contain - * equal values and kernel misses (context 0, ie. the nucleus context) - * are excluded from the TSB miss handler, so it makes no sense - * to have separate TSBs for primary, secondary and nucleus contexts. - * Clearing the extension registers will ensure that the value of the - * TSB Base register will be used as an address of TSB, making the code - * compatible with the US port. - */ - itsb_primary_extension_write(0); - itsb_nucleus_extension_write(0); - dtsb_primary_extension_write(0); - dtsb_secondary_extension_write(0); - dtsb_nucleus_extension_write(0); -#endif -#endif -#endif + mmu_secondary_context_write(as->asid); } Index: kernel/arch/sparc64/src/mm/sun4v/tlb.c =================================================================== --- kernel/arch/sparc64/src/mm/sun4v/tlb.c (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/src/mm/sun4v/tlb.c (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -60,11 +60,11 @@ #endif -static void dtlb_pte_copy(pte_t *, size_t, bool); -static void itlb_pte_copy(pte_t *, size_t); +//static void dtlb_pte_copy(pte_t *, size_t, bool); +static void itlb_pte_copy(pte_t *); static void do_fast_instruction_access_mmu_miss_fault(istate_t *, const char *); -static void do_fast_data_access_mmu_miss_fault(istate_t *, tlb_tag_access_reg_t, - const char *); -static void do_fast_data_access_protection_fault(istate_t *, - tlb_tag_access_reg_t, const char *); +//static void do_fast_data_access_mmu_miss_fault(istate_t *, uint64_t, +// const char *); +//static void do_fast_data_access_protection_fault(istate_t *, +// uint64_t, const char *); char *context_encoding[] = { @@ -132,4 +132,5 @@ } +#if 0 /** Copy PTE to TLB. * @@ -141,5 +142,4 @@ void dtlb_pte_copy(pte_t *t, size_t index, bool ro) { -#if 0 tlb_tag_access_reg_t tag; tlb_data_t data; @@ -170,41 +170,30 @@ dtlb_data_in_write(data.value); -#endif -} +} +#endif /** Copy PTE to ITLB. * * @param t Page Table Entry to be copied. - * @param index Zero if lower 8K-subpage, one if higher 8K-subpage. - */ -void itlb_pte_copy(pte_t *t, size_t index) -{ -#if 0 - tlb_tag_access_reg_t tag; - tlb_data_t data; - page_address_t pg; - frame_address_t fr; - - pg.address = t->page + (index << MMU_PAGE_WIDTH); - fr.address = t->frame + (index << MMU_PAGE_WIDTH); - - tag.value = 0; - tag.context = t->as->asid; - tag.vpn = pg.vpn; - - itlb_tag_access_write(tag.value); + */ +void itlb_pte_copy(pte_t *t) +{ + tte_data_t data; data.value = 0; data.v = true; + data.nfo = false; + data.ra = (t->frame) >> FRAME_WIDTH; + data.ie = false; + data.e = false; + data.cp = t->c; + data.cv = false; + data.p = t->k; + data.x = true; + data.w = false; data.size = PAGESIZE_8K; - data.pfn = fr.pfn; - data.l = false; - data.cp = t->c; - data.p = t->k; /* p like privileged */ - data.w = false; - data.g = t->g; - itlb_data_in_write(data.value); -#endif + __hypercall_hyperfast( + t->page, t->as->asid, data.value, MMU_FLAG_ITLB, 0, MMU_MAP_ADDR); } @@ -212,11 +201,10 @@ void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate) { - asm volatile ("sethi 0x41906, %g0"); - uintptr_t page_16k = ALIGN_DOWN(istate->tpc, PAGE_SIZE); - size_t index = (istate->tpc >> MMU_PAGE_WIDTH) % MMU_PAGES_PER_PAGE; + uintptr_t va = ALIGN_DOWN(istate->tpc, PAGE_SIZE); pte_t *t; page_table_lock(AS, true); - t = page_mapping_find(AS, page_16k); + t = page_mapping_find(AS, va); + if (t && PTE_EXECUTABLE(t)) { /* @@ -225,7 +213,7 @@ */ t->a = true; - itlb_pte_copy(t, index); + itlb_pte_copy(t); #ifdef CONFIG_TSB - itsb_pte_copy(t, index); + itsb_pte_copy(t); #endif page_table_unlock(AS, true); @@ -236,6 +224,5 @@ */ page_table_unlock(AS, true); - if (as_page_fault(page_16k, PF_ACCESS_EXEC, istate) == - AS_PF_FAULT) { + if (as_page_fault(va, PF_ACCESS_EXEC, istate) == AS_PF_FAULT) { do_fast_instruction_access_mmu_miss_fault(istate, __func__); @@ -248,4 +235,58 @@ * Note that some faults (e.g. kernel faults) were already resolved by the * low-level, assembly language part of the fast_data_access_mmu_miss handler. + * + * @param page_and_ctx A 64-bit value describing the fault. The most + * significant 51 bits of the value contain the virtual + * address which caused the fault truncated to the page + * boundary. The least significant 13 bits of the value + * contain the number of the context in which the fault + * occurred. + * @param istate Interrupted state saved on the stack. + */ +void fast_data_access_mmu_miss(uint64_t page_and_ctx, istate_t *istate) +{ +#if 0 + pte_t *t; + uintptr_t va = DMISS_ADDRESS(page_and_ctx); + uint16_t ctx = DMISS_CONTEXT(page_and_ctx); + + if (ctx == ASID_KERNEL) { + if (va == 0) { + /* NULL access in kernel */ + do_fast_data_access_mmu_miss_fault(istate, page_and_ctx, + __func__); + } + do_fast_data_access_mmu_miss_fault(istate, page_and_ctx, "Unexpected " + "kernel page fault."); + } + + page_table_lock(AS, true); + t = page_mapping_find(AS, va); + if (t) { + /* + * The mapping was found in the software page hash table. + * Insert it into DTLB. + */ + t->a = true; + dtlb_pte_copy(t, true); +#ifdef CONFIG_TSB + dtsb_pte_copy(t, true); +#endif + page_table_unlock(AS, true); + } else { + /* + * Forward the page fault to the address space page fault + * handler. + */ + page_table_unlock(AS, true); + if (as_page_fault(va, PF_ACCESS_READ, istate) == AS_PF_FAULT) { + do_fast_data_access_mmu_miss_fault(istate, page_and_ctx, + __func__); + } + } +#endif +} + +/** DTLB protection fault handler. * * @param tag Content of the TLB Tag Access register as it existed @@ -255,82 +296,14 @@ * @param istate Interrupted state saved on the stack. */ -void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate) -{ - uintptr_t page_8k; - uintptr_t page_16k; - size_t index; +void fast_data_access_protection(uint64_t page_and_ctx, istate_t *istate) +{ +#if 0 pte_t *t; - page_8k = (uint64_t) tag.vpn << MMU_PAGE_WIDTH; - page_16k = ALIGN_DOWN(page_8k, PAGE_SIZE); - index = tag.vpn % MMU_PAGES_PER_PAGE; - - if (tag.context == ASID_KERNEL) { - if (!tag.vpn) { - /* NULL access in kernel */ - do_fast_data_access_mmu_miss_fault(istate, tag, - __func__); -//MH - } else { -// } else if (page_8k >= end_of_identity) { - /* - * The kernel is accessing the I/O space. - * We still do identity mapping for I/O, - * but without caching. - */ - dtlb_insert_mapping(page_8k, KA2PA(page_8k), - PAGESIZE_8K, false, false); - return; - } - do_fast_data_access_mmu_miss_fault(istate, tag, "Unexpected " - "kernel page fault."); - } + uintptr_t va = DMISS_ADDRESS(page_and_ctx); + uint16_t ctx = DMISS_CONTEXT(page_and_ctx); page_table_lock(AS, true); - t = page_mapping_find(AS, page_16k); - if (t) { - /* - * The mapping was found in the software page hash table. - * Insert it into DTLB. - */ - t->a = true; - dtlb_pte_copy(t, index, true); -#ifdef CONFIG_TSB - dtsb_pte_copy(t, index, true); -#endif - page_table_unlock(AS, true); - } else { - /* - * Forward the page fault to the address space page fault - * handler. - */ - page_table_unlock(AS, true); - if (as_page_fault(page_16k, PF_ACCESS_READ, istate) == - AS_PF_FAULT) { - do_fast_data_access_mmu_miss_fault(istate, tag, - __func__); - } - } -} - -/** DTLB protection fault handler. - * - * @param tag Content of the TLB Tag Access register as it existed - * when the trap happened. This is to prevent confusion - * created by clobbered Tag Access register during a nested - * DTLB miss. - * @param istate Interrupted state saved on the stack. - */ -void fast_data_access_protection(tlb_tag_access_reg_t tag, istate_t *istate) -{ - uintptr_t page_16k; - size_t index; - pte_t *t; - - page_16k = ALIGN_DOWN((uint64_t) tag.vpn << MMU_PAGE_WIDTH, PAGE_SIZE); - index = tag.vpn % MMU_PAGES_PER_PAGE; /* 16K-page emulation */ - - page_table_lock(AS, true); - t = page_mapping_find(AS, page_16k); + t = page_mapping_find(AS, va); if (t && PTE_WRITABLE(t)) { /* @@ -341,9 +314,8 @@ t->a = true; t->d = true; - dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_SECONDARY, - page_16k + index * MMU_PAGE_SIZE); - dtlb_pte_copy(t, index, false); + mmu_demap_page(va, ctx, MMU_FLAG_DTLB); + dtlb_pte_copy(t, false); #ifdef CONFIG_TSB - dtsb_pte_copy(t, index, false); + dtsb_pte_copy(t, false); #endif page_table_unlock(AS, true); @@ -360,4 +332,5 @@ } } +#endif } @@ -371,85 +344,8 @@ * @param d TLB entry data */ -static void print_tlb_entry(int i, tlb_tag_read_reg_t t, tlb_data_t d) -{ -#if 0 - printf("%d: vpn=%#llx, context=%d, v=%d, size=%d, nfo=%d, " - "ie=%d, soft2=%#x, pfn=%#x, soft=%#x, l=%d, " - "cp=%d, cv=%d, e=%d, p=%d, w=%d, g=%d\n", i, t.vpn, - t.context, d.v, d.size, d.nfo, d.ie, d.soft2, - d.pfn, d.soft, d.l, d.cp, d.cv, d.e, d.p, d.w, d.g); -#endif -} - -#if defined (US) - -/** Print contents of both TLBs. */ void tlb_print(void) { - int i; - tlb_data_t d; - tlb_tag_read_reg_t t; - - printf("I-TLB contents:\n"); - for (i = 0; i < ITLB_ENTRY_COUNT; i++) { - d.value = itlb_data_access_read(i); - t.value = itlb_tag_read_read(i); - print_tlb_entry(i, t, d); - } - - printf("D-TLB contents:\n"); - for (i = 0; i < DTLB_ENTRY_COUNT; i++) { - d.value = dtlb_data_access_read(i); - t.value = dtlb_tag_read_read(i); - print_tlb_entry(i, t, d); - } -} - -#elif defined (US3) - -/** Print contents of all TLBs. */ -void tlb_print(void) -{ - int i; - tlb_data_t d; - tlb_tag_read_reg_t t; - - printf("TLB_ISMALL contents:\n"); - for (i = 0; i < tlb_ismall_size(); i++) { - d.value = dtlb_data_access_read(TLB_ISMALL, i); - t.value = dtlb_tag_read_read(TLB_ISMALL, i); - print_tlb_entry(i, t, d); - } - - printf("TLB_IBIG contents:\n"); - for (i = 0; i < tlb_ibig_size(); i++) { - d.value = dtlb_data_access_read(TLB_IBIG, i); - t.value = dtlb_tag_read_read(TLB_IBIG, i); - print_tlb_entry(i, t, d); - } - - printf("TLB_DSMALL contents:\n"); - for (i = 0; i < tlb_dsmall_size(); i++) { - d.value = dtlb_data_access_read(TLB_DSMALL, i); - t.value = dtlb_tag_read_read(TLB_DSMALL, i); - print_tlb_entry(i, t, d); - } - - printf("TLB_DBIG_1 contents:\n"); - for (i = 0; i < tlb_dbig_size(); i++) { - d.value = dtlb_data_access_read(TLB_DBIG_0, i); - t.value = dtlb_tag_read_read(TLB_DBIG_0, i); - print_tlb_entry(i, t, d); - } - - printf("TLB_DBIG_2 contents:\n"); - for (i = 0; i < tlb_dbig_size(); i++) { - d.value = dtlb_data_access_read(TLB_DBIG_1, i); - t.value = dtlb_tag_read_read(TLB_DBIG_1, i); - print_tlb_entry(i, t, d); - } -} - -#endif + printf("Operation not possible on Niagara.\n"); +} void do_fast_instruction_access_mmu_miss_fault(istate_t *istate, @@ -461,13 +357,11 @@ } +#if 0 void do_fast_data_access_mmu_miss_fault(istate_t *istate, - tlb_tag_access_reg_t tag, const char *str) -{ - uintptr_t va; - - va = tag.vpn << MMU_PAGE_WIDTH; - if (tag.context) { - fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d).", str, va, - tag.context); + uint64_t page_and_ctx, const char *str) +{ + if (DMISS_CONTEXT(page_and_ctx)) { + fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, DMISS_ADDRESS(page_and_ctx), + DMISS_CONTEXT(page_and_ctx)); } dump_istate(istate); @@ -475,42 +369,44 @@ panic("%s.", str); } - +#endif + +#if 0 void do_fast_data_access_protection_fault(istate_t *istate, - tlb_tag_access_reg_t tag, const char *str) -{ - uintptr_t va; - - va = tag.vpn << MMU_PAGE_WIDTH; - - if (tag.context) { - fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d).", str, va, - tag.context); - } - printf("Faulting page: %p, ASID=%d\n", va, tag.context); + uint64_t page_and_ctx, const char *str) +{ + if (DMISS_CONTEXT(page_and_ctx)) { + fault_if_from_uspace(istate, "%s, Page=%p (ASID=%d)\n", str, DMISS_ADDRESS(page_and_ctx), + DMISS_CONTEXT(page_and_ctx)); + } + printf("Faulting page: %p, ASID=%d\n", DMISS_ADDRESS(page_and_ctx), DMISS_CONTEXT(page_and_ctx)); dump_istate(istate); panic("%s.", str); } - -void dump_sfsr_and_sfar(void) -{ - tlb_sfsr_reg_t sfsr; - uintptr_t sfar; - - sfsr.value = dtlb_sfsr_read(); - sfar = dtlb_sfar_read(); - -#if defined (US) - printf("DTLB SFSR: asi=%#x, ft=%#x, e=%d, ct=%d, pr=%d, w=%d, ow=%d, " - "fv=%d\n", sfsr.asi, sfsr.ft, sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, - sfsr.ow, sfsr.fv); -#elif defined (US3) - printf("DTLB SFSR: nf=%d, asi=%#x, tm=%d, ft=%#x, e=%d, ct=%d, pr=%d, " - "w=%d, ow=%d, fv=%d\n", sfsr.nf, sfsr.asi, sfsr.tm, sfsr.ft, - sfsr.e, sfsr.ct, sfsr.pr, sfsr.w, sfsr.ow, sfsr.fv); -#endif - - printf("DTLB SFAR: address=%p\n", sfar); - - dtlb_sfsr_write(0); +#endif + +/** + * Describes the exact condition which caused the last DMMU fault. + */ +void describe_dmmu_fault(void) +{ +#if 0 + uint64_t myid; + __hypercall_fast_ret1(0, 0, 0, 0, 0, CPU_MYID, &myid); + + ASSERT(mmu_fsas[myid].dft < 16); + + printf("condition which caused the fault: %s\n", + fault_types[mmu_fsas[myid].dft]); +} + +/** Invalidate all unlocked ITLB and DTLB entries. */ +void tlb_invalidate_all(void) +{ + uint64_t errno = __hypercall_fast3(MMU_DEMAP_ALL, 0, 0, + MMU_FLAG_DTLB | MMU_FLAG_ITLB); + if (errno != EOK) { + panic("Error code = %d.\n", errno); + } +#endif } Index: kernel/arch/sparc64/src/trap/exception.c =================================================================== --- kernel/arch/sparc64/src/trap/exception.c (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/src/trap/exception.c (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -162,5 +162,6 @@ fault_if_from_uspace(istate, "%s.", __func__); dump_istate(istate); - dump_sfsr_and_sfar(); +//MH +// dump_sfsr_and_sfar(); panic("%s.", __func__); } Index: kernel/arch/sparc64/src/trap/sun4v/trap_table.S =================================================================== --- kernel/arch/sparc64/src/trap/sun4v/trap_table.S (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/arch/sparc64/src/trap/sun4v/trap_table.S (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -1092,5 +1092,5 @@ and %g1, NWINDOWS - 1, %g1 wrpr %g1, 0, %cwp ! CWP-- - + .if \is_syscall done Index: kernel/generic/include/errno.h =================================================================== --- kernel/generic/include/errno.h (revision eb79d60443133ae1a59f38af667844b1683015c0) +++ kernel/generic/include/errno.h (revision ba50a34216214cc680c5b1c6995600cc6e1828e8) @@ -57,6 +57,11 @@ #define EADDRNOTAVAIL -12 /* Address not available. */ #define ETIMEOUT -13 /* Timeout expired */ +//MH +#ifndef EINVAL #define EINVAL -14 /* Invalid value */ +#endif +#ifndef EBUSY #define EBUSY -15 /* Resource is busy */ +#endif #define EOVERFLOW -16 /* The result does not fit its size. */ #define EINTR -17 /* Operation was interrupted. */