Index: kernel/arch/sparc64/include/mm/tlb.h
===================================================================
--- kernel/arch/sparc64/include/mm/tlb.h	(revision c711efe7b237311021ad0089a03837cd8362035e)
+++ kernel/arch/sparc64/include/mm/tlb.h	(revision 965dc18d04fb8a15431d34485959f4aa8e6ec4b5)
@@ -36,6 +36,14 @@
 #define KERN_sparc64_TLB_H_
 
+#if defined (US)
 #define ITLB_ENTRY_COUNT		64
 #define DTLB_ENTRY_COUNT		64
+#define DTLB_MAX_LOCKED_ENTRIES		DTLB_ENTRY_COUNT
+#endif
+
+/** TLB_DSMALL is the only of the three DMMUs that can hold locked entries. */
+#if defined (US3)
+#define DTLB_MAX_LOCKED_ENTRIES		16
+#endif
 
 #define MEM_CONTEXT_KERNEL		0
@@ -54,4 +62,7 @@
 #define TLB_DEMAP_PAGE		0
 #define TLB_DEMAP_CONTEXT	1
+#if defined (US3)
+#define TLB_DEMAP_ALL		2
+#endif
 
 #define TLB_DEMAP_TYPE_SHIFT	6
@@ -61,4 +72,16 @@
 #define TLB_DEMAP_SECONDARY	1
 #define TLB_DEMAP_NUCLEUS	2
+
+/* There are more TLBs in one MMU in US3, their codes are defined here. */
+#if defined (US3)
+/* D-MMU: one small (16-entry) TLB and two big (512-entry) TLBs */
+#define TLB_DSMALL	0
+#define TLB_DBIG_0	2
+#define TLB_DBIG_1	3
+	
+/* I-MMU: one small (16-entry) TLB and one big TLB */
+#define TLB_ISMALL	0
+#define TLB_IBIG	2
+#endif
 
 #define TLB_DEMAP_CONTEXT_SHIFT	4
@@ -77,4 +100,6 @@
 #include <arch/barrier.h>
 #include <arch/types.h>
+#include <arch/register.h>
+#include <arch/cpu.h>
 
 union tlb_context_reg {
@@ -91,4 +116,7 @@
 
 /** I-/D-TLB Data Access Address in Alternate Space. */
+
+#if defined (US)
+
 union tlb_data_access_addr {
 	uint64_t value;
@@ -99,6 +127,51 @@
 	} __attribute__ ((packed));
 };
-typedef union tlb_data_access_addr tlb_data_access_addr_t;
-typedef union tlb_data_access_addr tlb_tag_read_addr_t;
+typedef union tlb_data_access_addr dtlb_data_access_addr_t;
+typedef union tlb_data_access_addr dtlb_tag_read_addr_t;
+typedef union tlb_data_access_addr itlb_data_access_addr_t;
+typedef union tlb_data_access_addr itlb_tag_read_addr_t;
+
+#elif defined (US3)
+
+/*
+ * In US3, I-MMU and D-MMU have different formats of the data
+ * access register virtual address. In the corresponding
+ * structures the member variable for the entry number is
+ * called "local_tlb_entry" - it contrasts with the "tlb_entry"
+ * for the US data access register VA structure. The rationale
+ * behind this is to prevent careless mistakes in the code
+ * caused by setting only the entry number and not the TLB
+ * number in the US3 code (when taking the code from US). 
+ */
+
+union dtlb_data_access_addr {
+	uint64_t value;
+	struct {
+		uint64_t : 45;
+		unsigned : 1;
+		unsigned tlb_number : 2;
+		unsigned : 4;
+		unsigned local_tlb_entry : 9;
+		unsigned : 3;
+	} __attribute__ ((packed));
+};
+typedef union dtlb_data_access_addr dtlb_data_access_addr_t;
+typedef union dtlb_data_access_addr dtlb_tag_read_addr_t;
+
+union itlb_data_access_addr {
+	uint64_t value;
+	struct {
+		uint64_t : 45;
+		unsigned : 1;
+		unsigned tlb_number : 2;
+		unsigned : 6;
+		unsigned local_tlb_entry : 7;
+		unsigned : 3;
+	} __attribute__ ((packed));
+};
+typedef union itlb_data_access_addr itlb_data_access_addr_t;
+typedef union itlb_data_access_addr itlb_tag_read_addr_t;
+
+#endif
 
 /** I-/D-TLB Tag Read Register. */
@@ -119,6 +192,11 @@
 	struct {
 		uint64_t vpn: 51;	/**< Virtual Address bits 63:13. */
+#if defined (US)
 		unsigned : 6;		/**< Ignored. */
 		unsigned type : 1;	/**< The type of demap operation. */
+#elif defined (US3)
+		unsigned : 5;		/**< Ignored. */
+		unsigned type: 2;	/**< The type of demap operation. */
+#endif
 		unsigned context : 2;	/**< Context register selection. */
 		unsigned : 4;		/**< Zero. */
@@ -131,8 +209,17 @@
 	uint64_t value;
 	struct {
+#if defined (US)
 		unsigned long : 40;	/**< Implementation dependent. */
 		unsigned asi : 8;	/**< ASI. */
 		unsigned : 2;
 		unsigned ft : 7;	/**< Fault type. */
+#elif defined (US3)
+		unsigned long : 39;	/**< Implementation dependent. */
+		unsigned nf : 1;	/**< Non-faulting load. */
+		unsigned asi : 8;	/**< ASI. */
+		unsigned tm : 1;	/**< I-TLB miss. */
+		unsigned : 3;		/**< Reserved. */
+		unsigned ft : 5;	/**< Fault type. */
+#endif
 		unsigned e : 1;		/**< Side-effect bit. */
 		unsigned ct : 2;	/**< Context Register selection. */
@@ -145,7 +232,51 @@
 typedef union tlb_sfsr_reg tlb_sfsr_reg_t;
 
+#if defined (US3)
+
+/*
+ * Functions for determining the number of entries in TLBs. They either return
+ * a constant value or a value based on the CPU autodetection.
+ */
+
+/**
+ * Determine the number of entries in the DMMU's small TLB. 
+ */
+static inline uint16_t tlb_dsmall_size(void)
+{
+	return 16;
+}
+
+/**
+ * Determine the number of entries in each DMMU's big TLB. 
+ */
+static inline uint16_t tlb_dbig_size(void)
+{
+	return 512;
+}
+
+/**
+ * Determine the number of entries in the IMMU's small TLB. 
+ */
+static inline uint16_t tlb_ismall_size(void)
+{
+	return 16;
+}
+
+/**
+ * Determine the number of entries in the IMMU's big TLB. 
+ */
+static inline uint16_t tlb_ibig_size(void)
+{
+	if (((ver_reg_t) ver_read()).impl == IMPL_ULTRASPARCIV_PLUS)
+		return 512;
+	else
+		return 128;
+}
+
+#endif
+
 /** Read MMU Primary Context Register.
  *
- * @return Current value of Primary Context Register.
+ * @return		Current value of Primary Context Register.
  */
 static inline uint64_t mmu_primary_context_read(void)
@@ -156,5 +287,5 @@
 /** Write MMU Primary Context Register.
  *
- * @param v New value of Primary Context Register.
+ * @param v		New value of Primary Context Register.
  */
 static inline void mmu_primary_context_write(uint64_t v)
@@ -166,5 +297,5 @@
 /** Read MMU Secondary Context Register.
  *
- * @return Current value of Secondary Context Register.
+ * @return		Current value of Secondary Context Register.
  */
 static inline uint64_t mmu_secondary_context_read(void)
@@ -175,5 +306,5 @@
 /** Write MMU Primary Context Register.
  *
- * @param v New value of Primary Context Register.
+ * @param v		New value of Primary Context Register.
  */
 static inline void mmu_secondary_context_write(uint64_t v)
@@ -183,13 +314,16 @@
 }
 
+#if defined (US)
+
 /** Read IMMU TLB Data Access Register.
  *
- * @param entry TLB Entry index.
- *
- * @return Current value of specified IMMU TLB Data Access Register.
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified IMMU TLB Data Access
+ * 			Register.
  */
 static inline uint64_t itlb_data_access_read(index_t entry)
 {
-	tlb_data_access_addr_t reg;
+	itlb_data_access_addr_t reg;
 	
 	reg.value = 0;
@@ -200,10 +334,10 @@
 /** Write IMMU TLB Data Access Register.
  *
- * @param entry TLB Entry index.
- * @param value Value to be written.
+ * @param entry		TLB Entry index.
+ * @param value		Value to be written.
  */
 static inline void itlb_data_access_write(index_t entry, uint64_t value)
 {
-	tlb_data_access_addr_t reg;
+	itlb_data_access_addr_t reg;
 	
 	reg.value = 0;
@@ -215,11 +349,12 @@
 /** Read DMMU TLB Data Access Register.
  *
- * @param entry TLB Entry index.
- *
- * @return Current value of specified DMMU TLB Data Access Register.
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified DMMU TLB Data Access
+ * 			Register.
  */
 static inline uint64_t dtlb_data_access_read(index_t entry)
 {
-	tlb_data_access_addr_t reg;
+	dtlb_data_access_addr_t reg;
 	
 	reg.value = 0;
@@ -230,10 +365,10 @@
 /** Write DMMU TLB Data Access Register.
  *
- * @param entry TLB Entry index.
- * @param value Value to be written.
+ * @param entry		TLB Entry index.
+ * @param value		Value to be written.
  */
 static inline void dtlb_data_access_write(index_t entry, uint64_t value)
 {
-	tlb_data_access_addr_t reg;
+	dtlb_data_access_addr_t reg;
 	
 	reg.value = 0;
@@ -245,11 +380,11 @@
 /** Read IMMU TLB Tag Read Register.
  *
- * @param entry TLB Entry index.
- *
- * @return Current value of specified IMMU TLB Tag Read Register.
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified IMMU TLB Tag Read Register.
  */
 static inline uint64_t itlb_tag_read_read(index_t entry)
 {
-	tlb_tag_read_addr_t tag;
+	itlb_tag_read_addr_t tag;
 
 	tag.value = 0;
@@ -260,11 +395,11 @@
 /** Read DMMU TLB Tag Read Register.
  *
- * @param entry TLB Entry index.
- *
- * @return Current value of specified DMMU TLB Tag Read Register.
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified DMMU TLB Tag Read Register.
  */
 static inline uint64_t dtlb_tag_read_read(index_t entry)
 {
-	tlb_tag_read_addr_t tag;
+	dtlb_tag_read_addr_t tag;
 
 	tag.value = 0;
@@ -273,7 +408,118 @@
 }
 
+#elif defined (US3)
+
+
+/** Read IMMU TLB Data Access Register.
+ *
+ * @param tlb		TLB number (one of TLB_ISMALL or TLB_IBIG)
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified IMMU TLB Data Access
+ * 			Register.
+ */
+static inline uint64_t itlb_data_access_read(int tlb, index_t entry)
+{
+	itlb_data_access_addr_t reg;
+	
+	reg.value = 0;
+	reg.tlb_number = tlb;
+	reg.local_tlb_entry = entry;
+	return asi_u64_read(ASI_ITLB_DATA_ACCESS_REG, reg.value);
+}
+
+/** Write IMMU TLB Data Access Register.
+ * @param tlb		TLB number (one of TLB_ISMALL or TLB_IBIG)
+ * @param entry		TLB Entry index.
+ * @param value		Value to be written.
+ */
+static inline void itlb_data_access_write(int tlb, index_t entry,
+	uint64_t value)
+{
+	itlb_data_access_addr_t reg;
+	
+	reg.value = 0;
+	reg.tlb_number = tlb;
+	reg.local_tlb_entry = entry;
+	asi_u64_write(ASI_ITLB_DATA_ACCESS_REG, reg.value, value);
+	flush_pipeline();
+}
+
+/** Read DMMU TLB Data Access Register.
+ *
+ * @param tlb		TLB number (one of TLB_DSMALL, TLB_DBIG, TLB_DBIG) 
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified DMMU TLB Data Access
+ * 			Register.
+ */
+static inline uint64_t dtlb_data_access_read(int tlb, index_t entry)
+{
+	dtlb_data_access_addr_t reg;
+	
+	reg.value = 0;
+	reg.tlb_number = tlb;
+	reg.local_tlb_entry = entry;
+	return asi_u64_read(ASI_DTLB_DATA_ACCESS_REG, reg.value);
+}
+
+/** Write DMMU TLB Data Access Register.
+ *
+ * @param tlb		TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1)  
+ * @param entry		TLB Entry index.
+ * @param value		Value to be written.
+ */
+static inline void dtlb_data_access_write(int tlb, index_t entry,
+	uint64_t value)
+{
+	dtlb_data_access_addr_t reg;
+	
+	reg.value = 0;
+	reg.tlb_number = tlb;
+	reg.local_tlb_entry = entry;
+	asi_u64_write(ASI_DTLB_DATA_ACCESS_REG, reg.value, value);
+	membar();
+}
+
+/** Read IMMU TLB Tag Read Register.
+ *
+ * @param tlb		TLB number (one of TLB_ISMALL or TLB_IBIG) 
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified IMMU TLB Tag Read Register.
+ */
+static inline uint64_t itlb_tag_read_read(int tlb, index_t entry)
+{
+	itlb_tag_read_addr_t tag;
+
+	tag.value = 0;
+	tag.tlb_number = tlb;
+	tag.local_tlb_entry = entry;
+	return asi_u64_read(ASI_ITLB_TAG_READ_REG, tag.value);
+}
+
+/** Read DMMU TLB Tag Read Register.
+ *
+ * @param tlb		TLB number (one of TLB_DSMALL, TLB_DBIG_0, TLB_DBIG_1)
+ * @param entry		TLB Entry index.
+ *
+ * @return		Current value of specified DMMU TLB Tag Read Register.
+ */
+static inline uint64_t dtlb_tag_read_read(int tlb, index_t entry)
+{
+	dtlb_tag_read_addr_t tag;
+
+	tag.value = 0;
+	tag.tlb_number = tlb;
+	tag.local_tlb_entry = entry;
+	return asi_u64_read(ASI_DTLB_TAG_READ_REG, tag.value);
+}
+
+#endif
+
+
 /** Write IMMU TLB Tag Access Register.
  *
- * @param v Value to be written.
+ * @param v		Value to be written.
  */
 static inline void itlb_tag_access_write(uint64_t v)
@@ -285,5 +531,5 @@
 /** Read IMMU TLB Tag Access Register.
  *
- * @return Current value of IMMU TLB Tag Access Register.
+ * @return		Current value of IMMU TLB Tag Access Register.
  */
 static inline uint64_t itlb_tag_access_read(void)
@@ -294,5 +540,5 @@
 /** Write DMMU TLB Tag Access Register.
  *
- * @param v Value to be written.
+ * @param v		Value to be written.
  */
 static inline void dtlb_tag_access_write(uint64_t v)
@@ -304,5 +550,5 @@
 /** Read DMMU TLB Tag Access Register.
  *
- * @return Current value of DMMU TLB Tag Access Register.
+ * @return 		Current value of DMMU TLB Tag Access Register.
  */
 static inline uint64_t dtlb_tag_access_read(void)
@@ -314,5 +560,5 @@
 /** Write IMMU TLB Data in Register.
  *
- * @param v Value to be written.
+ * @param v		Value to be written.
  */
 static inline void itlb_data_in_write(uint64_t v)
@@ -324,5 +570,5 @@
 /** Write DMMU TLB Data in Register.
  *
- * @param v Value to be written.
+ * @param v		Value to be written.
  */
 static inline void dtlb_data_in_write(uint64_t v)
@@ -334,5 +580,5 @@
 /** Read ITLB Synchronous Fault Status Register.
  *
- * @return Current content of I-SFSR register.
+ * @return		Current content of I-SFSR register.
  */
 static inline uint64_t itlb_sfsr_read(void)
@@ -343,5 +589,5 @@
 /** Write ITLB Synchronous Fault Status Register.
  *
- * @param v New value of I-SFSR register.
+ * @param v		New value of I-SFSR register.
  */
 static inline void itlb_sfsr_write(uint64_t v)
@@ -353,5 +599,5 @@
 /** Read DTLB Synchronous Fault Status Register.
  *
- * @return Current content of D-SFSR register.
+ * @return		Current content of D-SFSR register.
  */
 static inline uint64_t dtlb_sfsr_read(void)
@@ -362,5 +608,5 @@
 /** Write DTLB Synchronous Fault Status Register.
  *
- * @param v New value of D-SFSR register.
+ * @param v		New value of D-SFSR register.
  */
 static inline void dtlb_sfsr_write(uint64_t v)
@@ -372,5 +618,5 @@
 /** Read DTLB Synchronous Fault Address Register.
  *
- * @return Current content of D-SFAR register.
+ * @return		Current content of D-SFAR register.
  */
 static inline uint64_t dtlb_sfar_read(void)
@@ -381,8 +627,9 @@
 /** Perform IMMU TLB Demap Operation.
  *
- * @param type Selects between context and page demap.
+ * @param type		Selects between context and page demap (and entire MMU
+ * 			demap on US3).
  * @param context_encoding Specifies which Context register has Context ID for
- * 	demap.
- * @param page Address which is on the page to be demapped.
+ * 			demap.
+ * @param page		Address which is on the page to be demapped.
  */
 static inline void itlb_demap(int type, int context_encoding, uintptr_t page)
@@ -398,7 +645,7 @@
 	da.vpn = pg.vpn;
 	
-	asi_u64_write(ASI_IMMU_DEMAP, da.value, 0);	/* da.value is the
-							 * address within the
-							 * ASI */ 
+	/* da.value is the address within the ASI */ 
+	asi_u64_write(ASI_IMMU_DEMAP, da.value, 0);
+
 	flush_pipeline();
 }
@@ -406,8 +653,9 @@
 /** Perform DMMU TLB Demap Operation.
  *
- * @param type Selects between context and page demap.
+ * @param type		Selects between context and page demap (and entire MMU
+ * 			demap on US3).
  * @param context_encoding Specifies which Context register has Context ID for
- *	 demap.
- * @param page Address which is on the page to be demapped.
+ * 			demap.
+ * @param page		Address which is on the page to be demapped.
  */
 static inline void dtlb_demap(int type, int context_encoding, uintptr_t page)
@@ -423,15 +671,15 @@
 	da.vpn = pg.vpn;
 	
-	asi_u64_write(ASI_DMMU_DEMAP, da.value, 0);	/* da.value is the
-							 * address within the
-							 * ASI */ 
+	/* da.value is the address within the ASI */ 
+	asi_u64_write(ASI_DMMU_DEMAP, da.value, 0);
+
 	membar();
 }
 
-extern void fast_instruction_access_mmu_miss(unative_t unused, istate_t *istate);
-extern void fast_data_access_mmu_miss(tlb_tag_access_reg_t tag, istate_t *istate);
-extern void fast_data_access_protection(tlb_tag_access_reg_t tag , istate_t *istate);
-
-extern void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize, bool locked, bool cacheable);
+extern void fast_instruction_access_mmu_miss(unative_t, istate_t *);
+extern void fast_data_access_mmu_miss(tlb_tag_access_reg_t, istate_t *);
+extern void fast_data_access_protection(tlb_tag_access_reg_t , istate_t *);
+
+extern void dtlb_insert_mapping(uintptr_t, uintptr_t, int, bool, bool);
 
 extern void dump_sfsr_and_sfar(void);
