Index: kernel/arch/ppc32/src/mm/tlb.c
===================================================================
--- kernel/arch/ppc32/src/mm/tlb.c	(revision 655f70bc5b9e0071345a88b9cbe16715a93be963)
+++ kernel/arch/ppc32/src/mm/tlb.c	(revision 7e752b2a0d66c871748e5fa9e8bbe3a27c70a202)
@@ -33,23 +33,337 @@
  */
 
+#include <mm/tlb.h>
 #include <arch/mm/tlb.h>
+#include <arch/interrupt.h>
 #include <interrupt.h>
-#include <typedefs.h>
-
-void tlb_refill(unsigned int n, istate_t *istate)
-{
-	uint32_t tlbmiss;
-	ptehi_t ptehi;
-	ptelo_t ptelo;
-	
-	asm volatile (
-		"mfspr %[tlbmiss], 980\n"
-		"mfspr %[ptehi], 981\n"
-		"mfspr %[ptelo], 982\n"
-		: [tlbmiss] "=r" (tlbmiss),
-		  [ptehi] "=r" (ptehi),
-		  [ptelo] "=r" (ptelo)
-	);
-	
+#include <mm/as.h>
+#include <mm/page.h>
+#include <arch.h>
+#include <print.h>
+#include <macros.h>
+#include <symtab.h>
+
+static unsigned int seed = 10;
+static unsigned int seed_real
+    __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
+
+/** Try to find PTE for faulting address
+ *
+ * @param as       Address space.
+ * @param lock     Lock/unlock the address space.
+ * @param badvaddr Faulting virtual address.
+ * @param access   Access mode that caused the fault.
+ * @param istate   Pointer to interrupted state.
+ * @param pfrc     Pointer to variable where as_page_fault() return code
+ *                 will be stored.
+ *
+ * @return PTE on success, NULL otherwise.
+ *
+ */
+static pte_t *find_mapping_and_check(as_t *as, uintptr_t badvaddr, int access,
+    istate_t *istate, int *pfrc)
+{
+	ASSERT(mutex_locked(&as->lock));
+
+	/*
+	 * Check if the mapping exists in page tables.
+	 */
+	pte_t *pte = page_mapping_find(as, badvaddr);
+	if ((pte) && (pte->present)) {
+		/*
+		 * Mapping found in page tables.
+		 * Immediately succeed.
+		 */
+		return pte;
+	} else {
+		/*
+		 * Mapping not found in page tables.
+		 * Resort to higher-level page fault handler.
+		 */
+		page_table_unlock(as, true);
+		
+		int rc = as_page_fault(badvaddr, access, istate);
+		switch (rc) {
+		case AS_PF_OK:
+			/*
+			 * The higher-level page fault handler succeeded,
+			 * The mapping ought to be in place.
+			 */
+			page_table_lock(as, true);
+			pte = page_mapping_find(as, badvaddr);
+			ASSERT((pte) && (pte->present));
+			*pfrc = 0;
+			return pte;
+		case AS_PF_DEFER:
+			page_table_lock(as, true);
+			*pfrc = rc;
+			return NULL;
+		case AS_PF_FAULT:
+			page_table_lock(as, true);
+			*pfrc = rc;
+			return NULL;
+		default:
+			panic("Unexpected rc (%d).", rc);
+		}
+	}
+}
+
+static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
+{
+	fault_if_from_uspace(istate, "PHT Refill Exception on %p.",
+	    (void *) badvaddr);
+	panic_memtrap(istate, PF_ACCESS_UNKNOWN, badvaddr,
+	    "PHT Refill Exception.");
+}
+
+static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
+{
+	uint32_t page = (vaddr >> 12) & 0xffff;
+	uint32_t api = (vaddr >> 22) & 0x3f;
+	
+	uint32_t vsid = sr_get(vaddr);
+	uint32_t sdr1 = sdr1_get();
+	
+	// FIXME: compute size of PHT exactly
+	phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
+	
+	/* Primary hash (xor) */
+	uint32_t h = 0;
+	uint32_t hash = vsid ^ page;
+	uint32_t base = (hash & 0x3ff) << 3;
+	uint32_t i;
+	bool found = false;
+	
+	/* Find colliding PTE in PTEG */
+	for (i = 0; i < 8; i++) {
+		if ((phte[base + i].v)
+		    && (phte[base + i].vsid == vsid)
+		    && (phte[base + i].api == api)
+		    && (phte[base + i].h == 0)) {
+			found = true;
+			break;
+		}
+	}
+	
+	if (!found) {
+		/* Find unused PTE in PTEG */
+		for (i = 0; i < 8; i++) {
+			if (!phte[base + i].v) {
+				found = true;
+				break;
+			}
+		}
+	}
+	
+	if (!found) {
+		/* Secondary hash (not) */
+		uint32_t base2 = (~hash & 0x3ff) << 3;
+		
+		/* Find colliding PTE in PTEG */
+		for (i = 0; i < 8; i++) {
+			if ((phte[base2 + i].v)
+			    && (phte[base2 + i].vsid == vsid)
+			    && (phte[base2 + i].api == api)
+			    && (phte[base2 + i].h == 1)) {
+				found = true;
+				base = base2;
+				h = 1;
+				break;
+			}
+		}
+		
+		if (!found) {
+			/* Find unused PTE in PTEG */
+			for (i = 0; i < 8; i++) {
+				if (!phte[base2 + i].v) {
+					found = true;
+					base = base2;
+					h = 1;
+					break;
+				}
+			}
+		}
+		
+		if (!found)
+			i = RANDI(seed) % 8;
+	}
+	
+	phte[base + i].v = 1;
+	phte[base + i].vsid = vsid;
+	phte[base + i].h = h;
+	phte[base + i].api = api;
+	phte[base + i].rpn = pte->pfn;
+	phte[base + i].r = 0;
+	phte[base + i].c = 0;
+	phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
+	phte[base + i].pp = 2; // FIXME
+}
+
+/** Process Instruction/Data Storage Exception
+ *
+ * @param n      Exception vector number.
+ * @param istate Interrupted register context.
+ *
+ */
+void pht_refill(unsigned int n, istate_t *istate)
+{
+	as_t *as = (AS == NULL) ? AS_KERNEL : AS;
+	uintptr_t badvaddr;
+	
+	if (n == VECTOR_DATA_STORAGE)
+		badvaddr = istate->dar;
+	else
+		badvaddr = istate->pc;
+	
+	page_table_lock(as, true);
+	
+	int pfrc;
+	pte_t *pte = find_mapping_and_check(as, badvaddr,
+	    PF_ACCESS_READ /* FIXME */, istate, &pfrc);
+	
+	if (!pte) {
+		switch (pfrc) {
+		case AS_PF_FAULT:
+			goto fail;
+			break;
+		case AS_PF_DEFER:
+			/*
+			 * The page fault came during copy_from_uspace()
+			 * or copy_to_uspace().
+			 */
+			page_table_unlock(as, true);
+			return;
+		default:
+			panic("Unexpected pfrc (%d).", pfrc);
+		}
+	}
+	
+	/* Record access to PTE */
+	pte->accessed = 1;
+	pht_insert(badvaddr, pte);
+	
+	page_table_unlock(as, true);
+	return;
+	
+fail:
+	page_table_unlock(as, true);
+	pht_refill_fail(badvaddr, istate);
+}
+
+/** Process Instruction/Data Storage Exception in Real Mode
+ *
+ * @param n      Exception vector number.
+ * @param istate Interrupted register context.
+ *
+ */
+bool pht_refill_real(unsigned int n, istate_t *istate)
+{
+	uintptr_t badvaddr;
+	
+	if (n == VECTOR_DATA_STORAGE)
+		badvaddr = istate->dar;
+	else
+		badvaddr = istate->pc;
+	
+	uint32_t physmem = physmem_top();
+	
+	if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
+		return false;
+	
+	uint32_t page = (badvaddr >> 12) & 0xffff;
+	uint32_t api = (badvaddr >> 22) & 0x3f;
+	
+	uint32_t vsid = sr_get(badvaddr);
+	uint32_t sdr1 = sdr1_get();
+	
+	// FIXME: compute size of PHT exactly
+	phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
+	
+	/* Primary hash (xor) */
+	uint32_t h = 0;
+	uint32_t hash = vsid ^ page;
+	uint32_t base = (hash & 0x3ff) << 3;
+	uint32_t i;
+	bool found = false;
+	
+	/* Find colliding PTE in PTEG */
+	for (i = 0; i < 8; i++) {
+		if ((phte_real[base + i].v)
+		    && (phte_real[base + i].vsid == vsid)
+		    && (phte_real[base + i].api == api)
+		    && (phte_real[base + i].h == 0)) {
+			found = true;
+			break;
+		}
+	}
+	
+	if (!found) {
+		/* Find unused PTE in PTEG */
+		for (i = 0; i < 8; i++) {
+			if (!phte_real[base + i].v) {
+				found = true;
+				break;
+			}
+		}
+	}
+	
+	if (!found) {
+		/* Secondary hash (not) */
+		uint32_t base2 = (~hash & 0x3ff) << 3;
+		
+		/* Find colliding PTE in PTEG */
+		for (i = 0; i < 8; i++) {
+			if ((phte_real[base2 + i].v)
+			    && (phte_real[base2 + i].vsid == vsid)
+			    && (phte_real[base2 + i].api == api)
+			    && (phte_real[base2 + i].h == 1)) {
+				found = true;
+				base = base2;
+				h = 1;
+				break;
+			}
+		}
+		
+		if (!found) {
+			/* Find unused PTE in PTEG */
+			for (i = 0; i < 8; i++) {
+				if (!phte_real[base2 + i].v) {
+					found = true;
+					base = base2;
+					h = 1;
+					break;
+				}
+			}
+		}
+		
+		if (!found) {
+			/* Use secondary hash to avoid collisions
+			   with usual PHT refill handler. */
+			i = RANDI(seed_real) % 8;
+			base = base2;
+			h = 1;
+		}
+	}
+	
+	phte_real[base + i].v = 1;
+	phte_real[base + i].vsid = vsid;
+	phte_real[base + i].h = h;
+	phte_real[base + i].api = api;
+	phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
+	phte_real[base + i].r = 0;
+	phte_real[base + i].c = 0;
+	phte_real[base + i].wimg = 0;
+	phte_real[base + i].pp = 2; // FIXME
+	
+	return true;
+}
+
+/** Process ITLB/DTLB Miss Exception in Real Mode
+ *
+ *
+ */
+void tlb_refill_real(unsigned int n, uint32_t tlbmiss, ptehi_t ptehi,
+    ptelo_t ptelo, istate_t *istate)
+{
 	uint32_t badvaddr = tlbmiss & 0xfffffffc;
 	uint32_t physmem = physmem_top();
@@ -81,19 +395,19 @@
 void tlb_invalidate_all(void)
 {
+	uint32_t index;
+	
 	asm volatile (
+		"li %[index], 0\n"
 		"sync\n"
-	);
-	
-	for (unsigned int i = 0; i < 0x00040000; i += 0x00001000) {
-		asm volatile (
-			"tlbie %[i]\n"
-			:: [i] "r" (i)
-		);
-	}
-	
-	asm volatile (
+		
+		".rept 64\n"
+		"	tlbie %[index]\n"
+		"	addi %[index], %[index], 0x1000\n"
+		".endr\n"
+		
 		"eieio\n"
 		"tlbsync\n"
 		"sync\n"
+		: [index] "=r" (index)
 	);
 }
@@ -101,4 +415,16 @@
 void tlb_invalidate_asid(asid_t asid)
 {
+	uint32_t sdr1 = sdr1_get();
+	
+	// FIXME: compute size of PHT exactly
+	phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
+	
+	size_t i;
+	for (i = 0; i < 8192; i++) {
+		if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
+		    (phte[i].vsid < ((asid << 4) + 16)))
+			phte[i].v = 0;
+	}
+	
 	tlb_invalidate_all();
 }
@@ -106,4 +432,5 @@
 void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
 {
+	// TODO
 	tlb_invalidate_all();
 }
