Changeset c520034 in mainline for kernel/generic/src

Timestamp:

2011-12-31T18:19:35Z (14 years ago)

Author:

Jakub Jermar <jakub@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

295f658, 77c2b02, 96cd5b4

Parents:

852052d (diff), 22f0561 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Support for kernel non-identity mappings, phase I.

• identity/non-identity kernel memory split on all architectures
• low/high physical memory split on all architectures
• frame allocator understands low/high memory
• high physical memory currently unused (Phase II)
• more compact frame_t
• zone conf frames, pte_t, kernel stacks allocated from low memory
• lockless TLB-miss handlers everywhere (newly sparc64, ia64)
• preallocate PTL1 page tables for non-identity and prevent their deallocation
• hw_map() unification
• new resource allocator used for allocating kernel virtual addresses

Breakage:

• sparc64/sun4v creates too large kernel identity; not fixed because of lack of testing hw
• ppc32's tlb_refill() seems wrong as it creates too large kernel identity, but appears unused and the architecture works normally

Not implemented yet (phase II):

• allow high memory to be used for common kernel allocations

Location:

kernel/generic/src

Files:

• cpu/cpu.c (modified) (1 diff)
• lib/ra.c (added)
• main/main.c (modified) (4 diffs)
• mm/frame.c (modified) (20 diffs)
• mm/km.c (added)
• mm/page.c (modified) (3 diffs)
• mm/reserve.c (modified) (5 diffs)
• proc/thread.c (modified) (1 diff)

kernel/generic/src/cpu/cpu.c

@@ -74 +74 @@
                 for (i = 0; i < config.cpu_count; i++) {
                         cpus[i].stack = (uint8_t *) frame_alloc(STACK_FRAMES,
-                            FRAME_KA | FRAME_ATOMIC);
+                            FRAME_LOWMEM | FRAME_KA | FRAME_ATOMIC);
                         cpus[i].id = i;
                         

kernel/generic/src/main/main.c

@@ -68 +68 @@
 #include <mm/page.h>
 #include <genarch/mm/page_pt.h>
+#include <mm/km.h>
 #include <mm/tlb.h>
 #include <mm/as.h>
@@ -86 +87 @@
 #include <sysinfo/sysinfo.h>
 #include <sysinfo/stats.h>
+#include <lib/ra.h>
 
 /** Global configuration structure. */
-config_t config;
+config_t config = {
+        .identity_configured = false,
+        .non_identity_configured = false,
+        .physmem_end = 0
+};
 
 /** Initial user-space tasks */
@@ -205 +211 @@
          */
         arch_pre_mm_init();
+        km_identity_init();
         frame_init();
-        
-        /* Initialize at least 1 memory segment big enough for slab to work. */
         slab_cache_init();
+        ra_init();      
         sysinfo_init();
         btree_init();
@@ -214 +220 @@
         page_init();
         tlb_init();
+        km_non_identity_init();
         ddi_init();
         arch_post_mm_init();

kernel/generic/src/mm/frame.c

@@ -240 +240 @@
 NO_TRACE static bool zone_can_alloc(zone_t *zone, uint8_t order)
 {
-        return (zone_flags_available(zone->flags)
-            && buddy_system_can_alloc(zone->buddy_system, order));
+        return ((zone->flags & ZONE_AVAILABLE) &&
+            buddy_system_can_alloc(zone->buddy_system, order));
 }
 
@@ -265 +265 @@
                  * Check whether the zone meets the search criteria.
                  */
-                if ((zones.info[i].flags & flags) == flags) {
+                if (ZONE_FLAGS_MATCH(zones.info[i].flags, flags)) {
                         /*
                          * Check if the zone has 2^order frames area available.
@@ -460 +460 @@
 NO_TRACE static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
 {
-        ASSERT(zone_flags_available(zone->flags));
+        ASSERT(zone->flags & ZONE_AVAILABLE);
         
         /* Allocate frames from zone buddy system */
@@ -490 +490 @@
 NO_TRACE static size_t zone_frame_free(zone_t *zone, size_t frame_idx)
 {
-        ASSERT(zone_flags_available(zone->flags));
+        ASSERT(zone->flags & ZONE_AVAILABLE);
         
         frame_t *frame = &zone->frames[frame_idx];
@@ -518 +518 @@
 NO_TRACE static void zone_mark_unavailable(zone_t *zone, size_t frame_idx)
 {
-        ASSERT(zone_flags_available(zone->flags));
+        ASSERT(zone->flags & ZONE_AVAILABLE);
         
         frame_t *frame = zone_get_frame(zone, frame_idx);
@@ -549 +549 @@
     buddy_system_t *buddy)
 {
-        ASSERT(zone_flags_available(zones.info[z1].flags));
-        ASSERT(zone_flags_available(zones.info[z2].flags));
+        ASSERT(zones.info[z1].flags & ZONE_AVAILABLE);
+        ASSERT(zones.info[z2].flags & ZONE_AVAILABLE);
         ASSERT(zones.info[z1].flags == zones.info[z2].flags);
         ASSERT(zones.info[z1].base < zones.info[z2].base);
@@ -645 +645 @@
 NO_TRACE static void return_config_frames(size_t znum, pfn_t pfn, size_t count)
 {
-        ASSERT(zone_flags_available(zones.info[znum].flags));
+        ASSERT(zones.info[znum].flags & ZONE_AVAILABLE);
         
         size_t cframes = SIZE2FRAMES(zone_conf_size(count));
@@ -681 +681 @@
     size_t count)
 {
-        ASSERT(zone_flags_available(zones.info[znum].flags));
+        ASSERT(zones.info[znum].flags & ZONE_AVAILABLE);
         ASSERT(frame_idx + count < zones.info[znum].count);
         
@@ -723 +723 @@
          * set of flags
          */
-        if ((z1 >= zones.count) || (z2 >= zones.count)
-            || (z2 - z1 != 1)
-            || (!zone_flags_available(zones.info[z1].flags))
-            || (!zone_flags_available(zones.info[z2].flags))
-            || (zones.info[z1].flags != zones.info[z2].flags)) {
+        if ((z1 >= zones.count) || (z2 >= zones.count) || (z2 - z1 != 1) ||
+            (zones.info[z1].flags != zones.info[z2].flags)) {
                 ret = false;
                 goto errout;
@@ -828 +825 @@
         zone->buddy_system = buddy;
         
-        if (zone_flags_available(flags)) {
+        if (flags & ZONE_AVAILABLE) {
                 /*
                  * Compute order for buddy system and initialize
@@ -865 +862 @@
 {
         return (count * sizeof(frame_t) + buddy_conf_size(fnzb(count)));
+}
+
+/** Allocate external configuration frames from low memory. */
+pfn_t zone_external_conf_alloc(size_t count)
+{
+        size_t size = zone_conf_size(count);
+        size_t order = ispwr2(size) ? fnzb(size) : (fnzb(size) + 1);
+
+        return ADDR2PFN((uintptr_t) frame_alloc(order - FRAME_WIDTH, FRAME_LOWMEM));
 }
 
@@ -888 +894 @@
         irq_spinlock_lock(&zones.lock, true);
         
-        if (zone_flags_available(flags)) {  /* Create available zone */
+        if (flags & ZONE_AVAILABLE) {  /* Create available zone */
                 /* Theoretically we could have NULL here, practically make sure
                  * nobody tries to do that. If some platform requires, remove
@@ -894 +900 @@
                  */
                 ASSERT(confframe != ADDR2PFN((uintptr_t ) NULL));
+
+                /* Update the known end of physical memory. */
+                config.physmem_end = max(config.physmem_end, PFN2ADDR(start + count));
                 
                 /* If confframe is supposed to be inside our zone, then make sure
@@ -1232 +1241 @@
         
         /* Tell the architecture to create some memory */
-        frame_arch_init();
+        frame_low_arch_init();
         if (config.cpu_active == 1) {
                 frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
@@ -1255 +1264 @@
                 frame_mark_unavailable(0, 1);
         }
+        frame_high_arch_init();
+}
+
+/** Adjust bounds of physical memory region according to low/high memory split.
+ *
+ * @param low[in]       If true, the adujstment is performed to make the region
+ *                      fit in the low memory. Otherwise the adjustment is
+ *                      performed to make the region fit in the high memory.
+ * @param basep[inout]  Pointer to a variable which contains the region's base
+ *                      address and which may receive the adjusted base address.
+ * @param sizep[inout]  Pointer to a variable which contains the region's size
+ *                      and which may receive the adjusted size.
+ * @retun               True if the region still exists even after the
+ *                      adjustment, false otherwise.
+ */
+bool frame_adjust_zone_bounds(bool low, uintptr_t *basep, size_t *sizep)
+{
+        uintptr_t limit = config.identity_size;
+
+        if (low) {
+                if (*basep > limit)
+                        return false;
+                if (*basep + *sizep > limit)
+                        *sizep = limit - *basep;
+        } else {
+                if (*basep + *sizep <= limit)
+                        return false;
+                if (*basep <= limit) {
+                        *sizep -= limit - *basep;
+                        *basep = limit;
+                }
+        }
+        return true;
 }
 
@@ -1293 +1335 @@
                 *total += (uint64_t) FRAMES2SIZE(zones.info[i].count);
                 
-                if (zone_flags_available(zones.info[i].flags)) {
+                if (zones.info[i].flags & ZONE_AVAILABLE) {
                         *busy += (uint64_t) FRAMES2SIZE(zones.info[i].busy_count);
                         *free += (uint64_t) FRAMES2SIZE(zones.info[i].free_count);
@@ -1344 +1386 @@
                 irq_spinlock_unlock(&zones.lock, true);
                 
-                bool available = zone_flags_available(flags);
+                bool available = ((flags & ZONE_AVAILABLE) != 0);
                 
                 printf("%-4zu", i);
@@ -1356 +1398 @@
 #endif
                 
-                printf(" %12zu %c%c%c      ", count,
-                    available ? 'A' : ' ',
-                    (flags & ZONE_RESERVED) ? 'R' : ' ',
-                    (flags & ZONE_FIRMWARE) ? 'F' : ' ');
+                printf(" %12zu %c%c%c%c%c    ", count,
+                    available ? 'A' : '-',
+                    (flags & ZONE_RESERVED) ? 'R' : '-',
+                    (flags & ZONE_FIRMWARE) ? 'F' : '-',
+                    (flags & ZONE_LOWMEM) ? 'L' : '-',
+                    (flags & ZONE_HIGHMEM) ? 'H' : '-');
                 
                 if (available)
@@ -1401 +1445 @@
         irq_spinlock_unlock(&zones.lock, true);
         
-        bool available = zone_flags_available(flags);
+        bool available = ((flags & ZONE_AVAILABLE) != 0);
         
         uint64_t size;
@@ -1411 +1455 @@
         printf("Zone size:         %zu frames (%" PRIu64 " %s)\n", count,
             size, size_suffix);
-        printf("Zone flags:        %c%c%c\n",
-            available ? 'A' : ' ',
-            (flags & ZONE_RESERVED) ? 'R' : ' ',
-            (flags & ZONE_FIRMWARE) ? 'F' : ' ');
+        printf("Zone flags:        %c%c%c%c%c\n",
+            available ? 'A' : '-',
+            (flags & ZONE_RESERVED) ? 'R' : '-',
+            (flags & ZONE_FIRMWARE) ? 'F' : '-',
+            (flags & ZONE_LOWMEM) ? 'L' : '-',
+            (flags & ZONE_HIGHMEM) ? 'H' : '-');
         
         if (available) {

kernel/generic/src/mm/page.c

@@ -65 +65 @@
 #include <arch/mm/asid.h>
 #include <mm/as.h>
+#include <mm/km.h>
 #include <mm/frame.h>
 #include <arch/barrier.h>
@@ -75 +76 @@
 #include <errno.h>
 #include <align.h>
+#include <macros.h>
+#include <bitops.h>
 
 /** Virtual operations for page subsystem. */
@@ -177 +180 @@
 }
 
+/** Make the mapping shared by all page tables (not address spaces).
+ * 
+ * @param base Starting virtual address of the range that is made global.
+ * @param size Size of the address range that is made global.
+ */
+void page_mapping_make_global(uintptr_t base, size_t size)
+{
+        ASSERT(page_mapping_operations);
+        ASSERT(page_mapping_operations->mapping_make_global);
+        
+        return page_mapping_operations->mapping_make_global(base, size);
+}
+
+uintptr_t hw_map(uintptr_t physaddr, size_t size)
+{
+        uintptr_t virtaddr;
+        size_t asize;
+        size_t align;
+        pfn_t i;
+
+        asize = ALIGN_UP(size, PAGE_SIZE);
+        align = ispwr2(size) ? size : (1U << (fnzb(size) + 1));
+        virtaddr = km_page_alloc(asize, align);
+
+        page_table_lock(AS_KERNEL, true);
+        for (i = 0; i < ADDR2PFN(asize); i++) {
+                uintptr_t addr = PFN2ADDR(i);
+                page_mapping_insert(AS_KERNEL, virtaddr + addr, physaddr + addr,
+                    PAGE_NOT_CACHEABLE | PAGE_WRITE);
+        }
+        page_table_unlock(AS_KERNEL, true);
+        
+        return virtaddr;
+}
+
 int page_find_mapping(uintptr_t virt, void **phys)
 {

kernel/generic/src/mm/reserve.c

@@ -42 +42 @@
 #include <typedefs.h>
 #include <arch/types.h>
+#include <debug.h>
+
+static bool reserve_initialized = false;
 
 IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(reserve_lock, "reserve_lock");
@@ -54 +57 @@
 {
         reserve = frame_total_free_get();
+        reserve_initialized = true;
 }
 
@@ -67 +71 @@
 {
         bool reserved = false;
+
+        ASSERT(reserve_initialized);
 
         irq_spinlock_lock(&reserve_lock, true);
@@ -111 +117 @@
 void reserve_force_alloc(size_t size)
 {
+        if (!reserve_initialized)
+                return;
+
         irq_spinlock_lock(&reserve_lock, true);
         reserve -= size;
@@ -122 +131 @@
 void reserve_free(size_t size)
 {
+        if (!reserve_initialized)
+                return;
+
         irq_spinlock_lock(&reserve_lock, true);
         reserve += size;

kernel/generic/src/proc/thread.c

@@ -173 +173 @@
 #endif /* CONFIG_FPU */
         
+        /*
+         * Allocate the kernel stack from the low-memory to prevent an infinite
+         * nesting of TLB-misses when accessing the stack from the part of the
+         * TLB-miss handler written in C.
+         *
+         * Note that low-memory is safe to be used for the stack as it will be
+         * covered by the kernel identity mapping, which guarantees not to
+         * nest TLB-misses infinitely (either via some hardware mechanism or
+         * by the construciton of the assembly-language part of the TLB-miss
+         * handler).
+         *
+         * This restriction can be lifted once each architecture provides
+         * a similar guarantee, for example by locking the kernel stack
+         * in the TLB whenever it is allocated from the high-memory and the
+         * thread is being scheduled to run.
+         */
+        kmflags |= FRAME_LOWMEM;
+        kmflags &= ~FRAME_HIGHMEM;
+
         thread->kstack = (uint8_t *) frame_alloc(STACK_FRAMES, FRAME_KA | kmflags);
         if (!thread->kstack) {