Changeset c520034 in mainline for kernel/generic


Ignore:
Timestamp:
2011-12-31T18:19:35Z (14 years ago)
Author:
Jakub Jermar <jakub@…>
Branches:
lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
Children:
295f658, 77c2b02, 96cd5b4
Parents:
852052d (diff), 22f0561 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Support for kernel non-identity mappings, phase I.

  • identity/non-identity kernel memory split on all architectures
  • low/high physical memory split on all architectures
  • frame allocator understands low/high memory
  • high physical memory currently unused (Phase II)
  • more compact frame_t
  • zone conf frames, pte_t, kernel stacks allocated from low memory
  • lockless TLB-miss handlers everywhere (newly sparc64, ia64)
  • preallocate PTL1 page tables for non-identity and prevent their deallocation
  • hw_map() unification
  • new resource allocator used for allocating kernel virtual addresses

Breakage:

  • sparc64/sun4v creates too large kernel identity; not fixed because of lack of testing hw
  • ppc32's tlb_refill() seems wrong as it creates too large kernel identity, but appears unused and the architecture works normally

Not implemented yet (phase II):

  • allow high memory to be used for common kernel allocations
Location:
kernel/generic
Files:
4 added
11 edited

Legend:

Unmodified
Added
Removed

  • kernel/generic/include/align.h

    r852052d rc520034  
    4242 *
    4343 * @param s Address or size to be aligned.
    44  * @param a Size of alignment, must be power of 2.
     44 * @param a Size of alignment, must be a power of 2.
    4545 */
    4646#define ALIGN_DOWN(s, a)  ((s) & ~((a) - 1))
    … …  
    5050 *
    5151 * @param s Address or size to be aligned.
    52  * @param a Size of alignment, must be power of 2.
     52 * @param a Size of alignment, must be a power of 2.
    5353 */
    5454#define ALIGN_UP(s, a)  (((s) + ((a) - 1)) & ~((a) - 1))
     55
     56/** Check alignment.
     57 *
     58 * @param s Address or size to be checked for alignment.
     59 * @param a Size of alignment, must be a power of 2.
     60 */
     61#define IS_ALIGNED(s, a)        (ALIGN_UP((s), (a)) == (s))
    5562
    5663#endif

  • kernel/generic/include/config.h

    r852052d rc520034  
    7474
    7575typedef struct {
    76         unsigned int cpu_count;      /**< Number of processors detected. */
    77         volatile size_t cpu_active;  /**< Number of processors that are up and running. */
     76        /** Number of processors detected. */
     77        unsigned int cpu_count;
     78        /** Number of processors that are up and running. */
     79        volatile size_t cpu_active;
    7880       
    7981        uintptr_t base;
    80         size_t kernel_size;          /**< Size of memory in bytes taken by kernel and stack */
     82        /** Size of memory in bytes taken by kernel and stack. */
     83        size_t kernel_size;
    8184       
    82         uintptr_t stack_base;        /**< Base adddress of initial stack */
    83         size_t stack_size;           /**< Size of initial stack */
     85        /** Base adddress of initial stack. */
     86        uintptr_t stack_base;
     87        /** Size of initial stack. */
     88        size_t stack_size;
     89
     90        bool identity_configured;
     91        /** Base address of the kernel identity mapped memory. */
     92        uintptr_t identity_base;
     93        /** Size of the kernel identity mapped memory. */
     94        size_t identity_size;
     95
     96        bool non_identity_configured;   
     97
     98        /** End of physical memory. */
     99        uint64_t physmem_end;
    84100} config_t;
    85101

  • kernel/generic/include/macros.h

    r852052d rc520034  
    7777#endif /* __ASM__ */
    7878
     79#define ispwr2(x)       (((x) & ((x) - 1)) == 0)
     80
    7981#define isdigit(d)     (((d) >= '0') && ((d) <= '9'))
    8082#define islower(c)     (((c) >= 'a') && ((c) <= 'z'))

  • kernel/generic/include/mm/frame.h

    r852052d rc520034  
    5050typedef uint8_t frame_flags_t;
    5151
     52#define FRAME_NONE        0x0
    5253/** Convert the frame address to kernel VA. */
    5354#define FRAME_KA          0x1
    … …  
    5859/** Do not reserve / unreserve memory. */
    5960#define FRAME_NO_RESERVE  0x8
     61/** Allocate a frame which can be identity-mapped. */
     62#define FRAME_LOWMEM      0x10
     63/** Allocate a frame which cannot be identity-mapped. */
     64#define FRAME_HIGHMEM     0x20
    6065
    6166typedef uint8_t zone_flags_t;
    6267
     68#define ZONE_NONE       0x0
    6369/** Available zone (free for allocation) */
    64 #define ZONE_AVAILABLE  0x0
     70#define ZONE_AVAILABLE  0x1
    6571/** Zone is reserved (not available for allocation) */
    66 #define ZONE_RESERVED   0x8
     72#define ZONE_RESERVED   0x2
    6773/** Zone is used by firmware (not available for allocation) */
    68 #define ZONE_FIRMWARE   0x10
     74#define ZONE_FIRMWARE   0x4
     75/** Zone contains memory that can be identity-mapped */
     76#define ZONE_LOWMEM     0x8
     77/** Zone contains memory that cannot be identity-mapped */
     78#define ZONE_HIGHMEM    0x10
    6979
    70 /** Currently there is no equivalent zone flags
    71     for frame flags */
    72 #define FRAME_TO_ZONE_FLAGS(frame_flags)  0
     80/** Mask of zone bits that must be matched exactly. */
     81#define ZONE_EF_MASK    0x7
     82
     83#define FRAME_TO_ZONE_FLAGS(ff) \
     84        ((((ff) & FRAME_LOWMEM) ? ZONE_LOWMEM : \
     85            (((ff) & FRAME_HIGHMEM) ? ZONE_HIGHMEM : ZONE_NONE)) | \
     86            (ZONE_AVAILABLE | ZONE_LOWMEM /* | ZONE_HIGHMEM */))
     87
     88#define ZONE_FLAGS_MATCH(zf, f) \
     89        (((((zf) & ZONE_EF_MASK)) == ((f) & ZONE_EF_MASK)) && \
     90            (((zf) & ~ZONE_EF_MASK) & (f)))
    7391
    7492typedef struct {
    7593        size_t refcount;      /**< Tracking of shared frames */
    76         uint8_t buddy_order;  /**< Buddy system block order */
    7794        link_t buddy_link;    /**< Link to the next free block inside
    7895                                   one order */
    7996        void *parent;         /**< If allocated by slab, this points there */
     97        uint8_t buddy_order;  /**< Buddy system block order */
    8098} frame_t;
    8199
    … …  
    129147}
    130148
    131 NO_TRACE static inline bool zone_flags_available(zone_flags_t flags)
    132 {
    133         return ((flags & (ZONE_RESERVED | ZONE_FIRMWARE)) == 0);
    134 }
    135 
    136149#define IS_BUDDY_ORDER_OK(index, order) \
    137150    ((~(((sysarg_t) -1) << (order)) & (index)) == 0)
    … …  
    146159
    147160extern void frame_init(void);
     161extern bool frame_adjust_zone_bounds(bool, uintptr_t *, size_t *);
    148162extern void *frame_alloc_generic(uint8_t, frame_flags_t, size_t *);
    149163extern void *frame_alloc(uint8_t, frame_flags_t);
    … …  
    161175extern void frame_mark_unavailable(pfn_t, size_t);
    162176extern size_t zone_conf_size(size_t);
     177extern pfn_t zone_external_conf_alloc(size_t);
    163178extern bool zone_merge(size_t, size_t);
    164179extern void zone_merge_all(void);

  • kernel/generic/include/mm/page.h

    r852052d rc520034  
    4949        void (* mapping_remove)(as_t *, uintptr_t);
    5050        pte_t *(* mapping_find)(as_t *, uintptr_t, bool);
     51        void (* mapping_make_global)(uintptr_t, size_t);
    5152} page_mapping_operations_t;
    5253
    … …  
    6061extern void page_mapping_remove(as_t *, uintptr_t);
    6162extern pte_t *page_mapping_find(as_t *, uintptr_t, bool);
     63extern void page_mapping_make_global(uintptr_t, size_t);
    6264extern pte_t *page_table_create(unsigned int);
    6365extern void page_table_destroy(pte_t *);

  • kernel/generic/src/cpu/cpu.c

    r852052d rc520034  
    7474                for (i = 0; i < config.cpu_count; i++) {
    7575                        cpus[i].stack = (uint8_t *) frame_alloc(STACK_FRAMES,
    76                             FRAME_KA | FRAME_ATOMIC);
     76                            FRAME_LOWMEM | FRAME_KA | FRAME_ATOMIC);
    7777                        cpus[i].id = i;
    7878                       

  • kernel/generic/src/main/main.c

    r852052d rc520034  
    6868#include <mm/page.h>
    6969#include <genarch/mm/page_pt.h>
     70#include <mm/km.h>
    7071#include <mm/tlb.h>
    7172#include <mm/as.h>
    … …  
    8687#include <sysinfo/sysinfo.h>
    8788#include <sysinfo/stats.h>
     89#include <lib/ra.h>
    8890
    8991/** Global configuration structure. */
    90 config_t config;
     92config_t config = {
     93        .identity_configured = false,
     94        .non_identity_configured = false,
     95        .physmem_end = 0
     96};
    9197
    9298/** Initial user-space tasks */
    … …  
    205211         */
    206212        arch_pre_mm_init();
     213        km_identity_init();
    207214        frame_init();
    208        
    209         /* Initialize at least 1 memory segment big enough for slab to work. */
    210215        slab_cache_init();
     216        ra_init();     
    211217        sysinfo_init();
    212218        btree_init();
    … …  
    214220        page_init();
    215221        tlb_init();
     222        km_non_identity_init();
    216223        ddi_init();
    217224        arch_post_mm_init();

  • kernel/generic/src/mm/frame.c

    r852052d rc520034  
    240240NO_TRACE static bool zone_can_alloc(zone_t *zone, uint8_t order)
    241241{
    242         return (zone_flags_available(zone->flags)
    243             && buddy_system_can_alloc(zone->buddy_system, order));
     242        return ((zone->flags & ZONE_AVAILABLE) &&
     243            buddy_system_can_alloc(zone->buddy_system, order));
    244244}
    245245
    … …  
    265265                 * Check whether the zone meets the search criteria.
    266266                 */
    267                 if ((zones.info[i].flags & flags) == flags) {
     267                if (ZONE_FLAGS_MATCH(zones.info[i].flags, flags)) {
    268268                        /*
    269269                         * Check if the zone has 2^order frames area available.
    … …  
    460460NO_TRACE static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
    461461{
    462         ASSERT(zone_flags_available(zone->flags));
     462        ASSERT(zone->flags & ZONE_AVAILABLE);
    463463       
    464464        /* Allocate frames from zone buddy system */
    … …  
    490490NO_TRACE static size_t zone_frame_free(zone_t *zone, size_t frame_idx)
    491491{
    492         ASSERT(zone_flags_available(zone->flags));
     492        ASSERT(zone->flags & ZONE_AVAILABLE);
    493493       
    494494        frame_t *frame = &zone->frames[frame_idx];
    … …  
    518518NO_TRACE static void zone_mark_unavailable(zone_t *zone, size_t frame_idx)
    519519{
    520         ASSERT(zone_flags_available(zone->flags));
     520        ASSERT(zone->flags & ZONE_AVAILABLE);
    521521       
    522522        frame_t *frame = zone_get_frame(zone, frame_idx);
    … …  
    549549    buddy_system_t *buddy)
    550550{
    551         ASSERT(zone_flags_available(zones.info[z1].flags));
    552         ASSERT(zone_flags_available(zones.info[z2].flags));
     551        ASSERT(zones.info[z1].flags & ZONE_AVAILABLE);
     552        ASSERT(zones.info[z2].flags & ZONE_AVAILABLE);
    553553        ASSERT(zones.info[z1].flags == zones.info[z2].flags);
    554554        ASSERT(zones.info[z1].base < zones.info[z2].base);
    … …  
    645645NO_TRACE static void return_config_frames(size_t znum, pfn_t pfn, size_t count)
    646646{
    647         ASSERT(zone_flags_available(zones.info[znum].flags));
     647        ASSERT(zones.info[znum].flags & ZONE_AVAILABLE);
    648648       
    649649        size_t cframes = SIZE2FRAMES(zone_conf_size(count));
    … …  
    681681    size_t count)
    682682{
    683         ASSERT(zone_flags_available(zones.info[znum].flags));
     683        ASSERT(zones.info[znum].flags & ZONE_AVAILABLE);
    684684        ASSERT(frame_idx + count < zones.info[znum].count);
    685685       
    … …  
    723723         * set of flags
    724724         */
    725         if ((z1 >= zones.count) || (z2 >= zones.count)
    726             || (z2 - z1 != 1)
    727             || (!zone_flags_available(zones.info[z1].flags))
    728             || (!zone_flags_available(zones.info[z2].flags))
    729             || (zones.info[z1].flags != zones.info[z2].flags)) {
     725        if ((z1 >= zones.count) || (z2 >= zones.count) || (z2 - z1 != 1) ||
     726            (zones.info[z1].flags != zones.info[z2].flags)) {
    730727                ret = false;
    731728                goto errout;
    … …  
    828825        zone->buddy_system = buddy;
    829826       
    830         if (zone_flags_available(flags)) {
     827        if (flags & ZONE_AVAILABLE) {
    831828                /*
    832829                 * Compute order for buddy system and initialize
    … …  
    865862{
    866863        return (count * sizeof(frame_t) + buddy_conf_size(fnzb(count)));
     864}
     865
     866/** Allocate external configuration frames from low memory. */
     867pfn_t zone_external_conf_alloc(size_t count)
     868{
     869        size_t size = zone_conf_size(count);
     870        size_t order = ispwr2(size) ? fnzb(size) : (fnzb(size) + 1);
     871
     872        return ADDR2PFN((uintptr_t) frame_alloc(order - FRAME_WIDTH, FRAME_LOWMEM));
    867873}
    868874
    … …  
    888894        irq_spinlock_lock(&zones.lock, true);
    889895       
    890         if (zone_flags_available(flags)) {  /* Create available zone */
     896        if (flags & ZONE_AVAILABLE) {  /* Create available zone */
    891897                /* Theoretically we could have NULL here, practically make sure
    892898                 * nobody tries to do that. If some platform requires, remove
    … …  
    894900                 */
    895901                ASSERT(confframe != ADDR2PFN((uintptr_t ) NULL));
     902
     903                /* Update the known end of physical memory. */
     904                config.physmem_end = max(config.physmem_end, PFN2ADDR(start + count));
    896905               
    897906                /* If confframe is supposed to be inside our zone, then make sure
    … …  
    12321241       
    12331242        /* Tell the architecture to create some memory */
    1234         frame_arch_init();
     1243        frame_low_arch_init();
    12351244        if (config.cpu_active == 1) {
    12361245                frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
    … …  
    12551264                frame_mark_unavailable(0, 1);
    12561265        }
     1266        frame_high_arch_init();
     1267}
     1268
     1269/** Adjust bounds of physical memory region according to low/high memory split.
     1270 *
     1271 * @param low[in]       If true, the adujstment is performed to make the region
     1272 *                      fit in the low memory. Otherwise the adjustment is
     1273 *                      performed to make the region fit in the high memory.
     1274 * @param basep[inout]  Pointer to a variable which contains the region's base
     1275 *                      address and which may receive the adjusted base address.
     1276 * @param sizep[inout]  Pointer to a variable which contains the region's size
     1277 *                      and which may receive the adjusted size.
     1278 * @retun               True if the region still exists even after the
     1279 *                      adjustment, false otherwise.
     1280 */
     1281bool frame_adjust_zone_bounds(bool low, uintptr_t *basep, size_t *sizep)
     1282{
     1283        uintptr_t limit = config.identity_size;
     1284
     1285        if (low) {
     1286                if (*basep > limit)
     1287                        return false;
     1288                if (*basep + *sizep > limit)
     1289                        *sizep = limit - *basep;
     1290        } else {
     1291                if (*basep + *sizep <= limit)
     1292                        return false;
     1293                if (*basep <= limit) {
     1294                        *sizep -= limit - *basep;
     1295                        *basep = limit;
     1296                }
     1297        }
     1298        return true;
    12571299}
    12581300
    … …  
    12931335                *total += (uint64_t) FRAMES2SIZE(zones.info[i].count);
    12941336               
    1295                 if (zone_flags_available(zones.info[i].flags)) {
     1337                if (zones.info[i].flags & ZONE_AVAILABLE) {
    12961338                        *busy += (uint64_t) FRAMES2SIZE(zones.info[i].busy_count);
    12971339                        *free += (uint64_t) FRAMES2SIZE(zones.info[i].free_count);
    … …  
    13441386                irq_spinlock_unlock(&zones.lock, true);
    13451387               
    1346                 bool available = zone_flags_available(flags);
     1388                bool available = ((flags & ZONE_AVAILABLE) != 0);
    13471389               
    13481390                printf("%-4zu", i);
    … …  
    13561398#endif
    13571399               
    1358                 printf(" %12zu %c%c%c      ", count,
    1359                     available ? 'A' : ' ',
    1360                     (flags & ZONE_RESERVED) ? 'R' : ' ',
    1361                     (flags & ZONE_FIRMWARE) ? 'F' : ' ');
     1400                printf(" %12zu %c%c%c%c%c    ", count,
     1401                    available ? 'A' : '-',
     1402                    (flags & ZONE_RESERVED) ? 'R' : '-',
     1403                    (flags & ZONE_FIRMWARE) ? 'F' : '-',
     1404                    (flags & ZONE_LOWMEM) ? 'L' : '-',
     1405                    (flags & ZONE_HIGHMEM) ? 'H' : '-');
    13621406               
    13631407                if (available)
    … …  
    14011445        irq_spinlock_unlock(&zones.lock, true);
    14021446       
    1403         bool available = zone_flags_available(flags);
     1447        bool available = ((flags & ZONE_AVAILABLE) != 0);
    14041448       
    14051449        uint64_t size;
    … …  
    14111455        printf("Zone size:         %zu frames (%" PRIu64 " %s)\n", count,
    14121456            size, size_suffix);
    1413         printf("Zone flags:        %c%c%c\n",
    1414             available ? 'A' : ' ',
    1415             (flags & ZONE_RESERVED) ? 'R' : ' ',
    1416             (flags & ZONE_FIRMWARE) ? 'F' : ' ');
     1457        printf("Zone flags:        %c%c%c%c%c\n",
     1458            available ? 'A' : '-',
     1459            (flags & ZONE_RESERVED) ? 'R' : '-',
     1460            (flags & ZONE_FIRMWARE) ? 'F' : '-',
     1461            (flags & ZONE_LOWMEM) ? 'L' : '-',
     1462            (flags & ZONE_HIGHMEM) ? 'H' : '-');
    14171463       
    14181464        if (available) {

  • kernel/generic/src/mm/page.c

    r852052d rc520034  
    6565#include <arch/mm/asid.h>
    6666#include <mm/as.h>
     67#include <mm/km.h>
    6768#include <mm/frame.h>
    6869#include <arch/barrier.h>
    … …  
    7576#include <errno.h>
    7677#include <align.h>
     78#include <macros.h>
     79#include <bitops.h>
    7780
    7881/** Virtual operations for page subsystem. */
    … …  
    177180}
    178181
     182/** Make the mapping shared by all page tables (not address spaces).
     183 *
     184 * @param base Starting virtual address of the range that is made global.
     185 * @param size Size of the address range that is made global.
     186 */
     187void page_mapping_make_global(uintptr_t base, size_t size)
     188{
     189        ASSERT(page_mapping_operations);
     190        ASSERT(page_mapping_operations->mapping_make_global);
     191       
     192        return page_mapping_operations->mapping_make_global(base, size);
     193}
     194
     195uintptr_t hw_map(uintptr_t physaddr, size_t size)
     196{
     197        uintptr_t virtaddr;
     198        size_t asize;
     199        size_t align;
     200        pfn_t i;
     201
     202        asize = ALIGN_UP(size, PAGE_SIZE);
     203        align = ispwr2(size) ? size : (1U << (fnzb(size) + 1));
     204        virtaddr = km_page_alloc(asize, align);
     205
     206        page_table_lock(AS_KERNEL, true);
     207        for (i = 0; i < ADDR2PFN(asize); i++) {
     208                uintptr_t addr = PFN2ADDR(i);
     209                page_mapping_insert(AS_KERNEL, virtaddr + addr, physaddr + addr,
     210                    PAGE_NOT_CACHEABLE | PAGE_WRITE);
     211        }
     212        page_table_unlock(AS_KERNEL, true);
     213       
     214        return virtaddr;
     215}
     216
    179217int page_find_mapping(uintptr_t virt, void **phys)
    180218{

  • kernel/generic/src/mm/reserve.c

    r852052d rc520034  
    4242#include <typedefs.h>
    4343#include <arch/types.h>
     44#include <debug.h>
     45
     46static bool reserve_initialized = false;
    4447
    4548IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(reserve_lock, "reserve_lock");
    … …  
    5457{
    5558        reserve = frame_total_free_get();
     59        reserve_initialized = true;
    5660}
    5761
    … …  
    6771{
    6872        bool reserved = false;
     73
     74        ASSERT(reserve_initialized);
    6975
    7076        irq_spinlock_lock(&reserve_lock, true);
    … …  
    111117void reserve_force_alloc(size_t size)
    112118{
     119        if (!reserve_initialized)
     120                return;
     121
    113122        irq_spinlock_lock(&reserve_lock, true);
    114123        reserve -= size;
    … …  
    122131void reserve_free(size_t size)
    123132{
     133        if (!reserve_initialized)
     134                return;
     135
    124136        irq_spinlock_lock(&reserve_lock, true);
    125137        reserve += size;

  • kernel/generic/src/proc/thread.c

    r852052d rc520034  
    173173#endif /* CONFIG_FPU */
    174174       
     175        /*
     176         * Allocate the kernel stack from the low-memory to prevent an infinite
     177         * nesting of TLB-misses when accessing the stack from the part of the
     178         * TLB-miss handler written in C.
     179         *
     180         * Note that low-memory is safe to be used for the stack as it will be
     181         * covered by the kernel identity mapping, which guarantees not to
     182         * nest TLB-misses infinitely (either via some hardware mechanism or
     183         * by the construciton of the assembly-language part of the TLB-miss
     184         * handler).
     185         *
     186         * This restriction can be lifted once each architecture provides
     187         * a similar guarantee, for example by locking the kernel stack
     188         * in the TLB whenever it is allocated from the high-memory and the
     189         * thread is being scheduled to run.
     190         */
     191        kmflags |= FRAME_LOWMEM;
     192        kmflags &= ~FRAME_HIGHMEM;
     193
    175194        thread->kstack = (uint8_t *) frame_alloc(STACK_FRAMES, FRAME_KA | kmflags);
    176195        if (!thread->kstack) {
Note: See TracChangeset for help on using the changeset viewer.