source: mainline/kernel/arch/arm64/src/start.S

/*
 * Copyright (c) 2015 Petr Pavlu
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <abi/asmtool.h>
#include <arch/boot/boot.h>
#include <arch/mm/km.h>
#include <arch/mm/page.h>
#include <arch/regutils.h>

.section K_TEXT_START, "ax"

.macro dcache_flush addr size temp0 temp1
        mov \temp0, \addr
        mov \temp1, xzr

        0:
                /* Data or Unified Cache Line Clean */
                dc cvau, \temp0
                add \temp0, \temp0, #4
                add \temp1, \temp1, #4
                cmp \temp1, \size
                blo 0b

        dsb ish
        isb
.endm

/** Kernel entry
 *
 * MMU must be disabled at this point.
 *
 * @param x0 Kernel entry point (kernel_image_start).
 * @param x1 Pointer to the bootinfo structure.
 *
 */
SYMBOL(kernel_image_start)
        /* Get address of the main memory and remember it. */
        adrp x20, kernel_image_start - BOOT_OFFSET
        adrp x2, physmem_base
        str x20, [x2]

        /* Flush the data cache of physmem_base. */
        mov x28, #8
        dcache_flush x2 x28 x29 x30

        /*
         * Set up address translation that identity maps the 1 GiB area that
         * is holding the current execution page.
         */

        /* Prepare the level 0 page table. */
        adrp x2, lower_page_table_level0
        lsr x3, x20, #PTL0_VA_SHIFT
        and x3, x3, #PTL0_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            PTE_L012_TYPE_TABLE << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        adrp x4, lower_page_table_level1
        lsr x4, x4, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_NEXT_LEVEL_ADDRESS_SHIFT
        str x3, [x2]

        /* Prepare the level 1 page table. */
        adrp x2, lower_page_table_level1
        lsr x3, x20, #PTL1_VA_SHIFT
        and x3, x3, #PTL1_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
            PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        lsr x4, x20, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
        str x3, [x2]

        /*
         * Set up address translation that maps the first 4 GiB of the kernel
         * identity virtual address space to the first 4 GiB of the physical
         * memory.
         */

        mov x21, #KM_ARM64_IDENTITY_START
        ldr x22, =(1024 * 1024 * 1024)

        /* Prepare the level 0 page table. */
        adrp x2, upper_page_table_level0
        lsr x3, x21, #PTL0_VA_SHIFT
        and x3, x3, #PTL0_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            PTE_L012_TYPE_TABLE << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        adrp x4, upper_page_table_level1
        lsr x4, x4, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_NEXT_LEVEL_ADDRESS_SHIFT
        str x3, [x2]

        /* Prepare the level 1 page table. */
        adrp x2, upper_page_table_level1
        lsr x3, x21, #PTL1_VA_SHIFT
        and x3, x3, #PTL1_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
            PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        lsr x4, x20, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
        str x3, [x2]

        /* 2nd GiB */
        add x23, x20, x22
        add x24, x21, x22

        adrp x2, upper_page_table_level1
        lsr x3, x24, #PTL1_VA_SHIFT
        and x3, x3, #PTL1_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
            PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        lsr x4, x23, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
        str x3, [x2]

        /* 3rd GiB */
        add x23, x23, x22
        add x24, x24, x22

        adrp x2, upper_page_table_level1
        lsr x3, x24, #PTL1_VA_SHIFT
        and x3, x3, #PTL1_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
            PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        lsr x4, x23, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
        str x3, [x2]

        /* 4th GiB */
        add x23, x23, x22
        add x24, x24, x22

        adrp x2, upper_page_table_level1
        lsr x3, x24, #PTL1_VA_SHIFT
        and x3, x3, #PTL1_VA_MASK
        add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
        mov x3, #( \
            1 << PTE_ACCESS_SHIFT | \
            MAIR_EL1_DEVICE_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
            PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
            1 << PTE_PRESENT_SHIFT)
        lsr x4, x23, #FRAME_WIDTH
        orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
        str x3, [x2]

        /* Flush the data cache of page tables. */
        adrp x27, lower_page_table_level0
        mov x28, #4096
        dcache_flush x27 x28 x29 x30

        adrp x27, lower_page_table_level1
        mov x28, #4096
        dcache_flush x27 x28 x29 x30

        adrp x27, upper_page_table_level0
        mov x28, #4096
        dcache_flush x27 x28 x29 x30

        adrp x27, upper_page_table_level1
        mov x28, #4096
        dcache_flush x27 x28 x29 x30

        /* Make sure there are not any stale TLB entries. */
        tlbi vmalle1is
        dsb ish

        /*
         * Set TCR_EL1:
         * [63:39] - Reserved 0.
         * [38]    - TBI1=0, top byte of an address is used in the address
         *           calculation for the TTBR1_EL1 region.
         * [37]    - TBI0=0, top byte of an address is used in the address
         *           calculation for the TTBR0_EL1 region.
         * [36]    - AS=1, the upper 16 bits of TTBR0_EL1 and TTBR1_EL1 are used
         *           for allocation and matching in the TLB.
         * [35]    - Reserved 0.
         * [34:32] - IPS=101, intermediate physical address size is 48 bits,
         *           256TB.
         * [31:30] - TG1=10, TTBR1_EL1 granule size is 4KB.
         * [29:28] - SH1=11, memory associated with translation table walks
         *           using TTBR1_EL1 is inner shareable.
         * [27:26] - ORGN1=01, memory associated with translation table walks
         *           using TTBR1_EL1 is normal memory, outer write-through
         *           cacheable.
         * [25:24] - IRGN1=01, memory associated with translation table walks
         *           using TTBR1_EL1 is normal memory, inner write-back
         *           write-allocate cacheable.
         * [23]    - EPD1=0, perform translation table walks using TTBR1_EL1.
         * [22]    - A1=0, TTBR0_EL1.ASID defines the ASID.
         * [21:16] - T1SZ=010000, size of the memory region addressed by
         *           TTBR1_EL1 is 2^(64 - 16) bytes.
         * [15:14] - TG0=00, TTBR0_EL1 granule size is 4KB.
         * [13:12] - SH0=11, memory associated with translation table walks
         *           using TTBR0_EL1 is inner shareable.
         * [11:10] - ORGN0=01, memory associated with translation table walks
         *           using TTBR0_EL1 is normal memory, outer write-through
         *           cacheable.
         * [9:8]   - IRGN0=01, memory associated with translation table walks
         *           using TTBR0_EL1 is normal memory, inner write-back
         *           write-allocate cacheable.
         * [7]     - EPD0=0, perform translation table walks using TTBR0.
         * [6]     - Reserved 0.
         * [5:0]   - T0SZ=010000, size of the memory region addressed by
         *           TTBR0_EL1 is 2^(64 - 16) bytes.
         */
        ldr x2, =0x00000015b5103510
        msr tcr_el1, x2

        /* Initialize memory attributes. */
        ldr x2, =(MAIR_EL1_DEVICE_MEMORY_ATTR << \
            (MAIR_EL1_DEVICE_MEMORY_INDEX * MAIR_EL1_ATTR_SHIFT) | \
            MAIR_EL1_NORMAL_MEMORY_ATTR << \
            (MAIR_EL1_NORMAL_MEMORY_INDEX * MAIR_EL1_ATTR_SHIFT))
        msr mair_el1, x2

        /* Set translation tables. */
        adrp x2, lower_page_table_level0
        msr ttbr0_el1, x2
        adrp x2, upper_page_table_level0
        msr ttbr1_el1, x2
        isb

        /*
         * Set SCTLR_EL1:
         * [31:30] - Reserved 0.
         * [29:28] - Reserved 1.
         * [27]    - Reserved 0.
         * [26]    - UCI=0, any attempt to execute cache maintenance
         *           instructions at EL0 is trapped to EL1.
         * [25]    - EE=0, explicit data accesses at EL1, and stage 1
         *           translation table walks in the EL1&0 translation regime are
         *           little-endian.
         * [24]    - E0E=0, explicit data accesses at EL1 are little-endian.
         * [23:22] - Reserved 1.
         * [21]    - Reserved 0.
         * [20]    - Reserved 1.
         * [19]    - WXN=0, regions with write permission are not forced to
         *           Execute Never.
         * [18]    - nTWE=0, any attempt to execute WFE at EL0 is trapped to
         *           EL1.
         * [17]    - Reserved 0.
         * [16]    - nTWI=0, any attempt to execute WFI at EL0 is trapped to
         *           EL1.
         * [15]    - UCT=0, accesses to CTR_EL0 from EL0 are trapped to EL1.
         * [14]    - DZE=0, any attempt to execute DC ZVA at EL0 is trapped to
         *           EL1.
         * [13]    - Reserved 0.
         * [12]    - I=1, this control has no effect on the cacheability of
         *           instruction access to normal memory.
         * [11]    - Reserved 1.
         * [10]    - Reserved 0.
         * [9]     - UMA=0, any attempt to execute MSR/MRS that accesses DAIF at
         *           EL0 is trapped to EL1.
         * [8]     - SED=1, SETEND is undefined at EL0 using AArch32.
         * [7]     - ITD=1, disables some uses of IT at EL0 using AArch32.
         * [6]     - Reserved 0.
         * [5]     - CP15BEN=0, CP15DMB/DSB/ISB is undefined at EL0 using
         *           AArch32.
         * [4]     - SA0=1, use of stack pointer with load/store at EL0 must be
         *           aligned to a 16-byte boundary.
         * [3]     - SA=1, use of stack pointer with load/store at EL1 must be
         *           aligned to a 16-byte boundary.
         * [2]     - C=1, this control has no effect on the cacheability of data
         *           access to normal memory from EL0 and EL1, and normal memory
         *           accesses to the EL1&0 stage 1 translation tables.
         * [1]     - A=0, instructions that load/store registers (other than
         *           load/store exclusive and load-acquire/store-release) do not
         *           check that the address being accessed is aligned to the
         *           size of the data element(s) being accessed.
         * [0]     - M=1, EL1 and EL0 stage 1 address translation enabled.
         */
        ldr w2, =0x30d0199d
        msr sctlr_el1, x2
        isb

        /*
         * MMU is enabled at this point (SCTLR_EL1.M=1), switch to the kernel
         * mapping.
         */
        ldr x2, =1f
        br x2
1:

        /* Disable access to low addresses. */
        mov x2, #0
        msr ttbr0_el1, x2
        isb
        tlbi vmalle1is
        dsb ish

        /* Jump on a temporary stack. */
        ldr x2, =temp_stack
        mov sp, x2

        /* Create the first stack frame. */
        mov x29, #0
        mov x30, #0
        stp x29, x30, [sp, #-16]!
        mov x29, sp

        /* PA2KA(bootinfo). */
        sub x1, x1, x20
        ldr x2, =KM_ARM64_IDENTITY_START
        add x1, x1, x2

        bl arm64_pre_main
        bl main_bsp

.section K_DATA_START, "ax"

        /* Page tables. */
.align 12
lower_page_table_level0:
        .space 4096

lower_page_table_level1:
        .space 4096

upper_page_table_level0:
        .space 4096

upper_page_table_level1:
        .space 4096

        /* Physical memory base address. */
.align 12
SYMBOL(physmem_base)
        .quad 0

        /* Temporary stack. */
.align 10
        .space 1024
temp_stack: