source: mainline/kernel/arch/ia64/src/ivt.S@ 86018c1

#
# Copyright (c) 2005 Jakub Vana
# Copyright (c) 2005 Jakub Jermar
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# - The name of the author may not be used to endorse or promote products
#   derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

#include <arch/stack.h>
#include <arch/register.h>
#include <arch/mm/page.h>
#include <align.h>

#define FRS_TO_SAVE 30
#define STACK_ITEMS             (21 + FRS_TO_SAVE * 2)
#define STACK_FRAME_SIZE        ALIGN_UP((STACK_ITEMS*STACK_ITEM_SIZE) + STACK_SCRATCH_AREA_SIZE, STACK_ALIGNMENT)

#if (STACK_ITEMS % 2 == 0)
#       define STACK_FRAME_BIAS 8
#else
#       define STACK_FRAME_BIAS 16
#endif

/** Partitioning of bank 0 registers. */
#define R_OFFS          r16
#define R_HANDLER       r17
#define R_RET           r18
#define R_TMP           r19
#define R_KSTACK_BSP    r22     /* keep in sync with before_thread_runs_arch() */
#define R_KSTACK        r23     /* keep in sync with before_thread_runs_arch() */

/** Heavyweight interrupt handler
 *
 * This macro roughly follows steps from 1 to 19 described in
 * Intel Itanium Architecture Software Developer's Manual, Chapter 3.4.2.
 *
 * HEAVYWEIGHT_HANDLER macro must cram into 16 bundles (48 instructions).
 * This goal is achieved by using procedure calls after RSE becomes operational.
 *
 * Some steps are skipped (enabling and disabling interrupts).
 *
 * @param offs Offset from the beginning of IVT.
 * @param handler Interrupt handler address.
 */
.macro HEAVYWEIGHT_HANDLER offs, handler=universal_handler
    .org ivt + \offs
        mov R_OFFS = \offs
        movl R_HANDLER = \handler ;;
        br heavyweight_handler
.endm

.global heavyweight_handler
heavyweight_handler:
    /* 1. copy interrupt registers into bank 0 */
    
        /*
         * Note that r24-r31 from bank 0 can be used only as long as PSR.ic = 0.
         */
        
        /* Set up FPU as in interrupted context. */
        mov r24 = psr
        mov r25 = cr.ipsr 
        mov r26 = PSR_DFH_MASK
        mov r27 = ~PSR_DFH_MASK ;;
        and r26 = r25, r26
        and r24 = r24, r27;;
        or r24 = r24, r26;;
        mov psr.l = r24;;
        srlz.i
        srlz.d;;

        mov r24 = cr.iip
        mov r25 = cr.ipsr
        mov r26 = cr.iipa
        mov r27 = cr.isr
        mov r28 = cr.ifa
        
    /* 2. preserve predicate register into bank 0 */
        mov r29 = pr ;;
        
    /* 3. switch to kernel memory stack */
        mov r30 = cr.ipsr
        shr.u r31 = r12, VRN_SHIFT ;;

        shr.u r30 = r30, PSR_CPL_SHIFT ;;
        and r30 = PSR_CPL_MASK_SHIFTED, r30 ;;

        /*
         * Set p3 to true if the interrupted context executed in kernel mode.
         * Set p4 to false if the interrupted context didn't execute in kernel mode.
         */
        cmp.eq p3, p4 = r30, r0 ;;
        cmp.eq p1, p2 = r30, r0 ;;      /* remember IPSR setting in p1 and p2 */

        /*
         * Set p3 to true if the stack register references kernel address space.
         * Set p4 to false if the stack register doesn't reference kernel address space.
         */
(p3)    cmp.eq p3, p4 = VRN_KERNEL, r31 ;;
        
        /*
         * Now, p4 is true iff the stack needs to be switched to kernel stack.
         */
        mov r30 = r12
(p4)    mov r12 = R_KSTACK ;;
        
        add r31 = -STACK_FRAME_BIAS, r12 ;;
        add r12 = -STACK_FRAME_SIZE, r12

    /* 4. save registers in bank 0 into memory stack */

        /*
         * If this is break_instruction handler,
         * copy input parameters to stack.
         */
        mov R_TMP = 0x2c00 ;;
        cmp.eq p6, p5 = R_OFFS, R_TMP ;;
        
        /*
         * From now on, if this is break_instruction handler, p6 is true and p5
         * is false. Otherwise p6 is false and p5 is true.
         * Note that p5 is a preserved predicate register and we make use of it.
         */

(p6)    st8 [r31] = r38, -8 ;;          /* save in6 */
(p6)    st8 [r31] = r37, -8 ;;          /* save in5 */
(p6)    st8 [r31] = r36, -8 ;;          /* save in4 */  
(p6)    st8 [r31] = r35, -8 ;;          /* save in3 */
(p6)    st8 [r31] = r34, -8 ;;          /* save in2 */
(p6)    st8 [r31] = r33, -8 ;;          /* save in1 */
(p6)    st8 [r31] = r32, -8 ;;          /* save in0 */
(p5)    add r31 = -56, r31 ;;
    
        st8 [r31] = r30, -8 ;;          /* save old stack pointer */ 
        
        st8 [r31] = r29, -8 ;;          /* save predicate registers */

        st8 [r31] = r24, -8 ;;          /* save cr.iip */
        st8 [r31] = r25, -8 ;;          /* save cr.ipsr */
        st8 [r31] = r26, -8 ;;          /* save cr.iipa */
        st8 [r31] = r27, -8 ;;          /* save cr.isr */
        st8 [r31] = r28, -8 ;;          /* save cr.ifa */

    /* 5. RSE switch from interrupted context */
        mov r24 = ar.rsc
        mov r25 = ar.pfs
        cover
        mov r26 = cr.ifs
        
        st8 [r31] = r24, -8 ;;          /* save ar.rsc */
        st8 [r31] = r25, -8 ;;          /* save ar.pfs */
        st8 [r31] = r26, -8             /* save ar.ifs */
        
        and r24 = ~(RSC_PL_MASK), r24 ;;
        and r30 = ~(RSC_MODE_MASK), r24 ;;
        mov ar.rsc = r30 ;;             /* update RSE state */
        
        mov r27 = ar.rnat
        mov r28 = ar.bspstore ;;
        
        /*
         * Inspect BSPSTORE to figure out whether it is necessary to switch to
         * kernel BSPSTORE.
         */
(p1)    shr.u r30 = r28, VRN_SHIFT ;;
(p1)    cmp.eq p1, p2 = VRN_KERNEL, r30 ;;
        
        /*
         * If BSPSTORE needs to be switched, p1 is false and p2 is true.
         */
(p1)    mov r30 = r28
(p2)    mov r30 = R_KSTACK_BSP ;;
(p2)    mov ar.bspstore = r30 ;;
        
        mov r29 = ar.bsp
        
        st8 [r31] = r27, -8 ;;          /* save ar.rnat */
        st8 [r31] = r30, -8 ;;          /* save new value written to ar.bspstore */
        st8 [r31] = r28, -8 ;;          /* save ar.bspstore */
        st8 [r31] = r29, -8             /* save ar.bsp */
        
        mov ar.rsc = r24                /* restore RSE's setting + kernel privileges */
        
    /* steps 6 - 15 are done by heavyweight_handler_inner() */
        mov R_RET = b0                  /* save b0 belonging to interrupted context */
        br.call.sptk.many b0 = heavyweight_handler_inner
0:      mov b0 = R_RET                  /* restore b0 belonging to the interrupted context */

    /* 16. RSE switch to interrupted context */
        cover                           /* allocate zero size frame (step 1 (from Intel Docs)) */

        add r31 = (STACK_SCRATCH_AREA_SIZE + (FRS_TO_SAVE * 2 * 8)), r12 ;;

        ld8 r30 = [r31], +8 ;;          /* load ar.bsp */
        ld8 r29 = [r31], +8 ;;          /* load ar.bspstore */
        ld8 r28 = [r31], +8 ;;          /* load ar.bspstore_new */
        sub r27 = r30 , r28 ;;          /* calculate loadrs (step 2) */
        shl r27 = r27, 16

        mov r24 = ar.rsc ;;
        and r30 = ~3, r24 ;;
        or  r24 = r30 , r27 ;;     
        mov ar.rsc = r24 ;;             /* place RSE in enforced lazy mode */

        loadrs                          /* (step 3) */

        ld8 r27 = [r31], +8 ;;          /* load ar.rnat */
        ld8 r26 = [r31], +8 ;;          /* load cr.ifs */
        ld8 r25 = [r31], +8 ;;          /* load ar.pfs */
        ld8 r24 = [r31], +8 ;;          /* load ar.rsc */

        mov ar.bspstore = r29 ;;        /* (step 4) */
        mov ar.rnat = r27               /* (step 5) */

        mov ar.pfs = r25                /* (step 6) */
        mov cr.ifs = r26        

        mov ar.rsc = r24                /* (step 7) */

    /* 17. restore interruption state from memory stack */
        ld8 r28 = [r31], +8 ;;          /* load cr.ifa */               
        ld8 r27 = [r31], +8 ;;          /* load cr.isr */
        ld8 r26 = [r31], +8 ;;          /* load cr.iipa */
        ld8 r25 = [r31], +8 ;;          /* load cr.ipsr */
        ld8 r24 = [r31], +8 ;;          /* load cr.iip */

        mov cr.iip = r24;;
        mov cr.iipa = r26
        mov cr.isr = r27
        mov cr.ifa = r28

        /* Set up FPU as in exception. */
        mov r24 = psr
        mov r26 = PSR_DFH_MASK
        mov r27 = ~PSR_DFH_MASK ;;
        and r25 = r25, r27
        and r24 = r24, r26 ;;
        or r25 = r25, r24;;
        mov cr.ipsr = r25

    /* 18. restore predicate registers from memory stack */
        ld8 r29 = [r31], +8 ;;          /* load predicate registers */
        mov pr = r29
        
    /* 19. return from interruption */
        ld8 r12 = [r31]                 /* load stack pointer */ 
        rfi ;;

.global heavyweight_handler_inner
heavyweight_handler_inner:
        /*
         * From this point, the rest of the interrupted context
         * will be preserved in stacked registers and backing store.
         */
        alloc loc0 = ar.pfs, 0, 48, 2, 0 ;;
        
        /* bank 0 is going to be shadowed, copy essential data from there */
        mov loc1 = R_RET        /* b0 belonging to interrupted context */
        mov loc2 = R_HANDLER
        mov out0 = R_OFFS
        
        add out1 = STACK_SCRATCH_AREA_SIZE, r12

    /* 6. switch to bank 1 and reenable PSR.ic */
        ssm PSR_IC_MASK
        bsw.1 ;;
        srlz.d
        
    /* 7. preserve branch and application registers */
        mov loc3 = ar.unat
        mov loc4 = ar.lc
        mov loc5 = ar.ec
        mov loc6 = ar.ccv
        mov loc7 = ar.csd
        mov loc8 = ar.ssd
        
        mov loc9 = b0
        mov loc10 = b1
        mov loc11 = b2
        mov loc12 = b3
        mov loc13 = b4
        mov loc14 = b5
        mov loc15 = b6
        mov loc16 = b7
        
    /* 8. preserve general and floating-point registers */
        mov loc17 = r1
        mov loc18 = r2
        mov loc19 = r3
        mov loc20 = r4
        mov loc21 = r5
        mov loc22 = r6
        mov loc23 = r7
(p5)    mov loc24 = r8          /* only if not in break_instruction handler */
        mov loc25 = r9
        mov loc26 = r10
        mov loc27 = r11
        /* skip r12 (stack pointer) */
        mov loc28 = r13
        mov loc29 = r14
        mov loc30 = r15
        mov loc31 = r16
        mov loc32 = r17
        mov loc33 = r18
        mov loc34 = r19
        mov loc35 = r20
        mov loc36 = r21
        mov loc37 = r22
        mov loc38 = r23
        mov loc39 = r24
        mov loc40 = r25
        mov loc41 = r26
        mov loc42 = r27
        mov loc43 = r28
        mov loc44 = r29
        mov loc45 = r30
        mov loc46 = r31

        add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
        add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
        add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
        add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
        add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
        add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
        add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
        add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;
        
        stf.spill [r26] = f2, 0x80
        stf.spill [r27] = f3, 0x80
        stf.spill [r28] = f4, 0x80
        stf.spill [r29] = f5, 0x80
        stf.spill [r30] = f6, 0x80
        stf.spill [r31] = f7, 0x80 ;;

        stf.spill [r24] = f8, 0x80
        stf.spill [r25] = f9, 0x80
        stf.spill [r26] = f10, 0x80
        stf.spill [r27] = f11, 0x80
        stf.spill [r28] = f12, 0x80
        stf.spill [r29] = f13, 0x80
        stf.spill [r30] = f14, 0x80
        stf.spill [r31] = f15, 0x80 ;;

        stf.spill [r24] = f16, 0x80
        stf.spill [r25] = f17, 0x80
        stf.spill [r26] = f18, 0x80
        stf.spill [r27] = f19, 0x80
        stf.spill [r28] = f20, 0x80
        stf.spill [r29] = f21, 0x80
        stf.spill [r30] = f22, 0x80
        stf.spill [r31] = f23, 0x80 ;;

        stf.spill [r24] = f24, 0x80
        stf.spill [r25] = f25, 0x80
        stf.spill [r26] = f26, 0x80
        stf.spill [r27] = f27, 0x80
        stf.spill [r28] = f28, 0x80
        stf.spill [r29] = f29, 0x80
        stf.spill [r30] = f30, 0x80
        stf.spill [r31] = f31, 0x80 ;;

        mov loc47 = ar.fpsr     /* preserve floating point status register */
    
    /* 9. skipped (will not enable interrupts) */
        /*
         * ssm PSR_I_MASK
         * ;;
         * srlz.d
         */

    /* 10. call handler */
        movl r1 = kernel_image_start
    
        mov b1 = loc2
        br.call.sptk.many b0 = b1

    /* 11. return from handler */
0:
        
    /* 12. skipped (will not disable interrupts) */
        /*
         * rsm PSR_I_MASK
         * ;;
         * srlz.d
         */

    /* 13. restore general and floating-point registers */
        add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
        add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
        add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
        add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
        add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
        add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
        add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
        add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;

        ldf.fill f2 = [r26], 0x80
        ldf.fill f3 = [r27], 0x80
        ldf.fill f4 = [r28], 0x80
        ldf.fill f5 = [r29], 0x80
        ldf.fill f6 = [r30], 0x80
        ldf.fill f7 = [r31], 0x80 ;;

        ldf.fill f8 = [r24], 0x80
        ldf.fill f9 = [r25], 0x80
        ldf.fill f10 = [r26], 0x80
        ldf.fill f11 = [r27], 0x80
        ldf.fill f12 = [r28], 0x80
        ldf.fill f13 = [r29], 0x80
        ldf.fill f14 = [r30], 0x80
        ldf.fill f15 = [r31], 0x80 ;;

        ldf.fill f16 = [r24], 0x80
        ldf.fill f17 = [r25], 0x80
        ldf.fill f18 = [r26], 0x80
        ldf.fill f19 = [r27], 0x80
        ldf.fill f20 = [r28], 0x80
        ldf.fill f21 = [r29], 0x80
        ldf.fill f22 = [r30], 0x80
        ldf.fill f23 = [r31], 0x80 ;;

        ldf.fill f24 = [r24], 0x80
        ldf.fill f25 = [r25], 0x80
        ldf.fill f26 = [r26], 0x80
        ldf.fill f27 = [r27], 0x80
        ldf.fill f28 = [r28], 0x80
        ldf.fill f29 = [r29], 0x80
        ldf.fill f30 = [r30], 0x80
        ldf.fill f31 = [r31], 0x80 ;;
        
        mov r1 = loc17
        mov r2 = loc18
        mov r3 = loc19
        mov r4 = loc20
        mov r5 = loc21
        mov r6 = loc22
        mov r7 = loc23
(p5)    mov r8 = loc24          /* only if not in break_instruction handler */
        mov r9 = loc25
        mov r10 = loc26
        mov r11 = loc27
        /* skip r12 (stack pointer) */
        mov r13 = loc28
        mov r14 = loc29
        mov r15 = loc30
        mov r16 = loc31
        mov r17 = loc32
        mov r18 = loc33
        mov r19 = loc34
        mov r20 = loc35
        mov r21 = loc36
        mov r22 = loc37
        mov r23 = loc38
        mov r24 = loc39
        mov r25 = loc40
        mov r26 = loc41 
        mov r27 = loc42
        mov r28 = loc43
        mov r29 = loc44
        mov r30 = loc45
        mov r31 = loc46

        mov ar.fpsr = loc47     /* restore floating point status register */
        
    /* 14. restore branch and application registers */
        mov ar.unat = loc3
        mov ar.lc = loc4
        mov ar.ec = loc5
        mov ar.ccv = loc6
        mov ar.csd = loc7
        mov ar.ssd = loc8
        
        mov b0 = loc9
        mov b1 = loc10
        mov b2 = loc11
        mov b3 = loc12
        mov b4 = loc13
        mov b5 = loc14
        mov b6 = loc15
        mov b7 = loc16
        
    /* 15. disable PSR.ic and switch to bank 0 */
        rsm PSR_IC_MASK
        bsw.0 ;;
        srlz.d

        mov R_RET = loc1
        mov ar.pfs = loc0
        br.ret.sptk.many b0

.global ivt
.align 32768
ivt:
        HEAVYWEIGHT_HANDLER 0x0000
        HEAVYWEIGHT_HANDLER 0x0400
        HEAVYWEIGHT_HANDLER 0x0800
        HEAVYWEIGHT_HANDLER 0x0c00 alternate_instruction_tlb_fault
        HEAVYWEIGHT_HANDLER 0x1000 alternate_data_tlb_fault
        HEAVYWEIGHT_HANDLER 0x1400 data_nested_tlb_fault
        HEAVYWEIGHT_HANDLER 0x1800
        HEAVYWEIGHT_HANDLER 0x1c00
        HEAVYWEIGHT_HANDLER 0x2000 data_dirty_bit_fault
        HEAVYWEIGHT_HANDLER 0x2400 instruction_access_bit_fault
        HEAVYWEIGHT_HANDLER 0x2800 data_access_bit_fault
        HEAVYWEIGHT_HANDLER 0x2c00 break_instruction
        HEAVYWEIGHT_HANDLER 0x3000 external_interrupt   /* For external interrupt, heavyweight handler is used. */
        HEAVYWEIGHT_HANDLER 0x3400
        HEAVYWEIGHT_HANDLER 0x3800
        HEAVYWEIGHT_HANDLER 0x3c00
        HEAVYWEIGHT_HANDLER 0x4000
        HEAVYWEIGHT_HANDLER 0x4400
        HEAVYWEIGHT_HANDLER 0x4800
        HEAVYWEIGHT_HANDLER 0x4c00

        HEAVYWEIGHT_HANDLER 0x5000 page_not_present
        HEAVYWEIGHT_HANDLER 0x5100
        HEAVYWEIGHT_HANDLER 0x5200
        HEAVYWEIGHT_HANDLER 0x5300 data_access_rights_fault
        HEAVYWEIGHT_HANDLER 0x5400 general_exception
        HEAVYWEIGHT_HANDLER 0x5500 disabled_fp_register
        HEAVYWEIGHT_HANDLER 0x5600
        HEAVYWEIGHT_HANDLER 0x5700
        HEAVYWEIGHT_HANDLER 0x5800
        HEAVYWEIGHT_HANDLER 0x5900
        HEAVYWEIGHT_HANDLER 0x5a00
        HEAVYWEIGHT_HANDLER 0x5b00
        HEAVYWEIGHT_HANDLER 0x5c00
        HEAVYWEIGHT_HANDLER 0x5d00 
        HEAVYWEIGHT_HANDLER 0x5e00
        HEAVYWEIGHT_HANDLER 0x5f00
        
        HEAVYWEIGHT_HANDLER 0x6000
        HEAVYWEIGHT_HANDLER 0x6100
        HEAVYWEIGHT_HANDLER 0x6200
        HEAVYWEIGHT_HANDLER 0x6300
        HEAVYWEIGHT_HANDLER 0x6400
        HEAVYWEIGHT_HANDLER 0x6500
        HEAVYWEIGHT_HANDLER 0x6600
        HEAVYWEIGHT_HANDLER 0x6700
        HEAVYWEIGHT_HANDLER 0x6800
        HEAVYWEIGHT_HANDLER 0x6900
        HEAVYWEIGHT_HANDLER 0x6a00
        HEAVYWEIGHT_HANDLER 0x6b00
        HEAVYWEIGHT_HANDLER 0x6c00
        HEAVYWEIGHT_HANDLER 0x6d00
        HEAVYWEIGHT_HANDLER 0x6e00
        HEAVYWEIGHT_HANDLER 0x6f00

        HEAVYWEIGHT_HANDLER 0x7000
        HEAVYWEIGHT_HANDLER 0x7100
        HEAVYWEIGHT_HANDLER 0x7200
        HEAVYWEIGHT_HANDLER 0x7300
        HEAVYWEIGHT_HANDLER 0x7400
        HEAVYWEIGHT_HANDLER 0x7500
        HEAVYWEIGHT_HANDLER 0x7600
        HEAVYWEIGHT_HANDLER 0x7700
        HEAVYWEIGHT_HANDLER 0x7800
        HEAVYWEIGHT_HANDLER 0x7900
        HEAVYWEIGHT_HANDLER 0x7a00
        HEAVYWEIGHT_HANDLER 0x7b00
        HEAVYWEIGHT_HANDLER 0x7c00
        HEAVYWEIGHT_HANDLER 0x7d00
        HEAVYWEIGHT_HANDLER 0x7e00
        HEAVYWEIGHT_HANDLER 0x7f00