1 | /*
|
---|
2 | * Copyright (c) 2015 Petr Pavlu
|
---|
3 | * All rights reserved.
|
---|
4 | *
|
---|
5 | * Redistribution and use in source and binary forms, with or without
|
---|
6 | * modification, are permitted provided that the following conditions
|
---|
7 | * are met:
|
---|
8 | *
|
---|
9 | * - Redistributions of source code must retain the above copyright
|
---|
10 | * notice, this list of conditions and the following disclaimer.
|
---|
11 | * - Redistributions in binary form must reproduce the above copyright
|
---|
12 | * notice, this list of conditions and the following disclaimer in the
|
---|
13 | * documentation and/or other materials provided with the distribution.
|
---|
14 | * - The name of the author may not be used to endorse or promote products
|
---|
15 | * derived from this software without specific prior written permission.
|
---|
16 | *
|
---|
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
---|
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
---|
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
---|
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
---|
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
---|
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
---|
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
---|
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
---|
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
---|
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
---|
27 | */
|
---|
28 |
|
---|
29 | #include <abi/asmtool.h>
|
---|
30 | #include <arch/boot/boot.h>
|
---|
31 | #include <arch/mm/km.h>
|
---|
32 | #include <arch/mm/page.h>
|
---|
33 | #include <arch/regutils.h>
|
---|
34 |
|
---|
35 | .section K_TEXT_START, "ax"
|
---|
36 |
|
---|
37 | .macro dcache_flush addr size temp0 temp1
|
---|
38 | mov \temp0, \addr
|
---|
39 | mov \temp1, xzr
|
---|
40 |
|
---|
41 | 0:
|
---|
42 | /* Data or Unified Cache Line Clean */
|
---|
43 | dc cvau, \temp0
|
---|
44 | add \temp0, \temp0, #4
|
---|
45 | add \temp1, \temp1, #4
|
---|
46 | cmp \temp1, \size
|
---|
47 | blo 0b
|
---|
48 |
|
---|
49 | dsb ish
|
---|
50 | isb
|
---|
51 | .endm
|
---|
52 |
|
---|
53 | /** Kernel entry
|
---|
54 | *
|
---|
55 | * MMU must be disabled at this point.
|
---|
56 | *
|
---|
57 | * @param x0 Kernel entry point (kernel_image_start).
|
---|
58 | * @param x1 Pointer to the bootinfo structure.
|
---|
59 | *
|
---|
60 | */
|
---|
61 | SYMBOL(kernel_image_start)
|
---|
62 | /* Get address of the main memory and remember it. */
|
---|
63 | adrp x20, kernel_image_start - BOOT_OFFSET
|
---|
64 | adrp x2, physmem_base
|
---|
65 | str x20, [x2]
|
---|
66 |
|
---|
67 | /* Flush the data cache of physmem_base. */
|
---|
68 | mov x28, #8
|
---|
69 | dcache_flush x2 x28 x29 x30
|
---|
70 |
|
---|
71 | /*
|
---|
72 | * Set up address translation that identity maps the 1 GiB area that
|
---|
73 | * is holding the current execution page.
|
---|
74 | */
|
---|
75 |
|
---|
76 | /* Prepare the level 0 page table. */
|
---|
77 | adrp x2, lower_page_table_level0
|
---|
78 | lsr x3, x20, #PTL0_VA_SHIFT
|
---|
79 | and x3, x3, #PTL0_VA_MASK
|
---|
80 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
81 | mov x3, #( \
|
---|
82 | 1 << PTE_ACCESS_SHIFT | \
|
---|
83 | PTE_L012_TYPE_TABLE << PTE_TYPE_SHIFT | \
|
---|
84 | 1 << PTE_PRESENT_SHIFT)
|
---|
85 | adrp x4, lower_page_table_level1
|
---|
86 | lsr x4, x4, #FRAME_WIDTH
|
---|
87 | orr x3, x3, x4, lsl #PTE_NEXT_LEVEL_ADDRESS_SHIFT
|
---|
88 | str x3, [x2]
|
---|
89 |
|
---|
90 | /* Prepare the level 1 page table. */
|
---|
91 | adrp x2, lower_page_table_level1
|
---|
92 | lsr x3, x20, #PTL1_VA_SHIFT
|
---|
93 | and x3, x3, #PTL1_VA_MASK
|
---|
94 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
95 | mov x3, #( \
|
---|
96 | 1 << PTE_ACCESS_SHIFT | \
|
---|
97 | MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
|
---|
98 | PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
|
---|
99 | 1 << PTE_PRESENT_SHIFT)
|
---|
100 | lsr x4, x20, #FRAME_WIDTH
|
---|
101 | orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
|
---|
102 | str x3, [x2]
|
---|
103 |
|
---|
104 | /*
|
---|
105 | * Set up address translation that maps the first 4 GiB of the kernel
|
---|
106 | * identity virtual address space to the first 4 GiB of the physical
|
---|
107 | * memory.
|
---|
108 | */
|
---|
109 |
|
---|
110 | mov x21, #KM_ARM64_IDENTITY_START
|
---|
111 | ldr x22, =(1024 * 1024 * 1024)
|
---|
112 |
|
---|
113 | /* Prepare the level 0 page table. */
|
---|
114 | adrp x2, upper_page_table_level0
|
---|
115 | lsr x3, x21, #PTL0_VA_SHIFT
|
---|
116 | and x3, x3, #PTL0_VA_MASK
|
---|
117 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
118 | mov x3, #( \
|
---|
119 | 1 << PTE_ACCESS_SHIFT | \
|
---|
120 | PTE_L012_TYPE_TABLE << PTE_TYPE_SHIFT | \
|
---|
121 | 1 << PTE_PRESENT_SHIFT)
|
---|
122 | adrp x4, upper_page_table_level1
|
---|
123 | lsr x4, x4, #FRAME_WIDTH
|
---|
124 | orr x3, x3, x4, lsl #PTE_NEXT_LEVEL_ADDRESS_SHIFT
|
---|
125 | str x3, [x2]
|
---|
126 |
|
---|
127 | /* Prepare the level 1 page table. */
|
---|
128 | adrp x2, upper_page_table_level1
|
---|
129 | lsr x3, x21, #PTL1_VA_SHIFT
|
---|
130 | and x3, x3, #PTL1_VA_MASK
|
---|
131 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
132 | mov x3, #( \
|
---|
133 | 1 << PTE_ACCESS_SHIFT | \
|
---|
134 | MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
|
---|
135 | PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
|
---|
136 | 1 << PTE_PRESENT_SHIFT)
|
---|
137 | lsr x4, x20, #FRAME_WIDTH
|
---|
138 | orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
|
---|
139 | str x3, [x2]
|
---|
140 |
|
---|
141 | /* 2nd GiB */
|
---|
142 | add x23, x20, x22
|
---|
143 | add x24, x21, x22
|
---|
144 |
|
---|
145 | adrp x2, upper_page_table_level1
|
---|
146 | lsr x3, x24, #PTL1_VA_SHIFT
|
---|
147 | and x3, x3, #PTL1_VA_MASK
|
---|
148 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
149 | mov x3, #( \
|
---|
150 | 1 << PTE_ACCESS_SHIFT | \
|
---|
151 | MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
|
---|
152 | PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
|
---|
153 | 1 << PTE_PRESENT_SHIFT)
|
---|
154 | lsr x4, x23, #FRAME_WIDTH
|
---|
155 | orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
|
---|
156 | str x3, [x2]
|
---|
157 |
|
---|
158 | /* 3rd GiB */
|
---|
159 | add x23, x23, x22
|
---|
160 | add x24, x24, x22
|
---|
161 |
|
---|
162 | adrp x2, upper_page_table_level1
|
---|
163 | lsr x3, x24, #PTL1_VA_SHIFT
|
---|
164 | and x3, x3, #PTL1_VA_MASK
|
---|
165 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
166 | mov x3, #( \
|
---|
167 | 1 << PTE_ACCESS_SHIFT | \
|
---|
168 | MAIR_EL1_NORMAL_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
|
---|
169 | PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
|
---|
170 | 1 << PTE_PRESENT_SHIFT)
|
---|
171 | lsr x4, x23, #FRAME_WIDTH
|
---|
172 | orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
|
---|
173 | str x3, [x2]
|
---|
174 |
|
---|
175 | /* 4th GiB */
|
---|
176 | add x23, x23, x22
|
---|
177 | add x24, x24, x22
|
---|
178 |
|
---|
179 | adrp x2, upper_page_table_level1
|
---|
180 | lsr x3, x24, #PTL1_VA_SHIFT
|
---|
181 | and x3, x3, #PTL1_VA_MASK
|
---|
182 | add x2, x2, x3, lsl #PTL_ENTRY_SIZE_SHIFT
|
---|
183 | mov x3, #( \
|
---|
184 | 1 << PTE_ACCESS_SHIFT | \
|
---|
185 | MAIR_EL1_DEVICE_MEMORY_INDEX << PTE_ATTR_INDEX_SHIFT | \
|
---|
186 | PTE_L012_TYPE_BLOCK << PTE_TYPE_SHIFT | \
|
---|
187 | 1 << PTE_PRESENT_SHIFT)
|
---|
188 | lsr x4, x23, #FRAME_WIDTH
|
---|
189 | orr x3, x3, x4, lsl #PTE_OUTPUT_ADDRESS_SHIFT
|
---|
190 | str x3, [x2]
|
---|
191 |
|
---|
192 | /* Flush the data cache of page tables. */
|
---|
193 | adrp x27, lower_page_table_level0
|
---|
194 | mov x28, #4096
|
---|
195 | dcache_flush x27 x28 x29 x30
|
---|
196 |
|
---|
197 | adrp x27, lower_page_table_level1
|
---|
198 | mov x28, #4096
|
---|
199 | dcache_flush x27 x28 x29 x30
|
---|
200 |
|
---|
201 | adrp x27, upper_page_table_level0
|
---|
202 | mov x28, #4096
|
---|
203 | dcache_flush x27 x28 x29 x30
|
---|
204 |
|
---|
205 | adrp x27, upper_page_table_level1
|
---|
206 | mov x28, #4096
|
---|
207 | dcache_flush x27 x28 x29 x30
|
---|
208 |
|
---|
209 | /* Make sure there are not any stale TLB entries. */
|
---|
210 | tlbi vmalle1is
|
---|
211 | dsb ish
|
---|
212 |
|
---|
213 | /*
|
---|
214 | * Set TCR_EL1:
|
---|
215 | * [63:39] - Reserved 0.
|
---|
216 | * [38] - TBI1=0, top byte of an address is used in the address
|
---|
217 | * calculation for the TTBR1_EL1 region.
|
---|
218 | * [37] - TBI0=0, top byte of an address is used in the address
|
---|
219 | * calculation for the TTBR0_EL1 region.
|
---|
220 | * [36] - AS=1, the upper 16 bits of TTBR0_EL1 and TTBR1_EL1 are used
|
---|
221 | * for allocation and matching in the TLB.
|
---|
222 | * [35] - Reserved 0.
|
---|
223 | * [34:32] - IPS=101, intermediate physical address size is 48 bits,
|
---|
224 | * 256TB.
|
---|
225 | * [31:30] - TG1=10, TTBR1_EL1 granule size is 4KB.
|
---|
226 | * [29:28] - SH1=11, memory associated with translation table walks
|
---|
227 | * using TTBR1_EL1 is inner shareable.
|
---|
228 | * [27:26] - ORGN1=01, memory associated with translation table walks
|
---|
229 | * using TTBR1_EL1 is normal memory, outer write-through
|
---|
230 | * cacheable.
|
---|
231 | * [25:24] - IRGN1=01, memory associated with translation table walks
|
---|
232 | * using TTBR1_EL1 is normal memory, inner write-back
|
---|
233 | * write-allocate cacheable.
|
---|
234 | * [23] - EPD1=0, perform translation table walks using TTBR1_EL1.
|
---|
235 | * [22] - A1=0, TTBR0_EL1.ASID defines the ASID.
|
---|
236 | * [21:16] - T1SZ=010000, size of the memory region addressed by
|
---|
237 | * TTBR1_EL1 is 2^(64 - 16) bytes.
|
---|
238 | * [15:14] - TG0=00, TTBR0_EL1 granule size is 4KB.
|
---|
239 | * [13:12] - SH0=11, memory associated with translation table walks
|
---|
240 | * using TTBR0_EL1 is inner shareable.
|
---|
241 | * [11:10] - ORGN0=01, memory associated with translation table walks
|
---|
242 | * using TTBR0_EL1 is normal memory, outer write-through
|
---|
243 | * cacheable.
|
---|
244 | * [9:8] - IRGN0=01, memory associated with translation table walks
|
---|
245 | * using TTBR0_EL1 is normal memory, inner write-back
|
---|
246 | * write-allocate cacheable.
|
---|
247 | * [7] - EPD0=0, perform translation table walks using TTBR0.
|
---|
248 | * [6] - Reserved 0.
|
---|
249 | * [5:0] - T0SZ=010000, size of the memory region addressed by
|
---|
250 | * TTBR0_EL1 is 2^(64 - 16) bytes.
|
---|
251 | */
|
---|
252 | ldr x2, =0x00000015b5103510
|
---|
253 | msr tcr_el1, x2
|
---|
254 |
|
---|
255 | /* Initialize memory attributes. */
|
---|
256 | ldr x2, =(MAIR_EL1_DEVICE_MEMORY_ATTR << \
|
---|
257 | (MAIR_EL1_DEVICE_MEMORY_INDEX * MAIR_EL1_ATTR_SHIFT) | \
|
---|
258 | MAIR_EL1_NORMAL_MEMORY_ATTR << \
|
---|
259 | (MAIR_EL1_NORMAL_MEMORY_INDEX * MAIR_EL1_ATTR_SHIFT))
|
---|
260 | msr mair_el1, x2
|
---|
261 |
|
---|
262 | /* Set translation tables. */
|
---|
263 | adrp x2, lower_page_table_level0
|
---|
264 | msr ttbr0_el1, x2
|
---|
265 | adrp x2, upper_page_table_level0
|
---|
266 | msr ttbr1_el1, x2
|
---|
267 | isb
|
---|
268 |
|
---|
269 | /*
|
---|
270 | * Set SCTLR_EL1:
|
---|
271 | * [31:30] - Reserved 0.
|
---|
272 | * [29:28] - Reserved 1.
|
---|
273 | * [27] - Reserved 0.
|
---|
274 | * [26] - UCI=0, any attempt to execute cache maintenance
|
---|
275 | * instructions at EL0 is trapped to EL1.
|
---|
276 | * [25] - EE=0, explicit data accesses at EL1, and stage 1
|
---|
277 | * translation table walks in the EL1&0 translation regime are
|
---|
278 | * little-endian.
|
---|
279 | * [24] - E0E=0, explicit data accesses at EL1 are little-endian.
|
---|
280 | * [23:22] - Reserved 1.
|
---|
281 | * [21] - Reserved 0.
|
---|
282 | * [20] - Reserved 1.
|
---|
283 | * [19] - WXN=0, regions with write permission are not forced to
|
---|
284 | * Execute Never.
|
---|
285 | * [18] - nTWE=0, any attempt to execute WFE at EL0 is trapped to
|
---|
286 | * EL1.
|
---|
287 | * [17] - Reserved 0.
|
---|
288 | * [16] - nTWI=0, any attempt to execute WFI at EL0 is trapped to
|
---|
289 | * EL1.
|
---|
290 | * [15] - UCT=0, accesses to CTR_EL0 from EL0 are trapped to EL1.
|
---|
291 | * [14] - DZE=0, any attempt to execute DC ZVA at EL0 is trapped to
|
---|
292 | * EL1.
|
---|
293 | * [13] - Reserved 0.
|
---|
294 | * [12] - I=1, this control has no effect on the cacheability of
|
---|
295 | * instruction access to normal memory.
|
---|
296 | * [11] - Reserved 1.
|
---|
297 | * [10] - Reserved 0.
|
---|
298 | * [9] - UMA=0, any attempt to execute MSR/MRS that accesses DAIF at
|
---|
299 | * EL0 is trapped to EL1.
|
---|
300 | * [8] - SED=1, SETEND is undefined at EL0 using AArch32.
|
---|
301 | * [7] - ITD=1, disables some uses of IT at EL0 using AArch32.
|
---|
302 | * [6] - Reserved 0.
|
---|
303 | * [5] - CP15BEN=0, CP15DMB/DSB/ISB is undefined at EL0 using
|
---|
304 | * AArch32.
|
---|
305 | * [4] - SA0=1, use of stack pointer with load/store at EL0 must be
|
---|
306 | * aligned to a 16-byte boundary.
|
---|
307 | * [3] - SA=1, use of stack pointer with load/store at EL1 must be
|
---|
308 | * aligned to a 16-byte boundary.
|
---|
309 | * [2] - C=1, this control has no effect on the cacheability of data
|
---|
310 | * access to normal memory from EL0 and EL1, and normal memory
|
---|
311 | * accesses to the EL1&0 stage 1 translation tables.
|
---|
312 | * [1] - A=0, instructions that load/store registers (other than
|
---|
313 | * load/store exclusive and load-acquire/store-release) do not
|
---|
314 | * check that the address being accessed is aligned to the
|
---|
315 | * size of the data element(s) being accessed.
|
---|
316 | * [0] - M=1, EL1 and EL0 stage 1 address translation enabled.
|
---|
317 | */
|
---|
318 | ldr w2, =0x30d0199d
|
---|
319 | msr sctlr_el1, x2
|
---|
320 | isb
|
---|
321 |
|
---|
322 | /*
|
---|
323 | * MMU is enabled at this point (SCTLR_EL1.M=1), switch to the kernel
|
---|
324 | * mapping.
|
---|
325 | */
|
---|
326 | ldr x2, =1f
|
---|
327 | br x2
|
---|
328 | 1:
|
---|
329 |
|
---|
330 | /* Disable access to low addresses. */
|
---|
331 | mov x2, #0
|
---|
332 | msr ttbr0_el1, x2
|
---|
333 | isb
|
---|
334 | tlbi vmalle1is
|
---|
335 | dsb ish
|
---|
336 |
|
---|
337 | /* Jump on a temporary stack. */
|
---|
338 | ldr x2, =temp_stack
|
---|
339 | mov sp, x2
|
---|
340 |
|
---|
341 | /* Create the first stack frame. */
|
---|
342 | mov x29, #0
|
---|
343 | mov x30, #0
|
---|
344 | stp x29, x30, [sp, #-16]!
|
---|
345 | mov x29, sp
|
---|
346 |
|
---|
347 | /* PA2KA(bootinfo). */
|
---|
348 | sub x1, x1, x20
|
---|
349 | ldr x2, =KM_ARM64_IDENTITY_START
|
---|
350 | add x1, x1, x2
|
---|
351 |
|
---|
352 | bl arm64_pre_main
|
---|
353 | bl main_bsp
|
---|
354 |
|
---|
355 | .section K_DATA_START, "ax"
|
---|
356 |
|
---|
357 | /* Page tables. */
|
---|
358 | .align 12
|
---|
359 | lower_page_table_level0:
|
---|
360 | .space 4096
|
---|
361 |
|
---|
362 | lower_page_table_level1:
|
---|
363 | .space 4096
|
---|
364 |
|
---|
365 | upper_page_table_level0:
|
---|
366 | .space 4096
|
---|
367 |
|
---|
368 | upper_page_table_level1:
|
---|
369 | .space 4096
|
---|
370 |
|
---|
371 | /* Physical memory base address. */
|
---|
372 | .align 12
|
---|
373 | SYMBOL(physmem_base)
|
---|
374 | .quad 0
|
---|
375 |
|
---|
376 | /* Temporary stack. */
|
---|
377 | .align 10
|
---|
378 | .space 1024
|
---|
379 | temp_stack:
|
---|