Index: HelenOS.config =================================================================== --- HelenOS.config (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ HelenOS.config (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -362,4 +362,7 @@ ! CONFIG_TEST (y/n) +% Use link-time optimization +! [COMPILER=gcc_cross|COMPILER=gcc_native] CONFIG_LTO (n/y) + ## Hardware support @@ -521,2 +524,12 @@ % Bundle netif/nil network layer ! CONFIG_NETIF_NIL_BUNDLE (n/y) + +% Strip binaries +! CONFIG_STRIP_BINARIES (n/y) + +% Optimize for size +! CONFIG_OPTIMIZE_FOR_SIZE (n/y) + +% Barebone build with essential binaries only +! CONFIG_BAREBONE (n/y) + Index: boot/Makefile.build =================================================================== --- boot/Makefile.build (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/Makefile.build (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -41,9 +41,9 @@ -Wall -Wextra -Wno-unused-parameter -Wmissing-prototypes \ -Werror-implicit-function-declaration -Wwrite-strings \ - -Werror -pipe + -pipe ICC_CFLAGS = -I$(INCLUDES) -O$(OPTIMIZATION) -imacros $(CONFIG_HEADER) \ -ffreestanding -fno-builtin -nostdlib -nostdinc -Wall -Wmissing-prototypes \ - -Werror-implicit-function-declaration -Werror -wd170 + -Werror-implicit-function-declaration -wd170 SUNCC_CFLAGS = -I$(INCLUDES) -xO$(OPTIMIZATION) \ @@ -57,4 +57,9 @@ -Werror-implicit-function-declaration -Wwrite-strings \ -pipe -arch $(CLANG_ARCH) + +ifeq ($(CONFIG_DEBUG),y) + GCC_CFLAGS += -Werror + ICC_CFLAGS += -Werror +endif ifeq ($(COMPILER),gcc_native) Index: boot/Makefile.common =================================================================== --- boot/Makefile.common (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/Makefile.common (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -82,13 +82,15 @@ endif -RD_SRVS = \ +RD_SRVS_ESSENTIAL = \ + $(USPACE_PATH)/srv/hid/fb/fb \ + $(USPACE_PATH)/srv/hid/kbd/kbd \ + $(USPACE_PATH)/srv/hid/console/console \ + $(USPACE_PATH)/srv/fs/devfs/devfs + +RD_SRVS_NON_ESSENTIAL = \ $(USPACE_PATH)/srv/bd/file_bd/file_bd \ $(USPACE_PATH)/srv/bd/part/guid_part/g_part \ $(USPACE_PATH)/srv/bd/part/mbr_part/mbr_part \ $(USPACE_PATH)/srv/clip/clip \ - $(USPACE_PATH)/srv/hid/fb/fb \ - $(USPACE_PATH)/srv/hid/kbd/kbd \ - $(USPACE_PATH)/srv/hid/console/console \ - $(USPACE_PATH)/srv/fs/devfs/devfs \ $(USPACE_PATH)/srv/fs/tmpfs/tmpfs \ $(USPACE_PATH)/srv/fs/fat/fat \ @@ -105,14 +107,16 @@ ifneq ($(CONFIG_NETIF_NIL_BUNDLE),y) - RD_SRVS += \ + RD_SRVS_NON_ESSENTIAL += \ $(USPACE_PATH)/srv/net/nil/eth/eth \ $(USPACE_PATH)/srv/net/nil/nildummy/nildummy endif -RD_APPS = \ +RD_APPS_ESSENTIAL = \ $(USPACE_PATH)/app/bdsh/bdsh \ + $(USPACE_PATH)/app/getterm/getterm \ + $(USPACE_PATH)/app/klog/klog + +RD_APPS_NON_ESSENTIAL = \ $(USPACE_PATH)/app/edit/edit \ - $(USPACE_PATH)/app/getterm/getterm \ - $(USPACE_PATH)/app/klog/klog \ $(USPACE_PATH)/app/mkfat/mkfat \ $(USPACE_PATH)/app/sbi/sbi \ @@ -129,10 +133,11 @@ $(USPACE_PATH)/app/tasks/tasks \ $(USPACE_PATH)/app/top/top -# $(USPACE_PATH)/app/dummy_load/dummy_load +ifneq ($(CONFIG_BAREBONE),y) NET_CFG = \ $(USPACE_PATH)/srv/net/cfg/general \ $(USPACE_PATH)/srv/net/cfg/lo \ $(USPACE_PATH)/srv/net/cfg/ne2k +endif COMPONENTS = \ @@ -147,4 +152,12 @@ -include arch/$(BARCH)/Makefile.inc +ifeq ($(CONFIG_BAREBONE),y) +RD_SRVS = $(RD_SRVS_ESSENTIAL) +RD_APPS = $(RD_APPS_ESSENTIAL) +else +RD_SRVS = $(RD_SRVS_ESSENTIAL) $(RD_SRVS_NON_ESSENTIAL) +RD_APPS = $(RD_APPS_ESSENTIAL) $(RD_APPS_NON_ESSENTIAL) +endif + MODULES := $(notdir $(COMPONENTS)) Index: boot/arch/amd64/Makefile.inc =================================================================== --- boot/arch/amd64/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/amd64/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -29,7 +29,9 @@ PAGE_SIZE = 4096 -RD_SRVS += \ +RD_SRVS_ESSENTIAL += \ + $(USPACE_PATH)/srv/hw/char/i8042/i8042 + +RD_SRVS_NON_ESSENTIAL += \ $(USPACE_PATH)/srv/bd/ata_bd/ata_bd \ - $(USPACE_PATH)/srv/hw/char/i8042/i8042 \ $(USPACE_PATH)/srv/hw/bus/pci/pci \ $(USPACE_PATH)/srv/hid/char_mouse/char_ms Index: boot/arch/arm32/Makefile.inc =================================================================== --- boot/arch/arm32/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/arm32/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -35,5 +35,7 @@ PAGE_SIZE = 4096 -RD_SRVS += \ +RD_SRVS_ESSENTIAL += + +RD_SRVS_NON_ESSENTIAL += \ $(USPACE_PATH)/srv/bd/gxe_bd/gxe_bd Index: boot/arch/mips32/Makefile.inc =================================================================== --- boot/arch/mips32/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/mips32/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,5 +32,7 @@ EXTRA_CFLAGS = -mno-abicalls -G 0 -fno-zero-initialized-in-bss -mips3 -RD_SRVS += \ +RD_SRVS_ESSENTIAL += + +RD_SRVS_NON_ESSENTIAL += \ $(USPACE_PATH)/srv/bd/gxe_bd/gxe_bd Index: boot/arch/ppc32/Makefile.inc =================================================================== --- boot/arch/ppc32/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/ppc32/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -40,7 +40,9 @@ EXTRA_CFLAGS = -mcpu=powerpc -msoft-float -m32 -RD_SRVS += \ - $(USPACE_PATH)/srv/hid/adb_mouse/adb_ms \ +RD_SRVS_ESSENTIAL += \ $(USPACE_PATH)/srv/hw/bus/cuda_adb/cuda_adb + +RD_SRVS_NON_ESSENTIAL += \ + $(USPACE_PATH)/srv/hid/adb_mouse/adb_ms SOURCES = \ Index: boot/arch/ppc32/src/main.c =================================================================== --- boot/arch/ppc32/src/main.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/ppc32/src/main.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -167,5 +167,4 @@ printf("Booting the kernel...\n"); - ofw_quiesce(); jump_to_kernel(bootinfo_pa, transtable_pa, pages, real_mode_pa); } Index: boot/arch/sparc64/Makefile.inc =================================================================== --- boot/arch/sparc64/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/sparc64/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,7 +38,9 @@ ENDIANESS = BE PAGE_SIZE = 16384 -EXTRA_CFLAGS = -mcpu=ultrasparc -m64 -mno-fpu +EXTRA_CFLAGS = -mcpu=ultrasparc -m64 -mno-fpu -mcmodel=medlow -RD_SRVS += \ +RD_SRVS_NON_ESSENTIAL += + +RD_SRVS_ESSENTIAL += \ $(USPACE_PATH)/srv/hw/cir/fhc/fhc \ $(USPACE_PATH)/srv/hw/cir/obio/obio Index: boot/arch/sparc64/include/asm.h =================================================================== --- boot/arch/sparc64/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/sparc64/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,6 +33,6 @@ #include -extern void jump_to_kernel(void *entry, bootinfo_t *bootinfo, - uintptr_t physmem_start, uint8_t subarch) __attribute__((noreturn)); +extern void jump_to_kernel(uintptr_t physmem_start, bootinfo_t *bootinfo, + uint8_t subarch, void *entry) __attribute__((noreturn)); #endif Index: boot/arch/sparc64/src/asm.S =================================================================== --- boot/arch/sparc64/src/asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/sparc64/src/asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -182,5 +182,5 @@ * (subarch). */ - cmp %i2, SUBARCH_US3 + cmp %o2, SUBARCH_US3 be %xcc, 1f nop @@ -199,9 +199,5 @@ ! Jump to kernel - mov %o0, %l1 - mov %o1, %o0 - mov %o3, %o1 - - jmp %l1 + jmp %o3 nop Index: boot/arch/sparc64/src/main.c =================================================================== --- boot/arch/sparc64/src/main.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ boot/arch/sparc64/src/main.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -101,5 +101,6 @@ * an unknown value of the "compatible" property is encountered. */ - printf("Warning: Unknown architecture, assuming sun4u.\n"); + if (str_cmp(compatible, "sun4u") != 0) + printf("Warning: Unknown architecture, assuming sun4u.\n"); arch = ARCH_SUN4U; } else @@ -205,4 +206,5 @@ void *bootinfo_pa = ofw_translate(&bootinfo); + void *kernel_address_pa = ofw_translate((void *) KERNEL_ADDRESS); void *loader_address_pa = ofw_translate((void *) LOADER_ADDRESS); @@ -210,6 +212,6 @@ bootinfo.memmap.total >> 20, bootinfo.physmem_start); printf(" %p|%p: boot info structure\n", &bootinfo, bootinfo_pa); - printf(" %p|%p: kernel entry point\n", KERNEL_ADDRESS, KERNEL_ADDRESS); - printf(" %p|%p: loader entry pount\n", LOADER_ADDRESS, loader_address_pa); + printf(" %p|%p: kernel entry point\n", KERNEL_ADDRESS, kernel_address_pa); + printf(" %p|%p: loader entry point\n", LOADER_ADDRESS, loader_address_pa); size_t i; @@ -304,6 +306,5 @@ printf("Booting the kernel ...\n"); - ofw_quiesce(); - jump_to_kernel((void *) KERNEL_ADDRESS, &bootinfo, subarch, - bootinfo.physmem_start | BSP_PROCESSOR); -} + jump_to_kernel(bootinfo.physmem_start | BSP_PROCESSOR, &bootinfo, subarch, + (void *) KERNEL_ADDRESS); +} Index: defaults/sparc64/serengeti/Makefile.config =================================================================== --- defaults/sparc64/serengeti/Makefile.config (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ defaults/sparc64/serengeti/Makefile.config (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -1,2 +1,11 @@ # Machine type MACHINE = serengeti +RDFMT = tmpfs +CONFIG_SMP = n +CONFIG_DEBUG = n +CONFIG_UDEBUG = n +CONFIG_SYMTAB = n +CONFIG_TEST = n +CONFIG_STRIP_BINARIES = y +CONFIG_OPTIMIZE_FOR_SIZE = y +CONFIG_BAREBONE = y Index: kernel/Makefile =================================================================== --- kernel/Makefile (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/Makefile (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -82,5 +82,10 @@ INCLUDES = generic/include -OPTIMIZATION = 3 + +ifeq ($(CONFIG_OPTIMIZE_FOR_SIZE),y) + OPTIMIZATION = s +else + OPTIMIZATION = 3 +endif DEFS = -DKERNEL -DRELEASE=$(RELEASE) "-DNAME=$(NAME)" -D__$(BITS)_BITS__ -D__$(ENDIANESS)__ @@ -91,9 +96,9 @@ -Wall -Wextra -Wno-unused-parameter -Wmissing-prototypes \ -Werror-implicit-function-declaration -Wwrite-strings \ - -Werror -pipe + -pipe ICC_CFLAGS = -I$(INCLUDES) -O$(OPTIMIZATION) -imacros $(CONFIG_HEADER) \ -ffreestanding -fno-builtin -nostdlib -nostdinc -Wall -Wmissing-prototypes \ - -Werror-implicit-function-declaration -Werror -wd170 + -Werror-implicit-function-declaration -wd170 SUNCC_CFLAGS = -I$(INCLUDES) -xO$(OPTIMIZATION) \ @@ -108,4 +113,13 @@ -pipe -arch $(CLANG_ARCH) +ifeq ($(CONFIG_DEBUG),y) + GCC_CFLAGS += -Werror + ICC_CFLAGS += -Werror +endif + +ifeq ($(CONFIG_LTO),y) + GCC_CFLAGS += -flto +endif + -include arch/$(KARCH)/Makefile.inc -include genarch/Makefile.inc @@ -157,4 +171,7 @@ DEPEND_DEFS = $(DEFS) $(CONFIG_DEFS) endif + +AFLAGS = +LFLAGS = -N -T $(LINK) -M ## Generic kernel sources @@ -338,4 +355,6 @@ GENARCH_OBJECTS := $(addsuffix .o,$(basename $(GENARCH_SOURCES))) +LFLAGS_LTO := $(addprefix -Xlinker ,$(LFLAGS)) + ifeq ($(CONFIG_SYMTAB),y) SYMTAB_OBJECTS := generic/src/debug/real_map.o @@ -351,5 +370,12 @@ $(RAW): $(LINK) $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(SYMTAB_OBJECTS) - $(LD) -N $(LFLAGS) -T $(LINK) -M -Map $(MAP) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SYMTAB_OBJECTS) +ifeq ($(CONFIG_LTO),y) + $(GCC) $(LFLAGS_LTO) -Xlinker -Map -Xlinker $(MAP) $(DEFS) $(GCC_CFLAGS) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SYMTAB_OBJECTS) +else + $(LD) $(LFLAGS) -Map $(MAP) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SYMTAB_OBJECTS) +endif +ifeq ($(CONFIG_STRIP_BINARIES),y) + $(STRIP) $(RAW) +endif $(LINK): $(LINK).in $(DEPEND) @@ -357,5 +383,5 @@ %.o: %.S $(DEPEND) - $(GCC) $(DEFS) $(GCC_CFLAGS) -D__ASM__ -c $< -o $@ + $(GCC) $(DEFS) $(GCC_CFLAGS) -D__ASM__ -c -o $@ $< ifeq ($(PRECHECK),y) $(JOBFILE) $(JOB) $< $@ as asm/preproc $(DEFS) $(GCC_CFLAGS) -D__ASM__ @@ -374,5 +400,5 @@ test/fpu/%.o: test/fpu/%.c $(DEPEND) - $(CC) $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) -c $< -o $@ + $(CC) $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) -c -o $@ $< ifeq ($(PRECHECK),y) $(JOBFILE) $(JOB) $< $@ cc test $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) @@ -384,5 +410,5 @@ %.o: %.c $(DEPEND) - $(CC) $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) $(FPU_NO_CFLAGS) -c $< -o $@ + $(CC) $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) $(FPU_NO_CFLAGS) -c -o $@ $< ifeq ($(PRECHECK),y) $(JOBFILE) $(JOB) $< $@ cc core $(DEFS) $(CFLAGS) $(EXTRA_FLAGS) $(FPU_NO_CFLAGS) @@ -394,5 +420,9 @@ $(REAL_MAP).bin: $(LINK) $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) echo $(SYMTAB_SECTION) | $(AS) $(AFLAGS) -o $(EMPTY_MAP) - $(LD) -N $(LFLAGS) -T $(LINK) -M -Map $(MAP_PREV) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(EMPTY_MAP) +ifeq ($(CONFIG_LTO),y) + $(GCC) $(LFLAGS_LTO) -Xlinker -Map -Xlinker $(MAP_PREV) $(DEFS) $(GCC_CFLAGS) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(EMPTY_MAP) +else + $(LD) $(LFLAGS) -Map $(MAP_PREV) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(EMPTY_MAP) +endif $(OBJDUMP) -t $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) > $(DUMP) $(GENMAP) $(MAP_PREV) $(DUMP) $@ @@ -402,5 +432,9 @@ echo $(SYMTAB_SECTION)" .incbin \"$@\"" | $(AS) $(AFLAGS) -o $(SIZEOK_MAP) - $(LD) -N $(LFLAGS) -T $(LINK) -M -Map $(MAP_PREV) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SIZEOK_MAP) +ifeq ($(CONFIG_LTO),y) + $(GCC) $(LFLAGS_LTO) -Xlinker -Map -Xlinker $(MAP_PREV) $(DEFS) $(GCC_CFLAGS) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SIZEOK_MAP) +else + $(LD) $(LFLAGS) -Map $(MAP_PREV) -o $@ $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) $(EXTRA_OBJECTS) $(SIZEOK_MAP) +endif $(OBJDUMP) -t $(ARCH_OBJECTS) $(GENARCH_OBJECTS) $(GENERIC_OBJECTS) > $(DUMP) $(GENMAP) $(MAP_PREV) $(DUMP) $@ Index: kernel/arch/abs32le/include/asm.h =================================================================== --- kernel/arch/abs32le/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/abs32le/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -164,4 +164,11 @@ } +static inline bool interrupts_disabled(void) +{ + /* On real hardware the return value is true iff interrupts are + disabled. */ + return false; +} + static inline uintptr_t get_stack_base(void) { @@ -175,14 +182,4 @@ } -static inline uintptr_t *get_ip() -{ - /* On real hardware this returns the current instruction - pointer value. The value certainly changes with each - instruction, but it can be still used to identify - a specific function. */ - - return 0; -} - #endif Index: rnel/arch/abs32le/include/debug.h =================================================================== --- kernel/arch/abs32le/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,45 +1,0 @@ -/* - * Copyright (c) 2010 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup abs32ledebug - * @{ - */ -/** @file - */ - -#ifndef KERN_abs32le_DEBUG_H_ -#define KERN_abs32le_DEBUG_H_ - -#include - -#define HERE get_ip() - -#endif - -/** @} - */ Index: kernel/arch/amd64/include/asm.h =================================================================== --- kernel/arch/amd64/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,4 +38,5 @@ #include #include +#include extern void asm_delay_loop(uint32_t t); @@ -269,4 +270,23 @@ return v; } + +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +static inline bool interrupts_disabled(void) +{ + ipl_t v; + + asm volatile ( + "pushfq\n" + "popq %[v]\n" + : [v] "=r" (v) + ); + + return ((v & RFLAGS_IF) == 0); +} + /** Write to MSR */ @@ -310,16 +330,4 @@ ::: "%eax","%ecx","%edx" ); -} - -static inline uintptr_t * get_ip() -{ - uintptr_t *ip; - - asm volatile ( - "mov %%rip, %[ip]" - : [ip] "=r" (ip) - ); - - return ip; } Index: kernel/arch/amd64/include/ddi/ddi.h =================================================================== --- kernel/arch/amd64/include/ddi/ddi.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/include/ddi/ddi.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief amd64 specific DDI declarations and macros. + * @brief amd64 specific DDI declarations and macros. */ Index: rnel/arch/amd64/include/debug.h =================================================================== --- kernel/arch/amd64/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,1 +1,0 @@ -../../ia32/include/debug.h Index: kernel/arch/amd64/include/debugger.h =================================================================== --- kernel/arch/amd64/include/debugger.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/include/debugger.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,17 +38,17 @@ #include -#define BKPOINTS_MAX 4 +#define BKPOINTS_MAX 4 /* Flags that are passed to breakpoint_add function */ -#define BKPOINT_INSTR 0x1 -#define BKPOINT_WRITE 0x2 -#define BKPOINT_READ_WRITE 0x4 +#define BKPOINT_INSTR 0x1 +#define BKPOINT_WRITE 0x2 +#define BKPOINT_READ_WRITE 0x4 -#define BKPOINT_CHECK_ZERO 0x8 +#define BKPOINT_CHECK_ZERO 0x8 extern void debugger_init(void); -extern int breakpoint_add(const void *where, const int flags, int curidx); -extern void breakpoint_del(int slot); +extern int breakpoint_add(const void *, const unsigned int, int); +extern void breakpoint_del(int); #endif Index: kernel/arch/amd64/src/ddi/ddi.c =================================================================== --- kernel/arch/amd64/src/ddi/ddi.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/src/ddi/ddi.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -49,30 +49,27 @@ * Interrupts are disabled and task is locked. * - * @param task Task. + * @param task Task. * @param ioaddr Startign I/O space address. - * @param size Size of the enabled I/O range. + * @param size Size of the enabled I/O range. * * @return 0 on success or an error code from errno.h. + * */ int ddi_iospace_enable_arch(task_t *task, uintptr_t ioaddr, size_t size) { - size_t bits; - - bits = ioaddr + size; + size_t bits = ioaddr + size; if (bits > IO_PORTS) return ENOENT; if (task->arch.iomap.bits < bits) { - bitmap_t oldiomap; - uint8_t *newmap; - /* * The I/O permission bitmap is too small and needs to be grown. */ - newmap = (uint8_t *) malloc(BITS2BYTES(bits), FRAME_ATOMIC); + uint8_t *newmap = (uint8_t *) malloc(BITS2BYTES(bits), FRAME_ATOMIC); if (!newmap) return ENOMEM; + bitmap_t oldiomap; bitmap_initialize(&oldiomap, task->arch.iomap.map, task->arch.iomap.bits); @@ -115,23 +112,20 @@ * * Interrupts must be disabled prior this call. + * */ void io_perm_bitmap_install(void) { - size_t bits; - ptr_16_64_t cpugdtr; - descriptor_t *gdt_p; - tss_descriptor_t *tss_desc; - size_t ver; - /* First, copy the I/O Permission Bitmap. */ - spinlock_lock(&TASK->lock); - ver = TASK->arch.iomapver; - if ((bits = TASK->arch.iomap.bits)) { + irq_spinlock_lock(&TASK->lock, false); + size_t ver = TASK->arch.iomapver; + size_t bits = TASK->arch.iomap.bits; + if (bits) { + ASSERT(TASK->arch.iomap.map); + bitmap_t iomap; - - ASSERT(TASK->arch.iomap.map); bitmap_initialize(&iomap, CPU->arch.tss->iomap, TSS_IOMAP_SIZE * 8); bitmap_copy(&iomap, &TASK->arch.iomap, TASK->arch.iomap.bits); + /* * It is safe to set the trailing eight bits because of the @@ -140,5 +134,5 @@ bitmap_set_range(&iomap, ALIGN_UP(TASK->arch.iomap.bits, 8), 8); } - spinlock_unlock(&TASK->lock); + irq_spinlock_unlock(&TASK->lock, false); /* @@ -146,6 +140,8 @@ * Take the extra ending byte will all bits set into account. */ + ptr_16_64_t cpugdtr; gdtr_store(&cpugdtr); - gdt_p = (descriptor_t *) cpugdtr.base; + + descriptor_t *gdt_p = (descriptor_t *) cpugdtr.base; gdt_tss_setlimit(&gdt_p[TSS_DES], TSS_BASIC_SIZE + BITS2BYTES(bits)); gdtr_load(&cpugdtr); @@ -155,5 +151,5 @@ * type must be changed to describe inactive TSS. */ - tss_desc = (tss_descriptor_t *) &gdt_p[TSS_DES]; + tss_descriptor_t *tss_desc = (tss_descriptor_t *) &gdt_p[TSS_DES]; tss_desc->type = AR_TSS; tr_load(gdtselector(TSS_DES)); Index: kernel/arch/amd64/src/debugger.c =================================================================== --- kernel/arch/amd64/src/debugger.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/src/debugger.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,16 +46,27 @@ #include +#ifdef __64_BITS__ + #define getip(x) ((x)->rip) +#endif + +#ifdef __32_BITS__ + #define getip(x) ((x)->eip) +#endif + typedef struct { - uintptr_t address; /**< Breakpoint address */ - int flags; /**< Flags regarding breakpoint */ - int counter; /**< How many times the exception occured */ + uintptr_t address; /**< Breakpoint address */ + unsigned int flags; /**< Flags regarding breakpoint */ + size_t counter; /**< How many times the exception occured */ } bpinfo_t; static bpinfo_t breakpoints[BKPOINTS_MAX]; -SPINLOCK_INITIALIZE(bkpoint_lock); +IRQ_SPINLOCK_STATIC_INITIALIZE(bkpoint_lock); #ifdef CONFIG_KCONSOLE -static int cmd_print_breakpoints(cmd_arg_t *argv); +static int cmd_print_breakpoints(cmd_arg_t *); +static int cmd_del_breakpoint(cmd_arg_t *); +static int cmd_add_breakpoint(cmd_arg_t *); + static cmd_info_t bkpts_info = { .name = "bkpts", @@ -65,11 +76,11 @@ }; -static int cmd_del_breakpoint(cmd_arg_t *argv); static cmd_arg_t del_argv = { .type = ARG_TYPE_INT }; + static cmd_info_t delbkpt_info = { .name = "delbkpt", - .description = "delbkpt - Delete breakpoint.", + .description = "Delete breakpoint.", .func = cmd_del_breakpoint, .argc = 1, @@ -77,11 +88,11 @@ }; -static int cmd_add_breakpoint(cmd_arg_t *argv); static cmd_arg_t add_argv = { .type = ARG_TYPE_INT }; + static cmd_info_t addbkpt_info = { .name = "addbkpt", - .description = "addbkpt <&symbol> - new breakpoint.", + .description = "Add breakpoint.", .func = cmd_add_breakpoint, .argc = 1, @@ -92,7 +103,8 @@ .type = ARG_TYPE_INT }; + static cmd_info_t addwatchp_info = { .name = "addwatchp", - .description = "addbwatchp <&symbol> - new write watchpoint.", + .description = "Add write watchpoint.", .func = cmd_add_breakpoint, .argc = 1, @@ -102,16 +114,20 @@ #endif /* CONFIG_KCONSOLE */ -/* Setup DR register according to table */ +/** Setup DR register according to table + * + */ static void setup_dr(int curidx) { - unative_t dr7; + ASSERT(curidx >= 0); + bpinfo_t *cur = &breakpoints[curidx]; - int flags = breakpoints[curidx].flags; - + unsigned int flags = breakpoints[curidx].flags; + /* Disable breakpoint in DR7 */ - dr7 = read_dr7(); - dr7 &= ~(0x2 << (curidx*2)); - - if (cur->address) { /* Setup DR register */ + unative_t dr7 = read_dr7(); + dr7 &= ~(0x2 << (curidx * 2)); + + /* Setup DR register */ + if (cur->address) { /* Set breakpoint to debug registers */ switch (curidx) { @@ -129,15 +145,14 @@ break; } + /* Set type to requested breakpoint & length*/ - dr7 &= ~ (0x3 << (16 + 4*curidx)); - dr7 &= ~ (0x3 << (18 + 4*curidx)); - if ((flags & BKPOINT_INSTR)) { - ; - } else { + dr7 &= ~(0x3 << (16 + 4 * curidx)); + dr7 &= ~(0x3 << (18 + 4 * curidx)); + if (!(flags & BKPOINT_INSTR)) { #ifdef __32_BITS__ dr7 |= ((unative_t) 0x3) << (18 + 4 * curidx); #endif - + #ifdef __64_BITS__ dr7 |= ((unative_t) 0x2) << (18 + 4 * curidx); @@ -149,76 +164,70 @@ dr7 |= ((unative_t) 0x3) << (16 + 4 * curidx); } - + /* Enable global breakpoint */ dr7 |= 0x2 << (curidx * 2); - + write_dr7(dr7); - - } -} - + } +} + /** Enable hardware breakpoint * * @param where Address of HW breakpoint * @param flags Type of breakpoint (EXECUTE, WRITE) + * * @return Debug slot on success, -1 - no available HW breakpoint - */ -int breakpoint_add(const void *where, const int flags, int curidx) -{ - ipl_t ipl; - int i; - bpinfo_t *cur; - + * + */ +int breakpoint_add(const void *where, const unsigned int flags, int curidx) +{ ASSERT(flags & (BKPOINT_INSTR | BKPOINT_WRITE | BKPOINT_READ_WRITE)); - - ipl = interrupts_disable(); - spinlock_lock(&bkpoint_lock); + + irq_spinlock_lock(&bkpoint_lock, true); if (curidx == -1) { /* Find free space in slots */ - for (i = 0; i < BKPOINTS_MAX; i++) + unsigned int i; + for (i = 0; i < BKPOINTS_MAX; i++) { if (!breakpoints[i].address) { curidx = i; break; } + } + if (curidx == -1) { /* Too many breakpoints */ - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return -1; } } - cur = &breakpoints[curidx]; - + + bpinfo_t *cur = &breakpoints[curidx]; + cur->address = (uintptr_t) where; cur->flags = flags; cur->counter = 0; - + setup_dr(curidx); - - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); - + + irq_spinlock_unlock(&bkpoint_lock, true); + /* Send IPI */ // ipi_broadcast(VECTOR_DEBUG_IPI); - + return curidx; } -#ifdef __64_BITS__ - #define getip(x) ((x)->rip) -#else - #define getip(x) ((x)->eip) -#endif - static void handle_exception(int slot, istate_t *istate) { + ASSERT(slot >= 0); ASSERT(breakpoints[slot].address); - + /* Handle zero checker */ - if (! (breakpoints[slot].flags & BKPOINT_INSTR)) { + if (!(breakpoints[slot].flags & BKPOINT_INSTR)) { if ((breakpoints[slot].flags & BKPOINT_CHECK_ZERO)) { if (*((unative_t *) breakpoints[slot].address) != 0) return; + printf("*** Found ZERO on address %lx (slot %d) ***\n", breakpoints[slot].address, slot); @@ -228,8 +237,8 @@ } } - + printf("Reached breakpoint %d:%lx (%s)\n", slot, getip(istate), symtab_fmt_name_lookup(getip(istate))); - + #ifdef CONFIG_KCONSOLE atomic_set(&haltstate, 1); @@ -241,42 +250,37 @@ void breakpoint_del(int slot) { - bpinfo_t *cur; - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&bkpoint_lock); - - cur = &breakpoints[slot]; + ASSERT(slot >= 0); + + irq_spinlock_lock(&bkpoint_lock, true); + + bpinfo_t *cur = &breakpoints[slot]; if (!cur->address) { - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return; } - + cur->address = NULL; - + setup_dr(slot); - - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&bkpoint_lock, true); // ipi_broadcast(VECTOR_DEBUG_IPI); } - - static void debug_exception(int n __attribute__((unused)), istate_t *istate) { - unative_t dr6; - int i; - /* Set RF to restart the instruction */ #ifdef __64_BITS__ istate->rflags |= RFLAGS_RF; -#else +#endif + +#ifdef __32_BITS__ istate->eflags |= EFLAGS_RF; #endif - - dr6 = read_dr6(); - for (i=0; i < BKPOINTS_MAX; i++) { + + unative_t dr6 = read_dr6(); + + unsigned int i; + for (i = 0; i < BKPOINTS_MAX; i++) { if (dr6 & (1 << i)) { dr6 &= ~ (1 << i); @@ -289,38 +293,39 @@ #ifdef CONFIG_SMP -static void -debug_ipi(int n __attribute__((unused)), +static void debug_ipi(int n __attribute__((unused)), istate_t *istate __attribute__((unused))) { - int i; - - spinlock_lock(&bkpoint_lock); + irq_spinlock_lock(&bkpoint_lock, false); + + unsigned int i; for (i = 0; i < BKPOINTS_MAX; i++) setup_dr(i); - spinlock_unlock(&bkpoint_lock); -} -#endif - -/** Initialize debugger */ + + irq_spinlock_unlock(&bkpoint_lock, false); +} +#endif /* CONFIG_SMP */ + +/** Initialize debugger + * + */ void debugger_init() { - int i; - + unsigned int i; for (i = 0; i < BKPOINTS_MAX; i++) breakpoints[i].address = NULL; - + #ifdef CONFIG_KCONSOLE cmd_initialize(&bkpts_info); if (!cmd_register(&bkpts_info)) printf("Cannot register command %s\n", bkpts_info.name); - + cmd_initialize(&delbkpt_info); if (!cmd_register(&delbkpt_info)) printf("Cannot register command %s\n", delbkpt_info.name); - + cmd_initialize(&addbkpt_info); if (!cmd_register(&addbkpt_info)) printf("Cannot register command %s\n", addbkpt_info.name); - + cmd_initialize(&addwatchp_info); if (!cmd_register(&addwatchp_info)) @@ -331,18 +336,18 @@ #ifdef CONFIG_SMP exc_register(VECTOR_DEBUG_IPI, "debugger_smp", debug_ipi); -#endif +#endif /* CONFIG_SMP */ } #ifdef CONFIG_KCONSOLE -/** Print table of active breakpoints */ +/** Print table of active breakpoints + * + */ int cmd_print_breakpoints(cmd_arg_t *argv __attribute__((unused))) { - unsigned int i; - #ifdef __32_BITS__ printf("# Count Address In symbol\n"); printf("-- ----- ---------- ---------\n"); #endif - + #ifdef __64_BITS__ printf("# Count Address In symbol\n"); @@ -350,26 +355,30 @@ #endif - for (i = 0; i < BKPOINTS_MAX; i++) + unsigned int i; + for (i = 0; i < BKPOINTS_MAX; i++) { if (breakpoints[i].address) { const char *symbol = symtab_fmt_name_lookup( breakpoints[i].address); - + #ifdef __32_BITS__ - printf("%-2u %-5d %#10zx %s\n", i, + printf("%-2u %-5" PRIs " %p %s\n", i, breakpoints[i].counter, breakpoints[i].address, symbol); #endif - + #ifdef __64_BITS__ - printf("%-2u %-5d %#18zx %s\n", i, + printf("%-2u %-5" PRIs " %p %s\n", i, breakpoints[i].counter, breakpoints[i].address, symbol); #endif - - } + } + } + return 1; } -/** Remove breakpoint from table */ +/** Remove breakpoint from table + * + */ int cmd_del_breakpoint(cmd_arg_t *argv) { @@ -379,21 +388,23 @@ return 0; } + breakpoint_del(argv->intval); return 1; } -/** Add new breakpoint to table */ +/** Add new breakpoint to table + * + */ static int cmd_add_breakpoint(cmd_arg_t *argv) { - int flags; - int id; - - if (argv == &add_argv) { + unsigned int flags; + if (argv == &add_argv) flags = BKPOINT_INSTR; - } else { /* addwatchp */ + else flags = BKPOINT_WRITE; - } + printf("Adding breakpoint on address: %p\n", argv->intval); - id = breakpoint_add((void *)argv->intval, flags, -1); + + int id = breakpoint_add((void *)argv->intval, flags, -1); if (id < 0) printf("Add breakpoint failed.\n"); Index: kernel/arch/amd64/src/interrupt.c =================================================================== --- kernel/arch/amd64/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -106,14 +106,14 @@ } -/** General Protection Fault. */ +/** General Protection Fault. + * + */ static void gp_fault(int n, istate_t *istate) { if (TASK) { - size_t ver; - - spinlock_lock(&TASK->lock); - ver = TASK->arch.iomapver; - spinlock_unlock(&TASK->lock); - + irq_spinlock_lock(&TASK->lock, false); + size_t ver = TASK->arch.iomapver; + irq_spinlock_unlock(&TASK->lock, false); + if (CPU->arch.iomapver_copy != ver) { /* @@ -129,5 +129,5 @@ fault_if_from_uspace(istate, "General protection fault."); } - + decode_istate(n, istate); panic("General protection fault."); @@ -159,5 +159,7 @@ #endif -/** Handler of IRQ exceptions */ +/** Handler of IRQ exceptions. + * + */ static void irq_interrupt(int n, istate_t *istate) { @@ -174,5 +176,5 @@ * The IRQ handler was found. */ - + if (irq->preack) { /* Send EOI before processing the interrupt */ @@ -181,5 +183,5 @@ } irq->handler(irq); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { /* Index: kernel/arch/amd64/src/mm/page.c =================================================================== --- kernel/arch/amd64/src/mm/page.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/amd64/src/mm/page.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -88,4 +88,6 @@ if (config.cpu_active == 1) { page_mapping_operations = &pt_mapping_operations; + + page_table_lock(AS_KERNEL, true); /* @@ -112,4 +114,6 @@ } + page_table_unlock(AS_KERNEL, true); + exc_register(14, "page_fault", (iroutine) page_fault); write_cr3((uintptr_t) AS_KERNEL->genarch.page_table); @@ -208,6 +212,9 @@ uintptr_t virtaddr = PA2KA(last_frame); pfn_t i; + + page_table_lock(AS_KERNEL, true); for (i = 0; i < ADDR2PFN(ALIGN_UP(size, PAGE_SIZE)); i++) page_mapping_insert(AS_KERNEL, virtaddr + PFN2ADDR(i), physaddr + PFN2ADDR(i), PAGE_NOT_CACHEABLE | PAGE_WRITE); + page_table_unlock(AS_KERNEL, true); last_frame = ALIGN_UP(last_frame + size, FRAME_SIZE); Index: rnel/arch/arm32/include/debug.h =================================================================== --- kernel/arch/arm32/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,42 +1,0 @@ -/* - * Copyright (c) 2005 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup arm32debug - * @{ - */ -/** @file - * @brief Empty. - */ - -#ifndef KERN_arm32_DEBUG_H_ -#define KERN_arm32_DEBUG_H_ - -#endif - -/** @} - */ Index: kernel/arch/arm32/include/interrupt.h =================================================================== --- kernel/arch/arm32/include/interrupt.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/arm32/include/interrupt.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -52,4 +52,5 @@ extern void interrupts_restore(ipl_t ipl); extern ipl_t interrupts_read(void); +extern bool interrupts_disabled(void); Index: kernel/arch/arm32/src/interrupt.c =================================================================== --- kernel/arch/arm32/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/arm32/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -97,4 +97,14 @@ } +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +bool interrupts_disabled(void) +{ + return current_status_reg_read() & STATUS_REG_IRQ_DISABLED_BIT; +} + /** Initialize basic tables for exception dispatching * and starts the timer. Index: kernel/arch/arm32/src/mm/page.c =================================================================== --- kernel/arch/arm32/src/mm/page.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/arm32/src/mm/page.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -53,4 +53,6 @@ int flags = PAGE_CACHEABLE; page_mapping_operations = &pt_mapping_operations; + + page_table_lock(AS_KERNEL, true); uintptr_t cur; @@ -66,4 +68,6 @@ #error "Only high exception vector supported now" #endif + + page_table_unlock(AS_KERNEL, true); as_switch(NULL, AS_KERNEL); @@ -92,4 +96,6 @@ uintptr_t virtaddr = PA2KA(last_frame); pfn_t i; + + page_table_lock(AS_KERNEL, true); for (i = 0; i < ADDR2PFN(ALIGN_UP(size, PAGE_SIZE)); i++) { page_mapping_insert(AS_KERNEL, virtaddr + PFN2ADDR(i), @@ -97,4 +103,5 @@ PAGE_NOT_CACHEABLE | PAGE_READ | PAGE_WRITE | PAGE_KERNEL); } + page_table_unlock(AS_KERNEL, true); last_frame = ALIGN_UP(last_frame + size, FRAME_SIZE); Index: kernel/arch/ia32/include/asm.h =================================================================== --- kernel/arch/ia32/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,4 +38,5 @@ #include +#include #include #include @@ -299,4 +300,22 @@ } +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +static inline bool interrupts_disabled(void) +{ + ipl_t v; + + asm volatile ( + "pushf\n" + "popl %[v]\n" + : [v] "=r" (v) + ); + + return ((v & EFLAGS_IF) == 0); +} + /** Write to MSR */ static inline void write_msr(uint32_t msr, uint64_t value) @@ -343,17 +362,4 @@ } -/** Return current IP address */ -static inline uintptr_t * get_ip() -{ - uintptr_t *ip; - - asm volatile ( - "mov %%eip, %[ip]" - : [ip] "=r" (ip) - ); - - return ip; -} - /** Invalidate TLB Entry. * Index: kernel/arch/ia32/include/ddi/ddi.h =================================================================== --- kernel/arch/ia32/include/ddi/ddi.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/include/ddi/ddi.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,5 +32,5 @@ /** * @file - * @brief ia32 specific DDI declarations and macros. + * @brief ia32 specific DDI declarations and macros. */ Index: rnel/arch/ia32/include/debug.h =================================================================== --- kernel/arch/ia32/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,45 +1,0 @@ -/* - * Copyright (c) 2005 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup amd64debug - * @{ - */ -/** @file - */ - -#ifndef KERN_ia32_DEBUG_H_ -#define KERN_ia32_DEBUG_H_ - -#include - -#define HERE get_ip() - -#endif - -/** @} - */ Index: kernel/arch/ia32/include/smp/apic.h =================================================================== --- kernel/arch/ia32/include/smp/apic.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/include/smp/apic.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ia32 +/** @addtogroup ia32 * @{ */ @@ -39,87 +39,88 @@ #include -#define FIXED (0<<0) -#define LOPRI (1<<0) - -#define APIC_ID_COUNT 16 +#define FIXED (0 << 0) +#define LOPRI (1 << 0) + +#define APIC_ID_COUNT 16 /* local APIC macros */ -#define IPI_INIT 0 -#define IPI_STARTUP 0 +#define IPI_INIT 0 +#define IPI_STARTUP 0 /** Delivery modes. */ -#define DELMOD_FIXED 0x0 -#define DELMOD_LOWPRI 0x1 -#define DELMOD_SMI 0x2 +#define DELMOD_FIXED 0x0 +#define DELMOD_LOWPRI 0x1 +#define DELMOD_SMI 0x2 /* 0x3 reserved */ -#define DELMOD_NMI 0x4 -#define DELMOD_INIT 0x5 -#define DELMOD_STARTUP 0x6 -#define DELMOD_EXTINT 0x7 +#define DELMOD_NMI 0x4 +#define DELMOD_INIT 0x5 +#define DELMOD_STARTUP 0x6 +#define DELMOD_EXTINT 0x7 /** Destination modes. */ -#define DESTMOD_PHYS 0x0 -#define DESTMOD_LOGIC 0x1 +#define DESTMOD_PHYS 0x0 +#define DESTMOD_LOGIC 0x1 /** Trigger Modes. */ -#define TRIGMOD_EDGE 0x0 -#define TRIGMOD_LEVEL 0x1 +#define TRIGMOD_EDGE 0x0 +#define TRIGMOD_LEVEL 0x1 /** Levels. */ -#define LEVEL_DEASSERT 0x0 -#define LEVEL_ASSERT 0x1 +#define LEVEL_DEASSERT 0x0 +#define LEVEL_ASSERT 0x1 /** Destination Shorthands. */ -#define SHORTHAND_NONE 0x0 -#define SHORTHAND_SELF 0x1 -#define SHORTHAND_ALL_INCL 0x2 -#define SHORTHAND_ALL_EXCL 0x3 +#define SHORTHAND_NONE 0x0 +#define SHORTHAND_SELF 0x1 +#define SHORTHAND_ALL_INCL 0x2 +#define SHORTHAND_ALL_EXCL 0x3 /** Interrupt Input Pin Polarities. */ -#define POLARITY_HIGH 0x0 -#define POLARITY_LOW 0x1 +#define POLARITY_HIGH 0x0 +#define POLARITY_LOW 0x1 /** Divide Values. (Bit 2 is always 0) */ -#define DIVIDE_2 0x0 -#define DIVIDE_4 0x1 -#define DIVIDE_8 0x2 -#define DIVIDE_16 0x3 -#define DIVIDE_32 0x8 -#define DIVIDE_64 0x9 -#define DIVIDE_128 0xa -#define DIVIDE_1 0xb +#define DIVIDE_2 0x0 +#define DIVIDE_4 0x1 +#define DIVIDE_8 0x2 +#define DIVIDE_16 0x3 +#define DIVIDE_32 0x8 +#define DIVIDE_64 0x9 +#define DIVIDE_128 0xa +#define DIVIDE_1 0xb /** Timer Modes. */ -#define TIMER_ONESHOT 0x0 -#define TIMER_PERIODIC 0x1 +#define TIMER_ONESHOT 0x0 +#define TIMER_PERIODIC 0x1 /** Delivery status. */ -#define DELIVS_IDLE 0x0 -#define DELIVS_PENDING 0x1 +#define DELIVS_IDLE 0x0 +#define DELIVS_PENDING 0x1 /** Destination masks. */ -#define DEST_ALL 0xff +#define DEST_ALL 0xff /** Dest format models. */ -#define MODEL_FLAT 0xf -#define MODEL_CLUSTER 0x0 +#define MODEL_FLAT 0xf +#define MODEL_CLUSTER 0x0 /** Interrupt Command Register. */ -#define ICRlo (0x300 / sizeof(uint32_t)) -#define ICRhi (0x310 / sizeof(uint32_t)) +#define ICRlo (0x300 / sizeof(uint32_t)) +#define ICRhi (0x310 / sizeof(uint32_t)) + typedef struct { union { uint32_t lo; struct { - uint8_t vector; /**< Interrupt Vector. */ - unsigned delmod : 3; /**< Delivery Mode. */ - unsigned destmod : 1; /**< Destination Mode. */ - unsigned delivs : 1; /**< Delivery status (RO). */ - unsigned : 1; /**< Reserved. */ - unsigned level : 1; /**< Level. */ - unsigned trigger_mode : 1; /**< Trigger Mode. */ - unsigned : 2; /**< Reserved. */ - unsigned shorthand : 2; /**< Destination Shorthand. */ - unsigned : 12; /**< Reserved. */ + uint8_t vector; /**< Interrupt Vector. */ + unsigned int delmod : 3; /**< Delivery Mode. */ + unsigned int destmod : 1; /**< Destination Mode. */ + unsigned int delivs : 1; /**< Delivery status (RO). */ + unsigned int : 1; /**< Reserved. */ + unsigned int level : 1; /**< Level. */ + unsigned int trigger_mode : 1; /**< Trigger Mode. */ + unsigned int : 2; /**< Reserved. */ + unsigned int shorthand : 2; /**< Destination Shorthand. */ + unsigned int : 12; /**< Reserved. */ } __attribute__ ((packed)); }; @@ -127,6 +128,6 @@ uint32_t hi; struct { - unsigned : 24; /**< Reserved. */ - uint8_t dest; /**< Destination field. */ + unsigned int : 24; /**< Reserved. */ + uint8_t dest; /**< Destination field. */ } __attribute__ ((packed)); }; @@ -134,154 +135,165 @@ /* End Of Interrupt. */ -#define EOI (0x0b0 / sizeof(uint32_t)) +#define EOI (0x0b0 / sizeof(uint32_t)) /** Error Status Register. */ -#define ESR (0x280 / sizeof(uint32_t)) +#define ESR (0x280 / sizeof(uint32_t)) + typedef union { uint32_t value; uint8_t err_bitmap; struct { - unsigned send_checksum_error : 1; - unsigned receive_checksum_error : 1; - unsigned send_accept_error : 1; - unsigned receive_accept_error : 1; - unsigned : 1; - unsigned send_illegal_vector : 1; - unsigned received_illegal_vector : 1; - unsigned illegal_register_address : 1; - unsigned : 24; + unsigned int send_checksum_error : 1; + unsigned int receive_checksum_error : 1; + unsigned int send_accept_error : 1; + unsigned int receive_accept_error : 1; + unsigned int : 1; + unsigned int send_illegal_vector : 1; + unsigned int received_illegal_vector : 1; + unsigned int illegal_register_address : 1; + unsigned int : 24; } __attribute__ ((packed)); } esr_t; /* Task Priority Register */ -#define TPR (0x080 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - unsigned pri_sc : 4; /**< Task Priority Sub-Class. */ - unsigned pri : 4; /**< Task Priority. */ +#define TPR (0x080 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + unsigned int pri_sc : 4; /**< Task Priority Sub-Class. */ + unsigned int pri : 4; /**< Task Priority. */ } __attribute__ ((packed)); } tpr_t; /** Spurious-Interrupt Vector Register. */ -#define SVR (0x0f0 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - uint8_t vector; /**< Spurious Vector. */ - unsigned lapic_enabled : 1; /**< APIC Software Enable/Disable. */ - unsigned focus_checking : 1; /**< Focus Processor Checking. */ - unsigned : 22; /**< Reserved. */ +#define SVR (0x0f0 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + uint8_t vector; /**< Spurious Vector. */ + unsigned int lapic_enabled : 1; /**< APIC Software Enable/Disable. */ + unsigned int focus_checking : 1; /**< Focus Processor Checking. */ + unsigned int : 22; /**< Reserved. */ } __attribute__ ((packed)); } svr_t; /** Time Divide Configuration Register. */ -#define TDCR (0x3e0 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - unsigned div_value : 4; /**< Divide Value, bit 2 is always 0. */ - unsigned : 28; /**< Reserved. */ +#define TDCR (0x3e0 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + unsigned int div_value : 4; /**< Divide Value, bit 2 is always 0. */ + unsigned int : 28; /**< Reserved. */ } __attribute__ ((packed)); } tdcr_t; /* Initial Count Register for Timer */ -#define ICRT (0x380 / sizeof(uint32_t)) +#define ICRT (0x380 / sizeof(uint32_t)) /* Current Count Register for Timer */ -#define CCRT (0x390 / sizeof(uint32_t)) +#define CCRT (0x390 / sizeof(uint32_t)) /** LVT Timer register. */ -#define LVT_Tm (0x320 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - uint8_t vector; /**< Local Timer Interrupt vector. */ - unsigned : 4; /**< Reserved. */ - unsigned delivs : 1; /**< Delivery status (RO). */ - unsigned : 3; /**< Reserved. */ - unsigned masked : 1; /**< Interrupt Mask. */ - unsigned mode : 1; /**< Timer Mode. */ - unsigned : 14; /**< Reserved. */ +#define LVT_Tm (0x320 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + uint8_t vector; /**< Local Timer Interrupt vector. */ + unsigned int : 4; /**< Reserved. */ + unsigned int delivs : 1; /**< Delivery status (RO). */ + unsigned int : 3; /**< Reserved. */ + unsigned int masked : 1; /**< Interrupt Mask. */ + unsigned int mode : 1; /**< Timer Mode. */ + unsigned int : 14; /**< Reserved. */ } __attribute__ ((packed)); } lvt_tm_t; /** LVT LINT registers. */ -#define LVT_LINT0 (0x350 / sizeof(uint32_t)) -#define LVT_LINT1 (0x360 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - uint8_t vector; /**< LINT Interrupt vector. */ - unsigned delmod : 3; /**< Delivery Mode. */ - unsigned : 1; /**< Reserved. */ - unsigned delivs : 1; /**< Delivery status (RO). */ - unsigned intpol : 1; /**< Interrupt Input Pin Polarity. */ - unsigned irr : 1; /**< Remote IRR (RO). */ - unsigned trigger_mode : 1; /**< Trigger Mode. */ - unsigned masked : 1; /**< Interrupt Mask. */ - unsigned : 15; /**< Reserved. */ +#define LVT_LINT0 (0x350 / sizeof(uint32_t)) +#define LVT_LINT1 (0x360 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + uint8_t vector; /**< LINT Interrupt vector. */ + unsigned int delmod : 3; /**< Delivery Mode. */ + unsigned int : 1; /**< Reserved. */ + unsigned int delivs : 1; /**< Delivery status (RO). */ + unsigned int intpol : 1; /**< Interrupt Input Pin Polarity. */ + unsigned int irr : 1; /**< Remote IRR (RO). */ + unsigned int trigger_mode : 1; /**< Trigger Mode. */ + unsigned int masked : 1; /**< Interrupt Mask. */ + unsigned int : 15; /**< Reserved. */ } __attribute__ ((packed)); } lvt_lint_t; /** LVT Error register. */ -#define LVT_Err (0x370 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - uint8_t vector; /**< Local Timer Interrupt vector. */ - unsigned : 4; /**< Reserved. */ - unsigned delivs : 1; /**< Delivery status (RO). */ - unsigned : 3; /**< Reserved. */ - unsigned masked : 1; /**< Interrupt Mask. */ - unsigned : 15; /**< Reserved. */ +#define LVT_Err (0x370 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + uint8_t vector; /**< Local Timer Interrupt vector. */ + unsigned int : 4; /**< Reserved. */ + unsigned int delivs : 1; /**< Delivery status (RO). */ + unsigned int : 3; /**< Reserved. */ + unsigned int masked : 1; /**< Interrupt Mask. */ + unsigned int : 15; /**< Reserved. */ } __attribute__ ((packed)); } lvt_error_t; /** Local APIC ID Register. */ -#define L_APIC_ID (0x020 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - unsigned : 24; /**< Reserved. */ - uint8_t apic_id; /**< Local APIC ID. */ +#define L_APIC_ID (0x020 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + unsigned int : 24; /**< Reserved. */ + uint8_t apic_id; /**< Local APIC ID. */ } __attribute__ ((packed)); } l_apic_id_t; /** Local APIC Version Register */ -#define LAVR (0x030 / sizeof(uint32_t)) -#define LAVR_Mask 0xff -#define is_local_apic(x) (((x) & LAVR_Mask & 0xf0) == 0x1) -#define is_82489DX_apic(x) ((((x) & LAVR_Mask & 0xf0) == 0x0)) -#define is_local_xapic(x) (((x) & LAVR_Mask) == 0x14) +#define LAVR (0x030 / sizeof(uint32_t)) +#define LAVR_Mask 0xff + +#define is_local_apic(x) (((x) & LAVR_Mask & 0xf0) == 0x1) +#define is_82489DX_apic(x) ((((x) & LAVR_Mask & 0xf0) == 0x0)) +#define is_local_xapic(x) (((x) & LAVR_Mask) == 0x14) /** Logical Destination Register. */ -#define LDR (0x0d0 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - unsigned : 24; /**< Reserved. */ - uint8_t id; /**< Logical APIC ID. */ +#define LDR (0x0d0 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + unsigned int : 24; /**< Reserved. */ + uint8_t id; /**< Logical APIC ID. */ } __attribute__ ((packed)); } ldr_t; /** Destination Format Register. */ -#define DFR (0x0e0 / sizeof(uint32_t)) -typedef union { - uint32_t value; - struct { - unsigned : 28; /**< Reserved, all ones. */ - unsigned model : 4; /**< Model. */ +#define DFR (0x0e0 / sizeof(uint32_t)) + +typedef union { + uint32_t value; + struct { + unsigned int : 28; /**< Reserved, all ones. */ + unsigned int model : 4; /**< Model. */ } __attribute__ ((packed)); } dfr_t; /* IO APIC */ -#define IOREGSEL (0x00 / sizeof(uint32_t)) -#define IOWIN (0x10 / sizeof(uint32_t)) - -#define IOAPICID 0x00 -#define IOAPICVER 0x01 -#define IOAPICARB 0x02 -#define IOREDTBL 0x10 +#define IOREGSEL (0x00 / sizeof(uint32_t)) +#define IOWIN (0x10 / sizeof(uint32_t)) + +#define IOAPICID 0x00 +#define IOAPICVER 0x01 +#define IOAPICARB 0x02 +#define IOREDTBL 0x10 /** I/O Register Select Register. */ @@ -289,6 +301,6 @@ uint32_t value; struct { - uint8_t reg_addr; /**< APIC Register Address. */ - unsigned : 24; /**< Reserved. */ + uint8_t reg_addr; /**< APIC Register Address. */ + unsigned int : 24; /**< Reserved. */ } __attribute__ ((packed)); } io_regsel_t; @@ -299,13 +311,13 @@ uint32_t lo; struct { - uint8_t intvec; /**< Interrupt Vector. */ - unsigned delmod : 3; /**< Delivery Mode. */ - unsigned destmod : 1; /**< Destination mode. */ - unsigned delivs : 1; /**< Delivery status (RO). */ - unsigned intpol : 1; /**< Interrupt Input Pin Polarity. */ - unsigned irr : 1; /**< Remote IRR (RO). */ - unsigned trigger_mode : 1; /**< Trigger Mode. */ - unsigned masked : 1; /**< Interrupt Mask. */ - unsigned : 15; /**< Reserved. */ + uint8_t intvec; /**< Interrupt Vector. */ + unsigned int delmod : 3; /**< Delivery Mode. */ + unsigned int destmod : 1; /**< Destination mode. */ + unsigned int delivs : 1; /**< Delivery status (RO). */ + unsigned int intpol : 1; /**< Interrupt Input Pin Polarity. */ + unsigned int irr : 1; /**< Remote IRR (RO). */ + unsigned int trigger_mode : 1; /**< Trigger Mode. */ + unsigned int masked : 1; /**< Interrupt Mask. */ + unsigned int : 15; /**< Reserved. */ } __attribute__ ((packed)); }; @@ -313,6 +325,6 @@ uint32_t hi; struct { - unsigned : 24; /**< Reserved. */ - uint8_t dest : 8; /**< Destination Field. */ + unsigned int : 24; /**< Reserved. */ + uint8_t dest : 8; /**< Destination Field. */ } __attribute__ ((packed)); }; @@ -325,7 +337,7 @@ uint32_t value; struct { - unsigned : 24; /**< Reserved. */ - unsigned apic_id : 4; /**< IO APIC ID. */ - unsigned : 4; /**< Reserved. */ + unsigned int : 24; /**< Reserved. */ + unsigned int apic_id : 4; /**< IO APIC ID. */ + unsigned int : 4; /**< Reserved. */ } __attribute__ ((packed)); } io_apic_id_t; @@ -340,14 +352,14 @@ extern void l_apic_init(void); extern void l_apic_eoi(void); -extern int l_apic_broadcast_custom_ipi(uint8_t vector); -extern int l_apic_send_init_ipi(uint8_t apicid); +extern int l_apic_broadcast_custom_ipi(uint8_t); +extern int l_apic_send_init_ipi(uint8_t); extern void l_apic_debug(void); extern uint8_t l_apic_id(void); -extern uint32_t io_apic_read(uint8_t address); -extern void io_apic_write(uint8_t address , uint32_t x); -extern void io_apic_change_ioredtbl(uint8_t pin, uint8_t dest, uint8_t v, int flags); -extern void io_apic_disable_irqs(uint16_t irqmask); -extern void io_apic_enable_irqs(uint16_t irqmask); +extern uint32_t io_apic_read(uint8_t); +extern void io_apic_write(uint8_t, uint32_t); +extern void io_apic_change_ioredtbl(uint8_t pin, uint8_t dest, uint8_t v, unsigned int); +extern void io_apic_disable_irqs(uint16_t); +extern void io_apic_enable_irqs(uint16_t); #endif Index: kernel/arch/ia32/src/asm.S =================================================================== --- kernel/arch/ia32/src/asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -198,112 +198,114 @@ #define INTERRUPT_ALIGN 64 .macro handler i n - -.ifeq \i - 0x30 # Syscall handler - pushl %ds - pushl %es - pushl %fs - pushl %gs - - # - # Push syscall arguments onto the stack - # - # NOTE: The idea behind the order of arguments passed in registers is to - # use all scratch registers first and preserved registers next. - # An optimized libc syscall wrapper can make use of this setup. - # - pushl %eax - pushl %ebp - pushl %edi - pushl %esi - pushl %ebx - pushl %ecx - pushl %edx - - # we must fill the data segment registers - movw $16, %ax - movw %ax, %ds - movw %ax, %es - - cld - sti - # syscall_handler(edx, ecx, ebx, esi, edi, ebp, eax) - call syscall_handler - cli - addl $28, %esp # clean-up of parameters - - popl %gs - popl %fs - popl %es - popl %ds - - CLEAR_NT_FLAG - iret -.else - /* - * This macro distinguishes between two versions of ia32 exceptions. - * One version has error word and the other does not have it. - * The latter version fakes the error word on the stack so that the - * handlers and istate_t can be the same for both types. - */ - .iflt \i - 32 - .if (1 << \i) & ERROR_WORD_INTERRUPT_LIST - /* - * With error word, do nothing + + .ifeq \i - 0x30 # Syscall handler + pushl %ds + pushl %es + pushl %fs + pushl %gs + + # + # Push syscall arguments onto the stack + # + # NOTE: The idea behind the order of arguments passed in registers is to + # use all scratch registers first and preserved registers next. + # An optimized libc syscall wrapper can make use of this setup. + # + pushl %eax + pushl %ebp + pushl %edi + pushl %esi + pushl %ebx + pushl %ecx + pushl %edx + + # we must fill the data segment registers + movw $16, %ax + movw %ax, %ds + movw %ax, %es + + cld + sti + # syscall_handler(edx, ecx, ebx, esi, edi, ebp, eax) + call syscall_handler + cli + addl $28, %esp # clean-up of parameters + + popl %gs + popl %fs + popl %es + popl %ds + + CLEAR_NT_FLAG + iret + .else + /* + * This macro distinguishes between two versions of ia32 exceptions. + * One version has error word and the other does not have it. + * The latter version fakes the error word on the stack so that the + * handlers and istate_t can be the same for both types. + */ + .iflt \i - 32 + .if (1 << \i) & ERROR_WORD_INTERRUPT_LIST + /* + * With error word, do nothing + */ + .else + /* + * Version without error word + */ + subl $4, %esp + .endif + .else + /* + * Version without error word */ - .else - /* - * Version without error word, - */ subl $4, %esp - .endif - .else - /* - * Version without error word, - */ - subl $4, %esp + .endif + + pushl %ds + pushl %es + pushl %fs + pushl %gs + + pushl %ebp + pushl %edx + pushl %ecx + pushl %eax + + # we must fill the data segment registers + + movw $16, %ax + movw %ax, %ds + movw %ax, %es + + # stop stack traces here + xorl %ebp, %ebp + + pushl %esp # *istate + pushl $(\i) # intnum + call exc_dispatch # exc_dispatch(intnum, *istate) + addl $8, %esp # Clear arguments from stack + + CLEAR_NT_FLAG # Modifies %ecx + + popl %eax + popl %ecx + popl %edx + popl %ebp + + popl %gs + popl %fs + popl %es + popl %ds + + # skip error word, no matter whether real or fake + addl $4, %esp + iret .endif - pushl %ds - pushl %es - pushl %fs - pushl %gs - - pushl %ebp - pushl %edx - pushl %ecx - pushl %eax - - # we must fill the data segment registers - movw $16, %ax - movw %ax, %ds - movw %ax, %es - - # stop stack traces here - xorl %ebp, %ebp - - pushl %esp # *istate - pushl $(\i) # intnum - call exc_dispatch # excdispatch(intnum, *istate) - addl $8, %esp # Clear arguments from stack - - CLEAR_NT_FLAG # Modifies %ecx - - popl %eax - popl %ecx - popl %edx - popl %ebp - - popl %gs - popl %fs - popl %es - popl %ds - - addl $4, %esp # Skip error word, no matter whether real or fake. - iret -.endif - .align INTERRUPT_ALIGN .if (\n- \i) - 1 - handler "(\i + 1)", \n + handler "(\i + 1)", \n .endif .endm Index: kernel/arch/ia32/src/ddi/ddi.c =================================================================== --- kernel/arch/ia32/src/ddi/ddi.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/ddi/ddi.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -50,34 +50,31 @@ * Interrupts are disabled and task is locked. * - * @param task Task. + * @param task Task. * @param ioaddr Startign I/O space address. - * @param size Size of the enabled I/O range. + * @param size Size of the enabled I/O range. * * @return 0 on success or an error code from errno.h. + * */ int ddi_iospace_enable_arch(task_t *task, uintptr_t ioaddr, size_t size) { - size_t bits; - - bits = ioaddr + size; + size_t bits = ioaddr + size; if (bits > IO_PORTS) return ENOENT; - + if (task->arch.iomap.bits < bits) { - bitmap_t oldiomap; - uint8_t *newmap; - /* * The I/O permission bitmap is too small and needs to be grown. */ - newmap = (uint8_t *) malloc(BITS2BYTES(bits), FRAME_ATOMIC); + uint8_t *newmap = (uint8_t *) malloc(BITS2BYTES(bits), FRAME_ATOMIC); if (!newmap) return ENOMEM; + bitmap_t oldiomap; bitmap_initialize(&oldiomap, task->arch.iomap.map, task->arch.iomap.bits); bitmap_initialize(&task->arch.iomap, newmap, bits); - + /* * Mark the new range inaccessible. @@ -85,9 +82,9 @@ bitmap_set_range(&task->arch.iomap, oldiomap.bits, bits - oldiomap.bits); - + /* * In case there really existed smaller iomap, * copy its contents and deallocate it. - */ + */ if (oldiomap.bits) { bitmap_copy(&task->arch.iomap, &oldiomap, @@ -96,15 +93,15 @@ } } - + /* * Enable the range and we are done. */ bitmap_clear_range(&task->arch.iomap, (size_t) ioaddr, (size_t) size); - + /* * Increment I/O Permission bitmap generation counter. */ task->arch.iomapver++; - + return 0; } @@ -116,23 +113,20 @@ * * Interrupts must be disabled prior this call. + * */ void io_perm_bitmap_install(void) { - size_t bits; - ptr_16_32_t cpugdtr; - descriptor_t *gdt_p; - size_t ver; - /* First, copy the I/O Permission Bitmap. */ - spinlock_lock(&TASK->lock); - ver = TASK->arch.iomapver; - if ((bits = TASK->arch.iomap.bits)) { + irq_spinlock_lock(&TASK->lock, false); + size_t ver = TASK->arch.iomapver; + size_t bits = TASK->arch.iomap.bits; + if (bits) { + ASSERT(TASK->arch.iomap.map); + bitmap_t iomap; - task_t *task = TASK; - - ASSERT(TASK->arch.iomap.map); bitmap_initialize(&iomap, CPU->arch.tss->iomap, TSS_IOMAP_SIZE * 8); - bitmap_copy(&iomap, &task->arch.iomap, task->arch.iomap.bits); + bitmap_copy(&iomap, &TASK->arch.iomap, TASK->arch.iomap.bits); + /* * It is safe to set the trailing eight bits because of the @@ -141,15 +135,17 @@ bitmap_set_range(&iomap, ALIGN_UP(TASK->arch.iomap.bits, 8), 8); } - spinlock_unlock(&TASK->lock); - + irq_spinlock_unlock(&TASK->lock, false); + /* * Second, adjust TSS segment limit. * Take the extra ending byte with all bits set into account. */ + ptr_16_32_t cpugdtr; gdtr_store(&cpugdtr); - gdt_p = (descriptor_t *) cpugdtr.base; + + descriptor_t *gdt_p = (descriptor_t *) cpugdtr.base; gdt_setlimit(&gdt_p[TSS_DES], TSS_BASIC_SIZE + BITS2BYTES(bits)); gdtr_load(&cpugdtr); - + /* * Before we load new TSS limit, the current TSS descriptor Index: kernel/arch/ia32/src/drivers/i8254.c =================================================================== --- kernel/arch/ia32/src/drivers/i8254.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/drivers/i8254.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -54,9 +54,12 @@ #include -#define CLK_PORT1 ((ioport8_t *)0x40) -#define CLK_PORT4 ((ioport8_t *)0x43) +#define CLK_PORT1 ((ioport8_t *) 0x40) +#define CLK_PORT4 ((ioport8_t *) 0x43) -#define CLK_CONST 1193180 -#define MAGIC_NUMBER 1194 +#define CLK_CONST 1193180 +#define MAGIC_NUMBER 1194 + +#define LOOPS 150000 +#define SHIFT 11 static irq_t i8254_irq; @@ -75,7 +78,7 @@ * lock. We just release it, call clock() and then reacquire it again. */ - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); clock(); - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); } @@ -102,13 +105,6 @@ } -#define LOOPS 150000 -#define SHIFT 11 void i8254_calibrate_delay_loop(void) { - uint64_t clk1, clk2; - uint32_t t1, t2, o1, o2; - uint8_t not_ok; - - /* * One-shot timer. Count-down from 0xffff at 1193180Hz @@ -118,5 +114,9 @@ pio_write_8(CLK_PORT1, 0xff); pio_write_8(CLK_PORT1, 0xff); - + + uint8_t not_ok; + uint32_t t1; + uint32_t t2; + do { /* will read both status and count */ @@ -126,34 +126,34 @@ t1 |= pio_read_8(CLK_PORT1) << 8; } while (not_ok); - + asm_delay_loop(LOOPS); - + pio_write_8(CLK_PORT4, 0xd2); t2 = pio_read_8(CLK_PORT1); t2 |= pio_read_8(CLK_PORT1) << 8; - + /* * We want to determine the overhead of the calibrating mechanism. */ pio_write_8(CLK_PORT4, 0xd2); - o1 = pio_read_8(CLK_PORT1); + uint32_t o1 = pio_read_8(CLK_PORT1); o1 |= pio_read_8(CLK_PORT1) << 8; - + asm_fake_loop(LOOPS); - + pio_write_8(CLK_PORT4, 0xd2); - o2 = pio_read_8(CLK_PORT1); + uint32_t o2 = pio_read_8(CLK_PORT1); o2 |= pio_read_8(CLK_PORT1) << 8; - + CPU->delay_loop_const = ((MAGIC_NUMBER * LOOPS) / 1000) / ((t1 - t2) - (o1 - o2)) + (((MAGIC_NUMBER * LOOPS) / 1000) % ((t1 - t2) - (o1 - o2)) ? 1 : 0); - - clk1 = get_cycle(); + + uint64_t clk1 = get_cycle(); delay(1 << SHIFT); - clk2 = get_cycle(); + uint64_t clk2 = get_cycle(); CPU->frequency_mhz = (clk2 - clk1) >> SHIFT; - + return; } Index: kernel/arch/ia32/src/interrupt.c =================================================================== --- kernel/arch/ia32/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -94,5 +94,5 @@ { fault_if_from_uspace(istate, "Unserviced interrupt: %d.", n); - + decode_istate(istate); panic("Unserviced interrupt: %d.", n); @@ -102,5 +102,5 @@ { fault_if_from_uspace(istate, "Divide error."); - + decode_istate(istate); panic("Divide error."); @@ -111,10 +111,8 @@ { if (TASK) { - size_t ver; + irq_spinlock_lock(&TASK->lock, false); + size_t ver = TASK->arch.iomapver; + irq_spinlock_unlock(&TASK->lock, false); - spinlock_lock(&TASK->lock); - ver = TASK->arch.iomapver; - spinlock_unlock(&TASK->lock); - if (CPU->arch.iomapver_copy != ver) { /* @@ -130,5 +128,5 @@ fault_if_from_uspace(istate, "General protection fault."); } - + decode_istate(istate); panic("General protection fault."); @@ -138,5 +136,5 @@ { fault_if_from_uspace(istate, "Stack fault."); - + decode_istate(istate); panic("Stack fault."); @@ -146,8 +144,9 @@ { uint32_t mxcsr; - asm ( + asm volatile ( "stmxcsr %[mxcsr]\n" : [mxcsr] "=m" (mxcsr) ); + fault_if_from_uspace(istate, "SIMD FP exception(19), MXCSR: %#zx.", (unative_t) mxcsr); @@ -158,7 +157,8 @@ } -static void nm_fault(int n __attribute__((unused)), istate_t *istate __attribute__((unused))) -{ -#ifdef CONFIG_FPU_LAZY +static void nm_fault(int n __attribute__((unused)), + istate_t *istate __attribute__((unused))) +{ +#ifdef CONFIG_FPU_LAZY scheduler_fpu_lazy_request(); #else @@ -169,5 +169,6 @@ #ifdef CONFIG_SMP -static void tlb_shootdown_ipi(int n __attribute__((unused)), istate_t *istate __attribute__((unused))) +static void tlb_shootdown_ipi(int n __attribute__((unused)), + istate_t *istate __attribute__((unused))) { trap_virtual_eoi(); @@ -191,5 +192,5 @@ * The IRQ handler was found. */ - + if (irq->preack) { /* Send EOI before processing the interrupt */ @@ -198,5 +199,5 @@ } irq->handler(irq); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { /* Index: kernel/arch/ia32/src/mm/frame.c =================================================================== --- kernel/arch/ia32/src/mm/frame.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/mm/frame.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -55,4 +55,5 @@ { unsigned int i; + for (i = 0; i < e820counter; i++) { uint64_t base = e820table[i].base_address; @@ -60,5 +61,4 @@ #ifdef __32_BITS__ - /* Ignore physical memory above 4 GB */ if ((base >> 32) != 0) @@ -68,6 +68,6 @@ if (((base + size) >> 32) != 0) size = 0xffffffff - base; - #endif + pfn_t pfn; size_t count; @@ -118,5 +118,4 @@ }; - void physmem_print(void) { @@ -134,5 +133,5 @@ printf("%#18llx %#18llx %s\n", e820table[i].base_address, - e820table[i].size, name); + e820table[i].size, name); } } @@ -148,7 +147,8 @@ #ifdef CONFIG_SMP minconf = max(minconf, - ADDR2PFN(AP_BOOT_OFFSET + hardcoded_unmapped_ktext_size + - hardcoded_unmapped_kdata_size)); + ADDR2PFN(AP_BOOT_OFFSET + hardcoded_unmapped_ktext_size + + hardcoded_unmapped_kdata_size)); #endif + init_e820_memory(minconf); @@ -159,6 +159,6 @@ /* Reserve AP real mode bootstrap memory */ frame_mark_unavailable(AP_BOOT_OFFSET >> FRAME_WIDTH, - (hardcoded_unmapped_ktext_size + - hardcoded_unmapped_kdata_size) >> FRAME_WIDTH); + (hardcoded_unmapped_ktext_size + + hardcoded_unmapped_kdata_size) >> FRAME_WIDTH); #endif } Index: kernel/arch/ia32/src/mm/page.c =================================================================== --- kernel/arch/ia32/src/mm/page.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/mm/page.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -61,4 +61,5 @@ * PA2KA(identity) mapping for all frames until last_frame. */ + page_table_lock(AS_KERNEL, true); for (cur = 0; cur < last_frame; cur += FRAME_SIZE) { flags = PAGE_CACHEABLE | PAGE_WRITE; @@ -67,4 +68,5 @@ page_mapping_insert(AS_KERNEL, PA2KA(cur), cur, flags); } + page_table_unlock(AS_KERNEL, true); exc_register(14, "page_fault", (iroutine) page_fault); @@ -84,8 +86,10 @@ uintptr_t virtaddr = PA2KA(last_frame); pfn_t i; + page_table_lock(AS_KERNEL, true); for (i = 0; i < ADDR2PFN(ALIGN_UP(size, PAGE_SIZE)); i++) { uintptr_t addr = PFN2ADDR(i); page_mapping_insert(AS_KERNEL, virtaddr + addr, physaddr + addr, PAGE_NOT_CACHEABLE | PAGE_WRITE); } + page_table_unlock(AS_KERNEL, true); last_frame = ALIGN_UP(last_frame + size, FRAME_SIZE); Index: kernel/arch/ia32/src/proc/scheduler.c =================================================================== --- kernel/arch/ia32/src/proc/scheduler.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/proc/scheduler.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,5 +38,5 @@ #include #include -#include /* SP_DELTA */ +#include /* SP_DELTA */ #include #include Index: kernel/arch/ia32/src/smp/apic.c =================================================================== --- kernel/arch/ia32/src/smp/apic.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/smp/apic.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -53,11 +53,12 @@ * Advanced Programmable Interrupt Controller for SMP systems. * Tested on: - * Bochs 2.0.2 - Bochs 2.2.6 with 2-8 CPUs - * Simics 2.0.28 - Simics 2.2.19 2-15 CPUs - * VMware Workstation 5.5 with 2 CPUs - * QEMU 0.8.0 with 2-15 CPUs - * ASUS P/I-P65UP5 + ASUS C-P55T2D REV. 1.41 with 2x 200Mhz Pentium CPUs - * ASUS PCH-DL with 2x 3000Mhz Pentium 4 Xeon (HT) CPUs - * MSI K7D Master-L with 2x 2100MHz Athlon MP CPUs + * Bochs 2.0.2 - Bochs 2.2.6 with 2-8 CPUs + * Simics 2.0.28 - Simics 2.2.19 2-15 CPUs + * VMware Workstation 5.5 with 2 CPUs + * QEMU 0.8.0 with 2-15 CPUs + * ASUS P/I-P65UP5 + ASUS C-P55T2D REV. 1.41 with 2x 200Mhz Pentium CPUs + * ASUS PCH-DL with 2x 3000Mhz Pentium 4 Xeon (HT) CPUs + * MSI K7D Master-L with 2x 2100MHz Athlon MP CPUs + * */ @@ -69,4 +70,5 @@ * optimize the code too much and accesses to l_apic and io_apic, that must * always be 32-bit, would use byte oriented instructions. + * */ volatile uint32_t *l_apic = (uint32_t *) 0xfee00000; @@ -79,5 +81,5 @@ #ifdef LAPIC_VERBOSE -static char *delmod_str[] = { +static const char *delmod_str[] = { "Fixed", "Lowest Priority", @@ -90,30 +92,30 @@ }; -static char *destmod_str[] = { +static const char *destmod_str[] = { "Physical", "Logical" }; -static char *trigmod_str[] = { +static const char *trigmod_str[] = { "Edge", "Level" }; -static char *mask_str[] = { +static const char *mask_str[] = { "Unmasked", "Masked" }; -static char *delivs_str[] = { +static const char *delivs_str[] = { "Idle", "Send Pending" }; -static char *tm_mode_str[] = { +static const char *tm_mode_str[] = { "One-shot", "Periodic" }; -static char *intpol_str[] = { +static const char *intpol_str[] = { "Polarity High", "Polarity Low" @@ -123,8 +125,10 @@ /** APIC spurious interrupt handler. * - * @param n Interrupt vector. + * @param n Interrupt vector. * @param istate Interrupted state. - */ -static void apic_spurious(int n __attribute__((unused)), istate_t *istate __attribute__((unused))) + * + */ +static void apic_spurious(int n __attribute__((unused)), + istate_t *istate __attribute__((unused))) { #ifdef CONFIG_DEBUG @@ -145,7 +149,7 @@ * irq->lock so we just unlock it and then lock it again. */ - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); clock(); - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); } @@ -153,8 +157,6 @@ void apic_init(void) { - io_apic_id_t idreg; - exc_register(VECTOR_APIC_SPUR, "apic_spurious", (iroutine) apic_spurious); - + enable_irqs_function = io_apic_enable_irqs; disable_irqs_function = io_apic_disable_irqs; @@ -179,5 +181,5 @@ for (i = 0; i < IRQ_COUNT; i++) { int pin; - + if ((pin = smp_irq_to_pin(i)) != -1) io_apic_change_ioredtbl((uint8_t) pin, DEST_ALL, (uint8_t) (IVT_IRQBASE + i), LOPRI); @@ -187,6 +189,8 @@ * Ensure that io_apic has unique ID. */ + io_apic_id_t idreg; + idreg.value = io_apic_read(IOAPICID); - if ((1 << idreg.apic_id) & apic_id_mask) { /* see if IO APIC ID is used already */ + if ((1 << idreg.apic_id) & apic_id_mask) { /* See if IO APIC ID is used already */ for (i = 0; i < APIC_ID_COUNT; i++) { if (!((1 << i) & apic_id_mask)) { @@ -197,11 +201,10 @@ } } - + /* * Configure the BSP's lapic. */ l_apic_init(); - - l_apic_debug(); + l_apic_debug(); } @@ -211,4 +214,5 @@ * * @return 0 on error, 1 on success. + * */ int apic_poll_errors(void) @@ -232,5 +236,5 @@ if (esr.illegal_register_address) printf("Illegal Register Address\n"); - + return !esr.err_bitmap; } @@ -241,9 +245,10 @@ * * @return 0 on failure, 1 on success. + * */ int l_apic_broadcast_custom_ipi(uint8_t vector) { icr_t icr; - + icr.lo = l_apic[ICRlo]; icr.delmod = DELMOD_FIXED; @@ -253,7 +258,7 @@ icr.trigger_mode = TRIGMOD_LEVEL; icr.vector = vector; - + l_apic[ICRlo] = icr.lo; - + icr.lo = l_apic[ICRlo]; if (icr.delivs == DELIVS_PENDING) { @@ -262,5 +267,5 @@ #endif } - + return apic_poll_errors(); } @@ -271,13 +276,13 @@ * * @return 0 on failure, 1 on success. + * */ int l_apic_send_init_ipi(uint8_t apicid) { + /* + * Read the ICR register in and zero all non-reserved fields. + */ icr_t icr; - int i; - - /* - * Read the ICR register in and zero all non-reserved fields. - */ + icr.lo = l_apic[ICRlo]; icr.hi = l_apic[ICRhi]; @@ -293,5 +298,5 @@ l_apic[ICRhi] = icr.hi; l_apic[ICRlo] = icr.lo; - + /* * According to MP Specification, 20us should be enough to @@ -299,8 +304,8 @@ */ delay(20); - + if (!apic_poll_errors()) return 0; - + icr.lo = l_apic[ICRlo]; if (icr.delivs == DELIVS_PENDING) { @@ -309,5 +314,5 @@ #endif } - + icr.delmod = DELMOD_INIT; icr.destmod = DESTMOD_PHYS; @@ -317,15 +322,16 @@ icr.vector = 0; l_apic[ICRlo] = icr.lo; - + /* * Wait 10ms as MP Specification specifies. */ delay(10000); - + if (!is_82489DX_apic(l_apic[LAVR])) { /* * If this is not 82489DX-based l_apic we must send two STARTUP IPI's. */ - for (i = 0; i<2; i++) { + unsigned int i; + for (i = 0; i < 2; i++) { icr.lo = l_apic[ICRlo]; icr.vector = (uint8_t) (((uintptr_t) ap_boot) >> 12); /* calculate the reset vector */ @@ -346,31 +352,26 @@ void l_apic_init(void) { + /* Initialize LVT Error register. */ lvt_error_t error; - lvt_lint_t lint; - tpr_t tpr; - svr_t svr; - icr_t icr; - tdcr_t tdcr; - lvt_tm_t tm; - ldr_t ldr; - dfr_t dfr; - uint32_t t1, t2; - - /* Initialize LVT Error register. */ + error.value = l_apic[LVT_Err]; error.masked = true; l_apic[LVT_Err] = error.value; - + /* Initialize LVT LINT0 register. */ + lvt_lint_t lint; + lint.value = l_apic[LVT_LINT0]; lint.masked = true; l_apic[LVT_LINT0] = lint.value; - + /* Initialize LVT LINT1 register. */ lint.value = l_apic[LVT_LINT1]; lint.masked = true; l_apic[LVT_LINT1] = lint.value; - + /* Task Priority Register initialization. */ + tpr_t tpr; + tpr.value = l_apic[TPR]; tpr.pri_sc = 0; @@ -379,4 +380,6 @@ /* Spurious-Interrupt Vector Register initialization. */ + svr_t svr; + svr.value = l_apic[SVR]; svr.vector = VECTOR_APIC_SPUR; @@ -384,9 +387,11 @@ svr.focus_checking = true; l_apic[SVR] = svr.value; - + if (CPU->arch.family >= 6) enable_l_apic_in_msr(); /* Interrupt Command Register initialization. */ + icr_t icr; + icr.lo = l_apic[ICRlo]; icr.delmod = DELMOD_INIT; @@ -398,9 +403,13 @@ /* Timer Divide Configuration Register initialization. */ + tdcr_t tdcr; + tdcr.value = l_apic[TDCR]; tdcr.div_value = DIVIDE_1; l_apic[TDCR] = tdcr.value; - + /* Program local timer. */ + lvt_tm_t tm; + tm.value = l_apic[LVT_Tm]; tm.vector = VECTOR_CLK; @@ -408,23 +417,24 @@ tm.masked = false; l_apic[LVT_Tm] = tm.value; - + /* * Measure and configure the timer to generate timer * interrupt with period 1s/HZ seconds. */ + uint32_t t1 = l_apic[CCRT]; + l_apic[ICRT] = 0xffffffff; + + while (l_apic[CCRT] == t1); + t1 = l_apic[CCRT]; - l_apic[ICRT] = 0xffffffff; - - while (l_apic[CCRT] == t1) - ; - - t1 = l_apic[CCRT]; - delay(1000000/HZ); - t2 = l_apic[CCRT]; - - l_apic[ICRT] = t1-t2; + delay(1000000 / HZ); + uint32_t t2 = l_apic[CCRT]; + + l_apic[ICRT] = t1 - t2; /* Program Logical Destination Register. */ - ASSERT(CPU->id < 8) + ASSERT(CPU->id < 8); + ldr_t ldr; + ldr.value = l_apic[LDR]; ldr.id = (uint8_t) (1 << CPU->id); @@ -432,4 +442,6 @@ /* Program Destination Format Register for Flat mode. */ + dfr_t dfr; + dfr.value = l_apic[DFR]; dfr.model = MODEL_FLAT; @@ -447,16 +459,17 @@ { #ifdef LAPIC_VERBOSE + printf("LVT on cpu%" PRIs ", LAPIC ID: %" PRIu8 "\n", CPU->id, l_apic_id()); + lvt_tm_t tm; - lvt_lint_t lint; - lvt_error_t error; - - printf("LVT on cpu%d, LAPIC ID: %d\n", CPU->id, l_apic_id()); - tm.value = l_apic[LVT_Tm]; printf("LVT Tm: vector=%hhd, %s, %s, %s\n", tm.vector, delivs_str[tm.delivs], mask_str[tm.masked], tm_mode_str[tm.mode]); + + lvt_lint_t lint; lint.value = l_apic[LVT_LINT0]; printf("LVT LINT0: vector=%hhd, %s, %s, %s, irr=%d, %s, %s\n", tm.vector, delmod_str[lint.delmod], delivs_str[lint.delivs], intpol_str[lint.intpol], lint.irr, trigmod_str[lint.trigger_mode], mask_str[lint.masked]); lint.value = l_apic[LVT_LINT1]; printf("LVT LINT1: vector=%hhd, %s, %s, %s, irr=%d, %s, %s\n", tm.vector, delmod_str[lint.delmod], delivs_str[lint.delivs], intpol_str[lint.intpol], lint.irr, trigmod_str[lint.trigger_mode], mask_str[lint.masked]); + + lvt_error_t error; error.value = l_apic[LVT_Err]; printf("LVT Err: vector=%hhd, %s, %s\n", error.vector, delivs_str[error.delivs], mask_str[error.masked]); @@ -467,4 +480,5 @@ * * @return Local APIC ID. + * */ uint8_t l_apic_id(void) @@ -481,4 +495,5 @@ * * @return Content of the addressed IO APIC register. + * */ uint32_t io_apic_read(uint8_t address) @@ -495,7 +510,8 @@ * * @param address IO APIC register address. - * @param x Content to be written to the addressed IO APIC register. - */ -void io_apic_write(uint8_t address, uint32_t x) + * @param val Content to be written to the addressed IO APIC register. + * + */ +void io_apic_write(uint8_t address, uint32_t val) { io_regsel_t regsel; @@ -504,22 +520,26 @@ regsel.reg_addr = address; io_apic[IOREGSEL] = regsel.value; - io_apic[IOWIN] = x; + io_apic[IOWIN] = val; } /** Change some attributes of one item in I/O Redirection Table. * - * @param pin IO APIC pin number. - * @param dest Interrupt destination address. - * @param v Interrupt vector to trigger. + * @param pin IO APIC pin number. + * @param dest Interrupt destination address. + * @param vec Interrupt vector to trigger. * @param flags Flags. - */ -void io_apic_change_ioredtbl(uint8_t pin, uint8_t dest, uint8_t v, int flags) -{ - io_redirection_reg_t reg; - int dlvr = DELMOD_FIXED; + * + */ +void io_apic_change_ioredtbl(uint8_t pin, uint8_t dest, uint8_t vec, + unsigned int flags) +{ + unsigned int dlvr; if (flags & LOPRI) dlvr = DELMOD_LOWPRI; - + else + dlvr = DELMOD_FIXED; + + io_redirection_reg_t reg; reg.lo = io_apic_read((uint8_t) (IOREDTBL + pin * 2)); reg.hi = io_apic_read((uint8_t) (IOREDTBL + pin * 2 + 1)); @@ -530,6 +550,6 @@ reg.intpol = POLARITY_HIGH; reg.delmod = dlvr; - reg.intvec = v; - + reg.intvec = vec; + io_apic_write((uint8_t) (IOREDTBL + pin * 2), reg.lo); io_apic_write((uint8_t) (IOREDTBL + pin * 2 + 1), reg.hi); @@ -539,11 +559,9 @@ * * @param irqmask Bitmask of IRQs to be masked (0 = do not mask, 1 = mask). + * */ void io_apic_disable_irqs(uint16_t irqmask) { - io_redirection_reg_t reg; unsigned int i; - int pin; - for (i = 0; i < 16; i++) { if (irqmask & (1 << i)) { @@ -552,6 +570,8 @@ * mapping for the respective IRQ number. */ - pin = smp_irq_to_pin(i); + int pin = smp_irq_to_pin(i); if (pin != -1) { + io_redirection_reg_t reg; + reg.lo = io_apic_read((uint8_t) (IOREDTBL + pin * 2)); reg.masked = true; @@ -566,11 +586,9 @@ * * @param irqmask Bitmask of IRQs to be unmasked (0 = do not unmask, 1 = unmask). + * */ void io_apic_enable_irqs(uint16_t irqmask) { unsigned int i; - int pin; - io_redirection_reg_t reg; - for (i = 0; i < 16; i++) { if (irqmask & (1 << i)) { @@ -579,6 +597,8 @@ * mapping for the respective IRQ number. */ - pin = smp_irq_to_pin(i); + int pin = smp_irq_to_pin(i); if (pin != -1) { + io_redirection_reg_t reg; + reg.lo = io_apic_read((uint8_t) (IOREDTBL + pin * 2)); reg.masked = false; Index: kernel/arch/ia32/src/smp/smp.c =================================================================== --- kernel/arch/ia32/src/smp/smp.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia32/src/smp/smp.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -83,9 +83,11 @@ panic("Cannot allocate address for io_apic."); - if (config.cpu_count > 1) { + if (config.cpu_count > 1) { + page_table_lock(AS_KERNEL, true); page_mapping_insert(AS_KERNEL, l_apic_address, (uintptr_t) l_apic, PAGE_NOT_CACHEABLE | PAGE_WRITE); page_mapping_insert(AS_KERNEL, io_apic_address, (uintptr_t) io_apic, PAGE_NOT_CACHEABLE | PAGE_WRITE); + page_table_unlock(AS_KERNEL, true); l_apic = (uint32_t *) l_apic_address; Index: kernel/arch/ia64/Makefile.inc =================================================================== --- kernel/arch/ia64/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia64/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -34,6 +34,6 @@ ICC_CFLAGS += $(CMN1) -LFLAGS = -EL -AFLAGS = -mconstant-gp +AFLAGS += -mconstant-gp +LFLAGS += -EL BITS = 64 Index: kernel/arch/ia64/include/asm.h =================================================================== --- kernel/arch/ia64/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia64/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -417,4 +417,14 @@ } +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +static inline bool interrupts_disabled(void) +{ + return !(psr_read() & PSR_I_MASK); +} + /** Disable protection key checking. */ static inline void pk_disable(void) Index: rnel/arch/ia64/include/debug.h =================================================================== --- kernel/arch/ia64/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,42 +1,0 @@ -/* - * Copyright (c) 2005 Ondrej Palkovsky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ia64debug ia64 - * @ingroup debug - * @{ - */ -/** @file - */ - -#ifndef KERN_ia64_DEBUG_H_ -#define KERN_ia64_DEBUG_H_ - -#endif - -/** @} - */ Index: kernel/arch/ia64/src/drivers/it.c =================================================================== --- kernel/arch/ia64/src/drivers/it.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia64/src/drivers/it.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -34,5 +34,5 @@ /** Interval Timer driver. */ - + #include #include @@ -45,11 +45,11 @@ #include -#define IT_SERVICE_CLOCKS 64 +#define IT_SERVICE_CLOCKS 64 -#define FREQ_NUMERATOR_SHIFT 32 -#define FREQ_NUMERATOR_MASK 0xffffffff00000000ULL +#define FREQ_NUMERATOR_SHIFT 32 +#define FREQ_NUMERATOR_MASK 0xffffffff00000000ULL -#define FREQ_DENOMINATOR_SHIFT 0 -#define FREQ_DENOMINATOR_MASK 0xffffffffULL +#define FREQ_DENOMINATOR_SHIFT 0 +#define FREQ_DENOMINATOR_MASK 0xffffffffULL uint64_t it_delta; @@ -63,6 +63,4 @@ void it_init(void) { - cr_itv_t itv; - if (config.cpu_active == 1) { irq_initialize(&it_irq); @@ -83,17 +81,19 @@ } - /* initialize Interval Timer external interrupt vector */ + /* Initialize Interval Timer external interrupt vector */ + cr_itv_t itv; + itv.value = itv_read(); itv.vector = INTERRUPT_TIMER; itv.m = 0; itv_write(itv.value); - - /* set Interval Timer Counter to zero */ + + /* Set Interval Timer Counter to zero */ itc_write(0); - /* generate first Interval Timer interrupt in IT_DELTA ticks */ + /* Generate first Interval Timer interrupt in IT_DELTA ticks */ itm_write(IT_DELTA); - - /* propagate changes */ + + /* Propagate changes */ srlz_d(); } @@ -104,4 +104,5 @@ * * @return Always IRQ_ACCEPT. + * */ irq_ownership_t it_claim(irq_t *irq) @@ -113,17 +114,14 @@ void it_interrupt(irq_t *irq) { - int64_t c; - int64_t m; - eoi_write(EOI); - m = itm_read(); + int64_t itm = itm_read(); - while (1) { - c = itc_read(); - c += IT_SERVICE_CLOCKS; - - m += IT_DELTA; - if (m - c < 0) + while (true) { + int64_t itc = itc_read(); + itc += IT_SERVICE_CLOCKS; + + itm += IT_DELTA; + if (itm - itc < 0) CPU->missed_clock_ticks++; else @@ -131,14 +129,14 @@ } - itm_write(m); - srlz_d(); /* propagate changes */ - + itm_write(itm); + srlz_d(); /* Propagate changes */ + /* * We are holding a lock which prevents preemption. * Release the lock, call clock() and reacquire the lock again. */ - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); clock(); - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); } Index: kernel/arch/ia64/src/interrupt.c =================================================================== --- kernel/arch/ia64/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ia64/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -57,10 +57,11 @@ #include -#define VECTORS_64_BUNDLE 20 -#define VECTORS_16_BUNDLE 48 -#define VECTORS_16_BUNDLE_START 0x5000 -#define VECTOR_MAX 0x7f00 - -#define BUNDLE_SIZE 16 +#define VECTORS_64_BUNDLE 20 +#define VECTORS_16_BUNDLE 48 +#define VECTORS_16_BUNDLE_START 0x5000 + +#define VECTOR_MAX 0x7f00 + +#define BUNDLE_SIZE 16 static const char *vector_names_64_bundle[VECTORS_64_BUNDLE] = { @@ -134,8 +135,4 @@ static void dump_interrupted_context(istate_t *istate) { - const char *ifa = symtab_fmt_name_lookup(istate->cr_ifa); - const char *iipa = symtab_fmt_name_lookup(istate->cr_iipa); - const char *iip = symtab_fmt_name_lookup(istate->cr_iip); - putchar('\n'); printf("Interrupted context dump:\n"); @@ -149,8 +146,10 @@ istate->cr_ipsr); - printf("cr.iip=%#018llx, #%d\t(%s)\n", istate->cr_iip, - istate->cr_isr.ei, iip); - printf("cr.iipa=%#018llx\t(%s)\n", istate->cr_iipa, iipa); - printf("cr.ifa=%#018llx\t(%s)\n", istate->cr_ifa, ifa); + printf("cr.iip=%#018llx, #%d\t(%s)\n", istate->cr_iip, istate->cr_isr.ei, + symtab_fmt_name_lookup(istate->cr_iip)); + printf("cr.iipa=%#018llx\t(%s)\n", istate->cr_iipa, + symtab_fmt_name_lookup(istate->cr_iipa)); + printf("cr.ifa=%#018llx\t(%s)\n", istate->cr_ifa, + symtab_fmt_name_lookup(istate->cr_ifa)); } @@ -218,5 +217,5 @@ istate->cr_ipsr.ri++; } - + return syscall_handler(istate->in0, istate->in1, istate->in2, istate->in3, istate->in4, istate->in5, istate->in6); @@ -234,16 +233,18 @@ static void end_of_local_irq(void) { - asm volatile ("mov cr.eoi=r0;;"); -} - + asm volatile ( + "mov cr.eoi=r0;;" + ); +} void external_interrupt(uint64_t vector, istate_t *istate) { cr_ivr_t ivr; - irq_t *irq; ivr.value = ivr_read(); srlz_d(); - + + irq_t *irq; + switch (ivr.vector) { case INTERRUPT_SPURIOUS: @@ -252,5 +253,5 @@ #endif break; - + #ifdef CONFIG_SMP case VECTOR_TLB_SHOOTDOWN_IPI: @@ -259,10 +260,10 @@ break; #endif - + case INTERRUPT_TIMER: irq = irq_dispatch_and_lock(ivr.vector); if (irq) { irq->handler(irq); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { panic("Unhandled Internal Timer Interrupt (%d).", @@ -283,5 +284,5 @@ if (!irq->preack) end_of_local_irq(); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { /* Index: kernel/arch/mips32/include/asm.h =================================================================== --- kernel/arch/mips32/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32 +/** @addtogroup mips32 * @{ */ @@ -39,5 +39,4 @@ #include - static inline void cpu_sleep(void) { @@ -47,20 +46,21 @@ /** Return base address of current stack - * + * * Return the base address of the current stack. * The stack is assumed to be STACK_SIZE bytes long. * The stack must start on page boundary. + * */ static inline uintptr_t get_stack_base(void) { - uintptr_t v; + uintptr_t base; asm volatile ( - "and %0, $29, %1\n" - : "=r" (v) - : "r" (~(STACK_SIZE-1)) + "and %[base], $29, %[mask]\n" + : [base] "=r" (base) + : [mask] "r" (~(STACK_SIZE - 1)) ); - return v; + return base; } @@ -74,19 +74,19 @@ extern void interrupts_restore(ipl_t ipl); extern ipl_t interrupts_read(void); -extern void asm_delay_loop(uint32_t t); +extern bool interrupts_disabled(void); static inline void pio_write_8(ioport8_t *port, uint8_t v) { - *port = v; + *port = v; } static inline void pio_write_16(ioport16_t *port, uint16_t v) { - *port = v; + *port = v; } static inline void pio_write_32(ioport32_t *port, uint32_t v) { - *port = v; + *port = v; } Index: kernel/arch/mips32/include/cp0.h =================================================================== --- kernel/arch/mips32/include/cp0.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/include/cp0.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,10 +38,10 @@ #include -#define cp0_status_ie_enabled_bit (1 << 0) -#define cp0_status_exl_exception_bit (1 << 1) -#define cp0_status_erl_error_bit (1 << 2) -#define cp0_status_um_bit (1 << 4) -#define cp0_status_bev_bootstrap_bit (1 << 22) -#define cp0_status_fpu_bit (1 << 29) +#define cp0_status_ie_enabled_bit (1 << 0) +#define cp0_status_exl_exception_bit (1 << 1) +#define cp0_status_erl_error_bit (1 << 2) +#define cp0_status_um_bit (1 << 4) +#define cp0_status_bev_bootstrap_bit (1 << 22) +#define cp0_status_fpu_bit (1 << 29) #define cp0_status_im_shift 8 Index: kernel/arch/mips32/include/debug.h =================================================================== --- kernel/arch/mips32/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/include/debug.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,14 +36,18 @@ #define KERN_mips23_DEBUG_H_ -/** simulator enters the trace mode */ -#define ___traceon() asm volatile ( "\t.word\t0x39\n"); -/** simulator leaves the trace mode */ -#define ___traceoff() asm volatile ( "\t.word\t0x3d\n"); -/** register dump */ -#define ___regview() asm volatile ( "\t.word\t0x37\n"); -/** halt the simulator */ -#define ___halt() asm volatile ( "\t.word\t0x28\n"); -/** simulator enters interactive mode */ -#define ___intmode() asm volatile ( "\t.word\t0x29\n"); +/** Enter the simulator trace mode */ +#define ___traceon() asm volatile ( "\t.word\t0x39\n"); + +/** Leave the simulator trace mode */ +#define ___traceoff() asm volatile ( "\t.word\t0x3d\n"); + +/** Ask the simulator to dump registers */ +#define ___regview() asm volatile ( "\t.word\t0x37\n"); + +/** Halt the simulator */ +#define ___halt() asm volatile ( "\t.word\t0x28\n"); + +/** Enter the simulator interactive mode */ +#define ___intmode() asm volatile ( "\t.word\t0x29\n"); #endif Index: kernel/arch/mips32/include/debugger.h =================================================================== --- kernel/arch/mips32/include/debugger.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/include/debugger.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -41,25 +41,33 @@ #define BKPOINTS_MAX 10 -#define BKPOINT_INPROG (1 << 0) /**< Breakpoint was shot */ -#define BKPOINT_ONESHOT (1 << 1) /**< One-time breakpoint,mandatory for j/b - instructions */ -#define BKPOINT_REINST (1 << 2) /**< Breakpoint is set on the next - instruction, so that it could be - reinstalled on the previous one */ -#define BKPOINT_FUNCCALL (1 << 3) /**< Call a predefined function */ +/** Breakpoint was shot */ +#define BKPOINT_INPROG (1 << 0) + +/** One-time breakpoint, mandatory for j/b instructions */ +#define BKPOINT_ONESHOT (1 << 1) + +/** + * Breakpoint is set on the next instruction, so that it + * could be reinstalled on the previous one + */ +#define BKPOINT_REINST (1 << 2) + +/** Call a predefined function */ +#define BKPOINT_FUNCCALL (1 << 3) + typedef struct { - uintptr_t address; /**< Breakpoint address */ - unative_t instruction; /**< Original instruction */ + uintptr_t address; /**< Breakpoint address */ + unative_t instruction; /**< Original instruction */ unative_t nextinstruction; /**< Original instruction following break */ - int flags; /**< Flags regarding breakpoint */ + unsigned int flags; /**< Flags regarding breakpoint */ size_t counter; - void (*bkfunc)(void *b, istate_t *istate); + void (*bkfunc)(void *, istate_t *); } bpinfo_t; +extern bpinfo_t breakpoints[BKPOINTS_MAX]; + extern void debugger_init(void); -void debugger_bpoint(istate_t *istate); - -extern bpinfo_t breakpoints[BKPOINTS_MAX]; +extern void debugger_bpoint(istate_t *); #endif Index: kernel/arch/mips32/include/mm/as.h =================================================================== --- kernel/arch/mips32/include/mm/as.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/include/mm/as.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32mm +/** @addtogroup mips32mm * @{ */ @@ -36,5 +36,5 @@ #define KERN_mips32_AS_H_ -#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 +#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 #define KERNEL_ADDRESS_SPACE_START_ARCH (unsigned long) 0x80000000 Index: kernel/arch/mips32/src/context.S =================================================================== --- kernel/arch/mips32/src/context.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/context.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -28,6 +28,6 @@ #include - -.text + +.text .set noat @@ -38,16 +38,15 @@ .global context_restore_arch - context_save_arch: CONTEXT_SAVE_ARCH_CORE $a0 - + # context_save returns 1 j $31 - li $2, 1 - + li $2, 1 + context_restore_arch: CONTEXT_RESTORE_ARCH_CORE $a0 - + # context_restore returns 0 j $31 - xor $2, $2 + xor $2, $2 Index: kernel/arch/mips32/src/debugger.c =================================================================== --- kernel/arch/mips32/src/debugger.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/debugger.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,9 +46,12 @@ bpinfo_t breakpoints[BKPOINTS_MAX]; -SPINLOCK_INITIALIZE(bkpoint_lock); +IRQ_SPINLOCK_STATIC_INITIALIZE(bkpoint_lock); #ifdef CONFIG_KCONSOLE -static int cmd_print_breakpoints(cmd_arg_t *argv); +static int cmd_print_breakpoints(cmd_arg_t *); +static int cmd_del_breakpoint(cmd_arg_t *); +static int cmd_add_breakpoint(cmd_arg_t *); + static cmd_info_t bkpts_info = { .name = "bkpts", @@ -58,11 +61,11 @@ }; -static int cmd_del_breakpoint(cmd_arg_t *argv); static cmd_arg_t del_argv = { .type = ARG_TYPE_INT }; + static cmd_info_t delbkpt_info = { .name = "delbkpt", - .description = "delbkpt - Delete breakpoint.", + .description = "Delete breakpoint.", .func = cmd_del_breakpoint, .argc = 1, @@ -70,12 +73,11 @@ }; -static int cmd_add_breakpoint(cmd_arg_t *argv); static cmd_arg_t add_argv = { .type = ARG_TYPE_INT }; + static cmd_info_t addbkpt_info = { .name = "addbkpt", - .description = "addbkpt <&symbol> - new bkpoint. Break on J/Branch " - "insts unsupported.", + .description = "Add bkpoint (break on J/Branch insts unsupported).", .func = cmd_add_breakpoint, .argc = 1, @@ -89,6 +91,5 @@ static cmd_info_t addbkpte_info = { .name = "addbkpte", - .description = "addebkpte <&symbol> <&func> - new bkpoint. Call " - "func(or Nothing if 0).", + .description = "Add bkpoint with a trigger function.", .func = cmd_add_breakpoint, .argc = 2, @@ -100,156 +101,159 @@ uint32_t value; } jmpinstr[] = { - {0xf3ff0000, 0x41000000}, /* BCzF */ - {0xf3ff0000, 0x41020000}, /* BCzFL */ - {0xf3ff0000, 0x41010000}, /* BCzT */ - {0xf3ff0000, 0x41030000}, /* BCzTL */ - {0xfc000000, 0x10000000}, /* BEQ */ - {0xfc000000, 0x50000000}, /* BEQL */ - {0xfc1f0000, 0x04010000}, /* BEQL */ - {0xfc1f0000, 0x04110000}, /* BGEZAL */ - {0xfc1f0000, 0x04130000}, /* BGEZALL */ - {0xfc1f0000, 0x04030000}, /* BGEZL */ - {0xfc1f0000, 0x1c000000}, /* BGTZ */ - {0xfc1f0000, 0x5c000000}, /* BGTZL */ - {0xfc1f0000, 0x18000000}, /* BLEZ */ - {0xfc1f0000, 0x58000000}, /* BLEZL */ - {0xfc1f0000, 0x04000000}, /* BLTZ */ - {0xfc1f0000, 0x04100000}, /* BLTZAL */ - {0xfc1f0000, 0x04120000}, /* BLTZALL */ - {0xfc1f0000, 0x04020000}, /* BLTZL */ - {0xfc000000, 0x14000000}, /* BNE */ - {0xfc000000, 0x54000000}, /* BNEL */ - {0xfc000000, 0x08000000}, /* J */ - {0xfc000000, 0x0c000000}, /* JAL */ - {0xfc1f07ff, 0x00000009}, /* JALR */ - {0, 0} /* EndOfTable */ -}; - + {0xf3ff0000, 0x41000000}, /* BCzF */ + {0xf3ff0000, 0x41020000}, /* BCzFL */ + {0xf3ff0000, 0x41010000}, /* BCzT */ + {0xf3ff0000, 0x41030000}, /* BCzTL */ + {0xfc000000, 0x10000000}, /* BEQ */ + {0xfc000000, 0x50000000}, /* BEQL */ + {0xfc1f0000, 0x04010000}, /* BEQL */ + {0xfc1f0000, 0x04110000}, /* BGEZAL */ + {0xfc1f0000, 0x04130000}, /* BGEZALL */ + {0xfc1f0000, 0x04030000}, /* BGEZL */ + {0xfc1f0000, 0x1c000000}, /* BGTZ */ + {0xfc1f0000, 0x5c000000}, /* BGTZL */ + {0xfc1f0000, 0x18000000}, /* BLEZ */ + {0xfc1f0000, 0x58000000}, /* BLEZL */ + {0xfc1f0000, 0x04000000}, /* BLTZ */ + {0xfc1f0000, 0x04100000}, /* BLTZAL */ + {0xfc1f0000, 0x04120000}, /* BLTZALL */ + {0xfc1f0000, 0x04020000}, /* BLTZL */ + {0xfc000000, 0x14000000}, /* BNE */ + {0xfc000000, 0x54000000}, /* BNEL */ + {0xfc000000, 0x08000000}, /* J */ + {0xfc000000, 0x0c000000}, /* JAL */ + {0xfc1f07ff, 0x00000009}, /* JALR */ + {0, 0} /* end of table */ +}; /** Test, if the given instruction is a jump or branch instruction * * @param instr Instruction code - * @return true - it is jump instruction, false otherwise + * + * @return true if it is jump instruction, false otherwise * */ static bool is_jump(unative_t instr) { - int i; - + unsigned int i; + for (i = 0; jmpinstr[i].andmask; i++) { if ((instr & jmpinstr[i].andmask) == jmpinstr[i].value) return true; } - + return false; } -/** Add new breakpoint to table */ +/** Add new breakpoint to table + * + */ int cmd_add_breakpoint(cmd_arg_t *argv) { - bpinfo_t *cur = NULL; - ipl_t ipl; - int i; - if (argv->intval & 0x3) { printf("Not aligned instruction, forgot to use &symbol?\n"); return 1; } - ipl = interrupts_disable(); - spinlock_lock(&bkpoint_lock); - + + irq_spinlock_lock(&bkpoint_lock, true); + /* Check, that the breakpoints do not conflict */ + unsigned int i; for (i = 0; i < BKPOINTS_MAX; i++) { - if (breakpoints[i].address == (uintptr_t)argv->intval) { + if (breakpoints[i].address == (uintptr_t) argv->intval) { printf("Duplicate breakpoint %d.\n", i); - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return 0; - } else if (breakpoints[i].address == (uintptr_t)argv->intval + - sizeof(unative_t) || breakpoints[i].address == - (uintptr_t)argv->intval - sizeof(unative_t)) { + } else if ((breakpoints[i].address == (uintptr_t) argv->intval + + sizeof(unative_t)) || (breakpoints[i].address == + (uintptr_t) argv->intval - sizeof(unative_t))) { printf("Adjacent breakpoints not supported, conflict " "with %d.\n", i); - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return 0; } } - - for (i = 0; i < BKPOINTS_MAX; i++) + + bpinfo_t *cur = NULL; + + for (i = 0; i < BKPOINTS_MAX; i++) { if (!breakpoints[i].address) { cur = &breakpoints[i]; break; } + } + if (!cur) { printf("Too many breakpoints.\n"); - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return 0; } + + printf("Adding breakpoint on address %p\n", argv->intval); + cur->address = (uintptr_t) argv->intval; - printf("Adding breakpoint on address: %p\n", argv->intval); - cur->instruction = ((unative_t *)cur->address)[0]; - cur->nextinstruction = ((unative_t *)cur->address)[1]; + cur->instruction = ((unative_t *) cur->address)[0]; + cur->nextinstruction = ((unative_t *) cur->address)[1]; if (argv == &add_argv) { cur->flags = 0; - } else { /* We are add extended */ + } else { /* We are add extended */ cur->flags = BKPOINT_FUNCCALL; cur->bkfunc = (void (*)(void *, istate_t *)) argv[1].intval; } + if (is_jump(cur->instruction)) cur->flags |= BKPOINT_ONESHOT; + cur->counter = 0; - + /* Set breakpoint */ - *((unative_t *)cur->address) = 0x0d; + *((unative_t *) cur->address) = 0x0d; smc_coherence(cur->address); - - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); - + + irq_spinlock_unlock(&bkpoint_lock, true); + return 1; } -/** Remove breakpoint from table */ +/** Remove breakpoint from table + * + */ int cmd_del_breakpoint(cmd_arg_t *argv) { - bpinfo_t *cur; - ipl_t ipl; - if (argv->intval > BKPOINTS_MAX) { printf("Invalid breakpoint number.\n"); return 0; } - ipl = interrupts_disable(); - spinlock_lock(&bkpoint_lock); - - cur = &breakpoints[argv->intval]; + + irq_spinlock_lock(&bkpoint_lock, true); + + bpinfo_t *cur = &breakpoints[argv->intval]; if (!cur->address) { printf("Breakpoint does not exist.\n"); - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return 0; } + if ((cur->flags & BKPOINT_INPROG) && (cur->flags & BKPOINT_ONESHOT)) { printf("Cannot remove one-shot breakpoint in-progress\n"); - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&bkpoint_lock, true); return 0; } - ((uint32_t *)cur->address)[0] = cur->instruction; - smc_coherence(((uint32_t *)cur->address)[0]); - ((uint32_t *)cur->address)[1] = cur->nextinstruction; - smc_coherence(((uint32_t *)cur->address)[1]); - + + ((uint32_t *) cur->address)[0] = cur->instruction; + smc_coherence(((uint32_t *) cur->address)[0]); + ((uint32_t *) cur->address)[1] = cur->nextinstruction; + smc_coherence(((uint32_t *) cur->address)[1]); + cur->address = NULL; - - spinlock_unlock(&bkpoint_lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&bkpoint_lock, true); return 1; } -/** Print table of active breakpoints */ +/** Print table of active breakpoints + * + */ int cmd_print_breakpoints(cmd_arg_t *argv) { @@ -276,59 +280,64 @@ } -#endif - -/** Initialize debugger */ +#endif /* CONFIG_KCONSOLE */ + +/** Initialize debugger + * + */ void debugger_init() { - int i; - + unsigned int i; + for (i = 0; i < BKPOINTS_MAX; i++) breakpoints[i].address = NULL; - + #ifdef CONFIG_KCONSOLE cmd_initialize(&bkpts_info); if (!cmd_register(&bkpts_info)) printf("Cannot register command %s\n", bkpts_info.name); - + cmd_initialize(&delbkpt_info); if (!cmd_register(&delbkpt_info)) printf("Cannot register command %s\n", delbkpt_info.name); - + cmd_initialize(&addbkpt_info); if (!cmd_register(&addbkpt_info)) printf("Cannot register command %s\n", addbkpt_info.name); - + cmd_initialize(&addbkpte_info); if (!cmd_register(&addbkpte_info)) printf("Cannot register command %s\n", addbkpte_info.name); -#endif +#endif /* CONFIG_KCONSOLE */ } /** Handle breakpoint * - * Find breakpoint in breakpoint table. + * Find breakpoint in breakpoint table. * If found, call kconsole, set break on next instruction and reexecute. * If we are on "next instruction", set it back on the first and reexecute. * If breakpoint not found in breakpoint table, call kconsole and start * next instruction. + * */ void debugger_bpoint(istate_t *istate) { - bpinfo_t *cur = NULL; - uintptr_t fireaddr = istate->epc; - int i; - /* test branch delay slot */ if (cp0_cause_read() & 0x80000000) panic("Breakpoint in branch delay slot not supported."); - - spinlock_lock(&bkpoint_lock); + + irq_spinlock_lock(&bkpoint_lock, false); + + bpinfo_t *cur = NULL; + uintptr_t fireaddr = istate->epc; + unsigned int i; + for (i = 0; i < BKPOINTS_MAX; i++) { /* Normal breakpoint */ - if (fireaddr == breakpoints[i].address && - !(breakpoints[i].flags & BKPOINT_REINST)) { + if ((fireaddr == breakpoints[i].address) && + (!(breakpoints[i].flags & BKPOINT_REINST))) { cur = &breakpoints[i]; break; } + /* Reinst only breakpoint */ if ((breakpoints[i].flags & BKPOINT_REINST) && @@ -338,24 +347,28 @@ } } + if (cur) { if (cur->flags & BKPOINT_REINST) { /* Set breakpoint on first instruction */ - ((uint32_t *)cur->address)[0] = 0x0d; + ((uint32_t *) cur->address)[0] = 0x0d; smc_coherence(((uint32_t *)cur->address)[0]); + /* Return back the second */ - ((uint32_t *)cur->address)[1] = cur->nextinstruction; - smc_coherence(((uint32_t *)cur->address)[1]); + ((uint32_t *) cur->address)[1] = cur->nextinstruction; + smc_coherence(((uint32_t *) cur->address)[1]); + cur->flags &= ~BKPOINT_REINST; - spinlock_unlock(&bkpoint_lock); + irq_spinlock_unlock(&bkpoint_lock, false); return; - } + } + if (cur->flags & BKPOINT_INPROG) printf("Warning: breakpoint recursion\n"); if (!(cur->flags & BKPOINT_FUNCCALL)) { - printf("***Breakpoint %d: %p in %s.\n", i, fireaddr, - symtab_fmt_name_lookup(istate->epc)); - } - + printf("***Breakpoint %u: %p in %s.\n", i, fireaddr, + symtab_fmt_name_lookup(fireaddr)); + } + /* Return first instruction back */ ((uint32_t *)cur->address)[0] = cur->instruction; @@ -371,10 +384,12 @@ printf("***Breakpoint %d: %p in %s.\n", i, fireaddr, symtab_fmt_name_lookup(fireaddr)); - + /* Move on to next instruction */ istate->epc += 4; } + if (cur) cur->counter++; + if (cur && (cur->flags & BKPOINT_FUNCCALL)) { /* Allow zero bkfunc, just for counting */ @@ -383,26 +398,31 @@ } else { #ifdef CONFIG_KCONSOLE - /* This disables all other processors - we are not SMP, + /* + * This disables all other processors - we are not SMP, * actually this gets us to cpu_halt, if scheduler() is run * - we generally do not want scheduler to be run from debug, * so this is a good idea - */ + */ atomic_set(&haltstate, 1); - spinlock_unlock(&bkpoint_lock); + irq_spinlock_unlock(&bkpoint_lock, false); kconsole("debug", "Debug console ready.\n", false); - spinlock_lock(&bkpoint_lock); + irq_spinlock_lock(&bkpoint_lock, false); atomic_set(&haltstate, 0); #endif } - if (cur && cur->address == fireaddr && (cur->flags & BKPOINT_INPROG)) { + + if ((cur) && (cur->address == fireaddr) + && ((cur->flags & BKPOINT_INPROG))) { /* Remove one-shot breakpoint */ if ((cur->flags & BKPOINT_ONESHOT)) cur->address = NULL; + /* Remove in-progress flag */ cur->flags &= ~BKPOINT_INPROG; - } - spinlock_unlock(&bkpoint_lock); + } + + irq_spinlock_unlock(&bkpoint_lock, false); } Index: kernel/arch/mips32/src/exception.c =================================================================== --- kernel/arch/mips32/src/exception.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/exception.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32 +/** @addtogroup mips32 * @{ */ @@ -67,5 +67,5 @@ "Floating Point", NULL, NULL, NULL, NULL, NULL, NULL, NULL, - "WatchHi/WatchLo", /* 23 */ + "WatchHi/WatchLo", /* 23 */ NULL, NULL, NULL, NULL, NULL, NULL, NULL, "Virtual Coherency - data", @@ -74,9 +74,7 @@ static void print_regdump(istate_t *istate) { - const char *pcsymbol = symtab_fmt_name_lookup(istate->epc); - const char *rasymbol = symtab_fmt_name_lookup(istate->ra); - - printf("PC: %#x(%s) RA: %#x(%s), SP(%p)\n", istate->epc, pcsymbol, - istate->ra, rasymbol, istate->sp); + printf("PC: %#x(%s) RA: %#x(%s), SP(%p)\n", istate->epc, + symtab_fmt_name_lookup(istate->epc), istate->ra, + symtab_fmt_name_lookup(istate->ra), istate->sp); } @@ -135,10 +133,8 @@ static void interrupt_exception(int n, istate_t *istate) { - uint32_t cause; - int i; + /* Decode interrupt number and process the interrupt */ + uint32_t cause = (cp0_cause_read() >> 8) & 0xff; - /* decode interrupt number and process the interrupt */ - cause = (cp0_cause_read() >> 8) & 0xff; - + unsigned int i; for (i = 0; i < 8; i++) { if (cause & (1 << i)) { @@ -149,5 +145,5 @@ */ irq->handler(irq); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { /* @@ -172,5 +168,5 @@ { int i; - + /* Clear exception table */ for (i = 0; i < IVT_ITEMS; i++) Index: kernel/arch/mips32/src/interrupt.c =================================================================== --- kernel/arch/mips32/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -49,4 +49,10 @@ static irq_t timer_irq; +// TODO: This is SMP unsafe!!! + +uint32_t count_hi = 0; +static unsigned long nextcount; +static unsigned long lastcount; + /** Disable interrupts. * @@ -89,10 +95,17 @@ } -/* TODO: This is SMP unsafe!!! */ -uint32_t count_hi = 0; -static unsigned long nextcount; -static unsigned long lastcount; +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +bool interrupts_disabled(void) +{ + return !(cp0_status_read() & cp0_status_ie_enabled_bit); +} -/** Start hardware clock */ +/** Start hardware clock + * + */ static void timer_start(void) { @@ -109,16 +122,16 @@ static void timer_irq_handler(irq_t *irq) { - unsigned long drift; - if (cp0_count_read() < lastcount) /* Count overflow detected */ count_hi++; + lastcount = cp0_count_read(); - drift = cp0_count_read() - nextcount; + unsigned long drift = cp0_count_read() - nextcount; while (drift > cp0_compare_value) { drift -= cp0_compare_value; CPU->missed_clock_ticks++; } + nextcount = cp0_count_read() + cp0_compare_value - drift; cp0_compare_write(nextcount); @@ -128,7 +141,7 @@ * Release the lock, call clock() and reacquire the lock again. */ - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); clock(); - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); if (virtual_timer_fnc != NULL) Index: kernel/arch/mips32/src/mips32.c =================================================================== --- kernel/arch/mips32/src/mips32.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/mips32.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,4 +36,5 @@ #include #include +#include #include #include Index: kernel/arch/mips32/src/mm/tlb.c =================================================================== --- kernel/arch/mips32/src/mm/tlb.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/mm/tlb.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -353,6 +353,4 @@ /** Try to find PTE for faulting address. * - * The AS->lock must be held on entry to this function. - * * @param badvaddr Faulting virtual address. * @param access Access mode that caused the fault. @@ -369,4 +367,6 @@ entry_hi_t hi; pte_t *pte; + + ASSERT(mutex_locked(&AS->lock)); hi.value = cp0_entry_hi_read(); Index: kernel/arch/mips32/src/panic.S =================================================================== --- kernel/arch/mips32/src/panic.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/panic.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ # -.text +.text .set noat @@ -34,15 +34,10 @@ #include - + .global panic_printf -/* From printf return directly to halt() */ +/* From printf return directly to halt() */ panic_printf: - jal printf - nop - j halt - nop -/* This code does not work, god knows why */ -/* lui $ra, %hi(halt) + lui $ra, %hi(halt) j printf - ori $ra, %lo(halt) */ + ori $ra, %lo(halt) Index: kernel/arch/mips32/src/start.S =================================================================== --- kernel/arch/mips32/src/start.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/mips32/src/start.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -45,11 +45,16 @@ .global userspace_asm -# Which status bits should are thread-local -#define REG_SAVE_MASK 0x1f # KSU(UM), EXL, ERL, IE - -# Save registers to space defined by \r -# We will change status: Disable ERL,EXL,UM,IE -# These changes will be automatically reversed in REGISTER_LOAD -# SP is NOT saved as part of these registers +/* + * Which status bits should are thread-local: + * KSU(UM), EXL, ERL, IE + */ +#define REG_SAVE_MASK 0x1f + +/* + * Save registers to space defined by \r + * We will change status: Disable ERL, EXL, UM, IE + * These changes will be automatically reversed in REGISTER_LOAD + * %sp is NOT saved as part of these registers + */ .macro REGISTERS_STORE_AND_EXC_RESET r sw $at, EOFFSET_AT(\r) @@ -70,5 +75,5 @@ sw $t8, EOFFSET_T8(\r) sw $t9, EOFFSET_T9(\r) - + mflo $at sw $at, EOFFSET_LO(\r) @@ -79,27 +84,34 @@ sw $ra, EOFFSET_RA(\r) sw $k1, EOFFSET_K1(\r) - + mfc0 $t0, $status mfc0 $t1, $epc - and $t2, $t0, REG_SAVE_MASK # Save only KSU,EXL,ERL,IE - li $t3, ~(0x1f) - and $t0, $t0, $t3 # Clear KSU,EXL,ERL,IE - - sw $t2,EOFFSET_STATUS(\r) - sw $t1,EOFFSET_EPC(\r) + /* save only KSU, EXL, ERL, IE */ + and $t2, $t0, REG_SAVE_MASK + + /* clear KSU, EXL, ERL, IE */ + li $t3, ~(REG_SAVE_MASK) + and $t0, $t0, $t3 + + sw $t2, EOFFSET_STATUS(\r) + sw $t1, EOFFSET_EPC(\r) mtc0 $t0, $status .endm .macro REGISTERS_LOAD r - # Update only UM,EXR,IE from status, the rest - # is controlled by OS and not bound to task + /* + * Update only UM, EXR, IE from status, the rest + * is controlled by OS and not bound to task. + */ mfc0 $t0, $status lw $t1,EOFFSET_STATUS(\r) - - li $t2, ~REG_SAVE_MASK # Mask UM,EXL,ERL,IE + + /* mask UM, EXL, ERL, IE */ + li $t2, ~REG_SAVE_MASK and $t0, $t0, $t2 - or $t0, $t0, $t1 # Copy UM,EXL, ERL, IE from saved status + /* copy UM, EXL, ERL, IE from saved status */ + or $t0, $t0, $t1 mtc0 $t0, $status @@ -129,5 +141,5 @@ lw $at, EOFFSET_HI(\r) mthi $at - + lw $at, EOFFSET_EPC(\r) mtc0 $at, $epc @@ -137,9 +149,10 @@ .endm -# Move kernel stack pointer address to register K0 -# - if we are in user mode, load the appropriate stack -# address +/* + * Move kernel stack pointer address to register $k0. + * If we are in user mode, load the appropriate stack address. + */ .macro KERNEL_STACK_TO_K0 - # If we are in user mode + /* if we are in user mode */ mfc0 $k0, $status andi $k0, 0x10 @@ -148,24 +161,24 @@ add $k0, $sp, 0 - # Move $k0 pointer to kernel stack + /* move $k0 pointer to kernel stack */ lui $k0, %hi(supervisor_sp) ori $k0, $k0, %lo(supervisor_sp) - # Move $k0 (superveisor_sp) + + /* move $k0 (supervisor_sp) */ lw $k0, 0($k0) -1: + + 1: .endm .org 0x0 kernel_image_start: - /* Load temporary stack */ + /* load temporary stack */ lui $sp, %hi(end_stack) ori $sp, $sp, %lo(end_stack) - /* Not sure about this, but might - be needed for PIC code */ + /* not sure about this, but might be needed for PIC code */ lui $gp, 0x8000 /* $a1 contains physical address of bootinfo_t */ - jal arch_pre_main nop @@ -174,5 +187,5 @@ nop - .space TEMP_STACK_SIZE +.space TEMP_STACK_SIZE end_stack: @@ -191,4 +204,5 @@ exception_handler: KERNEL_STACK_TO_K0 + sub $k0, REGISTER_SPACE sw $sp, EOFFSET_SP($k0) @@ -197,75 +211,81 @@ mfc0 $k0, $cause - sra $k0, $k0, 0x2 # cp0_exc_cause() part 1 - andi $k0, $k0, 0x1f # cp0_exc_cause() part 2 - sub $k0, 8 # 8 = SYSCALL + sra $k0, $k0, 0x2 /* cp0_exc_cause() part 1 */ + andi $k0, $k0, 0x1f /* cp0_exc_cause() part 2 */ + sub $k0, 8 /* 8 = SYSCALL */ beqz $k0, syscall_shortcut - add $k0, 8 # Revert $k0 back to correct exc number + add $k0, 8 /* revert $k0 back to correct exc number */ REGISTERS_STORE_AND_EXC_RESET $sp move $a1, $sp - jal exc_dispatch # exc_dispatch(excno, register_space) + jal exc_dispatch /* exc_dispatch(excno, register_space) */ move $a0, $k0 - + REGISTERS_LOAD $sp - # The $sp is automatically restored to former value - eret - -## Syscall entry -# -# Registers: -# -# @param v0 Syscall number. -# @param a0 1st argument. -# @param a1 2nd argument. -# @param a2 3rd argument. -# @param a3 4th argument. -# @param t0 5th argument. -# @param t1 6th argument. -# -# @return The return value will be stored in v0. -# -#define SS_SP EOFFSET_SP -#define SS_STATUS EOFFSET_STATUS -#define SS_EPC EOFFSET_EPC -#define SS_K1 EOFFSET_K1 + /* the $sp is automatically restored to former value */ + eret + +#define SS_SP EOFFSET_SP +#define SS_STATUS EOFFSET_STATUS +#define SS_EPC EOFFSET_EPC +#define SS_K1 EOFFSET_K1 + +/** Syscall entry + * + * Registers: + * + * @param $v0 Syscall number. + * @param $a0 1st argument. + * @param $a1 2nd argument. + * @param $a2 3rd argument. + * @param $a3 4th argument. + * @param $t0 5th argument. + * @param $t1 6th argument. + * + * @return The return value will be stored in $v0. + * + */ syscall_shortcut: - # We have a lot of space on the stack, with free use + /* we have a lot of space on the stack, with free use */ mfc0 $t3, $epc mfc0 $t2, $status - sw $t3, SS_EPC($sp) # Save EPC - sw $k1, SS_K1($sp) # Save k1 not saved on context switch - - and $t4, $t2, REG_SAVE_MASK # Save only KSU, EXL, ERL, IE + sw $t3, SS_EPC($sp) /* save EPC */ + sw $k1, SS_K1($sp) /* save $k1 not saved on context switch */ + + and $t4, $t2, REG_SAVE_MASK /* save only KSU, EXL, ERL, IE */ li $t5, ~(0x1f) - and $t2, $t2, $t5 # Clear KSU, EXL, ERL - ori $t2, $t2, 0x1 # Set IE - + and $t2, $t2, $t5 /* clear KSU, EXL, ERL */ + ori $t2, $t2, 0x1 /* set IE */ + sw $t4, SS_STATUS($sp) mtc0 $t2, $status - - # - # Call the higher level system call handler - # We are going to reuse part of the unused exception stack frame - # - sw $t0, STACK_ARG4($sp) # save the 5th argument on the stack - sw $t1, STACK_ARG5($sp) # save the 6th argument on the stack + + /* + * Call the higher level system call handler. + * We are going to reuse part of the unused exception stack frame. + * + */ + sw $t0, STACK_ARG4($sp) /* save the 5th argument on the stack */ + sw $t1, STACK_ARG5($sp) /* save the 6th argument on the stack */ jal syscall_handler - sw $v0, STACK_ARG6($sp) # save the syscall number on the stack - - # restore status + sw $v0, STACK_ARG6($sp) /* save the syscall number on the stack */ + + /* restore status */ mfc0 $t2, $status lw $t3, SS_STATUS($sp) - - # Change back to EXL = 1 (from last exception), otherwise - # an interrupt could rewrite the CP0 - EPC - li $t4, ~REG_SAVE_MASK # Mask UM, EXL, ERL, IE + + /* + * Change back to EXL = 1 (from last exception), otherwise + * an interrupt could rewrite the CP0 - EPC. + * + */ + li $t4, ~REG_SAVE_MASK /* mask UM, EXL, ERL, IE */ and $t2, $t2, $t4 - or $t2, $t2, $t3 # Copy saved UM, EXL, ERL, IE + or $t2, $t2, $t3 /* copy saved UM, EXL, ERL, IE */ mtc0 $t2, $status - - # restore epc + 4 + + /* restore epc + 4 */ lw $t2, SS_EPC($sp) lw $k1, SS_K1($sp) @@ -273,8 +293,7 @@ mtc0 $t2, $epc - lw $sp, SS_SP($sp) # restore sp - - eret - + lw $sp, SS_SP($sp) /* restore $sp */ + eret + tlb_refill_handler: KERNEL_STACK_TO_K0 @@ -283,10 +302,9 @@ sw $sp,EOFFSET_SP($k0) add $sp, $k0, 0 - + jal tlb_refill add $a0, $sp, 0 - + REGISTERS_LOAD $sp - eret @@ -297,17 +315,16 @@ sw $sp,EOFFSET_SP($k0) add $sp, $k0, 0 - + jal cache_error - add $a0, $sp, 0 - + add $a0, $sp, 0 + REGISTERS_LOAD $sp - eret userspace_asm: add $sp, $a0, 0 - add $v0, $a1, 0 - add $t9, $a2, 0 # Set up correct entry into PIC code - xor $a0, $a0, $a0 # $a0 is defined to hold pcb_ptr - # set it to 0 - eret + add $v0, $a1, 0 + add $t9, $a2, 0 /* set up correct entry into PIC code */ + xor $a0, $a0, $a0 /* $a0 is defined to hold pcb_ptr */ + /* set it to 0 */ + eret Index: kernel/arch/ppc32/Makefile.inc =================================================================== --- kernel/arch/ppc32/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,6 +32,6 @@ GCC_CFLAGS += -mcpu=powerpc -msoft-float -m32 -AFLAGS = -a32 -LFLAGS = -no-check-sections -N +AFLAGS += -a32 +LFLAGS += -no-check-sections BITS = 32 Index: kernel/arch/ppc32/include/asm.h =================================================================== --- kernel/arch/ppc32/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,4 +38,60 @@ #include #include +#include +#include + +static inline uint32_t msr_read(void) +{ + uint32_t msr; + + asm volatile ( + "mfmsr %[msr]\n" + : [msr] "=r" (msr) + ); + + return msr; +} + +static inline void msr_write(uint32_t msr) +{ + asm volatile ( + "mtmsr %[msr]\n" + :: [msr] "r" (msr) + ); +} + +static inline void sr_set(uint32_t flags, asid_t asid, uint32_t sr) +{ + asm volatile ( + "mtsrin %[value], %[sr]\n" + :: [value] "r" ((flags << 16) + (asid << 4) + sr), + [sr] "r" (sr << 28) + ); +} + +static inline uint32_t sr_get(uint32_t vaddr) +{ + uint32_t vsid; + + asm volatile ( + "mfsrin %[vsid], %[vaddr]\n" + : [vsid] "=r" (vsid) + : [vaddr] "r" (vaddr) + ); + + return vsid; +} + +static inline uint32_t sdr1_get(void) +{ + uint32_t sdr1; + + asm volatile ( + "mfsdr1 %[sdr1]\n" + : [sdr1] "=r" (sdr1) + ); + + return sdr1; +} /** Enable interrupts. @@ -45,18 +101,11 @@ * * @return Old interrupt priority level. + * */ static inline ipl_t interrupts_enable(void) { - ipl_t v; - ipl_t tmp; - - asm volatile ( - "mfmsr %0\n" - "mfmsr %1\n" - "ori %1, %1, 1 << 15\n" - "mtmsr %1\n" - : "=r" (v), "=r" (tmp) - ); - return v; + ipl_t ipl = msr_read(); + msr_write(ipl | MSR_EE); + return ipl; } @@ -67,18 +116,11 @@ * * @return Old interrupt priority level. + * */ static inline ipl_t interrupts_disable(void) { - ipl_t v; - ipl_t tmp; - - asm volatile ( - "mfmsr %0\n" - "mfmsr %1\n" - "rlwinm %1, %1, 0, 17, 15\n" - "mtmsr %1\n" - : "=r" (v), "=r" (tmp) - ); - return v; + ipl_t ipl = msr_read(); + msr_write(ipl & (~MSR_EE)); + return ipl; } @@ -88,20 +130,9 @@ * * @param ipl Saved interrupt priority level. + * */ static inline void interrupts_restore(ipl_t ipl) { - ipl_t tmp; - - asm volatile ( - "mfmsr %1\n" - "rlwimi %0, %1, 0, 17, 15\n" - "cmpw 0, %0, %1\n" - "beq 0f\n" - "mtmsr %0\n" - "0:\n" - : "=r" (ipl), "=r" (tmp) - : "0" (ipl) - : "cr0" - ); + msr_write((msr_read() & (~MSR_EE)) | (ipl & MSR_EE)); } @@ -111,14 +142,19 @@ * * @return Current interrupt priority level. + * */ static inline ipl_t interrupts_read(void) { - ipl_t v; - - asm volatile ( - "mfmsr %0\n" - : "=r" (v) - ); - return v; + return msr_read(); +} + +/** Check whether interrupts are disabled. + * + * @return True if interrupts are disabled. + * + */ +static inline bool interrupts_disabled(void) +{ + return ((msr_read() & MSR_EE) == 0); } @@ -128,15 +164,17 @@ * The stack is assumed to be STACK_SIZE bytes long. * The stack must start on page boundary. + * */ static inline uintptr_t get_stack_base(void) { - uintptr_t v; - - asm volatile ( - "and %0, %%sp, %1\n" - : "=r" (v) - : "r" (~(STACK_SIZE - 1)) - ); - return v; + uintptr_t base; + + asm volatile ( + "and %[base], %%sp, %[mask]\n" + : [base] "=r" (base) + : [mask] "r" (~(STACK_SIZE - 1)) + ); + + return base; } Index: kernel/arch/ppc32/include/asm/regname.h =================================================================== --- kernel/arch/ppc32/include/asm/regname.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/asm/regname.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -217,17 +217,4 @@ #define hid0 1008 -/* MSR bits */ -#define msr_dr (1 << 4) -#define msr_ir (1 << 5) -#define msr_pr (1 << 14) -#define msr_ee (1 << 15) - -/* HID0 bits */ -#define hid0_sten (1 << 24) -#define hid0_ice (1 << 15) -#define hid0_dce (1 << 14) -#define hid0_icfi (1 << 11) -#define hid0_dci (1 << 10) - #endif Index: kernel/arch/ppc32/include/atomic.h =================================================================== --- kernel/arch/ppc32/include/atomic.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/atomic.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -42,11 +42,11 @@ asm volatile ( "1:\n" - "lwarx %0, 0, %2\n" - "addic %0, %0, 1\n" - "stwcx. %0, 0, %2\n" - "bne- 1b" - : "=&r" (tmp), + " lwarx %[tmp], 0, %[count_ptr]\n" + " addic %[tmp], %[tmp], 1\n" + " stwcx. %[tmp], 0, %[count_ptr]\n" + " bne- 1b" + : [tmp] "=&r" (tmp), "=m" (val->count) - : "r" (&val->count), + : [count_ptr] "r" (&val->count), "m" (val->count) : "cc" @@ -60,11 +60,11 @@ asm volatile ( "1:\n" - "lwarx %0, 0, %2\n" - "addic %0, %0, -1\n" - "stwcx. %0, 0, %2\n" - "bne- 1b" - : "=&r" (tmp), + " lwarx %[tmp], 0, %[count_ptr]\n" + " addic %[tmp], %[tmp], -1\n" + " stwcx. %[tmp], 0, %[count_ptr]\n" + " bne- 1b" + : [tmp] "=&r" (tmp), "=m" (val->count) - : "r" (&val->count), + : [count_ptr] "r" (&val->count), "m" (val->count) : "cc" Index: kernel/arch/ppc32/include/barrier.h =================================================================== --- kernel/arch/ppc32/include/barrier.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/barrier.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32 +/** @addtogroup ppc32 * @{ */ @@ -36,10 +36,18 @@ #define KERN_ppc32_BARRIER_H_ -#define CS_ENTER_BARRIER() asm volatile ("" ::: "memory") -#define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") +#define CS_ENTER_BARRIER() asm volatile ("" ::: "memory") +#define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") -#define memory_barrier() asm volatile ("sync" ::: "memory") -#define read_barrier() asm volatile ("sync" ::: "memory") -#define write_barrier() asm volatile ("eieio" ::: "memory") +#define memory_barrier() asm volatile ("sync" ::: "memory") +#define read_barrier() asm volatile ("sync" ::: "memory") +#define write_barrier() asm volatile ("eieio" ::: "memory") + +#define instruction_barrier() \ + asm volatile ( \ + "sync\n" \ + "isync\n" \ + ) + +#define COHERENCE_INVAL_MIN 4 /* @@ -53,33 +61,32 @@ { asm volatile ( - "dcbst 0, %0\n" + "dcbst 0, %[addr]\n" "sync\n" - "icbi 0, %0\n" + "icbi 0, %[addr]\n" "sync\n" "isync\n" - :: "r" (addr) + :: [addr] "r" (addr) ); } -#define COHERENCE_INVAL_MIN 4 - -static inline void smc_coherence_block(void *addr, unsigned long len) +static inline void smc_coherence_block(void *addr, unsigned int len) { - unsigned long i; - - for (i = 0; i < len; i += COHERENCE_INVAL_MIN) { - asm volatile ("dcbst 0, %0\n" :: "r" (addr + i)); - } - - asm volatile ("sync"); - - for (i = 0; i < len; i += COHERENCE_INVAL_MIN) { - asm volatile ("icbi 0, %0\n" :: "r" (addr + i)); - } - - asm volatile ( - "sync\n" - "isync\n" - ); + unsigned int i; + + for (i = 0; i < len; i += COHERENCE_INVAL_MIN) + asm volatile ( + "dcbst 0, %[addr]\n" + :: [addr] "r" (addr + i) + ); + + memory_barrier(); + + for (i = 0; i < len; i += COHERENCE_INVAL_MIN) + asm volatile ( + "icbi 0, %[addr]\n" + :: [addr] "r" (addr + i) + ); + + instruction_barrier(); } Index: kernel/arch/ppc32/include/cpu.h =================================================================== --- kernel/arch/ppc32/include/cpu.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/cpu.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,11 +36,36 @@ #define KERN_ppc32_CPU_H_ -#include +/* MSR bits */ +#define MSR_DR (1 << 4) +#define MSR_IR (1 << 5) +#define MSR_PR (1 << 14) +#define MSR_EE (1 << 15) + +/* HID0 bits */ +#define HID0_STEN (1 << 24) +#define HID0_ICE (1 << 15) +#define HID0_DCE (1 << 14) +#define HID0_ICFI (1 << 11) +#define HID0_DCI (1 << 10) + +#ifndef __ASM__ + +#include typedef struct { - int version; - int revision; -} cpu_arch_t; - + uint16_t version; + uint16_t revision; +} __attribute__ ((packed)) cpu_arch_t; + +static inline void cpu_version(cpu_arch_t *info) +{ + asm volatile ( + "mfpvr %[cpu_info]\n" + : [cpu_info] "=r" (*info) + ); +} + +#endif /* __ASM__ */ + #endif Index: rnel/arch/ppc32/include/cpuid.h =================================================================== --- kernel/arch/ppc32/include/cpuid.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,56 +1,0 @@ -/* - * Copyright (c) 2006 Martin Decky - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ppc32 - * @{ - */ -/** @file - */ - -#ifndef KERN_ppc32_CPUID_H_ -#define KERN_ppc32_CPUID_H_ - -#include - -typedef struct { - uint16_t version; - uint16_t revision; -} __attribute__ ((packed)) cpu_info_t; - -static inline void cpu_version(cpu_info_t *info) -{ - asm volatile ( - "mfpvr %0\n" - : "=r" (*info) - ); -} - -#endif - -/** @} - */ Index: kernel/arch/ppc32/include/cycle.h =================================================================== --- kernel/arch/ppc32/include/cycle.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/cycle.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -40,17 +40,16 @@ uint32_t lower; uint32_t upper; - uint32_t upper2; + uint32_t tmp; - asm volatile ( - "1: mftbu %0\n" - "mftb %1\n" - "mftbu %2\n" - "cmpw %0, %2\n" - "bne- 1b\n" - : "=r" (upper), - "=r" (lower), - "=r" (upper2) - :: "cr0" - ); + do { + asm volatile ( + "mftbu %[upper]\n" + "mftb %[lower]\n" + "mftbu %[tmp]\n" + : [upper] "=r" (upper), + [lower] "=r" (lower), + [tmp] "=r" (tmp) + ); + } while (upper != tmp); return ((uint64_t) upper << 32) + (uint64_t) lower; Index: rnel/arch/ppc32/include/debug.h =================================================================== --- kernel/arch/ppc32/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,41 +1,0 @@ -/* - * Copyright (c) 2005 - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ppc32debug - * @{ - */ -/** @file - */ - -#ifndef KERN_ppc32_DEBUG_H_ -#define KERN_ppc32_DEBUG_H_ - -#endif - -/** @} - */ Index: kernel/arch/ppc32/include/drivers/pic.h =================================================================== --- kernel/arch/ppc32/include/drivers/pic.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/drivers/pic.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -39,16 +39,16 @@ #include -#define PIC_PENDING_LOW 8 -#define PIC_PENDING_HIGH 4 -#define PIC_MASK_LOW 9 -#define PIC_MASK_HIGH 5 -#define PIC_ACK_LOW 10 -#define PIC_ACK_HIGH 6 +#define PIC_PENDING_LOW 8 +#define PIC_PENDING_HIGH 4 +#define PIC_MASK_LOW 9 +#define PIC_MASK_HIGH 5 +#define PIC_ACK_LOW 10 +#define PIC_ACK_HIGH 6 -extern void pic_init(uintptr_t base, size_t size, cir_t *cir, void **cir_arg); -extern void pic_enable_interrupt(inr_t intnum); -extern void pic_disable_interrupt(inr_t intnum); -extern void pic_ack_interrupt(void *arg, inr_t intnum); -extern int pic_get_pending(void); +extern void pic_init(uintptr_t, size_t, cir_t *, void **); +extern void pic_enable_interrupt(inr_t); +extern void pic_disable_interrupt(inr_t); +extern void pic_ack_interrupt(void *, inr_t); +extern uint8_t pic_get_pending(void); #endif Index: kernel/arch/ppc32/include/elf.h =================================================================== --- kernel/arch/ppc32/include/elf.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/elf.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32 +/** @addtogroup ppc32 * @{ */ @@ -36,7 +36,7 @@ #define KERN_ppc32_ELF_H_ -#define ELF_MACHINE EM_PPC -#define ELF_DATA_ENCODING ELFDATA2MSB -#define ELF_CLASS ELFCLASS32 +#define ELF_MACHINE EM_PPC +#define ELF_DATA_ENCODING ELFDATA2MSB +#define ELF_CLASS ELFCLASS32 #endif Index: kernel/arch/ppc32/include/exception.h =================================================================== --- kernel/arch/ppc32/include/exception.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/exception.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32 +/** @addtogroup ppc32 * @{ */ @@ -37,5 +37,5 @@ #include -#include +#include typedef struct istate { @@ -86,8 +86,11 @@ } -/** Return true if exception happened while in userspace */ +/** Return true if exception happened while in userspace + * + * The contexts of MSR register was stored in SRR1. + * + */ static inline int istate_from_uspace(istate_t *istate) { - /* true if privilege level PR (copied from MSR) == 1 */ return (istate->srr1 & MSR_PR) != 0; } Index: kernel/arch/ppc32/include/faddr.h =================================================================== --- kernel/arch/ppc32/include/faddr.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/faddr.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32 +/** @addtogroup ppc32 * @{ */ @@ -38,5 +38,5 @@ #include -#define FADDR(fptr) ((uintptr_t) (fptr)) +#define FADDR(fptr) ((uintptr_t) (fptr)) #endif Index: kernel/arch/ppc32/include/fpu_context.h =================================================================== --- kernel/arch/ppc32/include/fpu_context.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/fpu_context.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32 +/** @addtogroup ppc32 * @{ */ @@ -36,7 +36,5 @@ #define KERN_ppc32_FPU_CONTEXT_H_ -#ifndef KERN_ppc32_TYPES_H_ - #include -#endif +#include typedef struct { Index: kernel/arch/ppc32/include/interrupt.h =================================================================== --- kernel/arch/ppc32/include/interrupt.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/interrupt.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,15 +38,15 @@ #include -#define IVT_ITEMS 16 -#define IVT_FIRST 0 +#define IVT_ITEMS 16 +#define IVT_FIRST 0 -#define VECTOR_DATA_STORAGE 2 -#define VECTOR_INSTRUCTION_STORAGE 3 -#define VECTOR_EXTERNAL 4 -#define VECTOR_DECREMENTER 8 +#define VECTOR_DATA_STORAGE 2 +#define VECTOR_INSTRUCTION_STORAGE 3 +#define VECTOR_EXTERNAL 4 +#define VECTOR_DECREMENTER 8 extern void start_decrementer(void); extern void interrupt_init(void); -extern void extint_handler(int n, istate_t *istate); +extern void extint_handler(int, istate_t *); #endif Index: kernel/arch/ppc32/include/mm/as.h =================================================================== --- kernel/arch/ppc32/include/mm/as.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/mm/as.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32mm +/** @addtogroup ppc32mm * @{ */ @@ -36,12 +36,12 @@ #define KERN_ppc32_AS_H_ -#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 +#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 -#define KERNEL_ADDRESS_SPACE_START_ARCH ((unsigned long) 0x80000000) -#define KERNEL_ADDRESS_SPACE_END_ARCH ((unsigned long) 0xffffffff) -#define USER_ADDRESS_SPACE_START_ARCH ((unsigned long) 0x00000000) -#define USER_ADDRESS_SPACE_END_ARCH ((unsigned long) 0x7fffffff) +#define KERNEL_ADDRESS_SPACE_START_ARCH ((unsigned long) 0x80000000) +#define KERNEL_ADDRESS_SPACE_END_ARCH ((unsigned long) 0xffffffff) +#define USER_ADDRESS_SPACE_START_ARCH ((unsigned long) 0x00000000) +#define USER_ADDRESS_SPACE_END_ARCH ((unsigned long) 0x7fffffff) -#define USTACK_ADDRESS_ARCH (0x7fffffff - (PAGE_SIZE - 1)) +#define USTACK_ADDRESS_ARCH (0x7fffffff - (PAGE_SIZE - 1)) typedef struct { @@ -50,7 +50,7 @@ #include -#define as_constructor_arch(as, flags) (as != as) -#define as_destructor_arch(as) (as != as) -#define as_create_arch(as, flags) (as != as) +#define as_constructor_arch(as, flags) (as != as) +#define as_destructor_arch(as) (as != as) +#define as_create_arch(as, flags) (as != as) #define as_deinstall_arch(as) #define as_invalidate_translation_cache(as, page, cnt) Index: kernel/arch/ppc32/include/mm/asid.h =================================================================== --- kernel/arch/ppc32/include/mm/asid.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/mm/asid.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32mm +/** @addtogroup ppc32mm * @{ */ @@ -38,5 +38,5 @@ #include -#define ASID_MAX_ARCH 4096 +#define ASID_MAX_ARCH 4096 typedef uint32_t asid_t; Index: kernel/arch/ppc32/include/mm/frame.h =================================================================== --- kernel/arch/ppc32/include/mm/frame.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/mm/frame.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,4 +46,16 @@ extern uintptr_t last_frame; +static inline uint32_t physmem_top(void) +{ + uint32_t physmem; + + asm volatile ( + "mfsprg3 %[physmem]\n" + : [physmem] "=r" (physmem) + ); + + return physmem; +} + extern void frame_arch_init(void); extern void physmem_print(void); Index: kernel/arch/ppc32/include/mm/page.h =================================================================== --- kernel/arch/ppc32/include/mm/page.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/mm/page.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32mm +/** @addtogroup ppc32mm * @{ */ @@ -38,15 +38,15 @@ #include -#define PAGE_WIDTH FRAME_WIDTH -#define PAGE_SIZE FRAME_SIZE +#define PAGE_WIDTH FRAME_WIDTH +#define PAGE_SIZE FRAME_SIZE #ifdef KERNEL #ifndef __ASM__ -# define KA2PA(x) (((uintptr_t) (x)) - 0x80000000) -# define PA2KA(x) (((uintptr_t) (x)) + 0x80000000) + #define KA2PA(x) (((uintptr_t) (x)) - 0x80000000) + #define PA2KA(x) (((uintptr_t) (x)) + 0x80000000) #else -# define KA2PA(x) ((x) - 0x80000000) -# define PA2KA(x) ((x) + 0x80000000) + #define KA2PA(x) ((x) - 0x80000000) + #define PA2KA(x) ((x) + 0x80000000) #endif @@ -65,37 +65,43 @@ /* Number of entries in each level. */ -#define PTL0_ENTRIES_ARCH 1024 -#define PTL1_ENTRIES_ARCH 0 -#define PTL2_ENTRIES_ARCH 0 -#define PTL3_ENTRIES_ARCH 1024 +#define PTL0_ENTRIES_ARCH 1024 +#define PTL1_ENTRIES_ARCH 0 +#define PTL2_ENTRIES_ARCH 0 +#define PTL3_ENTRIES_ARCH 1024 /* Page table sizes for each level. */ -#define PTL0_SIZE_ARCH ONE_FRAME -#define PTL1_SIZE_ARCH 0 -#define PTL2_SIZE_ARCH 0 -#define PTL3_SIZE_ARCH ONE_FRAME +#define PTL0_SIZE_ARCH ONE_FRAME +#define PTL1_SIZE_ARCH 0 +#define PTL2_SIZE_ARCH 0 +#define PTL3_SIZE_ARCH ONE_FRAME /* Macros calculating indices into page tables on each level. */ -#define PTL0_INDEX_ARCH(vaddr) (((vaddr) >> 22) & 0x3ff) -#define PTL1_INDEX_ARCH(vaddr) 0 -#define PTL2_INDEX_ARCH(vaddr) 0 -#define PTL3_INDEX_ARCH(vaddr) (((vaddr) >> 12) & 0x3ff) +#define PTL0_INDEX_ARCH(vaddr) (((vaddr) >> 22) & 0x3ff) +#define PTL1_INDEX_ARCH(vaddr) 0 +#define PTL2_INDEX_ARCH(vaddr) 0 +#define PTL3_INDEX_ARCH(vaddr) (((vaddr) >> 12) & 0x3ff) /* Get PTE address accessors for each level. */ #define GET_PTL1_ADDRESS_ARCH(ptl0, i) \ (((pte_t *) (ptl0))[(i)].pfn << 12) + #define GET_PTL2_ADDRESS_ARCH(ptl1, i) \ (ptl1) + #define GET_PTL3_ADDRESS_ARCH(ptl2, i) \ (ptl2) -#define GET_FRAME_ADDRESS_ARCH(ptl3, i) \ + +#define GET_FRAME_ADDRESS_ARCH(ptl3, i) \ (((pte_t *) (ptl3))[(i)].pfn << 12) /* Set PTE address accessors for each level. */ #define SET_PTL0_ADDRESS_ARCH(ptl0) + #define SET_PTL1_ADDRESS_ARCH(ptl0, i, a) \ (((pte_t *) (ptl0))[(i)].pfn = (a) >> 12) + #define SET_PTL2_ADDRESS_ARCH(ptl1, i, a) #define SET_PTL3_ADDRESS_ARCH(ptl2, i, a) + #define SET_FRAME_ADDRESS_ARCH(ptl3, i, a) \ (((pte_t *) (ptl3))[(i)].pfn = (a) >> 12) @@ -104,25 +110,30 @@ #define GET_PTL1_FLAGS_ARCH(ptl0, i) \ get_pt_flags((pte_t *) (ptl0), (size_t) (i)) + #define GET_PTL2_FLAGS_ARCH(ptl1, i) \ PAGE_PRESENT + #define GET_PTL3_FLAGS_ARCH(ptl2, i) \ PAGE_PRESENT + #define GET_FRAME_FLAGS_ARCH(ptl3, i) \ get_pt_flags((pte_t *) (ptl3), (size_t) (i)) /* Set PTE flags accessors for each level. */ -#define SET_PTL1_FLAGS_ARCH(ptl0, i, x) \ +#define SET_PTL1_FLAGS_ARCH(ptl0, i, x) \ set_pt_flags((pte_t *) (ptl0), (size_t) (i), (x)) + #define SET_PTL2_FLAGS_ARCH(ptl1, i, x) #define SET_PTL3_FLAGS_ARCH(ptl2, i, x) + #define SET_FRAME_FLAGS_ARCH(ptl3, i, x) \ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) /* Macros for querying the last-level PTEs. */ -#define PTE_VALID_ARCH(pte) (*((uint32_t *) (pte)) != 0) -#define PTE_PRESENT_ARCH(pte) ((pte)->present != 0) -#define PTE_GET_FRAME_ARCH(pte) ((pte)->pfn << 12) -#define PTE_WRITABLE_ARCH(pte) 1 -#define PTE_EXECUTABLE_ARCH(pte) 1 +#define PTE_VALID_ARCH(pte) (*((uint32_t *) (pte)) != 0) +#define PTE_PRESENT_ARCH(pte) ((pte)->present != 0) +#define PTE_GET_FRAME_ARCH(pte) ((pte)->pfn << 12) +#define PTE_WRITABLE_ARCH(pte) 1 +#define PTE_EXECUTABLE_ARCH(pte) 1 #ifndef __ASM__ @@ -133,34 +144,34 @@ /** Page Table Entry. */ typedef struct { - unsigned present : 1; /**< Present bit. */ - unsigned page_write_through : 1; /**< Write thought caching. */ - unsigned page_cache_disable : 1; /**< No caching. */ - unsigned accessed : 1; /**< Accessed bit. */ - unsigned global : 1; /**< Global bit. */ - unsigned valid : 1; /**< Valid content even if not present. */ - unsigned pfn : 20; /**< Physical frame number. */ + unsigned int present : 1; /**< Present bit. */ + unsigned int page_write_through : 1; /**< Write thought caching. */ + unsigned int page_cache_disable : 1; /**< No caching. */ + unsigned int accessed : 1; /**< Accessed bit. */ + unsigned int global : 1; /**< Global bit. */ + unsigned int valid : 1; /**< Valid content even if not present. */ + unsigned int pfn : 20; /**< Physical frame number. */ } pte_t; static inline unsigned int get_pt_flags(pte_t *pt, size_t i) { - pte_t *p = &pt[i]; + pte_t *entry = &pt[i]; - return (((!p->page_cache_disable) << PAGE_CACHEABLE_SHIFT) | - ((!p->present) << PAGE_PRESENT_SHIFT) | + return (((!entry->page_cache_disable) << PAGE_CACHEABLE_SHIFT) | + ((!entry->present) << PAGE_PRESENT_SHIFT) | (1 << PAGE_USER_SHIFT) | (1 << PAGE_READ_SHIFT) | (1 << PAGE_WRITE_SHIFT) | (1 << PAGE_EXEC_SHIFT) | - (p->global << PAGE_GLOBAL_SHIFT)); + (entry->global << PAGE_GLOBAL_SHIFT)); } static inline void set_pt_flags(pte_t *pt, size_t i, int flags) { - pte_t *p = &pt[i]; + pte_t *entry = &pt[i]; - p->page_cache_disable = !(flags & PAGE_CACHEABLE); - p->present = !(flags & PAGE_NOT_PRESENT); - p->global = (flags & PAGE_GLOBAL) != 0; - p->valid = 1; + entry->page_cache_disable = !(flags & PAGE_CACHEABLE); + entry->present = !(flags & PAGE_NOT_PRESENT); + entry->global = (flags & PAGE_GLOBAL) != 0; + entry->valid = 1; } Index: kernel/arch/ppc32/include/mm/tlb.h =================================================================== --- kernel/arch/ppc32/include/mm/tlb.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/include/mm/tlb.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup ppc32mm +/** @addtogroup ppc32mm * @{ */ @@ -45,38 +45,41 @@ typedef struct { - unsigned v : 1; /**< Valid */ - unsigned vsid : 24; /**< Virtual Segment ID */ - unsigned h : 1; /**< Primary/secondary hash */ - unsigned api : 6; /**< Abbreviated Page Index */ - unsigned rpn : 20; /**< Real Page Number */ - unsigned reserved0 : 3; - unsigned r : 1; /**< Reference */ - unsigned c : 1; /**< Change */ - unsigned wimg : 4; /**< Access control */ - unsigned reserved1 : 1; - unsigned pp : 2; /**< Page protection */ + unsigned int v : 1; /**< Valid */ + unsigned int vsid : 24; /**< Virtual Segment ID */ + unsigned int h : 1; /**< Primary/secondary hash */ + unsigned int api : 6; /**< Abbreviated Page Index */ + unsigned int rpn : 20; /**< Real Page Number */ + unsigned int reserved0 : 3; + unsigned int r : 1; /**< Reference */ + unsigned int c : 1; /**< Change */ + unsigned int wimg : 4; /**< Access control */ + unsigned int reserved1 : 1; + unsigned int pp : 2; /**< Page protection */ } phte_t; typedef struct { - unsigned v : 1; - unsigned vsid : 24; - unsigned reserved0 : 1; - unsigned api : 6; + unsigned int v : 1; + unsigned int vsid : 24; + unsigned int reserved0 : 1; + unsigned int api : 6; } ptehi_t; typedef struct { - unsigned rpn : 20; - unsigned xpn : 3; - unsigned reserved0 : 1; - unsigned c : 1; - unsigned wimg : 4; - unsigned x : 1; - unsigned pp : 2; + unsigned int rpn : 20; + unsigned int xpn : 3; + unsigned int reserved0 : 1; + unsigned int c : 1; + unsigned int wimg : 4; + unsigned int x : 1; + unsigned int pp : 2; } ptelo_t; extern void pht_init(void); -extern void pht_refill(int n, istate_t *istate); -extern bool pht_refill_real(int n, istate_t *istate) __attribute__ ((section("K_UNMAPPED_TEXT_START"))); -extern void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate) __attribute__ ((section("K_UNMAPPED_TEXT_START"))); +extern void pht_refill(int, istate_t *); + +extern bool pht_refill_real(int, istate_t *) + __attribute__ ((section("K_UNMAPPED_TEXT_START"))); +extern void tlb_refill_real(int, uint32_t, ptehi_t, ptelo_t, istate_t *) + __attribute__ ((section("K_UNMAPPED_TEXT_START"))); #endif Index: rnel/arch/ppc32/include/regutils.h =================================================================== --- kernel/arch/ppc32/include/regutils.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,45 +1,0 @@ -/* - * Copyright (c) 2008 Jiri Svoboda - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup ppc32 - * @{ - */ -/** - * @file - * @brief Utilities for convenient manipulation with ppc32 registers. - */ - -#ifndef KERN_ppc32_REGUTILS_H_ -#define KERN_ppc32_REGUTILS_H_ - -#define MSR_PR (1 << 14) - -#endif - -/** @} - */ Index: kernel/arch/ppc32/src/asm.S =================================================================== --- kernel/arch/ppc32/src/asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -28,4 +28,5 @@ #include +#include .text @@ -43,5 +44,5 @@ userspace_asm: - + # r3 = uspace_uarg # r4 = stack @@ -49,5 +50,5 @@ # disable interrupts - + mfmsr r31 rlwinm r31, r31, 0, 17, 15 @@ -60,6 +61,6 @@ # set problem state, enable interrupts - ori r31, r31, msr_pr - ori r31, r31, msr_ee + ori r31, r31, MSR_PR + ori r31, r31, MSR_EE mtsrr1 r31 @@ -67,7 +68,7 @@ mr sp, r4 - + # %r6 is defined to hold pcb_ptr - set it to 0 - + xor r6, r6, r6 @@ -141,5 +142,5 @@ # reset decrementer - + li r31, 1000 mtdec r31 @@ -201,5 +202,5 @@ lwz r12, 156(sp) lwz sp, 160(sp) - + rfi @@ -213,9 +214,9 @@ memcpy_from_uspace: memcpy_to_uspace: - + srwi. r7, r5, 3 addi r6, r3, -4 addi r4, r4, -4 - beq 2f + beq 2f andi. r0, r6, 3 @@ -225,53 +226,53 @@ 1: - lwz r7, 4(r4) - lwzu r8, 8(r4) - stw r7, 4(r6) - stwu r8, 8(r6) - bdnz 1b - - andi. r5, r5, 7 + lwz r7, 4(r4) + lwzu r8, 8(r4) + stw r7, 4(r6) + stwu r8, 8(r6) + bdnz 1b + + andi. r5, r5, 7 2: - cmplwi 0, r5, 4 - blt 3f - - lwzu r0, 4(r4) - addi r5, r5, -4 - stwu r0, 4(r6) + cmplwi 0, r5, 4 + blt 3f + + lwzu r0, 4(r4) + addi r5, r5, -4 + stwu r0, 4(r6) 3: - cmpwi 0, r5, 0 - beqlr - mtctr r5 - addi r4, r4, 3 - addi r6, r6, 3 + cmpwi 0, r5, 0 + beqlr + mtctr r5 + addi r4, r4, 3 + addi r6, r6, 3 4: - lbzu r0, 1(r4) - stbu r0, 1(r6) - bdnz 4b - blr + lbzu r0, 1(r4) + stbu r0, 1(r6) + bdnz 4b + blr 5: - subfic r0, r0, 4 - mtctr r0 + subfic r0, r0, 4 + mtctr r0 6: - lbz r7, 4(r4) - addi r4, r4, 1 - stb r7, 4(r6) - addi r6, r6, 1 - bdnz 6b - subf r5, r0, r5 - rlwinm. r7, r5, 32-3, 3, 31 - beq 2b - mtctr r7 - b 1b + lbz r7, 4(r4) + addi r4, r4, 1 + stb r7, 4(r6) + addi r6, r6, 1 + bdnz 6b + subf r5, r0, r5 + rlwinm. r7, r5, 32-3, 3, 31 + beq 2b + mtctr r7 + b 1b memcpy_from_uspace_failover_address: Index: kernel/arch/ppc32/src/context.S =================================================================== --- kernel/arch/ppc32/src/context.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/context.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -29,5 +29,5 @@ #include -.text +.text .global context_save_arch @@ -46,5 +46,5 @@ li r3, 1 blr - + context_restore_arch: CONTEXT_RESTORE_ARCH_CORE r3 Index: kernel/arch/ppc32/src/cpu/cpu.c =================================================================== --- kernel/arch/ppc32/src/cpu/cpu.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/cpu/cpu.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -34,5 +34,4 @@ #include -#include #include #include @@ -45,33 +44,30 @@ void cpu_identify(void) { - cpu_info_t info; - - cpu_version(&info); - CPU->arch.version = info.version; - CPU->arch.revision = info.revision; + cpu_version(&CPU->arch); } -void cpu_print_report(cpu_t *m) +void cpu_print_report(cpu_t *cpu) { const char *name; - switch (m->arch.version) { + switch (cpu->arch.version) { case 8: - name = " (PowerPC 750)"; + name = "PowerPC 750"; break; case 9: - name = " (PowerPC 604e)"; + name = "PowerPC 604e"; break; case 0x81: - name = " (PowerPC 8260)"; + name = "PowerPC 8260"; break; case 0x8081: - name = " (PowerPC 826xA)"; + name = "PowerPC 826xA"; break; default: - name = ""; + name = "unknown"; } - printf("cpu%d: version=%d%s, revision=%d\n", m->id, m->arch.version, name, m->arch.revision); + printf("cpu%" PRIs ": version=%" PRIu16" (%s), revision=%" PRIu16 "\n", cpu->id, + cpu->arch.version, name, cpu->arch.revision); } Index: kernel/arch/ppc32/src/ddi/ddi.c =================================================================== --- kernel/arch/ppc32/src/ddi/ddi.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/ddi/ddi.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,4 +46,5 @@ * * @return 0 on success or an error code from errno.h. + * */ int ddi_iospace_enable_arch(task_t *task, uintptr_t ioaddr, size_t size) Index: kernel/arch/ppc32/src/debug/panic.s =================================================================== --- kernel/arch/ppc32/src/debug/panic.s (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/debug/panic.s (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -35,4 +35,4 @@ lis %r14, halt@ha addi %r14, %r14, halt@l - mtlr %r14 # fake stack to make printf return to halt + mtlr %r14 # fake stack to make printf return to halt b printf Index: kernel/arch/ppc32/src/debug/stacktrace.c =================================================================== --- kernel/arch/ppc32/src/debug/stacktrace.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/debug/stacktrace.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -37,32 +37,41 @@ #include +#define FRAME_OFFSET_FP_PREV 0 +#define FRAME_OFFSET_RA 1 + bool kernel_frame_pointer_validate(uintptr_t fp) { - return false; + return fp != 0; } bool kernel_frame_pointer_prev(uintptr_t fp, uintptr_t *prev) { - return false; + uint32_t *stack = (void *) fp; + *prev = stack[FRAME_OFFSET_FP_PREV]; + return true; } bool kernel_return_address_get(uintptr_t fp, uintptr_t *ra) { - return false; + uint32_t *stack = (void *) fp; + *ra = stack[FRAME_OFFSET_RA]; + return true; } bool uspace_frame_pointer_validate(uintptr_t fp) { - return false; + return fp != 0; } bool uspace_frame_pointer_prev(uintptr_t fp, uintptr_t *prev) { - return false; + return !copy_from_uspace((void *) prev, + (uint32_t *) fp + FRAME_OFFSET_FP_PREV, sizeof(*prev)); } bool uspace_return_address_get(uintptr_t fp, uintptr_t *ra) { - return false; + return !copy_from_uspace((void *) ra, (uint32_t *) fp + FRAME_OFFSET_RA, + sizeof(*ra)); } Index: kernel/arch/ppc32/src/debug/stacktrace_asm.S =================================================================== --- kernel/arch/ppc32/src/debug/stacktrace_asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/debug/stacktrace_asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,4 +27,6 @@ # +#include + .text @@ -33,6 +35,8 @@ frame_pointer_get: + mr r3, sp blr program_counter_get: + mflr r3 blr Index: kernel/arch/ppc32/src/drivers/pic.c =================================================================== --- kernel/arch/ppc32/src/drivers/pic.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/drivers/pic.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,5 +32,4 @@ /** @file */ - #include @@ -79,20 +78,22 @@ } -/** Return number of pending interrupt */ -int pic_get_pending(void) +/** Return number of pending interrupts + * + */ +uint8_t pic_get_pending(void) { if (pic) { - int pending; + uint32_t pending; pending = pic[PIC_PENDING_LOW]; - if (pending) + if (pending != 0) return fnzb32(pending); pending = pic[PIC_PENDING_HIGH]; - if (pending) + if (pending != 0) return fnzb32(pending) + 32; } - return -1; + return 255; } Index: kernel/arch/ppc32/src/exception.S =================================================================== --- kernel/arch/ppc32/src/exception.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/exception.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -28,4 +28,5 @@ #include +#include #include @@ -34,7 +35,7 @@ .macro CONTEXT_STORE - # save R12 in SPRG1, backup CR in R12 + # save r12 in SPRG1, backup CR in r12 # save SP in SPRG2 - + mtsprg1 r12 mfcr r12 @@ -288,5 +289,5 @@ mfmsr r12 - ori r12, r12, (msr_ir | msr_dr)@l + ori r12, r12, (MSR_IR | MSR_DR)@l mtsrr1 r12 @@ -307,5 +308,5 @@ mfmsr r12 - ori r12, r12, (msr_ir | msr_dr)@l + ori r12, r12, (MSR_IR | MSR_DR)@l mtsrr1 r12 Index: kernel/arch/ppc32/src/fpu_context.S =================================================================== --- kernel/arch/ppc32/src/fpu_context.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/fpu_context.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -30,5 +30,5 @@ #include -.text +.text .global fpu_context_save @@ -87,5 +87,5 @@ blr - + fpu_context_restore: // FPU_CONTEXT_LOAD r3 Index: kernel/arch/ppc32/src/interrupt.c =================================================================== --- kernel/arch/ppc32/src/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -44,21 +44,20 @@ #include - void start_decrementer(void) { asm volatile ( - "mtdec %0\n" - : - : "r" (1000) + "mtdec %[dec]\n" + :: [dec] "r" (1000) ); } - -/** Handler of external interrupts */ +/** External interrupts handler + * + */ static void exception_external(int n, istate_t *istate) { - int inum; + uint8_t inum; - while ((inum = pic_get_pending()) != -1) { + while ((inum = pic_get_pending()) != 255) { irq_t *irq = irq_dispatch_and_lock(inum); if (irq) { @@ -80,5 +79,5 @@ } - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } else { /* @@ -86,10 +85,10 @@ */ #ifdef CONFIG_DEBUG - printf("cpu%u: spurious interrupt (inum=%d)\n", CPU->id, inum); + printf("cpu%" PRIs ": spurious interrupt (inum=%" PRIu8 ")\n", + CPU->id, inum); #endif } } } - static void exception_decrementer(int n, istate_t *istate) @@ -98,5 +97,4 @@ clock(); } - /* Initialize basic tables for exception dispatching */ Index: kernel/arch/ppc32/src/mm/as.c =================================================================== --- kernel/arch/ppc32/src/mm/as.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/mm/as.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -55,26 +55,13 @@ void as_install_arch(as_t *as) { - asid_t asid; uint32_t sr; - - asid = as->asid; /* Lower 2 GB, user and supervisor access */ - for (sr = 0; sr < 8; sr++) { - asm volatile ( - "mtsrin %0, %1\n" - : - : "r" ((0x6000 << 16) + (asid << 4) + sr), "r" (sr << 28) - ); - } + for (sr = 0; sr < 8; sr++) + sr_set(0x6000, as->asid, sr); /* Upper 2 GB, only supervisor access */ - for (sr = 8; sr < 16; sr++) { - asm volatile ( - "mtsrin %0, %1\n" - : - : "r" ((0x4000 << 16) + (asid << 4) + sr), "r" (sr << 28) - ); - } + for (sr = 8; sr < 16; sr++) + sr_set(0x4000, as->asid, sr); } Index: kernel/arch/ppc32/src/mm/frame.c =================================================================== --- kernel/arch/ppc32/src/mm/frame.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/mm/frame.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -45,12 +45,11 @@ void physmem_print(void) { - unsigned int i; - printf("Base Size\n"); printf("---------- ----------\n"); - + + size_t i; for (i = 0; i < memmap.cnt; i++) { printf("%#10x %#10x\n", memmap.zones[i].start, - memmap.zones[i].size); + memmap.zones[i].size); } } @@ -60,11 +59,11 @@ pfn_t minconf = 2; size_t i; - pfn_t start, conf; - size_t size; for (i = 0; i < memmap.cnt; i++) { - start = ADDR2PFN(ALIGN_UP((uintptr_t) memmap.zones[i].start, FRAME_SIZE)); - size = SIZE2FRAMES(ALIGN_DOWN(memmap.zones[i].size, FRAME_SIZE)); + pfn_t start = ADDR2PFN(ALIGN_UP((uintptr_t) memmap.zones[i].start, + FRAME_SIZE)); + size_t size = SIZE2FRAMES(ALIGN_DOWN(memmap.zones[i].size, FRAME_SIZE)); + pfn_t conf; if ((minconf < start) || (minconf >= start + size)) conf = start; @@ -73,6 +72,8 @@ zone_create(start, size, conf, 0); - if (last_frame < ALIGN_UP((uintptr_t) memmap.zones[i].start + memmap.zones[i].size, FRAME_SIZE)) - last_frame = ALIGN_UP((uintptr_t) memmap.zones[i].start + memmap.zones[i].size, FRAME_SIZE); + if (last_frame < ALIGN_UP((uintptr_t) memmap.zones[i].start + + memmap.zones[i].size, FRAME_SIZE)) + last_frame = ALIGN_UP((uintptr_t) memmap.zones[i].start + + memmap.zones[i].size, FRAME_SIZE); } @@ -82,10 +83,8 @@ /* Mark the Page Hash Table frames as unavailable */ - uint32_t sdr1; - asm volatile ( - "mfsdr1 %0\n" - : "=r" (sdr1) - ); - frame_mark_unavailable(ADDR2PFN(sdr1 & 0xffff000), 16); // FIXME + uint32_t sdr1 = sdr1_get(); + + // FIXME: compute size of PHT exactly + frame_mark_unavailable(ADDR2PFN(sdr1 & 0xffff000), 16); } Index: kernel/arch/ppc32/src/mm/page.c =================================================================== --- kernel/arch/ppc32/src/mm/page.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/mm/page.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -54,7 +54,9 @@ uintptr_t virtaddr = PA2KA(last_frame); pfn_t i; + page_table_lock(AS_KERNEL, true); for (i = 0; i < ADDR2PFN(ALIGN_UP(size, PAGE_SIZE)); i++) page_mapping_insert(AS_KERNEL, virtaddr + PFN2ADDR(i), physaddr + PFN2ADDR(i), PAGE_NOT_CACHEABLE | PAGE_WRITE); + page_table_unlock(AS_KERNEL, true); last_frame = ALIGN_UP(last_frame + size, FRAME_SIZE); Index: kernel/arch/ppc32/src/mm/tlb.c =================================================================== --- kernel/arch/ppc32/src/mm/tlb.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/mm/tlb.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -45,30 +45,28 @@ static unsigned int seed = 10; -static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42; - +static unsigned int seed_real + __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42; /** Try to find PTE for faulting address * - * Try to find PTE for faulting address. - * The as->lock must be held on entry to this function - * if lock is true. - * - * @param as Address space. - * @param lock Lock/unlock the address space. - * @param badvaddr Faulting virtual address. - * @param access Access mode that caused the fault. - * @param istate Pointer to interrupted state. - * @param pfrc Pointer to variable where as_page_fault() return code - * will be stored. - * @return PTE on success, NULL otherwise. - * - */ -static pte_t * -find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access, + * @param as Address space. + * @param lock Lock/unlock the address space. + * @param badvaddr Faulting virtual address. + * @param access Access mode that caused the fault. + * @param istate Pointer to interrupted state. + * @param pfrc Pointer to variable where as_page_fault() return code + * will be stored. + * + * @return PTE on success, NULL otherwise. + * + */ +static pte_t *find_mapping_and_check(as_t *as, uintptr_t badvaddr, int access, istate_t *istate, int *pfrc) { + ASSERT(mutex_locked(&as->lock)); + /* * Check if the mapping exists in page tables. - */ + */ pte_t *pte = page_mapping_find(as, badvaddr); if ((pte) && (pte->present)) { @@ -79,12 +77,12 @@ return pte; } else { - int rc; - /* * Mapping not found in page tables. * Resort to higher-level page fault handler. */ - page_table_unlock(as, lock); - switch (rc = as_page_fault(badvaddr, access, istate)) { + page_table_unlock(as, true); + + int rc = as_page_fault(badvaddr, access, istate); + switch (rc) { case AS_PF_OK: /* @@ -92,5 +90,5 @@ * The mapping ought to be in place. */ - page_table_lock(as, lock); + page_table_lock(as, true); pte = page_mapping_find(as, badvaddr); ASSERT((pte) && (pte->present)); @@ -98,17 +96,16 @@ return pte; case AS_PF_DEFER: - page_table_lock(as, lock); + page_table_lock(as, true); *pfrc = rc; return NULL; case AS_PF_FAULT: - page_table_lock(as, lock); + page_table_lock(as, true); *pfrc = rc; return NULL; default: panic("Unexpected rc (%d).", rc); - } - } -} - + } + } +} static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate) @@ -123,5 +120,4 @@ } - static void pht_insert(const uintptr_t vaddr, const pte_t *pte) { @@ -129,16 +125,8 @@ uint32_t api = (vaddr >> 22) & 0x3f; - uint32_t vsid; - asm volatile ( - "mfsrin %0, %1\n" - : "=r" (vsid) - : "r" (vaddr) - ); - - uint32_t sdr1; - asm volatile ( - "mfsdr1 %0\n" - : "=r" (sdr1) - ); + uint32_t vsid = sr_get(vaddr); + uint32_t sdr1 = sdr1_get(); + + // FIXME: compute size of PHT exactly phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000); @@ -215,5 +203,4 @@ } - /** Process Instruction/Data Storage Exception * @@ -224,17 +211,6 @@ void pht_refill(int n, istate_t *istate) { + as_t *as = (AS == NULL) ? AS_KERNEL : AS; uintptr_t badvaddr; - pte_t *pte; - int pfrc; - as_t *as; - bool lock; - - if (AS == NULL) { - as = AS_KERNEL; - lock = false; - } else { - as = AS; - lock = true; - } if (n == VECTOR_DATA_STORAGE) @@ -242,9 +218,11 @@ else badvaddr = istate->pc; - - page_table_lock(as, lock); - - pte = find_mapping_and_check(as, lock, badvaddr, + + page_table_lock(as, true); + + int pfrc; + pte_t *pte = find_mapping_and_check(as, badvaddr, PF_ACCESS_READ /* FIXME */, istate, &pfrc); + if (!pte) { switch (pfrc) { @@ -257,5 +235,5 @@ * or copy_to_uspace(). */ - page_table_unlock(as, lock); + page_table_unlock(as, true); return; default: @@ -264,15 +242,15 @@ } - pte->accessed = 1; /* Record access to PTE */ + /* Record access to PTE */ + pte->accessed = 1; pht_insert(badvaddr, pte); - page_table_unlock(as, lock); + page_table_unlock(as, true); return; fail: - page_table_unlock(as, lock); + page_table_unlock(as, true); pht_refill_fail(badvaddr, istate); } - /** Process Instruction/Data Storage Exception in Real Mode @@ -291,9 +269,5 @@ badvaddr = istate->pc; - uint32_t physmem; - asm volatile ( - "mfsprg3 %0\n" - : "=r" (physmem) - ); + uint32_t physmem = physmem_top(); if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem))) @@ -303,16 +277,8 @@ uint32_t api = (badvaddr >> 22) & 0x3f; - uint32_t vsid; - asm volatile ( - "mfsrin %0, %1\n" - : "=r" (vsid) - : "r" (badvaddr) - ); - - uint32_t sdr1; - asm volatile ( - "mfsdr1 %0\n" - : "=r" (sdr1) - ); + uint32_t vsid = sr_get(badvaddr); + uint32_t sdr1 = sdr1_get(); + + // FIXME: compute size of PHT exactly phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000); @@ -396,5 +362,4 @@ } - /** Process ITLB/DTLB Miss Exception in Real Mode * @@ -404,10 +369,5 @@ { uint32_t badvaddr = tlbmiss & 0xfffffffc; - - uint32_t physmem; - asm volatile ( - "mfsprg3 %0\n" - : "=r" (physmem) - ); + uint32_t physmem = physmem_top(); if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem))) @@ -420,15 +380,14 @@ uint32_t index = 0; asm volatile ( - "mtspr 981, %0\n" - "mtspr 982, %1\n" - "tlbld %2\n" - "tlbli %2\n" - : "=r" (index) - : "r" (ptehi), - "r" (ptelo) + "mtspr 981, %[ptehi]\n" + "mtspr 982, %[ptelo]\n" + "tlbld %[index]\n" + "tlbli %[index]\n" + : [index] "=r" (index) + : [ptehi] "r" (ptehi), + [ptelo] "r" (ptelo) ); } - void tlb_arch_init(void) { @@ -436,15 +395,15 @@ } - void tlb_invalidate_all(void) { uint32_t index; + asm volatile ( - "li %0, 0\n" + "li %[index], 0\n" "sync\n" ".rept 64\n" - "tlbie %0\n" - "addi %0, %0, 0x1000\n" + " tlbie %[index]\n" + " addi %[index], %[index], 0x1000\n" ".endr\n" @@ -452,19 +411,16 @@ "tlbsync\n" "sync\n" - : "=r" (index) + : [index] "=r" (index) ); } - void tlb_invalidate_asid(asid_t asid) { - uint32_t sdr1; - asm volatile ( - "mfsdr1 %0\n" - : "=r" (sdr1) - ); + uint32_t sdr1 = sdr1_get(); + + // FIXME: compute size of PHT exactly phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000); - uint32_t i; + size_t i; for (i = 0; i < 8192; i++) { if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) && @@ -472,7 +428,7 @@ phte[i].v = 0; } + tlb_invalidate_all(); } - void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt) @@ -482,15 +438,17 @@ } - #define PRINT_BAT(name, ureg, lreg) \ asm volatile ( \ - "mfspr %0," #ureg "\n" \ - "mfspr %1," #lreg "\n" \ - : "=r" (upper), "=r" (lower) \ + "mfspr %[upper], " #ureg "\n" \ + "mfspr %[lower], " #lreg "\n" \ + : [upper] "=r" (upper), \ + [lower] "=r" (lower) \ ); \ + \ mask = (upper & 0x1ffc) >> 2; \ if (upper & 3) { \ uint32_t tmp = mask; \ length = 128; \ + \ while (tmp) { \ if ((tmp & 1) == 0) { \ @@ -503,4 +461,5 @@ } else \ length = 0; \ + \ printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \ sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \ @@ -515,10 +474,6 @@ for (sr = 0; sr < 16; sr++) { - uint32_t vsid; - asm volatile ( - "mfsrin %0, %1\n" - : "=r" (vsid) - : "r" (sr << 28) - ); + uint32_t vsid = sr_get(sr << 28); + printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr, sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4, Index: kernel/arch/ppc32/src/ppc32.c =================================================================== --- kernel/arch/ppc32/src/ppc32.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/ppc32.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -201,5 +201,5 @@ pic_init(assigned_address[0].addr, PAGE_SIZE, &pic_cir, &pic_cir_arg); - + #ifdef CONFIG_MAC_KBD uintptr_t pa = assigned_address[0].addr + 0x16000; @@ -223,5 +223,5 @@ } } - + /* * This is the necessary evil until the userspace driver is entirely @@ -284,5 +284,5 @@ { // TODO - while (1); + while (true); } Index: kernel/arch/ppc32/src/proc/scheduler.c =================================================================== --- kernel/arch/ppc32/src/proc/scheduler.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/ppc32/src/proc/scheduler.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -39,17 +39,21 @@ #include -/** Perform ppc32 specific tasks needed before the new task is run. */ +/** Perform ppc32 specific tasks needed before the new task is run. + * + */ void before_task_runs_arch(void) { } -/** Perform ppc32 specific tasks needed before the new thread is scheduled. */ +/** Perform ppc32 specific tasks needed before the new thread is scheduled. + * + */ void before_thread_runs_arch(void) { tlb_invalidate_all(); + asm volatile ( - "mtsprg0 %0\n" - : - : "r" (KA2PA(&THREAD->kstack[THREAD_STACK_SIZE - SP_DELTA])) + "mtsprg0 %[ksp]\n" + :: [ksp] "r" (KA2PA(&THREAD->kstack[THREAD_STACK_SIZE - SP_DELTA])) ); } Index: kernel/arch/sparc64/Makefile.inc =================================================================== --- kernel/arch/sparc64/Makefile.inc (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/Makefile.inc (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -31,8 +31,8 @@ BFD = binary -GCC_CFLAGS += -m64 -mcpu=ultrasparc +GCC_CFLAGS += -m64 -mcpu=ultrasparc -mcmodel=medlow SUNCC_CFLAGS += -m64 -xarch=sparc -xregs=appl,no%float -LFLAGS = -no-check-sections -N +LFLAGS += -no-check-sections BITS = 64 @@ -90,9 +90,4 @@ arch/$(KARCH)/src/trap/$(USARCH)/interrupt.c -ifeq ($(USARCH),sun4u) - ARCH_SOURCES += \ - arch/$(KARCH)/src/mm/cache.S -endif - ifeq ($(USARCH),sun4v) ARCH_SOURCES += \ Index: kernel/arch/sparc64/include/asm.h =================================================================== --- kernel/arch/sparc64/include/asm.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/asm.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -308,4 +308,17 @@ } +/** Check interrupts state. + * + * @return True if interrupts are disabled. + * + */ +static inline bool interrupts_disabled(void) +{ + pstate_reg_t pstate; + + pstate.value = pstate_read(); + return !pstate.ie; +} + /** Return base address of current stack. * @@ -433,4 +446,5 @@ extern void asm_delay_loop(const uint32_t usec); +extern uint64_t read_from_ag_g6(void); extern uint64_t read_from_ag_g7(void); extern void write_to_ag_g6(uint64_t val); Index: kernel/arch/sparc64/include/atomic.h =================================================================== --- kernel/arch/sparc64/include/atomic.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/atomic.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -135,6 +135,5 @@ "brz %2, 0b\n" "nop\n" - "ba %%xcc, 1b\n" - "nop\n" + "ba,a %%xcc, 1b\n" "2:\n" : "+m" (*((atomic_count_t *) ptr)), Index: kernel/arch/sparc64/include/barrier.h =================================================================== --- kernel/arch/sparc64/include/barrier.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/barrier.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,4 +36,10 @@ #define KERN_sparc64_BARRIER_H_ +#ifdef KERNEL +#include +#else +#include +#endif + /* * Our critical section barriers are prepared for the weakest RMO memory model. @@ -64,4 +70,6 @@ static inline void flush_pipeline(void) { + uint64_t pc; + /* * The FLUSH instruction takes address parameter. @@ -70,9 +78,13 @@ * The entire kernel text is mapped by a locked ITLB and * DTLB entries. Therefore, when this function is called, - * the %o7 register will always be in the range mapped by + * the %pc register will always be in the range mapped by * DTLB. */ - asm volatile ("flush %o7\n"); + asm volatile ( + "rd %%pc, %0\n" + "flush %0\n" + : "=&r" (pc) + ); } Index: kernel/arch/sparc64/include/boot/boot.h =================================================================== --- kernel/arch/sparc64/include/boot/boot.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/boot/boot.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -87,5 +87,4 @@ extern memmap_t memmap; -extern uintptr_t physmem_start; #endif Index: rnel/arch/sparc64/include/debug.h =================================================================== --- kernel/arch/sparc64/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,41 +1,0 @@ -/* - * Copyright (c) 2005 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup sparc64debug - * @{ - */ -/** @file - */ - -#ifndef KERN_sparc64_DEBUG_H_ -#define KERN_sparc64_DEBUG_H_ - -#endif - -/** @} - */ Index: kernel/arch/sparc64/include/drivers/scr.h =================================================================== --- kernel/arch/sparc64/include/drivers/scr.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/drivers/scr.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -44,5 +44,6 @@ SCR_FFB, SCR_CGSIX, - SCR_XVR + SCR_XVR, + SCR_QEMU_VGA } scr_type_t; Index: kernel/arch/sparc64/include/mm/cache_spec.h =================================================================== --- kernel/arch/sparc64/include/mm/cache_spec.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/mm/cache_spec.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64mm +/** @addtogroup sparc64mm * @{ */ @@ -39,17 +39,19 @@ * The following macros are valid for the following processors: * - * UltraSPARC, UltraSPARC II, UltraSPARC IIi, UltraSPARC III, - * UltraSPARC III+, UltraSPARC IV, UltraSPARC IV+ - * + * UltraSPARC, UltraSPARC II, UltraSPARC IIi, UltraSPARC III, + * UltraSPARC III+, UltraSPARC IV, UltraSPARC IV+ + * * Should we support other UltraSPARC processors, we need to make sure that * the macros are defined correctly for them. */ - + #if defined (US) -#define DCACHE_SIZE (16 * 1024) + #define DCACHE_SIZE (16 * 1024) #elif defined (US3) -#define DCACHE_SIZE (64 * 1024) + #define DCACHE_SIZE (64 * 1024) #endif -#define DCACHE_LINE_SIZE 32 + +#define DCACHE_LINE_SIZE 32 +#define DCACHE_TAG_SHIFT 2 #endif Index: kernel/arch/sparc64/include/trap/sun4v/mmu.h =================================================================== --- kernel/arch/sparc64/include/trap/sun4v/mmu.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/trap/sun4v/mmu.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -96,5 +96,5 @@ /* service by higher-level routine when context != 0 */ - brnz %g3, 0f + brnz %g3, 0f nop /* exclude page number 0 from installing the identity mapping */ @@ -107,8 +107,7 @@ * returns. */ - ba install_identity_mapping - nop + ba,a %xcc, install_identity_mapping - 0: +0: /* Index: kernel/arch/sparc64/include/trap/trap_table.h =================================================================== --- kernel/arch/sparc64/include/trap/trap_table.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/include/trap/trap_table.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -42,4 +42,26 @@ #define TRAP_TABLE_SIZE (TRAP_TABLE_ENTRY_COUNT * TRAP_TABLE_ENTRY_SIZE) +/* + * The following needs to be in sync with the definition of the istate + * structure. The one STACK_ITEM_SIZE is counted for space holding the 7th + * argument to syscall_handler (i.e. syscall number) and the other + * STACK_ITEM_SIZE is counted because of the required alignment. + */ +#define PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE \ + (STACK_WINDOW_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE + \ + (2 * STACK_ITEM_SIZE) + (12 * 8)) +#define SAVED_TSTATE -(1 * 8) +#define SAVED_TPC -(2 * 8) +#define SAVED_TNPC -(3 * 8) /* <-- istate_t begins here */ +#define SAVED_Y -(4 * 8) +#define SAVED_I0 -(5 * 8) +#define SAVED_I1 -(6 * 8) +#define SAVED_I2 -(7 * 8) +#define SAVED_I3 -(8 * 8) +#define SAVED_I4 -(9 * 8) +#define SAVED_I5 -(10 * 8) +#define SAVED_I6 -(11 * 8) +#define SAVED_I7 -(12 * 8) + #ifndef __ASM__ @@ -77,29 +99,7 @@ .endm -/* - * The following needs to be in sync with the definition of the istate - * structure. The one STACK_ITEM_SIZE is counted for space holding the 7th - * argument to syscall_handler (i.e. syscall number) and the other - * STACK_ITEM_SIZE is counted because of the required alignment. - */ -#define PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE \ - (STACK_WINDOW_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE + \ - (2 * STACK_ITEM_SIZE) + (12 * 8)) -#define SAVED_TSTATE -(1 * 8) -#define SAVED_TPC -(2 * 8) -#define SAVED_TNPC -(3 * 8) /* <-- istate_t begins here */ -#define SAVED_Y -(4 * 8) -#define SAVED_I0 -(5 * 8) -#define SAVED_I1 -(6 * 8) -#define SAVED_I2 -(7 * 8) -#define SAVED_I3 -(8 * 8) -#define SAVED_I4 -(9 * 8) -#define SAVED_I5 -(10 * 8) -#define SAVED_I6 -(11 * 8) -#define SAVED_I7 -(12 * 8) - .macro PREEMPTIBLE_HANDLER f sethi %hi(\f), %g1 - b preemptible_handler + ba %xcc, preemptible_handler or %g1, %lo(\f), %g1 .endm Index: kernel/arch/sparc64/src/debug/stacktrace.c =================================================================== --- kernel/arch/sparc64/src/debug/stacktrace.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/debug/stacktrace.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -37,17 +37,49 @@ #include +#include +#include +#include + +#if defined(SUN4V) +#include +#endif + +#define FRAME_OFFSET_FP_PREV 14 +#define FRAME_OFFSET_RA 15 + +extern void alloc_window_and_flush(void); + bool kernel_frame_pointer_validate(uintptr_t fp) { - return false; + uintptr_t kstack; + +#if defined(SUN4U) + kstack = read_from_ag_g6(); +#elif defined(SUN4V) + kstack = asi_u64_read(ASI_SCRATCHPAD, SCRATCHPAD_KSTACK); +#endif + + kstack += STACK_BIAS; + kstack -= PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE; + + if (THREAD && (fp == kstack)) + return false; + return fp != 0; } bool kernel_frame_pointer_prev(uintptr_t fp, uintptr_t *prev) { - return false; + uint64_t *stack = (void *) fp; + alloc_window_and_flush(); + *prev = stack[FRAME_OFFSET_FP_PREV] + STACK_BIAS; + return true; } bool kernel_return_address_get(uintptr_t fp, uintptr_t *ra) { - return false; + uint64_t *stack = (void *) fp; + alloc_window_and_flush(); + *ra = stack[FRAME_OFFSET_RA]; + return true; } Index: kernel/arch/sparc64/src/debug/stacktrace_asm.S =================================================================== --- kernel/arch/sparc64/src/debug/stacktrace_asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/debug/stacktrace_asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,15 +27,26 @@ # +#include + .text .global frame_pointer_get .global program_counter_get +.global alloc_window_and_flush frame_pointer_get: + # Add the stack bias to %sp to get the actual address. retl - nop + add %sp, STACK_BIAS, %o0 program_counter_get: retl - nop + mov %o7, %o0 +alloc_window_and_flush: + save %sp, -(STACK_WINDOW_SAVE_AREA_SIZE+STACK_ARG_SAVE_AREA_SIZE), %sp + # Flush all other windows to memory so that we can read their contents. + flushw + ret + restore + Index: kernel/arch/sparc64/src/drivers/scr.c =================================================================== --- kernel/arch/sparc64/src/drivers/scr.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/drivers/scr.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -77,4 +77,6 @@ else if (str_cmp(name, "cgsix") == 0) scr_type = SCR_CGSIX; + else if (str_cmp(name, "QEMU,VGA") == 0) + scr_type = SCR_QEMU_VGA; if (scr_type == SCR_UNKNOWN) { @@ -228,4 +230,47 @@ break; + + case SCR_QEMU_VGA: + if (prop->size / sizeof(ofw_pci_reg_t) < 2) { + printf("Too few screen registers.\n"); + return; + } + + pci_reg = &((ofw_pci_reg_t *) prop->value)[1]; + + if (!ofw_pci_reg_absolutize(node, pci_reg, &pci_abs_reg)) { + printf("Failed to absolutize fb register.\n"); + return; + } + + if (!ofw_pci_apply_ranges(node->parent, &pci_abs_reg, + &fb_addr)) { + printf("Failed to determine screen address.\n"); + return; + } + + switch (fb_depth) { + case 8: + fb_scanline = fb_linebytes * (fb_depth >> 3); + visual = VISUAL_INDIRECT_8; + break; + case 16: + fb_scanline = fb_linebytes * (fb_depth >> 3); + visual = VISUAL_RGB_5_6_5_BE; + break; + case 24: + fb_scanline = fb_linebytes * 4; + visual = VISUAL_BGR_8_8_8_0; + break; + case 32: + fb_scanline = fb_linebytes * (fb_depth >> 3); + visual = VISUAL_RGB_0_8_8_8; + break; + default: + printf("Unsupported bits per pixel.\n"); + return; + } + break; + default: panic("Unexpected type."); Index: kernel/arch/sparc64/src/drivers/sgcn.c =================================================================== --- kernel/arch/sparc64/src/drivers/sgcn.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/drivers/sgcn.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -132,5 +132,5 @@ static void init_sram_begin(void) { - ASSERT(instance) + ASSERT(instance); ofw_tree_node_t *chosen = ofw_tree_lookup("/chosen"); Index: rnel/arch/sparc64/src/mm/cache.S =================================================================== --- kernel/arch/sparc64/src/mm/cache.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,49 +1,0 @@ -/* - * Copyright (c) 2006 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#define DCACHE_TAG_SHIFT 2 - -.register %g2, #scratch -.register %g3, #scratch - -/** Flush the whole D-cache. */ -.global dcache_flush -dcache_flush: - set (DCACHE_SIZE - DCACHE_LINE_SIZE), %g1 - stxa %g0, [%g1] ASI_DCACHE_TAG -0: membar #Sync - subcc %g1, DCACHE_LINE_SIZE, %g1 - bnz,pt %xcc, 0b - stxa %g0, [%g1] ASI_DCACHE_TAG - membar #Sync - retl - ! beware SF Erratum #51, do not put the MEMBAR here - nop Index: kernel/arch/sparc64/src/mm/sun4u/as.c =================================================================== --- kernel/arch/sparc64/src/mm/sun4u/as.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/mm/sun4u/as.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -41,4 +41,5 @@ #ifdef CONFIG_TSB + #include #include @@ -47,4 +48,5 @@ #include #include + #endif /* CONFIG_TSB */ @@ -58,5 +60,5 @@ } -int as_constructor_arch(as_t *as, int flags) +int as_constructor_arch(as_t *as, unsigned int flags) { #ifdef CONFIG_TSB @@ -64,20 +66,22 @@ * The order must be calculated with respect to the emulated * 16K page size. - */ - int order = fnzb32(((ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * + * + */ + uint8_t order = fnzb32(((ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * sizeof(tsb_entry_t)) >> FRAME_WIDTH); - + uintptr_t tsb = (uintptr_t) frame_alloc(order, flags | FRAME_KA); - + if (!tsb) return -1; - + as->arch.itsb = (tsb_entry_t *) tsb; as->arch.dtsb = (tsb_entry_t *) (tsb + ITSB_ENTRY_COUNT * sizeof(tsb_entry_t)); - + memsetb(as->arch.itsb, (ITSB_ENTRY_COUNT + DTSB_ENTRY_COUNT) * sizeof(tsb_entry_t), 0); #endif + return 0; } @@ -93,4 +97,5 @@ sizeof(tsb_entry_t)) >> FRAME_WIDTH; frame_free(KA2PA((uintptr_t) as->arch.itsb)); + return cnt; #else @@ -99,9 +104,10 @@ } -int as_create_arch(as_t *as, int flags) +int as_create_arch(as_t *as, unsigned int flags) { #ifdef CONFIG_TSB tsb_invalidate(as, 0, (size_t) -1); #endif + return 0; } @@ -123,4 +129,5 @@ * * Moreover, the as->asid is protected by asidlock, which is being held. + * */ @@ -130,16 +137,17 @@ * secondary context register from the TL=1 code just before switch to * userspace. + * */ ctx.v = 0; ctx.context = as->asid; mmu_secondary_context_write(ctx.v); - -#ifdef CONFIG_TSB + +#ifdef CONFIG_TSB uintptr_t base = ALIGN_DOWN(config.base, 1 << KERNEL_PAGE_WIDTH); - + ASSERT(as->arch.itsb && as->arch.dtsb); - + uintptr_t tsb = (uintptr_t) as->arch.itsb; - + if (!overlaps(tsb, 8 * MMU_PAGE_SIZE, base, 1 << KERNEL_PAGE_WIDTH)) { /* @@ -147,18 +155,20 @@ * by the locked 4M kernel DTLB entry. We need * to map both TSBs explicitly. + * */ dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_NUCLEUS, tsb); dtlb_insert_mapping(tsb, KA2PA(tsb), PAGESIZE_64K, true, true); } - + /* * Setup TSB Base registers. + * */ tsb_base_reg_t tsb_base; - + tsb_base.value = 0; tsb_base.size = TSB_SIZE; tsb_base.split = 0; - + tsb_base.base = ((uintptr_t) as->arch.itsb) >> MMU_PAGE_WIDTH; itsb_base_write(tsb_base.value); @@ -175,5 +185,6 @@ * Clearing the extension registers will ensure that the value of the * TSB Base register will be used as an address of TSB, making the code - * compatible with the US port. + * compatible with the US port. + * */ itsb_primary_extension_write(0); @@ -195,5 +206,4 @@ void as_deinstall_arch(as_t *as) { - /* * Note that we don't and may not lock the address space. That's ok @@ -201,13 +211,14 @@ * * Moreover, the as->asid is protected by asidlock, which is being held. - */ - + * + */ + #ifdef CONFIG_TSB uintptr_t base = ALIGN_DOWN(config.base, 1 << KERNEL_PAGE_WIDTH); - + ASSERT(as->arch.itsb && as->arch.dtsb); - + uintptr_t tsb = (uintptr_t) as->arch.itsb; - + if (!overlaps(tsb, 8 * MMU_PAGE_SIZE, base, 1 << KERNEL_PAGE_WIDTH)) { /* Index: kernel/arch/sparc64/src/mm/sun4v/as.c =================================================================== --- kernel/arch/sparc64/src/mm/sun4v/as.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/mm/sun4v/as.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -44,4 +44,5 @@ #ifdef CONFIG_TSB + #include #include @@ -50,4 +51,5 @@ #include #include + #endif /* CONFIG_TSB */ @@ -61,15 +63,15 @@ } -int as_constructor_arch(as_t *as, int flags) +int as_constructor_arch(as_t *as, unsigned int flags) { #ifdef CONFIG_TSB - int order = fnzb32( + uint8_t order = fnzb32( (TSB_ENTRY_COUNT * sizeof(tsb_entry_t)) >> FRAME_WIDTH); - + uintptr_t tsb = (uintptr_t) frame_alloc(order, flags); - + if (!tsb) return -1; - + as->arch.tsb_description.page_size = PAGESIZE_8K; as->arch.tsb_description.associativity = 1; @@ -79,8 +81,9 @@ as->arch.tsb_description.reserved = 0; as->arch.tsb_description.context = 0; - + memsetb((void *) PA2KA(as->arch.tsb_description.tsb_base), TSB_ENTRY_COUNT * sizeof(tsb_entry_t), 0); #endif + return 0; } @@ -91,4 +94,5 @@ size_t cnt = (TSB_ENTRY_COUNT * sizeof(tsb_entry_t)) >> FRAME_WIDTH; frame_free((uintptr_t) as->arch.tsb_description.tsb_base); + return cnt; #else @@ -97,9 +101,10 @@ } -int as_create_arch(as_t *as, int flags) +int as_create_arch(as_t *as, unsigned int flags) { #ifdef CONFIG_TSB tsb_invalidate(as, 0, (size_t) -1); #endif + return 0; } @@ -111,14 +116,16 @@ * * @param as Address space. + * */ void as_install_arch(as_t *as) { mmu_secondary_context_write(as->asid); -#ifdef CONFIG_TSB + +#ifdef CONFIG_TSB uintptr_t base = ALIGN_DOWN(config.base, 1 << KERNEL_PAGE_WIDTH); - + ASSERT(as->arch.tsb_description.tsb_base); uintptr_t tsb = PA2KA(as->arch.tsb_description.tsb_base); - + if (!overlaps(tsb, 8 * MMU_PAGE_SIZE, base, 1 << KERNEL_PAGE_WIDTH)) { /* @@ -126,11 +133,11 @@ * by the locked 4M kernel DTLB entry. We need * to map both TSBs explicitly. + * */ mmu_demap_page(tsb, 0, MMU_FLAG_DTLB); dtlb_insert_mapping(tsb, KA2PA(tsb), PAGESIZE_64K, true, true); } - + __hypercall_fast2(MMU_TSB_CTXNON0, 1, KA2PA(&(as->arch.tsb_description))); - #endif } @@ -142,8 +149,8 @@ * * @param as Address space. + * */ void as_deinstall_arch(as_t *as) { - /* * Note that we don't and may not lock the address space. That's ok @@ -151,13 +158,14 @@ * * Moreover, the as->asid is protected by asidlock, which is being held. + * */ - + #ifdef CONFIG_TSB uintptr_t base = ALIGN_DOWN(config.base, 1 << KERNEL_PAGE_WIDTH); - + ASSERT(as->arch.tsb_description.tsb_base); - + uintptr_t tsb = PA2KA(as->arch.tsb_description.tsb_base); - + if (!overlaps(tsb, 8 * MMU_PAGE_SIZE, base, 1 << KERNEL_PAGE_WIDTH)) { /* @@ -165,4 +173,5 @@ * by the locked 4M kernel DTLB entry. We need * to demap the entry installed by as_install_arch(). + * */ __hypercall_fast3(MMU_UNMAP_PERM_ADDR, tsb, 0, MMU_FLAG_DTLB); Index: kernel/arch/sparc64/src/smp/sun4v/smp.c =================================================================== --- kernel/arch/sparc64/src/smp/sun4v/smp.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/smp/sun4v/smp.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -28,5 +28,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ @@ -292,5 +292,5 @@ exec_units[i].strand_count = 0; atomic_set(&(exec_units[i].nrdy), 0); - spinlock_initialize(&(exec_units[i].proposed_nrdy_lock), "proposed nrdy lock"); + spinlock_initialize(&(exec_units[i].proposed_nrdy_lock), "exec_units[].proposed_nrdy_lock"); exec_unit_count++; } @@ -330,5 +330,5 @@ exec_units[0].strand_count = cpu_count; exec_units[0].exec_unit_id = 1; - spinlock_initialize(&(exec_units[0].proposed_nrdy_lock), "proposed nrdy lock"); + spinlock_initialize(&(exec_units[0].proposed_nrdy_lock), "exec_units[0].proposed_nrdy_lock"); atomic_set(&(exec_units[0].nrdy), 0); max_core_strands = cpu_count; @@ -433,5 +433,5 @@ if (__hypercall_fast4(CPU_START, cpuid, (uint64_t) KA2PA(kernel_image_start), KA2PA(trap_table), - physmem_start) != EOK) + physmem_base) != EOK) return false; #endif Index: kernel/arch/sparc64/src/sun4u/asm.S =================================================================== --- kernel/arch/sparc64/src/sun4u/asm.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/sun4u/asm.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -67,4 +67,8 @@ WRITE_ALTERNATE_REGISTER %g6, PSTATE_IG_BIT +.global read_from_ag_g6 +read_from_ag_g6: + READ_ALTERNATE_REGISTER %g6, PSTATE_AG_BIT + .global read_from_ag_g7 read_from_ag_g7: Index: kernel/arch/sparc64/src/sun4u/start.S =================================================================== --- kernel/arch/sparc64/src/sun4u/start.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/sun4u/start.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,4 +36,5 @@ #include #include +#include #ifdef CONFIG_SMP @@ -62,9 +63,10 @@ * * The registers are expected to be in this state: - * - %o0 bootinfo structure address (BSP only) - * - %o1 starting address of physical memory + * - %o0 starting address of physical memory * + bootstrap processor flag * bits 63...1: physical memory starting address / 2 * bit 0: non-zero on BSP processor, zero on AP processors + * - %o1 bootinfo structure address (BSP only) + * * * Moreover, we depend on boot having established the following environment: @@ -77,6 +79,6 @@ kernel_image_start: mov BSP_FLAG, %l0 - and %o1, %l0, %l7 ! l7 <= bootstrap processor? - andn %o1, %l0, %l6 ! l6 <= start of physical memory + and %o0, %l0, %l7 ! l7 <= bootstrap processor? + andn %o0, %l0, %l6 ! l6 <= start of physical memory ! Get bits (PHYSMEM_ADDR_SIZE - 1):13 of physmem_base. @@ -268,11 +270,16 @@ or %l3, %l5, %l3 stx %l3, [%l4 + %lo(kernel_8k_tlb_data_template)] - - /* - * Flush D-Cache. - */ - call dcache_flush - nop - + + ! flush the whole D-cache + set (DCACHE_SIZE - DCACHE_LINE_SIZE), %g1 + stxa %g0, [%g1] ASI_DCACHE_TAG + +0: + membar #Sync + subcc %g1, DCACHE_LINE_SIZE, %g1 + bnz,pt %xcc, 0b + stxa %g0, [%g1] ASI_DCACHE_TAG + membar #Sync + /* * So far, we have not touched the stack. @@ -282,8 +289,18 @@ or %sp, %lo(temporary_boot_stack), %sp sub %sp, STACK_BIAS, %sp - + + /* + * Call arch_pre_main(bootinfo) + */ call arch_pre_main - nop - + mov %o1, %o0 + + /* + * Create the first stack frame. + */ + save %sp, -(STACK_WINDOW_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE), %sp + flushw + add %g0, -STACK_BIAS, %fp + call main_bsp nop @@ -292,6 +309,5 @@ 0: - ba %xcc, 0b - nop + ba,a %xcc, 0b @@ -343,4 +359,11 @@ ldx [%g1], %o6 + /* + * Create the first stack frame. + */ + save %sp, -(STACK_WINDOW_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE), %sp + flushw + add %g0, -STACK_BIAS, %fp + call main_ap nop @@ -350,6 +373,5 @@ 0: - ba %xcc, 0b - nop + ba,a %xcc, 0b Index: kernel/arch/sparc64/src/sun4v/start.S =================================================================== --- kernel/arch/sparc64/src/sun4v/start.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/sun4v/start.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -93,9 +93,10 @@ * * The registers are expected to be in this state: - * - %o0 bootinfo structure address (BSP only) - * - %o1 starting address of physical memory + * - %o0 starting address of physical memory * + bootstrap processor flag * bits 63...1: physical memory starting address / 2 * bit 0: non-zero on BSP processor, zero on AP processors + * - %o1 bootinfo structure address (BSP only) + * * * Moreover, we depend on boot having established the following environment: @@ -107,7 +108,7 @@ kernel_image_start: mov BSP_FLAG, %l0 - and %o1, %l0, %l7 ! l7 <= bootstrap processor? - andn %o1, %l0, %l6 ! l6 <= start of physical memory - or %o0, %g0, %l0 + and %o0, %l0, %l7 ! l7 <= bootstrap processor? + andn %o0, %l0, %l6 ! l6 <= start of physical memory + or %o1, %g0, %l1 ! Get bits (PHYSMEM_ADDR_SIZE - 1):13 of physmem_base. @@ -154,5 +155,4 @@ mov 0, %o5 ta 0xff - nop /* @@ -218,5 +218,5 @@ ! on APs skip executing the following code cmp %l7, 0 - be 1f + be %xcc, 1f nop @@ -246,8 +246,17 @@ sub %sp, STACK_BIAS, %sp - or %l0, %g0, %o0 + /* + * Call arch_pre_main(bootinfo) + */ call arch_pre_main - nop + or %l1, %g0, %o0 + /* + * Create the first stack frame. + */ + save %sp, -(STACK_WINDOW_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE), %sp + flushw + add %g0, -STACK_BIAS, %fp + call main_bsp nop @@ -256,6 +265,5 @@ 0: - ba 0b - nop + ba,a %xcc, 0b 1: @@ -269,8 +277,8 @@ */ - mov 1, %o0 ! MMU enable flag - set mmu_enabled, %o1 - mov MMU_ENABLE, %o5 ! MMU enable HV call - ta 0x80 ! call HV + mov 1, %o0 ! MMU enable flag + set mmu_enabled, %o1 + mov MMU_ENABLE, %o5 ! MMU enable HV call + ta 0x80 ! call HV mmu_enabled: @@ -285,4 +293,11 @@ ldx [%g1], %o6 + /* + * Create the first stack frame. + */ + save %sp, -(STACK_WINDWO_SAVE_AREA_SIZE + STACK_ARG_SAVE_AREA_SIZE), %sp + flushw + add %g0, -STACK_BIAS, %fp + call main_ap nop @@ -291,6 +306,5 @@ /* Not reached. */ 0: - ba 0b - nop + ba,a %xcc, 0b .align 8 @@ -310,6 +324,5 @@ before_ap_boots: setx 0x80400000, %g0, %o0 - ba kernel_image_start - nop + ba,a %xcc, kernel_image_start .section K_DATA_START, "aw", @progbits Index: kernel/arch/sparc64/src/trap/sun4u/interrupt.c =================================================================== --- kernel/arch/sparc64/src/trap/sun4u/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/trap/sun4u/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -55,18 +55,15 @@ void interrupt(int n, istate_t *istate) { - uint64_t status; - uint64_t intrcv; - uint64_t data0; - status = asi_u64_read(ASI_INTR_DISPATCH_STATUS, 0); + uint64_t status = asi_u64_read(ASI_INTR_DISPATCH_STATUS, 0); if (status & (!INTR_DISPATCH_STATUS_BUSY)) panic("Interrupt Dispatch Status busy bit not set\n"); - - intrcv = asi_u64_read(ASI_INTR_RECEIVE, 0); + + uint64_t intrcv = asi_u64_read(ASI_INTR_RECEIVE, 0); #if defined (US) - data0 = asi_u64_read(ASI_INTR_R, ASI_UDB_INTR_R_DATA_0); + uint64_t data0 = asi_u64_read(ASI_INTR_R, ASI_UDB_INTR_R_DATA_0); #elif defined (US3) - data0 = asi_u64_read(ASI_INTR_R, VA_INTR_R_DATA_0); + uint64_t data0 = asi_u64_read(ASI_INTR_R, VA_INTR_R_DATA_0); #endif - + irq_t *irq = irq_dispatch_and_lock(data0); if (irq) { @@ -75,11 +72,12 @@ */ irq->handler(irq); + /* * See if there is a clear-interrupt-routine and call it. */ - if (irq->cir) { + if (irq->cir) irq->cir(irq->cir_arg, irq->inr); - } - spinlock_unlock(&irq->lock); + + irq_spinlock_unlock(&irq->lock, false); } else if (data0 > config.base) { /* @@ -90,7 +88,6 @@ */ #ifdef CONFIG_SMP - if (data0 == (uintptr_t) tlb_shootdown_ipi_recv) { + if (data0 == (uintptr_t) tlb_shootdown_ipi_recv) tlb_shootdown_ipi_recv(); - } #endif } else { @@ -101,7 +98,9 @@ printf("cpu%u: spurious interrupt (intrcv=%#" PRIx64 ", data0=%#" PRIx64 ")\n", CPU->id, intrcv, data0); +#else + (void) intrcv; #endif } - + membar(); asi_u64_write(ASI_INTR_RECEIVE, 0, 0); Index: kernel/arch/sparc64/src/trap/sun4u/trap_table.S =================================================================== --- kernel/arch/sparc64/src/trap/sun4u/trap_table.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/trap/sun4u/trap_table.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -481,6 +481,6 @@ be %xcc, 1f nop -0: ba %xcc, 0b ! this is for debugging, if we ever get here - nop ! it will be easy to find + ! this is for debugging, if we ever get here it will be easy to find +0: ba,a %xcc, 0b 1: @@ -546,6 +546,5 @@ .if NOT(\is_syscall) - ba %xcc, 1f - nop + ba,a %xcc, 1f 0: save %sp, -PREEMPTIBLE_HANDLER_STACK_FRAME_SIZE, %sp Index: kernel/arch/sparc64/src/trap/sun4v/trap_table.S =================================================================== --- kernel/arch/sparc64/src/trap/sun4v/trap_table.S (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/arch/sparc64/src/trap/sun4v/trap_table.S (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -71,6 +71,5 @@ .org trap_table + TT_INSTRUCTION_ACCESS_MMU_MISS*ENTRY_SIZE .global instruction_access_mmu_miss_handler_tl0 - ba fast_instruction_access_mmu_miss_handler_tl0 - nop + ba,a %xcc, fast_instruction_access_mmu_miss_handler_tl0 /* TT = 0x0a, TL = 0, instruction_access_error */ @@ -187,6 +186,5 @@ .global data_access_mmu_miss_tl0 data_access_mmu_miss_tl0: - ba fast_data_access_mmu_miss_handler_tl0 - nop + ba,a %xcc, fast_data_access_mmu_miss_handler_tl0 /* TT = 0x32, TL = 0, data_access_error */ @@ -394,5 +392,5 @@ .global trap_instruction_\cur\()_tl0 trap_instruction_\cur\()_tl0: - ba trap_instruction_handler + ba %xcc, trap_instruction_handler mov \cur, %g2 .endr @@ -414,6 +412,5 @@ .global instruction_access_mmu_miss_handler_tl1 wrpr %g0, 1, %tl - ba fast_instruction_access_mmu_miss_handler_tl0 - nop + ba,a %xcc, fast_instruction_access_mmu_miss_handler_tl0 /* TT = 0x0a, TL > 0, instruction_access_error */ @@ -498,7 +495,5 @@ .global data_access_mmu_miss_tl1 data_access_mmu_miss_tl1: - ba fast_data_access_mmu_miss_handler_tl1 - nop - + ba,a %xcc, fast_data_access_mmu_miss_handler_tl1 /* TT = 0x32, TL > 0, data_access_error */ @@ -742,5 +737,5 @@ rdpr %cwp, %l1 cmp %l0, %l1 - bz 4f ! CWP is ok + bz %xcc, 4f ! CWP is ok nop @@ -785,8 +780,9 @@ rdpr %tl, %g3 cmp %g3, 1 - be 1f + be %xcc, 1f nop -0: ba 0b ! this is for debugging, if we ever get here - nop ! it will be easy to find + + ! this is for debugging, if we ever get here it will be easy to find +0: ba,a %xcc, 0b 1: @@ -957,5 +953,5 @@ clr %g4 5: andcc %g7, UWB_ALIGNMENT - 1, %g0 ! alignment check - bz 6f ! %g7 is UWB_ALIGNMENT-aligned, no more windows to refill + bz %xcc, 6f ! %g7 is UWB_ALIGNMENT-aligned, no more windows to refill nop @@ -982,5 +978,5 @@ wrpr %g3, 0, %cwp ! switch to the preceeding window - ba 5b + ba %xcc, 5b inc %g4 @@ -996,5 +992,5 @@ add %g4, %g2, %g2 cmp %g2, NWINDOWS - 2 - bg 8f ! fix the CANRESTORE=NWINDOWS-1 anomaly + bg %xcc, 8f ! fix the CANRESTORE=NWINDOWS-1 anomaly mov NWINDOWS - 2, %g1 ! use dealy slot for both cases sub %g1, %g2, %g1 @@ -1066,6 +1062,5 @@ PREEMPTIBLE_HANDLER_KERNEL - ba 101f - nop + ba,a %xcc, 101f 100: Index: rnel/doc/AUTHORS =================================================================== --- kernel/doc/AUTHORS (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,12 +1,0 @@ -Jakub Jermar -Martin Decky -Ondrej Palkovsky -Jiri Svoboda -Jakub Vana -Josef Cejka -Michal Kebrt -Sergey Bondari -Pavel Jancik -Petr Stepan -Michal Konopa -Vojtech Mencl Index: rnel/doc/arch/amd64 =================================================================== --- kernel/doc/arch/amd64 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,35 +1,0 @@ -amd64 port -========== - -The fifth port, amd64 port, was originally written by Ondrej Palkovsky. -The goal is to support AMD64 and Intel Extended Memory 64 Technology PC's. -The port makes use of portable parts of ia32. -Both uniprocessors and multiprocessors are supported. -The kernel runs on real hardware and in simulators too. - -HARDWARE REQUIREMENTS - o AMD64 architecture processor - o Intel Extended Memory 64 Technology processor - -CPU - o Intel Xeon with Intel Extended Memory 64 Technology - o AMD Athlon 64 - -SMP COMPATIBILITY - o Bochs 2.2.1 - 2.2.6 - o 2x-8x AMD64 CPU - o Simics 2.2.19 - o 2x-15x AMD hammer CPU - o QEMU 0.8.0 - QEMU 0.8.1 - o 2x-15x CPU - o HP ProLiant ML350 (HyperThreading) - -EMULATORS AND VIRTUALIZERS - o Bochs 2.2.6 - o Simics 2.2.19 - o QEMU 0.8.1 - -TOOLCHAIN REQUIREMENTS - o binutils 2.16, 2.16.1 - o gcc 4.0.1, 4.1.0, 4.1.1 - o older versions may do as well, but are now obsoleted Index: rnel/doc/arch/arm32 =================================================================== --- kernel/doc/arch/arm32 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,16 +1,0 @@ -arm32 port -========== - -arm32 port is the ninth port of SPARTAN, originally written by Michal Kebrt, -Petr Stepan, Pavel Jancik. The goal is to support 32-bit ARM architecture. -So far, it runs only in emulator. - -HARDWARE REQUIREMENTS - o no real hardware supported - -EMULATORS AND VIRTUALIZERS - o GXemul - -TOOLCHAIN REQUIREMENTS - o binutils 2.17 - o gcc 4.1.1 Index: rnel/doc/arch/ia32 =================================================================== --- kernel/doc/arch/ia32 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,40 +1,0 @@ -ia32 port -========= - -ia32 port is the oldest and the most advanced one. -It was originally written by Jakub Jermar. -It is meant to support ordinary PC's based on IA-32 architecture. -Both uniprocessor and multiprocessor modes are supported. -It runs both in emulated environment and on real hardware. - -HARDWARE REQUIREMENTS - o IA-32 processor (Pentium and successors) - -SMP COMPATIBILITY - o Bochs 2.0.2 - Bochs 2.2.6 - o 2x-8x 686 CPU - o Simics 2.0.28 - Simics 2.2.19 - o 2x-15x Pentium 4 CPU - o VMware Workstation 5.5 - o 2x CPU - o QEMU 0.8.0 - QEMU 0.8.1 - o 2x-15x CPU - o ASUS P/I-P65UP5 + ASUS C-P55T2D REV. 1.41 - o 2x 200Mhz Pentium CPU - o ASUS PCH-DL - o 2x 3000Mhz Pentium 4 Xeon (HT) CPU - o MSI K7D Master-L - o 2x 2100MHz Athlon MP CPU - o ECS 865PE-A REV : 2.0 - o 1x 2800MHz Pentium 4 Prescott (HT) CPU - -EMULATORS AND VIRTUALIZERS - o Bochs 2.0.2 - Bochs 2.2.6 - o VMware Workstation 4, VMware Workstation 5, VMware Workstation 5.5 - o Simics 2.2.19 - o QEMU 0.8.0 - QEMU 0.8.1 - -TOOLCHAIN REQUIREMENTS - o binutils 2.15, 2.16, 2.16.1 - o gcc 3.3.5, 4.0.1, 4.1.0, 4.1.1 - o older versions may do as well, but are now obsoleted Index: rnel/doc/arch/ia64 =================================================================== --- kernel/doc/arch/ia64 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,16 +1,0 @@ -ia64 port -========= - -ia64 port is the third port of SPARTAN originally written by Jakub Jermar. -It is still in its early stages. It runs on HP Ski simulator of IA-64 architecture. - -HARDWARE REQUIREMENTS - o no real hardware supported - -EMULATORS AND VIRTUALIZERS - o ski - -TOOLCHAIN REQUIREMENTS - o binutils 2.15, 2.16, 2.16.1 - o gcc 4.0.0, 4.0.1, 4.1.0, 4.1.1 - o older versions may do as well, but are now obsoleted Index: rnel/doc/arch/mips32 =================================================================== --- kernel/doc/arch/mips32 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,22 +1,0 @@ -mips32 port -=========== - -mips32 is the second port of SPARTAN kernel originally written by Jakub Jermar. -It was first developed to run on MIPS R4000 32-bit simulator. -It can be compiled and run either as little- or big-endian. - -HARDWARE REQUIREMENTS - o emulated MIPS 4K CPU - -CPU - o QED R4600 - -EMULATORS AND VIRTUALIZERS - o msim 1.2.12 - o gxemul - both big and little endian - o simics 2.2.19 - -TOOLCHAIN REQUIREMENTS - o binutils 2.16, 2.16.1 - o gcc 4.0.1, 4.1.0, 4.1.1 - o older versions may do as well, but are now obsoleted Index: rnel/doc/arch/ppc32 =================================================================== --- kernel/doc/arch/ppc32 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,15 +1,0 @@ -ppc32 port -========== - -ppc32 port is the fourth port of SPARTAN, originally written by Martin Decky. -The goal is to support 32-bit PowerPC architecture. - -HARDWARE REQUIREMENTS - o Apple iMac G4 - -EMULATORS AND VIRTUALIZERS - o QEMU - -TOOLCHAIN REQUIREMENTS - o binutils 2.16 - o gcc 4.0.1, 4.1.0, 4.1.1 Index: rnel/doc/arch/sparc64 =================================================================== --- kernel/doc/arch/sparc64 (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,25 +1,0 @@ -sparc64 port -============ - -Currently, this porting effort is subject to -Jakub Jermar's work on his master thesis. - -The goal is to provide support for UltraSPARC -implementation of SPARC V9 architecture. - -MACHINES - o Sun Ultra 5 - o Sun Ultra 60 - o Sun Enterprise E6500 (simulated) - -CPU - o UltraSPARC II - o UltraSPARC IIi - -SIMULATORS - o simics 2.2.19, simics 3.0.17, simics 3.0.21 - -TOOLCHAIN REQUIREMENTS - o binutils 2.17 - o gcc 4.1.1 - o older versions may do as well, but are now obsoleted Index: rnel/doc/build =================================================================== --- kernel/doc/build (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ (revision ) @@ -1,14 +1,0 @@ -Following make targets are supported: - -make, make all - - Check configuration, build - -make config - - Start kernel configuration program - -make clean - - Clean build temporary files - -make distclean - - Clean everything including configuration - Index: kernel/doc/synchronization =================================================================== --- kernel/doc/synchronization (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/doc/synchronization (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -5,4 +5,14 @@ | spinlock_t | +------------+ + | + +------------------------------+ + | + INTERRUPTS-DISABLED SPINNING LOCKS | + irq_spinlock_lock, irq_spinlock_trylock, irq_spinlock_unlock | + +----------------+ | + | irq_spinlock_t |<--------------------+ + +----------------+ + + WAIT QUEUES @@ -10,6 +20,6 @@ +---------+ | waitq_t | - +---------+ - / \ + +---------+ + / \ SEMAPHORES / \ CONDITION VARIABLES semaphore_down_timeout, semaphore_up condvar_wait_timeout, condvar_signal @@ -18,12 +28,12 @@ +--------------+ +-----------+ | ^ - | | + | | | +------+ - V / + V / MUTEXES / READERS/WRITERS LOCKS mutex_lock_timeout, mutex_unlock rwlock_reader/writer_lock_timeout, rwlock_unlock +---------+ / +----------+ - | mutex_t |------------------------------->| rwlock_t | - +---------+ / +----------+ - | / - +------------------------+ + | mutex_t |------------------------------->| rwlock_t | + +---------+ / +----------+ + | / + +------------------------+ Index: kernel/genarch/include/mm/page_ht.h =================================================================== --- kernel/genarch/include/mm/page_ht.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/include/mm/page_ht.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,5 +32,5 @@ /** * @file - * @brief This is the generic page hash table interface. + * @brief This is the generic page hash table interface. */ @@ -46,18 +46,18 @@ #include -#define PAGE_HT_KEYS 2 -#define KEY_AS 0 -#define KEY_PAGE 1 +#define PAGE_HT_KEYS 2 +#define KEY_AS 0 +#define KEY_PAGE 1 -#define PAGE_HT_ENTRIES_BITS 13 -#define PAGE_HT_ENTRIES (1 << PAGE_HT_ENTRIES_BITS) +#define PAGE_HT_ENTRIES_BITS 13 +#define PAGE_HT_ENTRIES (1 << PAGE_HT_ENTRIES_BITS) /* Macros for querying page hash table PTEs. */ -#define PTE_VALID(pte) ((pte) != NULL) -#define PTE_PRESENT(pte) ((pte)->p != 0) -#define PTE_GET_FRAME(pte) ((pte)->frame) -#define PTE_READABLE(pte) 1 -#define PTE_WRITABLE(pte) ((pte)->w != 0) -#define PTE_EXECUTABLE(pte) ((pte)->x != 0) +#define PTE_VALID(pte) ((pte) != NULL) +#define PTE_PRESENT(pte) ((pte)->p != 0) +#define PTE_GET_FRAME(pte) ((pte)->frame) +#define PTE_READABLE(pte) 1 +#define PTE_WRITABLE(pte) ((pte)->w != 0) +#define PTE_EXECUTABLE(pte) ((pte)->x != 0) extern as_operations_t as_ht_operations; Index: kernel/genarch/include/mm/page_pt.h =================================================================== --- kernel/genarch/include/mm/page_pt.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/include/mm/page_pt.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -37,4 +37,5 @@ * Architectures that use hierarchical page tables * are supposed to implement *_ARCH macros. + * */ @@ -52,76 +53,81 @@ * Number of entries in each level. */ -#define PTL0_ENTRIES PTL0_ENTRIES_ARCH -#define PTL1_ENTRIES PTL1_ENTRIES_ARCH -#define PTL2_ENTRIES PTL2_ENTRIES_ARCH -#define PTL3_ENTRIES PTL3_ENTRIES_ARCH +#define PTL0_ENTRIES PTL0_ENTRIES_ARCH +#define PTL1_ENTRIES PTL1_ENTRIES_ARCH +#define PTL2_ENTRIES PTL2_ENTRIES_ARCH +#define PTL3_ENTRIES PTL3_ENTRIES_ARCH /* Table sizes in each level */ -#define PTL0_SIZE PTL0_SIZE_ARCH -#define PTL1_SIZE PTL1_SIZE_ARCH -#define PTL2_SIZE PTL2_SIZE_ARCH -#define PTL3_SIZE PTL3_SIZE_ARCH +#define PTL0_SIZE PTL0_SIZE_ARCH +#define PTL1_SIZE PTL1_SIZE_ARCH +#define PTL2_SIZE PTL2_SIZE_ARCH +#define PTL3_SIZE PTL3_SIZE_ARCH /* * These macros process vaddr and extract those portions * of it that function as indices to respective page tables. + * */ -#define PTL0_INDEX(vaddr) PTL0_INDEX_ARCH(vaddr) -#define PTL1_INDEX(vaddr) PTL1_INDEX_ARCH(vaddr) -#define PTL2_INDEX(vaddr) PTL2_INDEX_ARCH(vaddr) -#define PTL3_INDEX(vaddr) PTL3_INDEX_ARCH(vaddr) +#define PTL0_INDEX(vaddr) PTL0_INDEX_ARCH(vaddr) +#define PTL1_INDEX(vaddr) PTL1_INDEX_ARCH(vaddr) +#define PTL2_INDEX(vaddr) PTL2_INDEX_ARCH(vaddr) +#define PTL3_INDEX(vaddr) PTL3_INDEX_ARCH(vaddr) -#define SET_PTL0_ADDRESS(ptl0) SET_PTL0_ADDRESS_ARCH(ptl0) +#define SET_PTL0_ADDRESS(ptl0) SET_PTL0_ADDRESS_ARCH(ptl0) /* * These macros traverse the 4-level tree of page tables, * each descending by one level. + * */ -#define GET_PTL1_ADDRESS(ptl0, i) GET_PTL1_ADDRESS_ARCH(ptl0, i) -#define GET_PTL2_ADDRESS(ptl1, i) GET_PTL2_ADDRESS_ARCH(ptl1, i) -#define GET_PTL3_ADDRESS(ptl2, i) GET_PTL3_ADDRESS_ARCH(ptl2, i) -#define GET_FRAME_ADDRESS(ptl3, i) GET_FRAME_ADDRESS_ARCH(ptl3, i) +#define GET_PTL1_ADDRESS(ptl0, i) GET_PTL1_ADDRESS_ARCH(ptl0, i) +#define GET_PTL2_ADDRESS(ptl1, i) GET_PTL2_ADDRESS_ARCH(ptl1, i) +#define GET_PTL3_ADDRESS(ptl2, i) GET_PTL3_ADDRESS_ARCH(ptl2, i) +#define GET_FRAME_ADDRESS(ptl3, i) GET_FRAME_ADDRESS_ARCH(ptl3, i) /* * These macros are provided to change the shape of the 4-level tree of page * tables on respective level. + * */ -#define SET_PTL1_ADDRESS(ptl0, i, a) SET_PTL1_ADDRESS_ARCH(ptl0, i, a) -#define SET_PTL2_ADDRESS(ptl1, i, a) SET_PTL2_ADDRESS_ARCH(ptl1, i, a) -#define SET_PTL3_ADDRESS(ptl2, i, a) SET_PTL3_ADDRESS_ARCH(ptl2, i, a) -#define SET_FRAME_ADDRESS(ptl3, i, a) SET_FRAME_ADDRESS_ARCH(ptl3, i, a) +#define SET_PTL1_ADDRESS(ptl0, i, a) SET_PTL1_ADDRESS_ARCH(ptl0, i, a) +#define SET_PTL2_ADDRESS(ptl1, i, a) SET_PTL2_ADDRESS_ARCH(ptl1, i, a) +#define SET_PTL3_ADDRESS(ptl2, i, a) SET_PTL3_ADDRESS_ARCH(ptl2, i, a) +#define SET_FRAME_ADDRESS(ptl3, i, a) SET_FRAME_ADDRESS_ARCH(ptl3, i, a) /* * These macros are provided to query various flags within the page tables. + * */ -#define GET_PTL1_FLAGS(ptl0, i) GET_PTL1_FLAGS_ARCH(ptl0, i) -#define GET_PTL2_FLAGS(ptl1, i) GET_PTL2_FLAGS_ARCH(ptl1, i) -#define GET_PTL3_FLAGS(ptl2, i) GET_PTL3_FLAGS_ARCH(ptl2, i) -#define GET_FRAME_FLAGS(ptl3, i) GET_FRAME_FLAGS_ARCH(ptl3, i) +#define GET_PTL1_FLAGS(ptl0, i) GET_PTL1_FLAGS_ARCH(ptl0, i) +#define GET_PTL2_FLAGS(ptl1, i) GET_PTL2_FLAGS_ARCH(ptl1, i) +#define GET_PTL3_FLAGS(ptl2, i) GET_PTL3_FLAGS_ARCH(ptl2, i) +#define GET_FRAME_FLAGS(ptl3, i) GET_FRAME_FLAGS_ARCH(ptl3, i) /* * These macros are provided to set/clear various flags within the page tables. + * */ -#define SET_PTL1_FLAGS(ptl0, i, x) SET_PTL1_FLAGS_ARCH(ptl0, i, x) -#define SET_PTL2_FLAGS(ptl1, i, x) SET_PTL2_FLAGS_ARCH(ptl1, i, x) -#define SET_PTL3_FLAGS(ptl2, i, x) SET_PTL3_FLAGS_ARCH(ptl2, i, x) -#define SET_FRAME_FLAGS(ptl3, i, x) SET_FRAME_FLAGS_ARCH(ptl3, i, x) +#define SET_PTL1_FLAGS(ptl0, i, x) SET_PTL1_FLAGS_ARCH(ptl0, i, x) +#define SET_PTL2_FLAGS(ptl1, i, x) SET_PTL2_FLAGS_ARCH(ptl1, i, x) +#define SET_PTL3_FLAGS(ptl2, i, x) SET_PTL3_FLAGS_ARCH(ptl2, i, x) +#define SET_FRAME_FLAGS(ptl3, i, x) SET_FRAME_FLAGS_ARCH(ptl3, i, x) /* * Macros for querying the last-level PTEs. + * */ -#define PTE_VALID(p) PTE_VALID_ARCH((p)) -#define PTE_PRESENT(p) PTE_PRESENT_ARCH((p)) -#define PTE_GET_FRAME(p) PTE_GET_FRAME_ARCH((p)) -#define PTE_READABLE(p) 1 -#define PTE_WRITABLE(p) PTE_WRITABLE_ARCH((p)) -#define PTE_EXECUTABLE(p) PTE_EXECUTABLE_ARCH((p)) +#define PTE_VALID(p) PTE_VALID_ARCH((p)) +#define PTE_PRESENT(p) PTE_PRESENT_ARCH((p)) +#define PTE_GET_FRAME(p) PTE_GET_FRAME_ARCH((p)) +#define PTE_READABLE(p) 1 +#define PTE_WRITABLE(p) PTE_WRITABLE_ARCH((p)) +#define PTE_EXECUTABLE(p) PTE_EXECUTABLE_ARCH((p)) extern as_operations_t as_pt_operations; extern page_mapping_operations_t pt_mapping_operations; -extern void page_mapping_insert_pt(as_t *as, uintptr_t page, uintptr_t frame, - int flags); -extern pte_t *page_mapping_find_pt(as_t *as, uintptr_t page); +extern void page_mapping_insert_pt(as_t *, uintptr_t, uintptr_t, unsigned int); +extern pte_t *page_mapping_find_pt(as_t *, uintptr_t); #endif Index: kernel/genarch/src/acpi/acpi.c =================================================================== --- kernel/genarch/src/acpi/acpi.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/acpi/acpi.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -99,6 +99,8 @@ static void map_sdt(struct acpi_sdt_header *sdt) { + page_table_lock(AS_KERNEL, true); page_mapping_insert(AS_KERNEL, (uintptr_t) sdt, (uintptr_t) sdt, PAGE_NOT_CACHEABLE | PAGE_WRITE); map_structure((uintptr_t) sdt, sdt->length); + page_table_unlock(AS_KERNEL, true); } Index: kernel/genarch/src/drivers/ega/ega.c =================================================================== --- kernel/genarch/src/drivers/ega/ega.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/drivers/ega/ega.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -63,5 +63,5 @@ typedef struct { - SPINLOCK_DECLARE(lock); + IRQ_SPINLOCK_DECLARE(lock); uint32_t cursor; @@ -71,6 +71,6 @@ } ega_instance_t; -static void ega_putchar(outdev_t *dev, wchar_t ch, bool silent); -static void ega_redraw(outdev_t *dev); +static void ega_putchar(outdev_t *, wchar_t, bool); +static void ega_redraw(outdev_t *); static outdev_operations_t egadev_ops = { @@ -540,6 +540,5 @@ ega_instance_t *instance = (ega_instance_t *) dev->data; - ipl_t ipl = interrupts_disable(); - spinlock_lock(&instance->lock); + irq_spinlock_lock(&instance->lock, true); switch (ch) { @@ -564,6 +563,5 @@ ega_move_cursor(instance, silent); - spinlock_unlock(&instance->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&instance->lock, true); } @@ -572,6 +570,5 @@ ega_instance_t *instance = (ega_instance_t *) dev->data; - ipl_t ipl = interrupts_disable(); - spinlock_lock(&instance->lock); + irq_spinlock_lock(&instance->lock, true); memcpy(instance->addr, instance->backbuf, EGA_VRAM_SIZE); @@ -579,6 +576,5 @@ ega_show_cursor(instance, silent); - spinlock_unlock(&instance->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&instance->lock, true); } @@ -598,5 +594,5 @@ egadev->data = instance; - spinlock_initialize(&instance->lock, "*ega_lock"); + irq_spinlock_initialize(&instance->lock, "*ega.instance.lock"); instance->base = base; Index: kernel/genarch/src/drivers/via-cuda/cuda.c =================================================================== --- kernel/genarch/src/drivers/via-cuda/cuda.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/drivers/via-cuda/cuda.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -99,5 +99,5 @@ instance->snd_bytes = 0; - spinlock_initialize(&instance->dev_lock, "cuda_dev"); + spinlock_initialize(&instance->dev_lock, "cuda.instance.dev_lock"); /* Disable all interrupts from CUDA. */ Index: kernel/genarch/src/fb/fb.c =================================================================== --- kernel/genarch/src/fb/fb.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/fb/fb.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -554,5 +554,5 @@ fbdev->data = instance; - spinlock_initialize(&instance->lock, "*fb_lock"); + spinlock_initialize(&instance->lock, "*fb.instance.lock"); instance->rgb_conv = rgb_conv; instance->pixelbytes = pixelbytes; Index: kernel/genarch/src/kbrd/kbrd.c =================================================================== --- kernel/genarch/src/kbrd/kbrd.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/kbrd/kbrd.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -169,5 +169,5 @@ indev_initialize("kbrd", &instance->raw, &kbrd_raw_ops); - spinlock_initialize(&instance->keylock, "instance_keylock"); + spinlock_initialize(&instance->keylock, "kbrd.instance.keylock"); instance->keyflags = 0; instance->lockflags = 0; Index: kernel/genarch/src/kbrd/kbrd_pl050.c =================================================================== --- kernel/genarch/src/kbrd/kbrd_pl050.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/kbrd/kbrd_pl050.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -181,5 +181,5 @@ indev_initialize("kbrd", &instance->raw, &kbrd_raw_ops); - spinlock_initialize(&instance->keylock, "instance_keylock"); + spinlock_initialize(&instance->keylock, "kbrd_pl050.instance.keylock"); instance->keyflags = 0; instance->lockflags = 0; Index: kernel/genarch/src/mm/as_ht.c =================================================================== --- kernel/genarch/src/mm/as_ht.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/mm/as_ht.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -30,8 +30,8 @@ * @{ */ - + /** * @file - * @brief Address space functions for global page hash table. + * @brief Address space functions for global page hash table. */ @@ -46,9 +46,10 @@ #include -static pte_t *ht_create(int flags); -static void ht_destroy(pte_t *page_table); +static pte_t *ht_create(unsigned int); +static void ht_destroy(pte_t *); -static void ht_lock(as_t *as, bool lock); -static void ht_unlock(as_t *as, bool unlock); +static void ht_lock(as_t *, bool); +static void ht_unlock(as_t *, bool); +static bool ht_locked(as_t *); as_operations_t as_ht_operations = { @@ -57,4 +58,5 @@ .page_table_lock = ht_lock, .page_table_unlock = ht_unlock, + .page_table_locked = ht_locked, }; @@ -68,6 +70,7 @@ * * @return Returns NULL. + * */ -pte_t *ht_create(int flags) +pte_t *ht_create(unsigned int flags) { if (flags & FLAG_AS_KERNEL) { @@ -75,4 +78,5 @@ mutex_initialize(&page_ht_lock, MUTEX_PASSIVE); } + return NULL; } @@ -83,4 +87,5 @@ * * @param page_table This parameter is ignored. + * */ void ht_destroy(pte_t *page_table) @@ -94,6 +99,7 @@ * Interrupts must be disabled. * - * @param as Address space. + * @param as Address space. * @param lock If false, do not attempt to lock the address space. + * */ void ht_lock(as_t *as, bool lock) @@ -101,4 +107,5 @@ if (lock) mutex_lock(&as->lock); + mutex_lock(&page_ht_lock); } @@ -109,14 +116,28 @@ * Interrupts must be disabled. * - * @param as Address space. + * @param as Address space. * @param unlock If false, do not attempt to lock the address space. + * */ void ht_unlock(as_t *as, bool unlock) { mutex_unlock(&page_ht_lock); + if (unlock) mutex_unlock(&as->lock); } +/** Test whether page tables are locked. + * + * @param as Address space where the page tables belong. + * + * @return True if the page tables belonging to the address soace + * are locked, otherwise false. + */ +bool ht_locked(as_t *as) +{ + return (mutex_locked(&page_ht_lock) && mutex_locked(&as->lock)); +} + /** @} */ Index: kernel/genarch/src/mm/as_pt.c =================================================================== --- kernel/genarch/src/mm/as_pt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/mm/as_pt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Address space functions for 4-level hierarchical pagetables. + * @brief Address space functions for 4-level hierarchical pagetables. */ @@ -47,9 +47,10 @@ #include -static pte_t *ptl0_create(int flags); -static void ptl0_destroy(pte_t *page_table); +static pte_t *ptl0_create(unsigned int); +static void ptl0_destroy(pte_t *); -static void pt_lock(as_t *as, bool lock); -static void pt_unlock(as_t *as, bool unlock); +static void pt_lock(as_t *, bool); +static void pt_unlock(as_t *, bool); +static bool pt_locked(as_t *); as_operations_t as_pt_operations = { @@ -57,5 +58,6 @@ .page_table_destroy = ptl0_destroy, .page_table_lock = pt_lock, - .page_table_unlock = pt_unlock + .page_table_unlock = pt_unlock, + .page_table_locked = pt_locked, }; @@ -67,36 +69,38 @@ * * @return New PTL0. + * */ -pte_t *ptl0_create(int flags) +pte_t *ptl0_create(unsigned int flags) { - pte_t *src_ptl0, *dst_ptl0; - ipl_t ipl; - int table_size; - - dst_ptl0 = (pte_t *) frame_alloc(PTL0_SIZE, FRAME_KA); - table_size = FRAME_SIZE << PTL0_SIZE; - - if (flags & FLAG_AS_KERNEL) { + pte_t *dst_ptl0 = (pte_t *) frame_alloc(PTL0_SIZE, FRAME_KA); + size_t table_size = FRAME_SIZE << PTL0_SIZE; + + if (flags & FLAG_AS_KERNEL) memsetb(dst_ptl0, table_size, 0); - } else { - uintptr_t src, dst; - + else { /* * Copy the kernel address space portion to new PTL0. + * */ - - ipl = interrupts_disable(); - mutex_lock(&AS_KERNEL->lock); - src_ptl0 = (pte_t *) PA2KA((uintptr_t) AS_KERNEL->genarch.page_table); - - src = (uintptr_t) &src_ptl0[PTL0_INDEX(KERNEL_ADDRESS_SPACE_START)]; - dst = (uintptr_t) &dst_ptl0[PTL0_INDEX(KERNEL_ADDRESS_SPACE_START)]; - + + ipl_t ipl = interrupts_disable(); + mutex_lock(&AS_KERNEL->lock); + + pte_t *src_ptl0 = + (pte_t *) PA2KA((uintptr_t) AS_KERNEL->genarch.page_table); + + uintptr_t src = + (uintptr_t) &src_ptl0[PTL0_INDEX(KERNEL_ADDRESS_SPACE_START)]; + uintptr_t dst = + (uintptr_t) &dst_ptl0[PTL0_INDEX(KERNEL_ADDRESS_SPACE_START)]; + memsetb(dst_ptl0, table_size, 0); - memcpy((void *) dst, (void *) src, table_size - (src - (uintptr_t) src_ptl0)); + memcpy((void *) dst, (void *) src, + table_size - (src - (uintptr_t) src_ptl0)); + mutex_unlock(&AS_KERNEL->lock); interrupts_restore(ipl); } - + return (pte_t *) KA2PA((uintptr_t) dst_ptl0); } @@ -107,8 +111,9 @@ * * @param page_table Physical address of PTL0. + * */ void ptl0_destroy(pte_t *page_table) { - frame_free((uintptr_t)page_table); + frame_free((uintptr_t) page_table); } @@ -118,6 +123,7 @@ * Interrupts must be disabled. * - * @param as Address space. + * @param as Address space. * @param lock If false, do not attempt to lock the address space. + * */ void pt_lock(as_t *as, bool lock) @@ -132,6 +138,7 @@ * Interrupts must be disabled. * - * @param as Address space. + * @param as Address space. * @param unlock If false, do not attempt to unlock the address space. + * */ void pt_unlock(as_t *as, bool unlock) @@ -141,4 +148,16 @@ } +/** Test whether page tables are locked. + * + * @param as Address space where the page tables belong. + * + * @return True if the page tables belonging to the address soace + * are locked, otherwise false. + */ +bool pt_locked(as_t *as) +{ + return mutex_locked(&as->lock); +} + /** @} */ Index: kernel/genarch/src/mm/asid.c =================================================================== --- kernel/genarch/src/mm/asid.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/mm/asid.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -70,7 +70,4 @@ /** Allocate free address space identifier. * - * Interrupts must be disabled and inactive_as_with_asid_lock must be held - * prior to this call - * * @return New ASID. */ @@ -80,4 +77,7 @@ link_t *tmp; as_t *as; + + ASSERT(interrupts_disabled()); + ASSERT(spinlock_locked(&asidlock)); /* Index: kernel/genarch/src/mm/page_ht.c =================================================================== --- kernel/genarch/src/mm/page_ht.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/mm/page_ht.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Virtual Address Translation (VAT) for global page hash table. + * @brief Virtual Address Translation (VAT) for global page hash table. */ @@ -52,12 +52,11 @@ #include -static size_t hash(unative_t key[]); -static bool compare(unative_t key[], size_t keys, link_t *item); -static void remove_callback(link_t *item); - -static void ht_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, - int flags); -static void ht_mapping_remove(as_t *as, uintptr_t page); -static pte_t *ht_mapping_find(as_t *as, uintptr_t page); +static size_t hash(unative_t[]); +static bool compare(unative_t[], size_t, link_t *); +static void remove_callback(link_t *); + +static void ht_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int); +static void ht_mapping_remove(as_t *, uintptr_t); +static pte_t *ht_mapping_find(as_t *, uintptr_t); /** @@ -65,10 +64,12 @@ * after address space lock and after any address space area * locks. + * */ mutex_t page_ht_lock; -/** - * Page hash table. +/** Page hash table. + * * The page hash table may be accessed only when page_ht_lock is held. + * */ hash_table_t page_ht; @@ -93,4 +94,5 @@ * * @return Index into page hash table. + * */ size_t hash(unative_t key[]) @@ -98,5 +100,4 @@ as_t *as = (as_t *) key[KEY_AS]; uintptr_t page = (uintptr_t) key[KEY_PAGE]; - size_t index; /* @@ -104,6 +105,7 @@ * of occurring. Least significant bits of VPN compose the * hash index. - */ - index = ((page >> PAGE_WIDTH) & (PAGE_HT_ENTRIES - 1)); + * + */ + size_t index = ((page >> PAGE_WIDTH) & (PAGE_HT_ENTRIES - 1)); /* @@ -111,4 +113,5 @@ * similar addresses. Least significant bits compose the * hash index. + * */ index |= ((unative_t) as) & (PAGE_HT_ENTRIES - 1); @@ -119,28 +122,28 @@ /** Compare page hash table item with page and/or address space. * - * @param key Array of one or two keys (i.e. page and/or address space). + * @param key Array of one or two keys (i.e. page and/or address space). * @param keys Number of keys passed. * @param item Item to compare the keys with. * * @return true on match, false otherwise. + * */ bool compare(unative_t key[], size_t keys, link_t *item) { - pte_t *t; - ASSERT(item); - ASSERT((keys > 0) && (keys <= PAGE_HT_KEYS)); - + ASSERT(keys > 0); + ASSERT(keys <= PAGE_HT_KEYS); + /* * Convert item to PTE. - */ - t = hash_table_get_instance(item, pte_t, link); - - if (keys == PAGE_HT_KEYS) { - return (key[KEY_AS] == (uintptr_t) t->as) && - (key[KEY_PAGE] == t->page); - } else { - return (key[KEY_AS] == (uintptr_t) t->as); - } + * + */ + pte_t *pte = hash_table_get_instance(item, pte_t, link); + + if (keys == PAGE_HT_KEYS) + return (key[KEY_AS] == (uintptr_t) pte->as) && + (key[KEY_PAGE] == pte->page); + + return (key[KEY_AS] == (uintptr_t) pte->as); } @@ -148,17 +151,17 @@ * * @param item Page hash table item being removed. + * */ void remove_callback(link_t *item) { - pte_t *t; - ASSERT(item); - + /* * Convert item to PTE. - */ - t = hash_table_get_instance(item, pte_t, link); - - free(t); + * + */ + pte_t *pte = hash_table_get_instance(item, pte_t, link); + + free(pte); } @@ -166,39 +169,41 @@ * * Map virtual address page to physical address frame - * using flags. - * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to which page belongs. - * @param page Virtual address of the page to be mapped. + * using flags. + * + * @param as Address space to which page belongs. + * @param page Virtual address of the page to be mapped. * @param frame Physical address of memory frame to which the mapping is done. * @param flags Flags to be used for mapping. - */ -void ht_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, int flags) -{ - pte_t *t; + * + */ +void ht_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, + unsigned int flags) +{ unative_t key[2] = { (uintptr_t) as, page = ALIGN_DOWN(page, PAGE_SIZE) }; + + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); if (!hash_table_find(&page_ht, key)) { - t = (pte_t *) malloc(sizeof(pte_t), FRAME_ATOMIC); - ASSERT(t != NULL); - - t->g = (flags & PAGE_GLOBAL) != 0; - t->x = (flags & PAGE_EXEC) != 0; - t->w = (flags & PAGE_WRITE) != 0; - t->k = !(flags & PAGE_USER); - t->c = (flags & PAGE_CACHEABLE) != 0; - t->p = !(flags & PAGE_NOT_PRESENT); - t->a = false; - t->d = false; - - t->as = as; - t->page = ALIGN_DOWN(page, PAGE_SIZE); - t->frame = ALIGN_DOWN(frame, FRAME_SIZE); - - hash_table_insert(&page_ht, key, &t->link); + pte_t *pte = (pte_t *) malloc(sizeof(pte_t), FRAME_ATOMIC); + ASSERT(pte != NULL); + + pte->g = (flags & PAGE_GLOBAL) != 0; + pte->x = (flags & PAGE_EXEC) != 0; + pte->w = (flags & PAGE_WRITE) != 0; + pte->k = !(flags & PAGE_USER); + pte->c = (flags & PAGE_CACHEABLE) != 0; + pte->p = !(flags & PAGE_NOT_PRESENT); + pte->a = false; + pte->d = false; + + pte->as = as; + pte->page = ALIGN_DOWN(page, PAGE_SIZE); + pte->frame = ALIGN_DOWN(frame, FRAME_SIZE); + + hash_table_insert(&page_ht, key, &pte->link); } } @@ -210,8 +215,7 @@ * this call visible. * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to wich page belongs. + * @param as Address space to wich page belongs. * @param page Virtual address of the page to be demapped. + * */ void ht_mapping_remove(as_t *as, uintptr_t page) @@ -221,4 +225,7 @@ page = ALIGN_DOWN(page, PAGE_SIZE) }; + + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); /* @@ -234,25 +241,25 @@ * Find mapping for virtual page. * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to wich page belongs. + * @param as Address space to wich page belongs. * @param page Virtual page. * * @return NULL if there is no such mapping; requested mapping otherwise. + * */ pte_t *ht_mapping_find(as_t *as, uintptr_t page) { - link_t *hlp; - pte_t *t = NULL; unative_t key[2] = { (uintptr_t) as, page = ALIGN_DOWN(page, PAGE_SIZE) }; - - hlp = hash_table_find(&page_ht, key); - if (hlp) - t = hash_table_get_instance(hlp, pte_t, link); - - return t; + + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + + link_t *cur = hash_table_find(&page_ht, key); + if (cur) + return hash_table_get_instance(cur, pte_t, link); + + return NULL; } Index: kernel/genarch/src/mm/page_pt.c =================================================================== --- kernel/genarch/src/mm/page_pt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/genarch/src/mm/page_pt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Virtual Address Translation for hierarchical 4-level page tables. + * @brief Virtual Address Translation for hierarchical 4-level page tables. */ @@ -46,7 +46,7 @@ #include -static void pt_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, int flags); -static void pt_mapping_remove(as_t *as, uintptr_t page); -static pte_t *pt_mapping_find(as_t *as, uintptr_t page); +static void pt_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int); +static void pt_mapping_remove(as_t *, uintptr_t); +static pte_t *pt_mapping_find(as_t *, uintptr_t); page_mapping_operations_t pt_mapping_operations = { @@ -61,45 +61,45 @@ * using flags. * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to wich page belongs. - * @param page Virtual address of the page to be mapped. + * @param as Address space to wich page belongs. + * @param page Virtual address of the page to be mapped. * @param frame Physical address of memory frame to which the mapping is done. * @param flags Flags to be used for mapping. - */ -void pt_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, int flags) + * + */ +void pt_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, + unsigned int flags) { - pte_t *ptl0, *ptl1, *ptl2, *ptl3; - pte_t *newpt; - - ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); - + pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); + + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) { - newpt = (pte_t *)frame_alloc(PTL1_SIZE, FRAME_KA); + pte_t *newpt = (pte_t *) frame_alloc(PTL1_SIZE, FRAME_KA); memsetb(newpt, FRAME_SIZE << PTL1_SIZE, 0); SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page), KA2PA(newpt)); SET_PTL1_FLAGS(ptl0, PTL0_INDEX(page), PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); } - - ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); - + + pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); + if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) { - newpt = (pte_t *)frame_alloc(PTL2_SIZE, FRAME_KA); + pte_t *newpt = (pte_t *) frame_alloc(PTL2_SIZE, FRAME_KA); memsetb(newpt, FRAME_SIZE << PTL2_SIZE, 0); SET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page), KA2PA(newpt)); SET_PTL2_FLAGS(ptl1, PTL1_INDEX(page), PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); } - - ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); - + + pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); + if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) { - newpt = (pte_t *)frame_alloc(PTL3_SIZE, FRAME_KA); + pte_t *newpt = (pte_t *) frame_alloc(PTL3_SIZE, FRAME_KA); memsetb(newpt, FRAME_SIZE << PTL3_SIZE, 0); SET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page), KA2PA(newpt)); SET_PTL3_FLAGS(ptl2, PTL2_INDEX(page), PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); } - - ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); - + + pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); + SET_FRAME_ADDRESS(ptl3, PTL3_INDEX(page), frame); SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags); @@ -114,44 +114,44 @@ * Empty page tables except PTL0 are freed. * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to wich page belongs. + * @param as Address space to wich page belongs. * @param page Virtual address of the page to be demapped. + * */ void pt_mapping_remove(as_t *as, uintptr_t page) { - pte_t *ptl0, *ptl1, *ptl2, *ptl3; - bool empty = true; - int i; + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); /* * First, remove the mapping, if it exists. + * */ - - ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); - + + pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) return; - - ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); - + + pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) return; - - ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); - + + pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) return; - - ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); - + + pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); + /* Destroy the mapping. Setting to PAGE_NOT_PRESENT is not sufficient. */ memsetb(&ptl3[PTL3_INDEX(page)], sizeof(pte_t), 0); - + /* * Second, free all empty tables along the way from PTL3 down to PTL0. + * */ - /* check PTL3 */ + /* Check PTL3 */ + bool empty = true; + + unsigned int i; for (i = 0; i < PTL3_ENTRIES; i++) { if (PTE_VALID(&ptl3[i])) { @@ -160,16 +160,19 @@ } } + if (empty) { /* * PTL3 is empty. * Release the frame and remove PTL3 pointer from preceding table. + * */ frame_free(KA2PA((uintptr_t) ptl3)); - if (PTL2_ENTRIES) - memsetb(&ptl2[PTL2_INDEX(page)], sizeof(pte_t), 0); - else if (PTL1_ENTRIES) - memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0); - else - memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); +#if (PTL2_ENTRIES != 0) + memsetb(&ptl2[PTL2_INDEX(page)], sizeof(pte_t), 0); +#elif (PTL1_ENTRIES != 0) + memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0); +#else + memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); +#endif } else { /* @@ -177,55 +180,60 @@ * Therefore, there must be a path from PTL0 to PTL3 and * thus nothing to free in higher levels. - */ - return; - } - - /* check PTL2, empty is still true */ - if (PTL2_ENTRIES) { - for (i = 0; i < PTL2_ENTRIES; i++) { - if (PTE_VALID(&ptl2[i])) { - empty = false; - break; - } + * + */ + return; + } + + /* Check PTL2, empty is still true */ +#if (PTL2_ENTRIES != 0) + for (i = 0; i < PTL2_ENTRIES; i++) { + if (PTE_VALID(&ptl2[i])) { + empty = false; + break; } - if (empty) { - /* - * PTL2 is empty. - * Release the frame and remove PTL2 pointer from preceding table. - */ - frame_free(KA2PA((uintptr_t) ptl2)); - if (PTL1_ENTRIES) - memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0); - else - memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); + } + + if (empty) { + /* + * PTL2 is empty. + * Release the frame and remove PTL2 pointer from preceding table. + * + */ + frame_free(KA2PA((uintptr_t) ptl2)); +#if (PTL1_ENTRIES != 0) + memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0); +#else + memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); +#endif + } else { + /* + * PTL2 is not empty. + * Therefore, there must be a path from PTL0 to PTL2 and + * thus nothing to free in higher levels. + * + */ + return; + } +#endif /* PTL2_ENTRIES != 0 */ + + /* check PTL1, empty is still true */ +#if (PTL1_ENTRIES != 0) + for (i = 0; i < PTL1_ENTRIES; i++) { + if (PTE_VALID(&ptl1[i])) { + empty = false; + break; } - else { - /* - * PTL2 is not empty. - * Therefore, there must be a path from PTL0 to PTL2 and - * thus nothing to free in higher levels. - */ - return; - } - } - - /* check PTL1, empty is still true */ - if (PTL1_ENTRIES) { - for (i = 0; i < PTL1_ENTRIES; i++) { - if (PTE_VALID(&ptl1[i])) { - empty = false; - break; - } - } - if (empty) { - /* - * PTL1 is empty. - * Release the frame and remove PTL1 pointer from preceding table. - */ - frame_free(KA2PA((uintptr_t) ptl1)); - memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); - } - } - + } + + if (empty) { + /* + * PTL1 is empty. + * Release the frame and remove PTL1 pointer from preceding table. + * + */ + frame_free(KA2PA((uintptr_t) ptl1)); + memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0); + } +#endif /* PTL1_ENTRIES != 0 */ } @@ -234,32 +242,30 @@ * Find mapping for virtual page. * - * The page table must be locked and interrupts must be disabled. - * - * @param as Address space to which page belongs. + * @param as Address space to which page belongs. * @param page Virtual page. * - * @return NULL if there is no such mapping; entry from PTL3 describing the mapping otherwise. + * @return NULL if there is no such mapping; entry from PTL3 describing + * the mapping otherwise. + * */ pte_t *pt_mapping_find(as_t *as, uintptr_t page) { - pte_t *ptl0, *ptl1, *ptl2, *ptl3; - - ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); - + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + + pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table); if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; - - ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); - + + pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; - - ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); - + + pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; - - ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); - + + pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); + return &ptl3[PTL3_INDEX(page)]; } Index: kernel/generic/include/arch.h =================================================================== --- kernel/generic/include/arch.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/arch.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ @@ -41,14 +41,14 @@ #include -#define DEFAULT_CONTEXT 0 +#define DEFAULT_CONTEXT 0 -#define CPU THE->cpu -#define THREAD THE->thread -#define TASK THE->task -#define AS THE->as -#define CONTEXT (THE->task ? THE->task->context : DEFAULT_CONTEXT) -#define PREEMPTION_DISABLED THE->preemption_disabled +#define CPU THE->cpu +#define THREAD THE->thread +#define TASK THE->task +#define AS THE->as +#define CONTEXT (THE->task ? THE->task->context : DEFAULT_CONTEXT) +#define PREEMPTION_DISABLED THE->preemption_disabled -#define context_check(ctx1, ctx2) ((ctx1) == (ctx2)) +#define context_check(ctx1, ctx2) ((ctx1) == (ctx2)) /** @@ -58,15 +58,21 @@ */ typedef struct { - size_t preemption_disabled; /**< Preemption disabled counter. */ - thread_t *thread; /**< Current thread. */ - task_t *task; /**< Current task. */ - cpu_t *cpu; /**< Executing cpu. */ - as_t *as; /**< Current address space. */ + size_t preemption_disabled; /**< Preemption disabled counter. */ + thread_t *thread; /**< Current thread. */ + task_t *task; /**< Current task. */ + cpu_t *cpu; /**< Executing cpu. */ + as_t *as; /**< Current address space. */ } the_t; +/* + * THE is not an abbreviation, but the English definite article written in + * capital letters. It means the current pointer to something, e.g. thread, + * processor or address space. Kind reader of this comment shall appreciate + * the wit of constructs like THE->thread and similar. + */ #define THE ((the_t * )(get_stack_base())) -extern void the_initialize(the_t *the); -extern void the_copy(the_t *src, the_t *dst); +extern void the_initialize(the_t *); +extern void the_copy(the_t *, the_t *); extern void arch_pre_mm_init(void); @@ -80,5 +86,5 @@ extern void reboot(void); extern void arch_reboot(void); -extern void *arch_construct_function(fncptr_t *fptr, void *addr, void *caller); +extern void *arch_construct_function(fncptr_t *, void *, void *); #endif Index: kernel/generic/include/bitops.h =================================================================== --- kernel/generic/include/bitops.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/bitops.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ @@ -36,13 +36,21 @@ #define KERN_BITOPS_H_ +#ifdef __32_BITS__ + #define fnzb(arg) fnzb32(arg) +#endif -/** Return position of first non-zero bit from left (i.e. [log_2(arg)]). +#ifdef __64_BITS__ + #define fnzb(arg) fnzb64(arg) +#endif + +/** Return position of first non-zero bit from left (32b variant). * - * If number is zero, it returns 0 + * @return 0 (if the number is zero) or [log_2(arg)]. + * */ -static inline int fnzb32(uint32_t arg) +static inline uint8_t fnzb32(uint32_t arg) { - int n = 0; - + uint8_t n = 0; + if (arg >> 16) { arg >>= 16; @@ -71,8 +79,13 @@ } -static inline int fnzb64(uint64_t arg) +/** Return position of first non-zero bit from left (64b variant). + * + * @return 0 (if the number is zero) or [log_2(arg)]. + * + */ +static inline uint8_t fnzb64(uint64_t arg) { - int n = 0; - + uint8_t n = 0; + if (arg >> 32) { arg >>= 32; @@ -83,6 +96,4 @@ } -#define fnzb(x) fnzb32(x) - #endif Index: kernel/generic/include/console/chardev.h =================================================================== --- kernel/generic/include/console/chardev.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/console/chardev.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -57,5 +57,5 @@ /** Protects everything below. */ - SPINLOCK_DECLARE(lock); + IRQ_SPINLOCK_DECLARE(lock); wchar_t buffer[INDEV_BUFLEN]; size_t counter; @@ -95,13 +95,13 @@ } outdev_t; -extern void indev_initialize(const char *name, indev_t *indev, - indev_operations_t *op); -extern void indev_push_character(indev_t *indev, wchar_t ch); -extern wchar_t indev_pop_character(indev_t *indev); +extern void indev_initialize(const char *, indev_t *, + indev_operations_t *); +extern void indev_push_character(indev_t *, wchar_t); +extern wchar_t indev_pop_character(indev_t *); -extern void outdev_initialize(const char *name, outdev_t *outdev, - outdev_operations_t *op); +extern void outdev_initialize(const char *, outdev_t *, + outdev_operations_t *); -extern bool check_poll(indev_t *indev); +extern bool check_poll(indev_t *); #endif /* KERN_CHARDEV_H_ */ Index: kernel/generic/include/cpu.h =================================================================== --- kernel/generic/include/cpu.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/cpu.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -42,5 +42,5 @@ #include -#define CPU_STACK_SIZE STACK_SIZE +#define CPU_STACK_SIZE STACK_SIZE /** CPU structure. @@ -49,40 +49,44 @@ */ typedef struct cpu { - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + tlb_shootdown_msg_t tlb_messages[TLB_MESSAGE_QUEUE_LEN]; size_t tlb_messages_count; context_t saved_context; - + atomic_t nrdy; runq_t rq[RQ_COUNT]; volatile size_t needs_relink; - - SPINLOCK_DECLARE(timeoutlock); + + IRQ_SPINLOCK_DECLARE(timeoutlock); link_t timeout_active_head; - - size_t missed_clock_ticks; /**< When system clock loses a tick, it is recorded here - so that clock() can react. This variable is - CPU-local and can be only accessed when interrupts - are disabled. */ - + + /** + * When system clock loses a tick, it is + * recorded here so that clock() can react. + * This variable is CPU-local and can be + * only accessed when interrupts are + * disabled. + */ + size_t missed_clock_ticks; + bool idle; uint64_t idle_ticks; uint64_t busy_ticks; - + /** * Processor ID assigned by kernel. */ - unsigned int id; + size_t id; bool active; - int tlb_active; - + bool tlb_active; + uint16_t frequency_mhz; uint32_t delay_loop_const; - + cpu_arch_t arch; - + struct thread *fpu_owner; @@ -100,5 +104,5 @@ extern void cpu_arch_init(void); extern void cpu_identify(void); -extern void cpu_print_report(cpu_t *m); +extern void cpu_print_report(cpu_t *); #endif Index: kernel/generic/include/ddi/ddi.h =================================================================== --- kernel/generic/include/ddi/ddi.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ddi/ddi.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -50,15 +50,14 @@ extern void ddi_init(void); -extern void ddi_parea_register(parea_t *parea); +extern void ddi_parea_register(parea_t *); -extern unative_t sys_physmem_map(unative_t phys_base, unative_t virt_base, - unative_t pages, unative_t flags); -extern unative_t sys_iospace_enable(ddi_ioarg_t *uspace_io_arg); -extern unative_t sys_preempt_control(int enable); +extern unative_t sys_physmem_map(unative_t, unative_t, unative_t, unative_t); +extern unative_t sys_iospace_enable(ddi_ioarg_t *); +extern unative_t sys_preempt_control(int); /* * Interface to be implemented by all architectures. */ -extern int ddi_iospace_enable_arch(task_t *task, uintptr_t ioaddr, size_t size); +extern int ddi_iospace_enable_arch(task_t *, uintptr_t, size_t); #endif Index: kernel/generic/include/ddi/ddi_arg.h =================================================================== --- kernel/generic/include/ddi/ddi_arg.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ddi/ddi_arg.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,8 +36,14 @@ #define KERN_DDI_ARG_H_ +#ifdef KERNEL + +#include + +#endif /* KERNEL */ + /** Structure encapsulating arguments for SYS_PHYSMEM_MAP syscall. */ typedef struct { /** ID of the destination task. */ - unsigned long long task_id; + uint64_t task_id; /** Physical address of starting frame. */ void *phys_base; @@ -45,14 +51,14 @@ void *virt_base; /** Number of pages to map. */ - unsigned long pages; + size_t pages; /** Address space area flags for the mapping. */ - int flags; + unsigned int flags; } ddi_memarg_t; /** Structure encapsulating arguments for SYS_ENABLE_IOSPACE syscall. */ typedef struct { - unsigned long long task_id; /**< ID of the destination task. */ - void *ioaddr; /**< Starting I/O space address. */ - unsigned long size; /**< Number of bytes. */ + uint64_t task_id; /**< ID of the destination task. */ + void *ioaddr; /**< Starting I/O space address. */ + size_t size; /**< Number of bytes. */ } ddi_ioarg_t; Index: kernel/generic/include/ddi/irq.h =================================================================== --- kernel/generic/include/ddi/irq.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ddi/irq.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -35,4 +35,15 @@ #ifndef KERN_IRQ_H_ #define KERN_IRQ_H_ + +#ifdef KERNEL + +#include +#include +#include +#include +#include +#include + +#endif /* KERNEL */ typedef enum { @@ -49,4 +60,5 @@ /** Write 4 bytes to the I/O space. */ CMD_PIO_WRITE_32, + /** * Perform a bit test on the source argument and store the result into @@ -54,4 +66,5 @@ */ CMD_BTEST, + /** * Predicate the execution of the following N commands by the boolean @@ -59,4 +72,5 @@ */ CMD_PREDICATE, + /** Accept the interrupt. */ CMD_ACCEPT, @@ -69,11 +83,11 @@ irq_cmd_type cmd; void *addr; - unsigned long long value; - unsigned int srcarg; - unsigned int dstarg; + uint32_t value; + uintptr_t srcarg; + uintptr_t dstarg; } irq_cmd_t; typedef struct { - unsigned int cmdcount; + size_t cmdcount; irq_cmd_t *cmds; } irq_code_t; @@ -81,14 +95,7 @@ #ifdef KERNEL -#include -#include -#include -#include -#include -#include - typedef enum { - IRQ_DECLINE, /**< Decline to service. */ - IRQ_ACCEPT /**< Accept to service. */ + IRQ_DECLINE, /**< Decline to service. */ + IRQ_ACCEPT /**< Accept to service. */ } irq_ownership_t; @@ -108,4 +115,5 @@ * Primarily, this structure is encapsulated in the irq_t structure. * It is protected by irq_t::lock. + * */ typedef struct { @@ -117,9 +125,10 @@ unative_t method; /** Arguments that will be sent if the IRQ is claimed. */ - unative_t scratch[IPC_CALL_LEN]; + uint32_t scratch[IPC_CALL_LEN]; /** Top-half pseudocode. */ irq_code_t *code; /** Counter. */ size_t counter; + /** * Link between IRQs that are notifying the same answerbox. The list is @@ -133,9 +142,10 @@ * If one device has multiple interrupts, there will be multiple irq_t * instantions with the same devno. + * */ typedef struct irq { /** Hash table link. */ link_t link; - + /** Lock protecting everything in this structure * except the link member. When both the IRQ @@ -143,5 +153,5 @@ * this lock must not be taken first. */ - SPINLOCK_DECLARE(lock); + IRQ_SPINLOCK_DECLARE(lock); /** Send EOI before processing the interrupt. @@ -152,8 +162,8 @@ */ bool preack; - + /** Unique device number. -1 if not yet assigned. */ devno_t devno; - + /** Actual IRQ number. -1 if not yet assigned. */ inr_t inr; @@ -166,15 +176,15 @@ /** Instance argument for the handler and the claim function. */ void *instance; - + /** Clear interrupt routine. */ cir_t cir; /** First argument to the clear interrupt routine. */ void *cir_arg; - + /** Notification configuration structure. */ ipc_notif_cfg_t notif_cfg; } irq_t; -SPINLOCK_EXTERN(irq_uspace_hash_table_lock); +IRQ_SPINLOCK_EXTERN(irq_uspace_hash_table_lock); extern hash_table_t irq_uspace_hash_table; @@ -184,8 +194,8 @@ extern irq_t *irq_dispatch_and_lock(inr_t); +#endif /* KERNEL */ + #endif -#endif - /** @} */ Index: kernel/generic/include/debug.h =================================================================== --- kernel/generic/include/debug.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/debug.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -37,12 +37,9 @@ #include -#include +#include -#define CALLER ((uintptr_t) __builtin_return_address(0)) +#define CALLER ((uintptr_t) __builtin_return_address(0)) -#ifndef HERE -/** Current Instruction Pointer address */ -# define HERE ((uintptr_t *) 0) -#endif +#ifdef CONFIG_DEBUG /** Debugging ASSERT macro @@ -55,12 +52,35 @@ * */ -#ifdef CONFIG_DEBUG -# define ASSERT(expr) \ - if (!(expr)) { \ - panic("Assertion failed (%s), caller=%p.", #expr, CALLER); \ - } -#else -# define ASSERT(expr) -#endif +#define ASSERT(expr) \ + do { \ + if (!(expr)) \ + panic("Assertion failed (%s)", #expr); \ + } while (0) + +/** Debugging verbose ASSERT macro + * + * If CONFIG_DEBUG is set, the ASSERT() macro + * evaluates expr and if it is false raises + * kernel panic. The panic message contains also + * the supplied message. + * + * @param expr Expression which is expected to be true. + * @param msg Additional message to show (string). + * + */ +#define ASSERT_VERBOSE(expr, msg) \ + do { \ + if (!(expr)) \ + panic("Assertion failed (%s, %s)", #expr, msg); \ + } while (0) + +#else /* CONFIG_DEBUG */ + +#define ASSERT(expr) +#define ASSERT_VERBOSE(expr, msg) + +#endif /* CONFIG_DEBUG */ + +#ifdef CONFIG_LOG /** Extensive logging output macro @@ -71,12 +91,9 @@ * */ - -#ifdef CONFIG_LOG -# define LOG(format, ...) \ - printf("%s() at %s:%u: " format "\n", __func__, __FILE__, \ - __LINE__, ##__VA_ARGS__); -#else -# define LOG(format, ...) -#endif +#define LOG(format, ...) \ + do { \ + printf("%s->%s() at %s:%u: " format "\n", symtab_fmt_name_lookup(CALLER), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__); \ + } while (0) /** Extensive logging execute macro @@ -87,16 +104,17 @@ * */ +#define LOG_EXEC(fnc) \ + do { \ + printf("%s->%s() at %s:%u: " #fnc "\n", symtab_fmt_name_lookup(CALLER), \ + __func__, __FILE__, __LINE__); \ + fnc; \ + } while (0) -#ifdef CONFIG_LOG -# define LOG_EXEC(fnc) \ - { \ - printf("%s() at %s:%u: " #fnc "\n", __func__, __FILE__, \ - __LINE__); \ - fnc; \ - } -#else -# define LOG_EXEC(fnc) fnc -#endif +#else /* CONFIG_LOG */ +#define LOG(format, ...) +#define LOG_EXEC(fnc) fnc + +#endif /* CONFOG_LOG */ #endif Index: kernel/generic/include/interrupt.h =================================================================== --- kernel/generic/include/interrupt.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/interrupt.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -44,12 +44,12 @@ #include -typedef void (* iroutine)(int n, istate_t *istate); +typedef void (* iroutine)(int, istate_t *); -extern void fault_if_from_uspace(istate_t *istate, const char *fmt, ...); -extern iroutine exc_register(int n, const char *name, iroutine f); -extern void exc_dispatch(int n, istate_t *t); -void exc_init(void); +extern void fault_if_from_uspace(istate_t *, const char *, ...); +extern iroutine exc_register(int, const char *, iroutine); +extern void exc_dispatch(int, istate_t *); +extern void exc_init(void); -extern void irq_initialize_arch(irq_t *irq); +extern void irq_initialize_arch(irq_t *); #endif Index: kernel/generic/include/ipc/ipc.h =================================================================== --- kernel/generic/include/ipc/ipc.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ipc/ipc.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -36,13 +36,16 @@ #define KERN_IPC_H_ -/* Length of data being transfered with IPC call */ -/* - the uspace may not be able to utilize full length */ -#define IPC_CALL_LEN 6 +/** Length of data being transfered with IPC call + * + * The uspace may not be able to utilize full length + * + */ +#define IPC_CALL_LEN 6 /** Maximum active async calls per thread */ #ifdef CONFIG_DEBUG -#define IPC_MAX_ASYNC_CALLS 4 + #define IPC_MAX_ASYNC_CALLS 4 #else -#define IPC_MAX_ASYNC_CALLS 4000 + #define IPC_MAX_ASYNC_CALLS 4000 #endif @@ -50,52 +53,61 @@ /** This is answer to a call */ -#define IPC_CALL_ANSWERED (1 << 0) +#define IPC_CALL_ANSWERED (1 << 0) + /** Answer will not be passed to userspace, will be discarded */ -#define IPC_CALL_DISCARD_ANSWER (1 << 1) +#define IPC_CALL_DISCARD_ANSWER (1 << 1) + /** Call was forwarded */ -#define IPC_CALL_FORWARDED (1 << 2) +#define IPC_CALL_FORWARDED (1 << 2) + /** Identify connect_me_to answer */ -#define IPC_CALL_CONN_ME_TO (1 << 3) +#define IPC_CALL_CONN_ME_TO (1 << 3) + /** Interrupt notification */ -#define IPC_CALL_NOTIF (1 << 4) - -/* - * Bits used in call hashes. +#define IPC_CALL_NOTIF (1 << 4) + + +/** Bits used in call hashes. + * * The addresses are aligned at least to 4 that is why we can use the 2 least * significant bits of the call address. - */ + * + */ + /** Type of this call is 'answer' */ -#define IPC_CALLID_ANSWERED 1 +#define IPC_CALLID_ANSWERED 1 + /** Type of this call is 'notification' */ -#define IPC_CALLID_NOTIFICATION 2 +#define IPC_CALLID_NOTIFICATION 2 /* Return values from sys_ipc_call_async(). */ -#define IPC_CALLRET_FATAL -1 -#define IPC_CALLRET_TEMPORARY -2 +#define IPC_CALLRET_FATAL -1 +#define IPC_CALLRET_TEMPORARY -2 /* Macros for manipulating calling data */ -#define IPC_SET_RETVAL(data, retval) ((data).args[0] = (retval)) -#define IPC_SET_METHOD(data, val) ((data).args[0] = (val)) -#define IPC_SET_ARG1(data, val) ((data).args[1] = (val)) -#define IPC_SET_ARG2(data, val) ((data).args[2] = (val)) -#define IPC_SET_ARG3(data, val) ((data).args[3] = (val)) -#define IPC_SET_ARG4(data, val) ((data).args[4] = (val)) -#define IPC_SET_ARG5(data, val) ((data).args[5] = (val)) - -#define IPC_GET_METHOD(data) ((data).args[0]) -#define IPC_GET_RETVAL(data) ((data).args[0]) - -#define IPC_GET_ARG1(data) ((data).args[1]) -#define IPC_GET_ARG2(data) ((data).args[2]) -#define IPC_GET_ARG3(data) ((data).args[3]) -#define IPC_GET_ARG4(data) ((data).args[4]) -#define IPC_GET_ARG5(data) ((data).args[5]) +#define IPC_SET_RETVAL(data, retval) ((data).args[0] = (retval)) +#define IPC_SET_METHOD(data, val) ((data).args[0] = (val)) +#define IPC_SET_ARG1(data, val) ((data).args[1] = (val)) +#define IPC_SET_ARG2(data, val) ((data).args[2] = (val)) +#define IPC_SET_ARG3(data, val) ((data).args[3] = (val)) +#define IPC_SET_ARG4(data, val) ((data).args[4] = (val)) +#define IPC_SET_ARG5(data, val) ((data).args[5] = (val)) + +#define IPC_GET_METHOD(data) ((data).args[0]) +#define IPC_GET_RETVAL(data) ((data).args[0]) + +#define IPC_GET_ARG1(data) ((data).args[1]) +#define IPC_GET_ARG2(data) ((data).args[2]) +#define IPC_GET_ARG3(data) ((data).args[3]) +#define IPC_GET_ARG4(data) ((data).args[4]) +#define IPC_GET_ARG5(data) ((data).args[5]) /* Well known phone descriptors */ -#define PHONE_NS 0 +#define PHONE_NS 0 /* Forwarding flags. */ -#define IPC_FF_NONE 0 +#define IPC_FF_NONE 0 + /** * The call will be routed as though it was initially sent via the phone used to @@ -104,10 +116,13 @@ * calls that were initially sent by the forwarder to the same destination. This * flag has no imapct on routing replies. - */ -#define IPC_FF_ROUTE_FROM_ME (1 << 0) - -/* System-specific methods - only through special syscalls + * + */ +#define IPC_FF_ROUTE_FROM_ME (1 << 0) + +/** System-specific methods - only through special syscalls * These methods have special behaviour - */ + * + */ + /** Clone connection. * @@ -115,19 +130,24 @@ * * - ARG1 - The caller sets ARG1 to the phone of the cloned connection. - * - The callee gets the new phone from ARG1. + * - The callee gets the new phone from ARG1. + * * - on answer, the callee acknowledges the new connection by sending EOK back * or the kernel closes it - */ -#define IPC_M_CONNECTION_CLONE 1 + * + */ +#define IPC_M_CONNECTION_CLONE 1 + /** Protocol for CONNECT - ME * * Through this call, the recipient learns about the new cloned connection. - * + * * - ARG5 - the kernel sets ARG5 to contain the hash of the used phone * - on asnwer, the callee acknowledges the new connection by sending EOK back * or the kernel closes it - */ -#define IPC_M_CONNECT_ME 2 -/** Protocol for CONNECT - TO - ME + * + */ +#define IPC_M_CONNECT_ME 2 + +/** Protocol for CONNECT - TO - ME * * Calling process asks the callee to create a callback connection, @@ -144,6 +164,8 @@ * - the allocated phoneid is passed to userspace * (on the receiving side) as ARG5 of the call. - */ -#define IPC_M_CONNECT_TO_ME 3 + * + */ +#define IPC_M_CONNECT_TO_ME 3 + /** Protocol for CONNECT - ME - TO * @@ -163,9 +185,10 @@ * */ -#define IPC_M_CONNECT_ME_TO 4 -/** This message is sent to answerbox when the phone - * is hung up - */ -#define IPC_M_PHONE_HUNGUP 5 +#define IPC_M_CONNECT_ME_TO 4 + +/** This message is sent to answerbox when the phone is hung up + * + */ +#define IPC_M_PHONE_HUNGUP 5 /** Send as_area over IPC. @@ -173,9 +196,10 @@ * - ARG2 - size of source as_area (filled automatically by kernel) * - ARG3 - flags of the as_area being sent - * + * * on answer, the recipient must set: * - ARG1 - dst as_area base adress - */ -#define IPC_M_SHARE_OUT 6 + * + */ +#define IPC_M_SHARE_OUT 6 /** Receive as_area over IPC. @@ -183,11 +207,12 @@ * - ARG2 - destination as_area size * - ARG3 - user defined argument - * + * * on answer, the recipient must set: * * - ARG1 - source as_area base address * - ARG2 - flags that will be used for sharing - */ -#define IPC_M_SHARE_IN 7 + * + */ +#define IPC_M_SHARE_IN 7 /** Send data to another address space over IPC. @@ -199,6 +224,7 @@ * - ARG1 - final destination address space virtual address * - ARG2 - final size of data to be copied - */ -#define IPC_M_DATA_WRITE 8 + * + */ +#define IPC_M_DATA_WRITE 8 /** Receive data from another address space over IPC. @@ -210,18 +236,21 @@ * - ARG1 - source virtual address in the destination address space * - ARG2 - final size of data to be copied - */ -#define IPC_M_DATA_READ 9 + * + */ +#define IPC_M_DATA_READ 9 /** Debug the recipient. * - ARG1 - specifies the debug method (from udebug_method_t) * - other arguments are specific to the debug method - */ -#define IPC_M_DEBUG_ALL 10 + * + */ +#define IPC_M_DEBUG_ALL 10 /* Well-known methods */ -#define IPC_M_LAST_SYSTEM 511 -#define IPC_M_PING 512 +#define IPC_M_LAST_SYSTEM 511 +#define IPC_M_PING 512 + /* User methods */ -#define IPC_FIRST_USER_METHOD 1024 +#define IPC_FIRST_USER_METHOD 1024 #ifdef KERNEL @@ -259,23 +288,24 @@ typedef struct answerbox { - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + struct task *task; - + waitq_t wq; - + /** Linkage for the list of task's synchronous answerboxes. */ link_t sync_box_link; - + /** Phones connected to this answerbox. */ link_t connected_phones; /** Received calls. */ - link_t calls; - link_t dispatched_calls; /* Should be hash table in the future */ - + link_t calls; + link_t dispatched_calls; /* Should be hash table in the future */ + /** Answered calls. */ link_t answers; - - SPINLOCK_DECLARE(irq_lock); + + IRQ_SPINLOCK_DECLARE(irq_lock); + /** Notifications from IRQ handlers. */ link_t irq_notifs; @@ -291,43 +321,46 @@ typedef struct { link_t link; - - int flags; - + + unsigned int flags; + /** Identification of the caller. */ struct task *sender; - /** The caller box is different from sender->answerbox for synchronous - * calls. */ + + /* + * The caller box is different from sender->answerbox + * for synchronous calls. + * + */ answerbox_t *callerbox; - + /** Private data to internal IPC. */ unative_t priv; - + /** Data passed from/to userspace. */ ipc_data_t data; - + /** Buffer for IPC_M_DATA_WRITE and IPC_M_DATA_READ. */ uint8_t *buffer; - + /* * The forward operation can masquerade the caller phone. For those * cases, we must keep it aside so that the answer is processed * correctly. + * */ phone_t *caller_phone; } call_t; - extern answerbox_t *ipc_phone_0; - extern void ipc_init(void); -extern call_t * ipc_call_alloc(int); +extern call_t *ipc_call_alloc(unsigned int); extern void ipc_call_free(call_t *); extern int ipc_call(phone_t *, call_t *); extern int ipc_call_sync(phone_t *, call_t *); -extern call_t * ipc_wait_for_call(answerbox_t *, uint32_t, int); -extern int ipc_forward(call_t *, phone_t *, answerbox_t *, int); +extern call_t * ipc_wait_for_call(answerbox_t *, uint32_t, unsigned int); +extern int ipc_forward(call_t *, phone_t *, answerbox_t *, unsigned int); extern void ipc_answer(answerbox_t *, call_t *); @@ -345,8 +378,8 @@ extern void ipc_print_task(task_id_t); +#endif /* KERNEL */ + #endif -#endif - /** @} */ Index: kernel/generic/include/ipc/irq.h =================================================================== --- kernel/generic/include/ipc/irq.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ipc/irq.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -37,5 +37,5 @@ /** Maximum length of IPC IRQ program */ -#define IRQ_MAX_PROG_SIZE 20 +#define IRQ_MAX_PROG_SIZE 20 #include @@ -58,15 +58,20 @@ */ #define ipc_irq_send_msg_0(irq) \ - ipc_irq_send_msg((irq), 0, 0, 0, 0, 0) + ipc_irq_send_msg((irq), 0, 0, 0, 0, 0) + #define ipc_irq_send_msg_1(irq, a1) \ - ipc_irq_send_msg((irq), (a1), 0, 0, 0, 0) + ipc_irq_send_msg((irq), (a1), 0, 0, 0, 0) + #define ipc_irq_send_msg_2(irq, a1, a2) \ - ipc_irq_send_msg((irq), (a1), (a2), 0, 0, 0) + ipc_irq_send_msg((irq), (a1), (a2), 0, 0, 0) + #define ipc_irq_send_msg_3(irq, a1, a2, a3) \ - ipc_irq_send_msg((irq), (a1), (a2), (a3), 0, 0) + ipc_irq_send_msg((irq), (a1), (a2), (a3), 0, 0) + #define ipc_irq_send_msg_4(irq, a1, a2, a3, a4) \ - ipc_irq_send_msg((irq), (a1), (a2), (a3), (a4), 0) + ipc_irq_send_msg((irq), (a1), (a2), (a3), (a4), 0) + #define ipc_irq_send_msg_5(irq, a1, a2, a3, a4, a5) \ - ipc_irq_send_msg((irq), (a1), (a2), (a3), (a4), (a5)) + ipc_irq_send_msg((irq), (a1), (a2), (a3), (a4), (a5)) extern void ipc_irq_send_msg(irq_t *, unative_t, unative_t, unative_t, unative_t, Index: kernel/generic/include/ipc/sysipc.h =================================================================== --- kernel/generic/include/ipc/sysipc.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/ipc/sysipc.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -40,26 +40,26 @@ #include -unative_t sys_ipc_call_sync_fast(unative_t phoneid, unative_t method, +extern unative_t sys_ipc_call_sync_fast(unative_t phoneid, unative_t method, unative_t arg1, unative_t arg2, unative_t arg3, ipc_data_t *data); -unative_t sys_ipc_call_sync_slow(unative_t phoneid, ipc_data_t *question, +extern unative_t sys_ipc_call_sync_slow(unative_t phoneid, ipc_data_t *question, ipc_data_t *reply); -unative_t sys_ipc_call_async_fast(unative_t phoneid, unative_t method, +extern unative_t sys_ipc_call_async_fast(unative_t phoneid, unative_t method, unative_t arg1, unative_t arg2, unative_t arg3, unative_t arg4); -unative_t sys_ipc_call_async_slow(unative_t phoneid, ipc_data_t *data); -unative_t sys_ipc_answer_fast(unative_t callid, unative_t retval, +extern unative_t sys_ipc_call_async_slow(unative_t phoneid, ipc_data_t *data); +extern unative_t sys_ipc_answer_fast(unative_t callid, unative_t retval, unative_t arg1, unative_t arg2, unative_t arg3, unative_t arg4); -unative_t sys_ipc_answer_slow(unative_t callid, ipc_data_t *data); -unative_t sys_ipc_wait_for_call(ipc_data_t *calldata, uint32_t usec, - int nonblocking); -unative_t sys_ipc_poke(void); -unative_t sys_ipc_forward_fast(unative_t callid, unative_t phoneid, - unative_t method, unative_t arg1, unative_t arg2, int mode); -unative_t sys_ipc_forward_slow(unative_t callid, unative_t phoneid, - ipc_data_t *data, int mode); -unative_t sys_ipc_hangup(unative_t phoneid); -unative_t sys_ipc_register_irq(inr_t inr, devno_t devno, unative_t method, +extern unative_t sys_ipc_answer_slow(unative_t callid, ipc_data_t *data); +extern unative_t sys_ipc_wait_for_call(ipc_data_t *calldata, uint32_t usec, + unsigned int nonblocking); +extern unative_t sys_ipc_poke(void); +extern unative_t sys_ipc_forward_fast(unative_t callid, unative_t phoneid, + unative_t method, unative_t arg1, unative_t arg2, unsigned int mode); +extern unative_t sys_ipc_forward_slow(unative_t callid, unative_t phoneid, + ipc_data_t *data, unsigned int mode); +extern unative_t sys_ipc_hangup(unative_t phoneid); +extern unative_t sys_ipc_register_irq(inr_t inr, devno_t devno, unative_t method, irq_code_t *ucode); -unative_t sys_ipc_unregister_irq(inr_t inr, devno_t devno); -unative_t sys_ipc_connect_kbox(sysarg64_t *task_id); +extern unative_t sys_ipc_unregister_irq(inr_t inr, devno_t devno); +extern unative_t sys_ipc_connect_kbox(sysarg64_t *task_id); #endif Index: kernel/generic/include/lib/elf.h =================================================================== --- kernel/generic/include/lib/elf.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/lib/elf.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ @@ -42,150 +42,150 @@ * current ELF version */ -#define EV_CURRENT 1 +#define EV_CURRENT 1 /** * ELF types */ -#define ET_NONE 0 /* No type */ -#define ET_REL 1 /* Relocatable file */ -#define ET_EXEC 2 /* Executable */ -#define ET_DYN 3 /* Shared object */ -#define ET_CORE 4 /* Core */ -#define ET_LOPROC 0xff00 /* Processor specific */ -#define ET_HIPROC 0xffff /* Processor specific */ - -/** +#define ET_NONE 0 /* No type */ +#define ET_REL 1 /* Relocatable file */ +#define ET_EXEC 2 /* Executable */ +#define ET_DYN 3 /* Shared object */ +#define ET_CORE 4 /* Core */ +#define ET_LOPROC 0xff00 /* Processor specific */ +#define ET_HIPROC 0xffff /* Processor specific */ + +/** * ELF machine types */ -#define EM_NO 0 /* No machine */ -#define EM_SPARC 2 /* SPARC */ -#define EM_386 3 /* i386 */ -#define EM_MIPS 8 /* MIPS RS3000 */ -#define EM_MIPS_RS3_LE 10 /* MIPS RS3000 LE */ -#define EM_PPC 20 /* PPC32 */ -#define EM_PPC64 21 /* PPC64 */ -#define EM_ARM 40 /* ARM */ -#define EM_SPARCV9 43 /* SPARC64 */ -#define EM_IA_64 50 /* IA-64 */ -#define EM_X86_64 62 /* AMD64/EMT64 */ +#define EM_NO 0 /* No machine */ +#define EM_SPARC 2 /* SPARC */ +#define EM_386 3 /* i386 */ +#define EM_MIPS 8 /* MIPS RS3000 */ +#define EM_MIPS_RS3_LE 10 /* MIPS RS3000 LE */ +#define EM_PPC 20 /* PPC32 */ +#define EM_PPC64 21 /* PPC64 */ +#define EM_ARM 40 /* ARM */ +#define EM_SPARCV9 43 /* SPARC64 */ +#define EM_IA_64 50 /* IA-64 */ +#define EM_X86_64 62 /* AMD64/EMT64 */ /** * ELF identification indexes */ -#define EI_MAG0 0 -#define EI_MAG1 1 -#define EI_MAG2 2 -#define EI_MAG3 3 -#define EI_CLASS 4 /* File class */ -#define EI_DATA 5 /* Data encoding */ -#define EI_VERSION 6 /* File version */ -#define EI_OSABI 7 -#define EI_ABIVERSION 8 -#define EI_PAD 9 /* Start of padding bytes */ -#define EI_NIDENT 16 /* ELF identification table size */ +#define EI_MAG0 0 +#define EI_MAG1 1 +#define EI_MAG2 2 +#define EI_MAG3 3 +#define EI_CLASS 4 /* File class */ +#define EI_DATA 5 /* Data encoding */ +#define EI_VERSION 6 /* File version */ +#define EI_OSABI 7 +#define EI_ABIVERSION 8 +#define EI_PAD 9 /* Start of padding bytes */ +#define EI_NIDENT 16 /* ELF identification table size */ /** * ELF magic number */ -#define ELFMAG0 0x7f -#define ELFMAG1 'E' -#define ELFMAG2 'L' -#define ELFMAG3 'F' +#define ELFMAG0 0x7f +#define ELFMAG1 'E' +#define ELFMAG2 'L' +#define ELFMAG3 'F' /** * ELF file classes */ -#define ELFCLASSNONE 0 -#define ELFCLASS32 1 -#define ELFCLASS64 2 +#define ELFCLASSNONE 0 +#define ELFCLASS32 1 +#define ELFCLASS64 2 /** * ELF data encoding types */ -#define ELFDATANONE 0 -#define ELFDATA2LSB 1 /* Least significant byte first (little endian) */ -#define ELFDATA2MSB 2 /* Most signigicant byte first (big endian) */ +#define ELFDATANONE 0 +#define ELFDATA2LSB 1 /* Least significant byte first (little endian) */ +#define ELFDATA2MSB 2 /* Most signigicant byte first (big endian) */ /** * ELF error return codes */ -#define EE_OK 0 /* No error */ -#define EE_INVALID 1 /* Invalid ELF image */ -#define EE_MEMORY 2 /* Cannot allocate address space */ -#define EE_INCOMPATIBLE 3 /* ELF image is not compatible with current architecture */ -#define EE_UNSUPPORTED 4 /* Non-supported ELF (e.g. dynamic ELFs) */ -#define EE_LOADER 5 /* The image is actually a program loader */ -#define EE_IRRECOVERABLE 6 +#define EE_OK 0 /* No error */ +#define EE_INVALID 1 /* Invalid ELF image */ +#define EE_MEMORY 2 /* Cannot allocate address space */ +#define EE_INCOMPATIBLE 3 /* ELF image is not compatible with current architecture */ +#define EE_UNSUPPORTED 4 /* Non-supported ELF (e.g. dynamic ELFs) */ +#define EE_LOADER 5 /* The image is actually a program loader */ +#define EE_IRRECOVERABLE 6 /** * ELF section types */ -#define SHT_NULL 0 -#define SHT_PROGBITS 1 -#define SHT_SYMTAB 2 -#define SHT_STRTAB 3 -#define SHT_RELA 4 -#define SHT_HASH 5 -#define SHT_DYNAMIC 6 -#define SHT_NOTE 7 -#define SHT_NOBITS 8 -#define SHT_REL 9 -#define SHT_SHLIB 10 -#define SHT_DYNSYM 11 -#define SHT_LOOS 0x60000000 -#define SHT_HIOS 0x6fffffff -#define SHT_LOPROC 0x70000000 -#define SHT_HIPROC 0x7fffffff -#define SHT_LOUSER 0x80000000 -#define SHT_HIUSER 0xffffffff +#define SHT_NULL 0 +#define SHT_PROGBITS 1 +#define SHT_SYMTAB 2 +#define SHT_STRTAB 3 +#define SHT_RELA 4 +#define SHT_HASH 5 +#define SHT_DYNAMIC 6 +#define SHT_NOTE 7 +#define SHT_NOBITS 8 +#define SHT_REL 9 +#define SHT_SHLIB 10 +#define SHT_DYNSYM 11 +#define SHT_LOOS 0x60000000 +#define SHT_HIOS 0x6fffffff +#define SHT_LOPROC 0x70000000 +#define SHT_HIPROC 0x7fffffff +#define SHT_LOUSER 0x80000000 +#define SHT_HIUSER 0xffffffff /** * ELF section flags */ -#define SHF_WRITE 0x1 -#define SHF_ALLOC 0x2 -#define SHF_EXECINSTR 0x4 -#define SHF_TLS 0x400 -#define SHF_MASKPROC 0xf0000000 +#define SHF_WRITE 0x1 +#define SHF_ALLOC 0x2 +#define SHF_EXECINSTR 0x4 +#define SHF_TLS 0x400 +#define SHF_MASKPROC 0xf0000000 /** * Symbol binding */ -#define STB_LOCAL 0 -#define STB_GLOBAL 1 -#define STB_WEAK 2 -#define STB_LOPROC 13 -#define STB_HIPROC 15 +#define STB_LOCAL 0 +#define STB_GLOBAL 1 +#define STB_WEAK 2 +#define STB_LOPROC 13 +#define STB_HIPROC 15 /** * Symbol types */ -#define STT_NOTYPE 0 -#define STT_OBJECT 1 -#define STT_FUNC 2 -#define STT_SECTION 3 -#define STT_FILE 4 -#define STT_LOPROC 13 -#define STT_HIPROC 15 +#define STT_NOTYPE 0 +#define STT_OBJECT 1 +#define STT_FUNC 2 +#define STT_SECTION 3 +#define STT_FILE 4 +#define STT_LOPROC 13 +#define STT_HIPROC 15 /** * Program segment types */ -#define PT_NULL 0 -#define PT_LOAD 1 -#define PT_DYNAMIC 2 -#define PT_INTERP 3 -#define PT_NOTE 4 -#define PT_SHLIB 5 -#define PT_PHDR 6 -#define PT_LOPROC 0x70000000 -#define PT_HIPROC 0x7fffffff +#define PT_NULL 0 +#define PT_LOAD 1 +#define PT_DYNAMIC 2 +#define PT_INTERP 3 +#define PT_NOTE 4 +#define PT_SHLIB 5 +#define PT_PHDR 6 +#define PT_LOPROC 0x70000000 +#define PT_HIPROC 0x7fffffff /** * Program segment attributes. */ -#define PF_X 1 -#define PF_W 2 -#define PF_R 4 +#define PF_X 1 +#define PF_W 2 +#define PF_R 4 /** @@ -195,4 +195,5 @@ * ELF object file specifications. They are the only types used * in ELF header. + * */ typedef uint64_t elf_xword; @@ -206,4 +207,5 @@ * * These types are specific for 32-bit format. + * */ typedef uint32_t elf32_addr; @@ -214,4 +216,5 @@ * * These types are specific for 64-bit format. + * */ typedef uint64_t elf64_addr; @@ -235,4 +238,5 @@ elf_half e_shstrndx; }; + struct elf64_header { uint8_t e_ident[EI_NIDENT]; @@ -252,5 +256,5 @@ }; -/* +/** * ELF segment header. * Segments headers are also known as program headers. @@ -266,4 +270,5 @@ elf_word p_align; }; + struct elf64_segment_header { elf_word p_type; @@ -277,5 +282,5 @@ }; -/* +/** * ELF section header */ @@ -292,4 +297,5 @@ elf_word sh_entsize; }; + struct elf64_section_header { elf_word sh_name; @@ -305,5 +311,5 @@ }; -/* +/** * ELF symbol table entry */ @@ -316,4 +322,5 @@ elf_half st_shndx; }; + struct elf64_symbol { elf_word st_name; @@ -325,5 +332,5 @@ }; -#ifdef __32_BITS__ +#ifdef __32_BITS__ typedef struct elf32_header elf_header_t; typedef struct elf32_segment_header elf_segment_header_t; @@ -331,4 +338,5 @@ typedef struct elf32_symbol elf_symbol_t; #endif + #ifdef __64_BITS__ typedef struct elf64_header elf_header_t; @@ -340,6 +348,6 @@ extern const char *elf_error(unsigned int rc); -/* Interpreter string used to recognize the program loader */ -#define ELF_INTERP_ZSTR "kernel" +/** Interpreter string used to recognize the program loader */ +#define ELF_INTERP_ZSTR "kernel" #define ELF_INTERP_ZLEN sizeof(ELF_INTERP_ZSTR) Index: kernel/generic/include/mm/as.h =================================================================== --- kernel/generic/include/mm/as.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/mm/as.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -43,8 +43,8 @@ /** Address space area flags. */ -#define AS_AREA_READ 1 -#define AS_AREA_WRITE 2 -#define AS_AREA_EXEC 4 -#define AS_AREA_CACHEABLE 8 +#define AS_AREA_READ 1 +#define AS_AREA_WRITE 2 +#define AS_AREA_EXEC 4 +#define AS_AREA_CACHEABLE 8 /** Address space area info exported to userspace. */ @@ -52,10 +52,10 @@ /** Starting address */ uintptr_t start_addr; - + /** Area size */ size_t size; - + /** Area flags */ - int flags; + unsigned int flags; } as_area_info_t; @@ -75,27 +75,30 @@ * Defined to be true if user address space and kernel address space shadow each * other. - */ -#define KERNEL_ADDRESS_SPACE_SHADOWED KERNEL_ADDRESS_SPACE_SHADOWED_ARCH - -#define KERNEL_ADDRESS_SPACE_START KERNEL_ADDRESS_SPACE_START_ARCH -#define KERNEL_ADDRESS_SPACE_END KERNEL_ADDRESS_SPACE_END_ARCH -#define USER_ADDRESS_SPACE_START USER_ADDRESS_SPACE_START_ARCH -#define USER_ADDRESS_SPACE_END USER_ADDRESS_SPACE_END_ARCH - -#define USTACK_ADDRESS USTACK_ADDRESS_ARCH + * + */ +#define KERNEL_ADDRESS_SPACE_SHADOWED KERNEL_ADDRESS_SPACE_SHADOWED_ARCH + +#define KERNEL_ADDRESS_SPACE_START KERNEL_ADDRESS_SPACE_START_ARCH +#define KERNEL_ADDRESS_SPACE_END KERNEL_ADDRESS_SPACE_END_ARCH +#define USER_ADDRESS_SPACE_START USER_ADDRESS_SPACE_START_ARCH +#define USER_ADDRESS_SPACE_END USER_ADDRESS_SPACE_END_ARCH + +#define USTACK_ADDRESS USTACK_ADDRESS_ARCH /** Kernel address space. */ -#define FLAG_AS_KERNEL (1 << 0) +#define FLAG_AS_KERNEL (1 << 0) /* Address space area attributes. */ -#define AS_AREA_ATTR_NONE 0 -#define AS_AREA_ATTR_PARTIAL 1 /**< Not fully initialized area. */ +#define AS_AREA_ATTR_NONE 0 +#define AS_AREA_ATTR_PARTIAL 1 /**< Not fully initialized area. */ /** The page fault was not resolved by as_page_fault(). */ -#define AS_PF_FAULT 0 +#define AS_PF_FAULT 0 + /** The page fault was resolved by as_page_fault(). */ -#define AS_PF_OK 1 +#define AS_PF_OK 1 + /** The page fault was caused by memcpy_from_uspace() or memcpy_to_uspace(). */ -#define AS_PF_DEFER 2 +#define AS_PF_DEFER 2 /** Address space structure. @@ -105,8 +108,10 @@ * supposed to figure in the list as they are shared by all tasks and * set up during system initialization. + * */ typedef struct as { /** Protected by asidlock. */ link_t inactive_as_with_asid_link; + /** * Number of processors on wich is this address space active. @@ -114,4 +119,5 @@ */ size_t cpu_refcount; + /** * Address space identifier. @@ -120,10 +126,10 @@ */ asid_t asid; - + /** Number of references (i.e tasks that reference this as). */ atomic_t refcount; - + mutex_t lock; - + /** B+tree of address space areas. */ btree_t as_area_btree; @@ -131,5 +137,5 @@ /** Non-generic content. */ as_genarch_t genarch; - + /** Architecture specific content. */ as_arch_t arch; @@ -137,8 +143,9 @@ typedef struct { - pte_t *(* page_table_create)(int flags); - void (* page_table_destroy)(pte_t *page_table); - void (* page_table_lock)(as_t *as, bool lock); - void (* page_table_unlock)(as_t *as, bool unlock); + pte_t *(* page_table_create)(unsigned int); + void (* page_table_destroy)(pte_t *); + void (* page_table_lock)(as_t *, bool); + void (* page_table_unlock)(as_t *, bool); + bool (* page_table_locked)(as_t *); } as_operations_t; @@ -146,10 +153,12 @@ * This structure contains information associated with the shared address space * area. + * */ typedef struct { /** This lock must be acquired only when the as_area lock is held. */ - mutex_t lock; + mutex_t lock; /** This structure can be deallocated if refcount drops to 0. */ size_t refcount; + /** * B+tree containing complete map of anonymous pages of the shared area. @@ -169,9 +178,12 @@ /** Backend data stored in address space area. */ typedef union mem_backend_data { - struct { /**< elf_backend members */ + /** elf_backend members */ + struct { elf_header_t *elf; elf_segment_header_t *segment; }; - struct { /**< phys_backend members */ + + /** phys_backend members */ + struct { uintptr_t base; size_t frames; @@ -182,15 +194,18 @@ * * Each as_area_t structure describes one contiguous area of virtual memory. + * */ typedef struct { mutex_t lock; /** Containing address space. */ - as_t *as; + as_t *as; + /** * Flags related to the memory represented by the address space area. */ - int flags; + unsigned int flags; + /** Attributes related to the address space area itself. */ - int attributes; + unsigned int attributes; /** Size of this area in multiples of PAGE_SIZE. */ size_t pages; @@ -199,5 +214,5 @@ /** Map of used space. */ btree_t used_space; - + /** * If the address space area has been shared, this pointer will @@ -205,8 +220,8 @@ */ share_info_t *sh_info; - + /** Memory backend backing this address space area. */ struct mem_backend *backend; - + /** Data to be used by the backend. */ mem_backend_data_t backend_data; @@ -215,7 +230,7 @@ /** Address space area backend structure. */ typedef struct mem_backend { - int (* page_fault)(as_area_t *area, uintptr_t addr, pf_access_t access); - void (* frame_free)(as_area_t *area, uintptr_t page, uintptr_t frame); - void (* share)(as_area_t *area); + int (* page_fault)(as_area_t *, uintptr_t, pf_access_t); + void (* frame_free)(as_area_t *, uintptr_t, uintptr_t); + void (* share)(as_area_t *); } mem_backend_t; @@ -227,5 +242,5 @@ extern void as_init(void); -extern as_t *as_create(int); +extern as_t *as_create(unsigned int); extern void as_destroy(as_t *); extern void as_hold(as_t *); @@ -234,12 +249,13 @@ extern int as_page_fault(uintptr_t, pf_access_t, istate_t *); -extern as_area_t *as_area_create(as_t *, int, size_t, uintptr_t, int, - mem_backend_t *, mem_backend_data_t *); +extern as_area_t *as_area_create(as_t *, unsigned int, size_t, uintptr_t, + unsigned int, mem_backend_t *, mem_backend_data_t *); extern int as_area_destroy(as_t *, uintptr_t); -extern int as_area_resize(as_t *, uintptr_t, size_t, int); -extern int as_area_share(as_t *, uintptr_t, size_t, as_t *, uintptr_t, int); -extern int as_area_change_flags(as_t *, int, uintptr_t); - -extern int as_area_get_flags(as_area_t *); +extern int as_area_resize(as_t *, uintptr_t, size_t, unsigned int); +extern int as_area_share(as_t *, uintptr_t, size_t, as_t *, uintptr_t, + unsigned int); +extern int as_area_change_flags(as_t *, unsigned int, uintptr_t); + +extern unsigned int as_area_get_flags(as_area_t *); extern bool as_area_check_access(as_area_t *, pf_access_t); extern size_t as_area_get_size(uintptr_t); @@ -249,16 +265,21 @@ /* Interface to be implemented by architectures. */ + #ifndef as_constructor_arch -extern int as_constructor_arch(as_t *, int); +extern int as_constructor_arch(as_t *, unsigned int); #endif /* !def as_constructor_arch */ + #ifndef as_destructor_arch extern int as_destructor_arch(as_t *); #endif /* !def as_destructor_arch */ + #ifndef as_create_arch -extern int as_create_arch(as_t *, int); +extern int as_create_arch(as_t *, unsigned int); #endif /* !def as_create_arch */ + #ifndef as_install_arch extern void as_install_arch(as_t *); #endif /* !def as_install_arch */ + #ifndef as_deinstall_arch extern void as_deinstall_arch(as_t *); @@ -270,17 +291,18 @@ extern mem_backend_t phys_backend; -/** +/** * This flags is passed when running the loader, otherwise elf_load() * would return with a EE_LOADER error code. - */ -#define ELD_F_NONE 0 -#define ELD_F_LOADER 1 - -extern unsigned int elf_load(elf_header_t *, as_t *, int); + * + */ +#define ELD_F_NONE 0 +#define ELD_F_LOADER 1 + +extern unsigned int elf_load(elf_header_t *, as_t *, unsigned int); /* Address space area related syscalls. */ -extern unative_t sys_as_area_create(uintptr_t, size_t, int); -extern unative_t sys_as_area_resize(uintptr_t, size_t, int); -extern unative_t sys_as_area_change_flags(uintptr_t, int); +extern unative_t sys_as_area_create(uintptr_t, size_t, unsigned int); +extern unative_t sys_as_area_resize(uintptr_t, size_t, unsigned int); +extern unative_t sys_as_area_change_flags(uintptr_t, unsigned int); extern unative_t sys_as_area_destroy(uintptr_t); Index: kernel/generic/include/mm/frame.h =================================================================== --- kernel/generic/include/mm/frame.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/mm/frame.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -81,5 +81,5 @@ typedef struct { - size_t refcount; /**< Tracking of shared frames */ + size_t refcount; /**< Tracking of shared frames */ uint8_t buddy_order; /**< Buddy system block order */ link_t buddy_link; /**< Link to the next free block inside @@ -91,8 +91,8 @@ pfn_t base; /**< Frame_no of the first frame in the frames array */ - size_t count; /**< Size of zone */ - size_t free_count; /**< Number of free frame_t + size_t count; /**< Size of zone */ + size_t free_count; /**< Number of free frame_t structures */ - size_t busy_count; /**< Number of busy frame_t + size_t busy_count; /**< Number of busy frame_t structures */ zone_flags_t flags; /**< Type of the zone */ @@ -108,5 +108,5 @@ */ typedef struct { - SPINLOCK_DECLARE(lock); + IRQ_SPINLOCK_DECLARE(lock); size_t count; zone_t info[ZONES_MAX]; Index: kernel/generic/include/mm/page.h =================================================================== --- kernel/generic/include/mm/page.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/mm/page.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -42,8 +42,7 @@ /** Operations to manipulate page mappings. */ typedef struct { - void (* mapping_insert)(as_t *as, uintptr_t page, uintptr_t frame, - int flags); - void (* mapping_remove)(as_t *as, uintptr_t page); - pte_t *(* mapping_find)(as_t *as, uintptr_t page); + void (* mapping_insert)(as_t *, uintptr_t, uintptr_t, unsigned int); + void (* mapping_remove)(as_t *, uintptr_t); + pte_t *(* mapping_find)(as_t *, uintptr_t); } page_mapping_operations_t; @@ -51,15 +50,15 @@ extern void page_init(void); -extern void page_table_lock(as_t *as, bool lock); -extern void page_table_unlock(as_t *as, bool unlock); -extern void page_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, - int flags); -extern void page_mapping_remove(as_t *as, uintptr_t page); -extern pte_t *page_mapping_find(as_t *as, uintptr_t page); -extern pte_t *page_table_create(int flags); -extern void page_table_destroy(pte_t *page_table); -extern void map_structure(uintptr_t s, size_t size); +extern void page_table_lock(as_t *, bool); +extern void page_table_unlock(as_t *, bool); +extern bool page_table_locked(as_t *); +extern void page_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int); +extern void page_mapping_remove(as_t *, uintptr_t); +extern pte_t *page_mapping_find(as_t *, uintptr_t); +extern pte_t *page_table_create(unsigned int); +extern void page_table_destroy(pte_t *); +extern void map_structure(uintptr_t, size_t); -extern uintptr_t hw_map(uintptr_t physaddr, size_t size); +extern uintptr_t hw_map(uintptr_t, size_t); #endif Index: kernel/generic/include/mm/slab.h =================================================================== --- kernel/generic/include/mm/slab.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/mm/slab.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -84,5 +84,4 @@ } slab_mag_cache_t; - typedef struct { const char *name; @@ -94,13 +93,13 @@ size_t size; - int (*constructor)(void *obj, int kmflag); - int (*destructor)(void *obj); + int (*constructor)(void *obj, unsigned int kmflag); + size_t (*destructor)(void *obj); /** Flags changing behaviour of cache */ - int flags; + unsigned int flags; /* Computed values */ - uint8_t order; /**< Order of frames to be allocated */ - unsigned int objects; /**< Number of objects that fit in */ + uint8_t order; /**< Order of frames to be allocated */ + size_t objects; /**< Number of objects that fit in */ /* Statistics */ @@ -109,5 +108,5 @@ atomic_t cached_objs; /** How many magazines in magazines list */ - atomic_t magazine_counter; + atomic_t magazine_counter; /* Slabs */ @@ -124,10 +123,10 @@ extern slab_cache_t *slab_cache_create(const char *, size_t, size_t, - int (*)(void *, int), int (*)(void *), int); + int (*)(void *, unsigned int), size_t (*)(void *), unsigned int); extern void slab_cache_destroy(slab_cache_t *); -extern void * slab_alloc(slab_cache_t *, int); +extern void * slab_alloc(slab_cache_t *, unsigned int); extern void slab_free(slab_cache_t *, void *); -extern size_t slab_reclaim(int); +extern size_t slab_reclaim(unsigned int); /* slab subsytem initialization */ @@ -139,6 +138,6 @@ /* malloc support */ -extern void *malloc(unsigned int, int); -extern void *realloc(void *, unsigned int, int); +extern void *malloc(size_t, unsigned int); +extern void *realloc(void *, size_t, unsigned int); extern void free(void *); Index: kernel/generic/include/panic.h =================================================================== --- kernel/generic/include/panic.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/panic.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -41,20 +41,25 @@ #ifdef CONFIG_DEBUG -# define panic(format, ...) \ - do { \ - silent = false; \ - printf("Kernel panic in %s() at %s:%u.\n", \ - __func__, __FILE__, __LINE__); \ - stack_trace(); \ - panic_printf("Panic message: " format "\n", \ - ##__VA_ARGS__);\ - } while (0) -#else -# define panic(format, ...) \ - do { \ - silent = false; \ - panic_printf("Kernel panic: " format "\n", ##__VA_ARGS__); \ - } while (0) -#endif + +#define panic(format, ...) \ + do { \ + silent = false; \ + printf("Kernel panic in %s() at %s:%u\n", \ + __func__, __FILE__, __LINE__); \ + stack_trace(); \ + panic_printf("Panic message: " format "\n", \ + ##__VA_ARGS__);\ + } while (0) + +#else /* CONFIG_DEBUG */ + +#define panic(format, ...) \ + do { \ + silent = false; \ + panic_printf("Kernel panic: " format "\n", ##__VA_ARGS__); \ + stack_trace(); \ + } while (0) + +#endif /* CONFIG_DEBUG */ extern bool silent; Index: kernel/generic/include/preemption.h =================================================================== --- kernel/generic/include/preemption.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/preemption.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,5 +27,5 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ Index: kernel/generic/include/proc/scheduler.h =================================================================== --- kernel/generic/include/proc/scheduler.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/proc/scheduler.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -47,7 +47,7 @@ /** Scheduler run queue structure. */ typedef struct { - SPINLOCK_DECLARE(lock); - link_t rq_head; /**< List of ready threads. */ - size_t n; /**< Number of threads in rq_ready. */ + IRQ_SPINLOCK_DECLARE(lock); + link_t rq_head; /**< List of ready threads. */ + size_t n; /**< Number of threads in rq_ready. */ } runq_t; Index: kernel/generic/include/proc/task.h =================================================================== --- kernel/generic/include/proc/task.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/proc/task.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -70,6 +70,6 @@ * threads. */ - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + char name[TASK_NAME_BUFLEN]; /** List of threads contained in this task. */ @@ -81,13 +81,13 @@ /** Task security context. */ context_id_t context; - + /** Number of references (i.e. threads). */ atomic_t refcount; /** Number of threads that haven't exited yet. */ atomic_t lifecount; - + /** Task capabilities. */ cap_t capabilities; - + /* IPC stuff */ answerbox_t answerbox; /**< Communication endpoint */ @@ -101,13 +101,13 @@ /** List of synchronous answerboxes. */ link_t sync_box_head; - + #ifdef CONFIG_UDEBUG /** Debugging stuff. */ udebug_task_t udebug; - + /** Kernel answerbox. */ kbox_t kb; -#endif - +#endif /* CONFIG_UDEBUG */ + /** Architecture specific task data. */ task_arch_t arch; @@ -126,5 +126,5 @@ } task_t; -SPINLOCK_EXTERN(tasks_lock); +IRQ_SPINLOCK_EXTERN(tasks_lock); extern avltree_t tasks_tree; Index: kernel/generic/include/proc/thread.h =================================================================== --- kernel/generic/include/proc/thread.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/proc/thread.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -50,6 +50,6 @@ #include -#define THREAD_STACK_SIZE STACK_SIZE -#define THREAD_NAME_BUFLEN 20 +#define THREAD_STACK_SIZE STACK_SIZE +#define THREAD_NAME_BUFLEN 20 extern const char *thread_states[]; @@ -61,19 +61,23 @@ * When using this flag, the caller must set cpu in the thread_t * structure manually before calling thread_ready (even on uniprocessor). - */ -#define THREAD_FLAG_WIRED (1 << 0) + * + */ +#define THREAD_FLAG_WIRED (1 << 0) + /** Thread was migrated to another CPU and has not run yet. */ -#define THREAD_FLAG_STOLEN (1 << 1) +#define THREAD_FLAG_STOLEN (1 << 1) + /** Thread executes in userspace. */ -#define THREAD_FLAG_USPACE (1 << 2) +#define THREAD_FLAG_USPACE (1 << 2) + /** Thread will be attached by the caller. */ -#define THREAD_FLAG_NOATTACH (1 << 3) +#define THREAD_FLAG_NOATTACH (1 << 3) /** Thread structure. There is one per thread. */ typedef struct thread { - link_t rq_link; /**< Run queue link. */ - link_t wq_link; /**< Wait queue link. */ - link_t th_link; /**< Links to threads within containing task. */ - + link_t rq_link; /**< Run queue link. */ + link_t wq_link; /**< Wait queue link. */ + link_t th_link; /**< Links to threads within containing task. */ + /** Threads linkage to the threads_tree. */ avltree_node_t threads_tree_node; @@ -83,13 +87,13 @@ * Protects the whole thread structure except list links above. */ - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + char name[THREAD_NAME_BUFLEN]; - + /** Function implementing the thread. */ void (* thread_code)(void *); /** Argument passed to thread_code() function. */ void *thread_arg; - + /** * From here, the stored context is restored when the thread is @@ -107,5 +111,5 @@ */ context_t sleep_interruption_context; - + /** If true, the thread can be interrupted from sleep. */ bool sleep_interruptible; @@ -115,6 +119,6 @@ timeout_t sleep_timeout; /** Flag signalling sleep timeout in progress. */ - volatile int timeout_pending; - + volatile bool timeout_pending; + /** * True if this thread is executing copy_from_uspace(). @@ -132,5 +136,5 @@ * thread_exit() before returning to userspace. */ - bool interrupted; + bool interrupted; /** If true, thread_join_timeout() cannot be used on this thread. */ @@ -140,8 +144,8 @@ /** Link used in the joiner_head list. */ link_t joiner_link; - + fpu_context_t *saved_fpu_context; int fpu_context_exists; - + /* * Defined only if thread doesn't run. @@ -150,16 +154,16 @@ */ int fpu_context_engaged; - + rwlock_type_t rwlock_holder_type; - + /** Callback fired in scheduler before the thread is put asleep. */ void (* call_me)(void *); /** Argument passed to call_me(). */ void *call_me_with; - + /** Thread's state. */ state_t state; /** Thread's flags. */ - int flags; + unsigned int flags; /** Thread's CPU. */ @@ -167,5 +171,5 @@ /** Containing task. */ task_t *task; - + /** Ticks before preemption. */ uint64_t ticks; @@ -176,7 +180,7 @@ /** Last sampled cycle. */ uint64_t last_cycle; - /** Thread doesn't affect accumulated accounting. */ + /** Thread doesn't affect accumulated accounting. */ bool uncounted; - + /** Thread's priority. Implemented as index to CPU->rq */ int priority; @@ -186,13 +190,12 @@ /** Architecture-specific data. */ thread_arch_t arch; - + /** Thread's kernel stack. */ uint8_t *kstack; - + #ifdef CONFIG_UDEBUG /** Debugging stuff */ udebug_thread_t udebug; -#endif - +#endif /* CONFIG_UDEBUG */ } thread_t; @@ -203,5 +206,5 @@ * */ -SPINLOCK_EXTERN(threads_lock); +IRQ_SPINLOCK_EXTERN(threads_lock); /** AVL tree containing all threads. */ @@ -209,6 +212,6 @@ extern void thread_init(void); -extern thread_t *thread_create(void (*)(void *), void *, task_t *, int, - const char *, bool); +extern thread_t *thread_create(void (*)(void *), void *, task_t *, + unsigned int, const char *, bool); extern void thread_attach(thread_t *, task_t *); extern void thread_ready(thread_t *); @@ -218,7 +221,9 @@ extern void thread_create_arch(thread_t *); #endif + #ifndef thr_constructor_arch extern void thr_constructor_arch(thread_t *); #endif + #ifndef thr_destructor_arch extern void thr_destructor_arch(thread_t *); @@ -230,10 +235,11 @@ #define thread_join(t) \ thread_join_timeout((t), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE) -extern int thread_join_timeout(thread_t *, uint32_t, int); + +extern int thread_join_timeout(thread_t *, uint32_t, unsigned int); extern void thread_detach(thread_t *); extern void thread_register_call_me(void (*)(void *), void *); extern void thread_print_list(void); -extern void thread_destroy(thread_t *); +extern void thread_destroy(thread_t *, bool); extern thread_t *thread_find_by_id(thread_id_t); extern void thread_update_accounting(bool); Index: kernel/generic/include/stacktrace.h =================================================================== --- kernel/generic/include/stacktrace.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/stacktrace.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,5 +38,5 @@ #include -/* Forward declarations. */ +/* Forward declaration. */ struct istate; Index: kernel/generic/include/symtab.h =================================================================== --- kernel/generic/include/symtab.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/symtab.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,5 +38,5 @@ #include -#define MAX_SYMBOL_NAME 64 +#define MAX_SYMBOL_NAME 64 struct symtab_entry { @@ -53,8 +53,10 @@ #ifdef CONFIG_SYMTAB -/* Symtable linked together by build process */ +/** Symtable linked together by build process + * + */ extern struct symtab_entry symbol_table[]; -#endif +#endif /* CONFIG_SYMTAB */ #endif Index: kernel/generic/include/synch/mutex.h =================================================================== --- kernel/generic/include/synch/mutex.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/synch/mutex.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -50,13 +50,16 @@ } mutex_t; -#define mutex_lock(mtx) \ +#define mutex_lock(mtx) \ _mutex_lock_timeout((mtx), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE) -#define mutex_trylock(mtx) \ + +#define mutex_trylock(mtx) \ _mutex_lock_timeout((mtx), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING) -#define mutex_lock_timeout(mtx, usec) \ + +#define mutex_lock_timeout(mtx, usec) \ _mutex_lock_timeout((mtx), (usec), SYNCH_FLAGS_NON_BLOCKING) extern void mutex_initialize(mutex_t *, mutex_type_t); -extern int _mutex_lock_timeout(mutex_t *, uint32_t, int); +extern bool mutex_locked(mutex_t *); +extern int _mutex_lock_timeout(mutex_t *, uint32_t, unsigned int); extern void mutex_unlock(mutex_t *); Index: kernel/generic/include/synch/rwlock.h =================================================================== --- kernel/generic/include/synch/rwlock.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/synch/rwlock.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -48,9 +48,12 @@ typedef struct { - SPINLOCK_DECLARE(lock); + IRQ_SPINLOCK_DECLARE(lock); + /** * Mutex for writers, readers can bypass it if readers_in is positive. + * */ mutex_t exclusive; + /** Number of readers in critical section. */ size_t readers_in; @@ -59,22 +62,27 @@ #define rwlock_write_lock(rwl) \ _rwlock_write_lock_timeout((rwl), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE) + #define rwlock_read_lock(rwl) \ _rwlock_read_lock_timeout((rwl), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE) + #define rwlock_write_trylock(rwl) \ _rwlock_write_lock_timeout((rwl), SYNCH_NO_TIMEOUT, \ SYNCH_FLAGS_NON_BLOCKING) + #define rwlock_read_trylock(rwl) \ _rwlock_read_lock_timeout((rwl), SYNCH_NO_TIMEOUT, \ SYNCH_FLAGS_NON_BLOCKING) + #define rwlock_write_lock_timeout(rwl, usec) \ _rwlock_write_lock_timeout((rwl), (usec), SYNCH_FLAGS_NONE) + #define rwlock_read_lock_timeout(rwl, usec) \ _rwlock_read_lock_timeout((rwl), (usec), SYNCH_FLAGS_NONE) -extern void rwlock_initialize(rwlock_t *rwl); -extern void rwlock_read_unlock(rwlock_t *rwl); -extern void rwlock_write_unlock(rwlock_t *rwl); -extern int _rwlock_read_lock_timeout(rwlock_t *rwl, uint32_t usec, int flags); -extern int _rwlock_write_lock_timeout(rwlock_t *rwl, uint32_t usec, int flags); +extern void rwlock_initialize(rwlock_t *); +extern void rwlock_read_unlock(rwlock_t *); +extern void rwlock_write_unlock(rwlock_t *); +extern int _rwlock_read_lock_timeout(rwlock_t *, uint32_t, unsigned int); +extern int _rwlock_write_lock_timeout(rwlock_t *, uint32_t, unsigned int); #endif Index: kernel/generic/include/synch/semaphore.h =================================================================== --- kernel/generic/include/synch/semaphore.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/synch/semaphore.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,12 +46,15 @@ #define semaphore_down(s) \ _semaphore_down_timeout((s), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE) + #define semaphore_trydown(s) \ _semaphore_down_timeout((s), SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING) + #define semaphore_down_timeout(s, usec) \ _semaphore_down_timeout((s), (usec), SYNCH_FLAGS_NONE) -extern void semaphore_initialize(semaphore_t *s, int val); -extern int _semaphore_down_timeout(semaphore_t *s, uint32_t usec, int flags); -extern void semaphore_up(semaphore_t *s); +extern void semaphore_initialize(semaphore_t *, int); +extern int _semaphore_down_timeout(semaphore_t *, uint32_t, unsigned int); +extern void semaphore_up(semaphore_t *); +extern int semaphore_count_get(semaphore_t *); #endif Index: kernel/generic/include/synch/spinlock.h =================================================================== --- kernel/generic/include/synch/spinlock.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/synch/spinlock.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -41,4 +41,5 @@ #include #include +#include #ifdef CONFIG_SMP @@ -49,5 +50,5 @@ #ifdef CONFIG_DEBUG_SPINLOCK const char *name; -#endif +#endif /* CONFIG_DEBUG_SPINLOCK */ } spinlock_t; @@ -60,6 +61,7 @@ /* - * SPINLOCK_INITIALIZE is to be used for statically allocated spinlocks. - * It declares and initializes the lock. + * SPINLOCK_INITIALIZE and SPINLOCK_STATIC_INITIALIZE are to be used + * for statically allocated spinlocks. They declare (either as global + * or static) symbol and initialize the lock. */ #ifdef CONFIG_DEBUG_SPINLOCK @@ -77,8 +79,11 @@ } -#define spinlock_lock(lock) spinlock_lock_debug((lock)) -#define spinlock_unlock(lock) spinlock_unlock_debug((lock)) - -#else +#define ASSERT_SPINLOCK(expr, lock) \ + ASSERT_VERBOSE(expr, (lock)->name) + +#define spinlock_lock(lock) spinlock_lock_debug((lock)) +#define spinlock_unlock(lock) spinlock_unlock_debug((lock)) + +#else /* CONFIG_DEBUG_SPINLOCK */ #define SPINLOCK_INITIALIZE_NAME(lock_name, desc_name) \ @@ -92,8 +97,11 @@ } -#define spinlock_lock(lock) atomic_lock_arch(&(lock)->val) -#define spinlock_unlock(lock) spinlock_unlock_nondebug((lock)) - -#endif +#define ASSERT_SPINLOCK(expr, lock) \ + ASSERT(expr) + +#define spinlock_lock(lock) atomic_lock_arch(&(lock)->val) +#define spinlock_unlock(lock) spinlock_unlock_nondebug((lock)) + +#endif /* CONFIG_DEBUG_SPINLOCK */ #define SPINLOCK_INITIALIZE(lock_name) \ @@ -103,8 +111,9 @@ SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, #lock_name) -extern void spinlock_initialize(spinlock_t *lock, const char *name); -extern int spinlock_trylock(spinlock_t *lock); -extern void spinlock_lock_debug(spinlock_t *lock); -extern void spinlock_unlock_debug(spinlock_t *lock); +extern void spinlock_initialize(spinlock_t *, const char *); +extern int spinlock_trylock(spinlock_t *); +extern void spinlock_lock_debug(spinlock_t *); +extern void spinlock_unlock_debug(spinlock_t *); +extern bool spinlock_locked(spinlock_t *); /** Unlock spinlock @@ -113,4 +122,5 @@ * * @param sl Pointer to spinlock_t structure. + * */ static inline void spinlock_unlock_nondebug(spinlock_t *lock) @@ -141,10 +151,10 @@ } -#else +#else /* CONFIG_DEBUG_SPINLOCK */ #define DEADLOCK_PROBE_INIT(pname) #define DEADLOCK_PROBE(pname, value) -#endif +#endif /* CONFIG_DEBUG_SPINLOCK */ #else /* CONFIG_SMP */ @@ -160,4 +170,6 @@ #define SPINLOCK_INITIALIZE_NAME(name, desc_name) #define SPINLOCK_STATIC_INITIALIZE_NAME(name, desc_name) + +#define ASSERT_SPINLOCK(expr, lock) ASSERT(expr) #define spinlock_initialize(lock, name) @@ -166,12 +178,116 @@ #define spinlock_trylock(lock) (preemption_disable(), 1) #define spinlock_unlock(lock) preemption_enable() +#define spinlock_locked(lock) 1 +#define spinlock_unlocked(lock) 1 #define DEADLOCK_PROBE_INIT(pname) #define DEADLOCK_PROBE(pname, value) +#endif /* CONFIG_SMP */ + +typedef struct { + SPINLOCK_DECLARE(lock); /**< Spinlock */ + bool guard; /**< Flag whether ipl is valid */ + ipl_t ipl; /**< Original interrupt level */ +} irq_spinlock_t; + +#define IRQ_SPINLOCK_DECLARE(lock_name) irq_spinlock_t lock_name +#define IRQ_SPINLOCK_EXTERN(lock_name) extern irq_spinlock_t lock_name + +#ifdef CONFIG_SMP + +#define ASSERT_IRQ_SPINLOCK(expr, irq_lock) \ + ASSERT_SPINLOCK(expr, &((irq_lock)->lock)) + +/* + * IRQ_SPINLOCK_INITIALIZE and IRQ_SPINLOCK_STATIC_INITIALIZE are to be used + * for statically allocated interrupts-disabled spinlocks. They declare (either + * as global or static symbol) and initialize the lock. + */ +#ifdef CONFIG_DEBUG_SPINLOCK + +#define IRQ_SPINLOCK_INITIALIZE_NAME(lock_name, desc_name) \ + irq_spinlock_t lock_name = { \ + .lock = { \ + .name = desc_name, \ + .val = { 0 } \ + }, \ + .guard = false, \ + .ipl = 0 \ + } + +#define IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, desc_name) \ + static irq_spinlock_t lock_name = { \ + .lock = { \ + .name = desc_name, \ + .val = { 0 } \ + }, \ + .guard = false, \ + .ipl = 0 \ + } + +#else /* CONFIG_DEBUG_SPINLOCK */ + +#define IRQ_SPINLOCK_INITIALIZE_NAME(lock_name, desc_name) \ + irq_spinlock_t lock_name = { \ + .lock = { \ + .val = { 0 } \ + }, \ + .guard = false, \ + .ipl = 0 \ + } + +#define IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, desc_name) \ + static irq_spinlock_t lock_name = { \ + .lock = { \ + .val = { 0 } \ + }, \ + .guard = false, \ + .ipl = 0 \ + } + +#endif /* CONFIG_DEBUG_SPINLOCK */ + +#else /* CONFIG_SMP */ + +/* + * Since the spinlocks are void on UP systems, we also need + * to have a special variant of interrupts-disabled spinlock + * macros which take this into account. + */ + +#define ASSERT_IRQ_SPINLOCK(expr, irq_lock) \ + ASSERT_SPINLOCK(expr, NULL) + +#define IRQ_SPINLOCK_INITIALIZE_NAME(lock_name, desc_name) \ + irq_spinlock_t lock_name = { \ + .guard = false, \ + .ipl = 0 \ + } + +#define IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, desc_name) \ + static irq_spinlock_t lock_name = { \ + .guard = false, \ + .ipl = 0 \ + } + +#endif /* CONFIG_SMP */ + +#define IRQ_SPINLOCK_INITIALIZE(lock_name) \ + IRQ_SPINLOCK_INITIALIZE_NAME(lock_name, #lock_name) + +#define IRQ_SPINLOCK_STATIC_INITIALIZE(lock_name) \ + IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(lock_name, #lock_name) + +extern void irq_spinlock_initialize(irq_spinlock_t *, const char *); +extern void irq_spinlock_lock(irq_spinlock_t *, bool); +extern void irq_spinlock_unlock(irq_spinlock_t *, bool); +extern int irq_spinlock_trylock(irq_spinlock_t *); +extern void irq_spinlock_pass(irq_spinlock_t *, irq_spinlock_t *); +extern void irq_spinlock_exchange(irq_spinlock_t *, irq_spinlock_t *); +extern bool irq_spinlock_locked(irq_spinlock_t *); + #endif -#endif - /** @} */ Index: kernel/generic/include/synch/waitq.h =================================================================== --- kernel/generic/include/synch/waitq.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/synch/waitq.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -46,17 +46,20 @@ } wakeup_mode_t; -/** Wait queue structure. */ +/** Wait queue structure. + * + */ typedef struct { - /** Lock protecting wait queue structure. * * Must be acquired before T.lock for each T of type thread_t. */ - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + /** * Number of waitq_wakeup() calls that didn't find a thread to wake up. + * */ int missed_wakeups; + /** List of sleeping threads for wich there was no missed_wakeup. */ link_t head; @@ -69,7 +72,7 @@ extern void waitq_initialize(waitq_t *); -extern int waitq_sleep_timeout(waitq_t *, uint32_t, int); +extern int waitq_sleep_timeout(waitq_t *, uint32_t, unsigned int); extern ipl_t waitq_sleep_prepare(waitq_t *); -extern int waitq_sleep_timeout_unsafe(waitq_t *, uint32_t, int); +extern int waitq_sleep_timeout_unsafe(waitq_t *, uint32_t, unsigned int); extern void waitq_sleep_finish(waitq_t *, int, ipl_t); extern void waitq_wakeup(waitq_t *, wakeup_mode_t); @@ -77,4 +80,6 @@ extern void waitq_interrupt_sleep(struct thread *); extern void waitq_unsleep(waitq_t *); +extern int waitq_count_get(waitq_t *); +extern void waitq_count_set(waitq_t *, int val); #endif Index: kernel/generic/include/sysinfo/abi.h =================================================================== --- kernel/generic/include/sysinfo/abi.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/sysinfo/abi.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -38,5 +38,5 @@ /** Number of load components */ -#define LOAD_STEPS 3 +#define LOAD_STEPS 3 /** Maximum task name size */ @@ -65,5 +65,5 @@ */ typedef struct { - unsigned int id; /**< CPU ID as stored by kernel */ + size_t id; /**< CPU ID as stored by kernel */ bool active; /**< CPU is activate */ uint16_t frequency_mhz; /**< Frequency in MHz */ Index: kernel/generic/include/time/timeout.h =================================================================== --- kernel/generic/include/time/timeout.h (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/include/time/timeout.h (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -43,9 +43,9 @@ typedef struct { - SPINLOCK_DECLARE(lock); - + IRQ_SPINLOCK_DECLARE(lock); + /** Link to the list of active timeouts on THE->cpu */ link_t link; - /** Timeout will be activated in this amount of clock() ticks. */ + /** Timeout will be activated in this amount of clock() ticks. */ uint64_t ticks; /** Function that will be called on timeout activation. */ @@ -57,12 +57,11 @@ } timeout_t; -#define us2ticks(us) ((uint64_t) (((uint32_t) (us) / (1000000 / HZ)))) +#define us2ticks(us) ((uint64_t) (((uint32_t) (us) / (1000000 / HZ)))) extern void timeout_init(void); -extern void timeout_initialize(timeout_t *t); -extern void timeout_reinitialize(timeout_t *t); -extern void timeout_register(timeout_t *t, uint64_t usec, timeout_handler_t f, - void *arg); -extern bool timeout_unregister(timeout_t *t); +extern void timeout_initialize(timeout_t *); +extern void timeout_reinitialize(timeout_t *); +extern void timeout_register(timeout_t *, uint64_t, timeout_handler_t, void *); +extern bool timeout_unregister(timeout_t *); #endif Index: kernel/generic/src/console/chardev.c =================================================================== --- kernel/generic/src/console/chardev.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/console/chardev.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -52,5 +52,5 @@ indev->name = name; waitq_initialize(&indev->wq); - spinlock_initialize(&indev->lock, "indev"); + irq_spinlock_initialize(&indev->lock, "chardev.indev.lock"); indev->counter = 0; indev->index = 0; @@ -68,8 +68,8 @@ ASSERT(indev); - spinlock_lock(&indev->lock); + irq_spinlock_lock(&indev->lock, true); if (indev->counter == INDEV_BUFLEN - 1) { /* Buffer full */ - spinlock_unlock(&indev->lock); + irq_spinlock_unlock(&indev->lock, true); return; } @@ -81,5 +81,5 @@ indev->index = indev->index % INDEV_BUFLEN; waitq_wakeup(&indev->wq, WAKEUP_FIRST); - spinlock_unlock(&indev->lock); + irq_spinlock_unlock(&indev->lock, true); } @@ -114,10 +114,8 @@ waitq_sleep(&indev->wq); - ipl_t ipl = interrupts_disable(); - spinlock_lock(&indev->lock); + irq_spinlock_lock(&indev->lock, true); wchar_t ch = indev->buffer[(indev->index - indev->counter) % INDEV_BUFLEN]; indev->counter--; - spinlock_unlock(&indev->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&indev->lock, true); return ch; @@ -134,5 +132,5 @@ { outdev->name = name; - spinlock_initialize(&outdev->lock, "outdev"); + spinlock_initialize(&outdev->lock, "chardev.outdev.lock"); link_initialize(&outdev->link); list_initialize(&outdev->list); Index: kernel/generic/src/console/cmd.c =================================================================== --- kernel/generic/src/console/cmd.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/console/cmd.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -510,5 +510,5 @@ void cmd_initialize(cmd_info_t *cmd) { - spinlock_initialize(&cmd->lock, "cmd"); + spinlock_initialize(&cmd->lock, "cmd.lock"); link_initialize(&cmd->link); } @@ -658,8 +658,12 @@ printf("Duplicate symbol, be more specific.\n"); } else if (rc == EOK) { + ipl_t ipl; + + ipl = interrupts_disable(); fnc = (unative_t (*)(void)) arch_construct_function(&fptr, (void *) symaddr, (void *) cmd_call0); printf("Calling %s() (%p)\n", symbol, symaddr); printf("Result: %#" PRIxn "\n", fnc()); + interrupts_restore(ipl); } else { printf("No symbol information available.\n"); @@ -681,15 +685,18 @@ continue; - thread_t *t; - if ((t = thread_create((void (*)(void *)) cmd_call0, (void *) argv, TASK, THREAD_FLAG_WIRED, "call0", false))) { - spinlock_lock(&t->lock); - t->cpu = &cpus[i]; - spinlock_unlock(&t->lock); - printf("cpu%u: ", i); - thread_ready(t); - thread_join(t); - thread_detach(t); + thread_t *thread; + if ((thread = thread_create((void (*)(void *)) cmd_call0, + (void *) argv, TASK, THREAD_FLAG_WIRED, "call0", false))) { + irq_spinlock_lock(&thread->lock, true); + thread->cpu = &cpus[i]; + irq_spinlock_unlock(&thread->lock, true); + + printf("cpu%" PRIs ": ", i); + + thread_ready(thread); + thread_join(thread); + thread_detach(thread); } else - printf("Unable to create thread for cpu%u\n", i); + printf("Unable to create thread for cpu%" PRIs "\n", i); } @@ -716,7 +723,11 @@ printf("Duplicate symbol, be more specific.\n"); } else if (rc == EOK) { + ipl_t ipl; + + ipl = interrupts_disable(); fnc = (unative_t (*)(unative_t, ...)) arch_construct_function(&fptr, (void *) symaddr, (void *) cmd_call1); printf("Calling f(%#" PRIxn "): %p: %s\n", arg1, symaddr, symbol); printf("Result: %#" PRIxn "\n", fnc(arg1)); + interrupts_restore(ipl); } else { printf("No symbol information available.\n"); @@ -746,8 +757,12 @@ printf("Duplicate symbol, be more specific.\n"); } else if (rc == EOK) { + ipl_t ipl; + + ipl = interrupts_disable(); fnc = (unative_t (*)(unative_t, unative_t, ...)) arch_construct_function(&fptr, (void *) symaddr, (void *) cmd_call2); printf("Calling f(%#" PRIxn ", %#" PRIxn "): %p: %s\n", arg1, arg2, symaddr, symbol); printf("Result: %#" PRIxn "\n", fnc(arg1, arg2)); + interrupts_restore(ipl); } else { printf("No symbol information available.\n"); @@ -777,8 +792,12 @@ printf("Duplicate symbol, be more specific.\n"); } else if (rc == EOK) { + ipl_t ipl; + + ipl = interrupts_disable(); fnc = (unative_t (*)(unative_t, unative_t, unative_t, ...)) arch_construct_function(&fptr, (void *) symaddr, (void *) cmd_call3); printf("Calling f(%#" PRIxn ",%#" PRIxn ", %#" PRIxn "): %p: %s\n", arg1, arg2, arg3, symaddr, symbol); printf("Result: %#" PRIxn "\n", fnc(arg1, arg2, arg3)); + interrupts_restore(ipl); } else { printf("No symbol information available.\n"); @@ -1049,10 +1068,8 @@ /* Update and read thread accounting for benchmarking */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); uint64_t ucycles0, kcycles0; task_get_accounting(TASK, &ucycles0, &kcycles0); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&TASK->lock, true); /* Execute the test */ @@ -1061,10 +1078,8 @@ /* Update and read thread accounting */ - uint64_t ucycles1, kcycles1; - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + uint64_t ucycles1, kcycles1; + irq_spinlock_lock(&TASK->lock, true); task_get_accounting(TASK, &ucycles1, &kcycles1); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&TASK->lock, true); uint64_t ucycles, kcycles; @@ -1072,7 +1087,7 @@ order_suffix(ucycles1 - ucycles0, &ucycles, &usuffix); order_suffix(kcycles1 - kcycles0, &kcycles, &ksuffix); - + printf("Time: %" PRIu64 "%c user cycles, %" PRIu64 "%c kernel cycles\n", - ucycles, usuffix, kcycles, ksuffix); + ucycles, usuffix, kcycles, ksuffix); if (ret == NULL) { @@ -1080,5 +1095,5 @@ return true; } - + printf("%s\n", ret); return false; @@ -1106,10 +1121,8 @@ /* Update and read thread accounting for benchmarking */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); uint64_t ucycles0, kcycles0; task_get_accounting(TASK, &ucycles0, &kcycles0); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&TASK->lock, true); /* Execute the test */ @@ -1118,11 +1131,9 @@ /* Update and read thread accounting */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); uint64_t ucycles1, kcycles1; task_get_accounting(TASK, &ucycles1, &kcycles1); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&TASK->lock, true); + if (ret != NULL) { printf("%s\n", ret); @@ -1135,5 +1146,5 @@ order_suffix(kcycles1 - kcycles0, &kcycles, &ksuffix); printf("OK (%" PRIu64 "%c user cycles, %" PRIu64 "%c kernel cycles)\n", - ucycles, usuffix, kcycles, ksuffix); + ucycles, usuffix, kcycles, ksuffix); } Index: kernel/generic/src/console/console.c =================================================================== --- kernel/generic/src/console/console.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/console/console.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -62,10 +62,14 @@ /** Kernel log initialized */ static bool klog_inited = false; + /** First kernel log characters */ static size_t klog_start = 0; + /** Number of valid kernel log characters */ static size_t klog_len = 0; + /** Number of stored (not printed) kernel log characters */ static size_t klog_stored = 0; + /** Number of stored kernel log characters for uspace */ static size_t klog_uspace = 0; @@ -84,6 +88,6 @@ }; -static void stdout_write(outdev_t *dev, wchar_t ch, bool silent); -static void stdout_redraw(outdev_t *dev); +static void stdout_write(outdev_t *, wchar_t, bool); +static void stdout_redraw(outdev_t *); static outdev_operations_t stdout_ops = { @@ -174,7 +178,10 @@ stdout->op->redraw(stdout); - /* Force the console to print the prompt */ - if ((stdin) && (prev)) + if ((stdin) && (prev)) { + /* + * Force the console to print the prompt. + */ indev_push_character(stdin, '\n'); + } } Index: kernel/generic/src/cpu/cpu.c =================================================================== --- kernel/generic/src/cpu/cpu.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/cpu/cpu.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,7 +33,7 @@ /** * @file - * @brief CPU subsystem initialization and listing. + * @brief CPU subsystem initialization and listing. */ - + #include #include @@ -58,26 +58,27 @@ */ void cpu_init(void) { - unsigned int i, j; - #ifdef CONFIG_SMP if (config.cpu_active == 1) { #endif /* CONFIG_SMP */ + cpus = (cpu_t *) malloc(sizeof(cpu_t) * config.cpu_count, - FRAME_ATOMIC); + FRAME_ATOMIC); if (!cpus) panic("Cannot allocate CPU structures."); - - /* initialize everything */ + + /* Initialize everything */ memsetb(cpus, sizeof(cpu_t) * config.cpu_count, 0); - + + size_t i; for (i = 0; i < config.cpu_count; i++) { - cpus[i].stack = (uint8_t *) frame_alloc(STACK_FRAMES, FRAME_KA | FRAME_ATOMIC); - + cpus[i].stack = (uint8_t *) frame_alloc(STACK_FRAMES, + FRAME_KA | FRAME_ATOMIC); cpus[i].id = i; - spinlock_initialize(&cpus[i].lock, "cpu_t.lock"); - + irq_spinlock_initialize(&cpus[i].lock, "cpus[].lock"); + + unsigned int j; for (j = 0; j < RQ_COUNT; j++) { - spinlock_initialize(&cpus[i].rq[j].lock, "rq_t.lock"); + irq_spinlock_initialize(&cpus[i].rq[j].lock, "cpus[].rq[].lock"); list_initialize(&cpus[i].rq[j].rq_head); } @@ -87,9 +88,9 @@ } #endif /* CONFIG_SMP */ - + CPU = &cpus[config.cpu_active - 1]; - CPU->active = 1; - CPU->tlb_active = 1; + CPU->active = true; + CPU->tlb_active = true; cpu_identify(); @@ -100,6 +101,6 @@ void cpu_list(void) { - unsigned int i; - + size_t i; + for (i = 0; i < config.cpu_count; i++) { if (cpus[i].active) Index: kernel/generic/src/ddi/ddi.c =================================================================== --- kernel/generic/src/ddi/ddi.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ddi/ddi.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -59,5 +59,7 @@ static btree_t parea_btree; -/** Initialize DDI. */ +/** Initialize DDI. + * + */ void ddi_init(void) { @@ -97,5 +99,6 @@ * */ -static int ddi_physmem_map(uintptr_t pf, uintptr_t vp, size_t pages, int flags) +static int ddi_physmem_map(uintptr_t pf, uintptr_t vp, size_t pages, + unsigned int flags) { ASSERT(TASK); @@ -114,8 +117,6 @@ backend_data.frames = pages; - ipl_t ipl = interrupts_disable(); - /* Find the zone of the physical memory */ - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = find_zone(ADDR2PFN(pf), pages, 0); @@ -124,5 +125,5 @@ * -> assume it is hardware device and allow mapping */ - spinlock_unlock(&zones.lock); + irq_spinlock_unlock(&zones.lock, true); goto map; } @@ -130,13 +131,14 @@ if (zones.info[znum].flags & ZONE_FIRMWARE) { /* Frames are part of firmware */ - spinlock_unlock(&zones.lock); + irq_spinlock_unlock(&zones.lock, true); goto map; } if (zone_flags_available(zones.info[znum].flags)) { - /* Frames are part of physical memory, check if the memory + /* + * Frames are part of physical memory, check if the memory * region is enabled for mapping. */ - spinlock_unlock(&zones.lock); + irq_spinlock_unlock(&zones.lock, true); mutex_lock(&parea_lock); @@ -154,12 +156,10 @@ } - spinlock_unlock(&zones.lock); + irq_spinlock_unlock(&zones.lock, true); + err: - interrupts_restore(ipl); return ENOENT; map: - interrupts_restore(ipl); - if (!as_area_create(TASK->as, flags, pages * PAGE_SIZE, vp, AS_AREA_ATTR_NONE, &phys_backend, &backend_data)) { @@ -196,6 +196,5 @@ return EPERM; - ipl_t ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); task_t *task = task_find_by_id(id); @@ -207,17 +206,14 @@ * context. */ - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return ENOENT; } /* Lock the task and release the lock protecting tasks_btree. */ - spinlock_lock(&task->lock); - spinlock_unlock(&tasks_lock); + irq_spinlock_exchange(&tasks_lock, &task->lock); int rc = ddi_iospace_enable_arch(task, ioaddr, size); - spinlock_unlock(&task->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&task->lock, true); return rc; Index: kernel/generic/src/ddi/irq.c =================================================================== --- kernel/generic/src/ddi/irq.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ddi/irq.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,5 +32,5 @@ /** * @file - * @brief IRQ dispatcher. + * @brief IRQ dispatcher. * * This file provides means of connecting IRQs with particular @@ -78,20 +78,24 @@ #include -#define KEY_INR 0 -#define KEY_DEVNO 1 - -/** - * Spinlock protecting the kernel IRQ hash table. +#define KEY_INR 0 +#define KEY_DEVNO 1 + +/** Spinlock protecting the kernel IRQ hash table. + * * This lock must be taken only when interrupts are disabled. - */ -SPINLOCK_INITIALIZE(irq_kernel_hash_table_lock); + * + */ +IRQ_SPINLOCK_STATIC_INITIALIZE(irq_kernel_hash_table_lock); + /** The kernel IRQ hash table. */ static hash_table_t irq_kernel_hash_table; -/** - * Spinlock protecting the uspace IRQ hash table. +/** Spinlock protecting the uspace IRQ hash table. + * * This lock must be taken only when interrupts are disabled. - */ -SPINLOCK_INITIALIZE(irq_uspace_hash_table_lock); + * + */ +IRQ_SPINLOCK_INITIALIZE(irq_uspace_hash_table_lock); + /** The uspace IRQ hash table. */ hash_table_t irq_uspace_hash_table; @@ -100,4 +104,5 @@ * Hash table operations for cases when we know that * there will be collisions between different keys. + * */ static size_t irq_ht_hash(unative_t *key); @@ -116,4 +121,5 @@ * However, there might be still collisions among * elements with single key (sharing of one IRQ). + * */ static size_t irq_lin_hash(unative_t *key); @@ -132,6 +138,7 @@ /** Initialize IRQ subsystem. * - * @param inrs Numbers of unique IRQ numbers or INRs. + * @param inrs Numbers of unique IRQ numbers or INRs. * @param chains Number of chains in the hash table. + * */ void irq_init(size_t inrs, size_t chains) @@ -166,9 +173,9 @@ memsetb(irq, sizeof(irq_t), 0); link_initialize(&irq->link); - spinlock_initialize(&irq->lock, "irq.lock"); + irq_spinlock_initialize(&irq->lock, "irq.lock"); link_initialize(&irq->notif_cfg.link); irq->inr = -1; irq->devno = -1; - + irq_initialize_arch(irq); } @@ -180,10 +187,11 @@ * function pointer and handler() function pointer. * - * @param irq IRQ structure belonging to a device. - * @return True on success, false on failure. + * @param irq IRQ structure belonging to a device. + * + * @return True on success, false on failure. + * */ void irq_register(irq_t *irq) { - ipl_t ipl; unative_t key[] = { (unative_t) irq->inr, @@ -191,11 +199,9 @@ }; - ipl = interrupts_disable(); - spinlock_lock(&irq_kernel_hash_table_lock); - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq_kernel_hash_table_lock, true); + irq_spinlock_lock(&irq->lock, false); hash_table_insert(&irq_kernel_hash_table, key, &irq->link); - spinlock_unlock(&irq->lock); - spinlock_unlock(&irq_kernel_hash_table_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&irq->lock, false); + irq_spinlock_unlock(&irq_kernel_hash_table_lock, true); } @@ -208,17 +214,15 @@ unative_t key[] = { (unative_t) inr, - (unative_t) -1 /* search will use claim() instead of devno */ + (unative_t) -1 /* Search will use claim() instead of devno */ }; - spinlock_lock(&irq_uspace_hash_table_lock); + irq_spinlock_lock(&irq_uspace_hash_table_lock, false); lnk = hash_table_find(&irq_uspace_hash_table, key); if (lnk) { - irq_t *irq; - - irq = hash_table_get_instance(lnk, irq_t, link); - spinlock_unlock(&irq_uspace_hash_table_lock); + irq_t *irq = hash_table_get_instance(lnk, irq_t, link); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, false); return irq; } - spinlock_unlock(&irq_uspace_hash_table_lock); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, false); return NULL; @@ -233,17 +237,15 @@ unative_t key[] = { (unative_t) inr, - (unative_t) -1 /* search will use claim() instead of devno */ + (unative_t) -1 /* Search will use claim() instead of devno */ }; - spinlock_lock(&irq_kernel_hash_table_lock); + irq_spinlock_lock(&irq_kernel_hash_table_lock, false); lnk = hash_table_find(&irq_kernel_hash_table, key); if (lnk) { - irq_t *irq; - - irq = hash_table_get_instance(lnk, irq_t, link); - spinlock_unlock(&irq_kernel_hash_table_lock); + irq_t *irq = hash_table_get_instance(lnk, irq_t, link); + irq_spinlock_unlock(&irq_kernel_hash_table_lock, false); return irq; } - spinlock_unlock(&irq_kernel_hash_table_lock); + irq_spinlock_unlock(&irq_kernel_hash_table_lock, false); return NULL; @@ -263,9 +265,8 @@ * * @return IRQ structure of the respective device or NULL. + * */ irq_t *irq_dispatch_and_lock(inr_t inr) { - irq_t *irq; - /* * If the kernel console is silenced, @@ -277,13 +278,15 @@ */ if (silent) { - irq = irq_dispatch_and_lock_uspace(inr); + irq_t *irq = irq_dispatch_and_lock_uspace(inr); if (irq) return irq; + return irq_dispatch_and_lock_kernel(inr); } - irq = irq_dispatch_and_lock_kernel(inr); + irq_t *irq = irq_dispatch_and_lock_kernel(inr); if (irq) return irq; + return irq_dispatch_and_lock_uspace(inr); } @@ -301,4 +304,5 @@ * * @return Index into the hash table. + * */ size_t irq_ht_hash(unative_t key[]) @@ -322,9 +326,10 @@ * This function assumes interrupts are already disabled. * - * @param key Keys (i.e. inr and devno). + * @param key Keys (i.e. inr and devno). * @param keys This is 2. * @param item The item to compare the key with. * * @return True on match or false otherwise. + * */ bool irq_ht_compare(unative_t key[], size_t keys, link_t *item) @@ -333,8 +338,8 @@ inr_t inr = (inr_t) key[KEY_INR]; devno_t devno = (devno_t) key[KEY_DEVNO]; - + bool rv; - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); if (devno == -1) { /* Invoked by irq_dispatch_and_lock(). */ @@ -348,6 +353,6 @@ /* unlock only on non-match */ if (!rv) - spinlock_unlock(&irq->lock); - + irq_spinlock_unlock(&irq->lock, false); + return rv; } @@ -361,5 +366,5 @@ irq_t *irq __attribute__((unused)) = hash_table_get_instance(lnk, irq_t, link); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } @@ -374,4 +379,5 @@ * * @return Index into the hash table. + * */ size_t irq_lin_hash(unative_t key[]) @@ -395,9 +401,10 @@ * This function assumes interrupts are already disabled. * - * @param key Keys (i.e. inr and devno). + * @param key Keys (i.e. inr and devno). * @param keys This is 2. * @param item The item to compare the key with. * * @return True on match or false otherwise. + * */ bool irq_lin_compare(unative_t key[], size_t keys, link_t *item) @@ -407,5 +414,5 @@ bool rv; - spinlock_lock(&irq->lock); + irq_spinlock_lock(&irq->lock, false); if (devno == -1) { /* Invoked by irq_dispatch_and_lock() */ @@ -418,5 +425,5 @@ /* unlock only on non-match */ if (!rv) - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); return rv; @@ -425,5 +432,6 @@ /** Unlock IRQ structure after hash_table_remove(). * - * @param lnk Link in the removed and locked IRQ structure. + * @param lnk Link in the removed and locked IRQ structure. + * */ void irq_lin_remove(link_t *lnk) @@ -431,5 +439,5 @@ irq_t *irq __attribute__((unused)) = hash_table_get_instance(lnk, irq_t, link); - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); } Index: kernel/generic/src/interrupt/interrupt.c =================================================================== --- kernel/generic/src/interrupt/interrupt.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/interrupt/interrupt.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -32,9 +32,10 @@ /** * @file - * @brief Interrupt redirector. + * @brief Interrupt redirector. * * This file provides means of registering interrupt handlers * by kernel functions and calling the handlers when interrupts * occur. + * */ @@ -61,19 +62,18 @@ /** Register exception handler - * - * @param n Exception number - * @param name Description - * @param f Exception handler - */ -iroutine exc_register(int n, const char *name, iroutine f) + * + * @param n Exception number + * @param name Description + * @param handler Exception handler + * + */ +iroutine exc_register(int n, const char *name, iroutine handler) { ASSERT(n < IVT_ITEMS); - iroutine old; - spinlock_lock(&exctbl_lock); - old = exc_table[n].f; - exc_table[n].f = f; + iroutine old = exc_table[n].f; + exc_table[n].f = handler; exc_table[n].name = name; @@ -87,32 +87,43 @@ * Called directly from the assembler code. * CPU is interrupts_disable()'d. + * */ void exc_dispatch(int n, istate_t *istate) { ASSERT(n < IVT_ITEMS); - + /* Account user cycles */ + if (THREAD) { + irq_spinlock_lock(&THREAD->lock, false); + thread_update_accounting(true); + irq_spinlock_unlock(&THREAD->lock, false); + } + +#ifdef CONFIG_UDEBUG if (THREAD) - thread_update_accounting(true); - + THREAD->udebug.uspace_state = istate; +#endif + + exc_table[n].f(n + IVT_FIRST, istate); + #ifdef CONFIG_UDEBUG - if (THREAD) THREAD->udebug.uspace_state = istate; -#endif - - exc_table[n].f(n + IVT_FIRST, istate); - -#ifdef CONFIG_UDEBUG - if (THREAD) THREAD->udebug.uspace_state = NULL; -#endif - + if (THREAD) + THREAD->udebug.uspace_state = NULL; +#endif + /* This is a safe place to exit exiting thread */ - if (THREAD && THREAD->interrupted && istate_from_uspace(istate)) + if ((THREAD) && (THREAD->interrupted) && (istate_from_uspace(istate))) thread_exit(); - - if (THREAD) + + if (THREAD) { + irq_spinlock_lock(&THREAD->lock, false); thread_update_accounting(false); -} - -/** Default 'null' exception handler */ + irq_spinlock_unlock(&THREAD->lock, false); + } +} + +/** Default 'null' exception handler + * + */ static void exc_undef(int n, istate_t *istate) { @@ -121,25 +132,26 @@ } -/** Terminate thread and task if exception came from userspace. */ +/** Terminate thread and task if exception came from userspace. + * + */ void fault_if_from_uspace(istate_t *istate, const char *fmt, ...) { - task_t *task = TASK; - va_list args; - if (!istate_from_uspace(istate)) return; - + printf("Task %s (%" PRIu64 ") killed due to an exception at " - "program counter %p.\n", task->name, task->taskid, + "program counter %p.\n", TASK->name, TASK->taskid, istate_get_pc(istate)); - + stack_trace_istate(istate); - + printf("Kill message: "); + + va_list args; va_start(args, fmt); vprintf(fmt, args); va_end(args); printf("\n"); - + /* * Userspace can subscribe for FAULT events to take action @@ -152,5 +164,5 @@ event_notify_3(EVENT_FAULT, LOWER32(TASK->taskid), UPPER32(TASK->taskid), (unative_t) THREAD); - + #ifdef CONFIG_UDEBUG /* Wait for a debugging session. */ @@ -158,6 +170,6 @@ #endif } - - task_kill(task->taskid); + + task_kill(TASK->taskid); thread_exit(); } @@ -165,17 +177,19 @@ #ifdef CONFIG_KCONSOLE -/** kconsole cmd - print all exceptions */ +/** Print all exceptions + * + */ static int cmd_exc_print(cmd_arg_t *argv) { #if (IVT_ITEMS > 0) unsigned int i; - + spinlock_lock(&exctbl_lock); - + #ifdef __32_BITS__ printf("Exc Description Handler Symbol\n"); printf("--- -------------------- ---------- --------\n"); #endif - + #ifdef __64_BITS__ printf("Exc Description Handler Symbol\n"); @@ -185,10 +199,10 @@ for (i = 0; i < IVT_ITEMS; i++) { const char *symbol = symtab_fmt_name_lookup((unative_t) exc_table[i].f); - + #ifdef __32_BITS__ printf("%-3u %-20s %10p %s\n", i + IVT_FIRST, exc_table[i].name, exc_table[i].f, symbol); #endif - + #ifdef __64_BITS__ printf("%-3u %-20s %18p %s\n", i + IVT_FIRST, exc_table[i].name, @@ -210,5 +224,4 @@ return 1; } - static cmd_info_t exc_info = { @@ -221,14 +234,20 @@ }; -#endif - -/** Initialize generic exception handling support */ +#endif /* CONFIG_KCONSOLE */ + +/** Initialize generic exception handling support + * + */ void exc_init(void) { - int i; - + (void) exc_undef; + +#if (IVT_ITEMS > 0) + unsigned int i; + for (i = 0; i < IVT_ITEMS; i++) exc_register(i, "undef", (iroutine) exc_undef); - +#endif + #ifdef CONFIG_KCONSOLE cmd_initialize(&exc_info); Index: kernel/generic/src/ipc/event.c =================================================================== --- kernel/generic/src/ipc/event.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/event.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -137,9 +137,7 @@ IPC_SET_ARG5(call->data, a5); - ipl_t ipl = interrupts_disable(); - spinlock_lock(&events[evno].answerbox->irq_lock); + irq_spinlock_lock(&events[evno].answerbox->irq_lock, true); list_append(&call->link, &events[evno].answerbox->irq_notifs); - spinlock_unlock(&events[evno].answerbox->irq_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&events[evno].answerbox->irq_lock, true); waitq_wakeup(&events[evno].answerbox->wq, WAKEUP_FIRST); Index: kernel/generic/src/ipc/ipc.c =================================================================== --- kernel/generic/src/ipc/ipc.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/ipc.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -66,5 +66,6 @@ /** Initialize a call structure. * - * @param call Call structure to be initialized. + * @param call Call structure to be initialized. + * */ static void _ipc_call_init(call_t *call) @@ -77,21 +78,20 @@ /** Allocate and initialize a call structure. - * + * * The call is initialized, so that the reply will be directed to * TASK->answerbox. * - * @param flags Parameters for slab_alloc (e.g FRAME_ATOMIC). - * - * @return If flags permit it, return NULL, or initialized kernel - * call structure. - */ -call_t *ipc_call_alloc(int flags) -{ - call_t *call; - - call = slab_alloc(ipc_call_slab, flags); + * @param flags Parameters for slab_alloc (e.g FRAME_ATOMIC). + * + * @return If flags permit it, return NULL, or initialized kernel + * call structure. + * + */ +call_t *ipc_call_alloc(unsigned int flags) +{ + call_t *call = slab_alloc(ipc_call_slab, flags); if (call) _ipc_call_init(call); - + return call; } @@ -99,5 +99,6 @@ /** Deallocate a call structure. * - * @param call Call structure to be freed. + * @param call Call structure to be freed. + * */ void ipc_call_free(call_t *call) @@ -111,11 +112,12 @@ /** Initialize an answerbox structure. * - * @param box Answerbox structure to be initialized. - * @param task Task to which the answerbox belongs. + * @param box Answerbox structure to be initialized. + * @param task Task to which the answerbox belongs. + * */ void ipc_answerbox_init(answerbox_t *box, task_t *task) { - spinlock_initialize(&box->lock, "ipc_box_lock"); - spinlock_initialize(&box->irq_lock, "ipc_box_irqlock"); + irq_spinlock_initialize(&box->lock, "ipc.box.lock"); + irq_spinlock_initialize(&box->irq_lock, "ipc.box.irqlock"); waitq_initialize(&box->wq); link_initialize(&box->sync_box_link); @@ -131,18 +133,19 @@ /** Connect a phone to an answerbox. * - * @param phone Initialized phone structure. - * @param box Initialized answerbox structure. + * @param phone Initialized phone structure. + * @param box Initialized answerbox structure. + * */ void ipc_phone_connect(phone_t *phone, answerbox_t *box) { mutex_lock(&phone->lock); - + phone->state = IPC_PHONE_CONNECTED; phone->callee = box; - - spinlock_lock(&box->lock); + + irq_spinlock_lock(&box->lock, true); list_append(&phone->link, &box->connected_phones); - spinlock_unlock(&box->lock); - + irq_spinlock_unlock(&box->lock, true); + mutex_unlock(&phone->lock); } @@ -150,5 +153,6 @@ /** Initialize a phone structure. * - * @param phone Phone structure to be initialized. + * @param phone Phone structure to be initialized. + * */ void ipc_phone_init(phone_t *phone) @@ -162,47 +166,41 @@ /** Helper function to facilitate synchronous calls. * - * @param phone Destination kernel phone structure. - * @param request Call structure with request. - * - * @return EOK on success or EINTR if the sleep was interrupted. + * @param phone Destination kernel phone structure. + * @param request Call structure with request. + * + * @return EOK on success or EINTR if the sleep was interrupted. + * */ int ipc_call_sync(phone_t *phone, call_t *request) { - answerbox_t *sync_box; - ipl_t ipl; - - sync_box = slab_alloc(ipc_answerbox_slab, 0); + answerbox_t *sync_box = slab_alloc(ipc_answerbox_slab, 0); ipc_answerbox_init(sync_box, TASK); - + /* * Put the answerbox on the TASK's list of synchronous answerboxes so * that it can be cleaned up if the call is interrupted. */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); list_append(&sync_box->sync_box_link, &TASK->sync_box_head); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&TASK->lock, true); + /* We will receive data in a special box. */ request->callerbox = sync_box; - + ipc_call(phone, request); if (!ipc_wait_for_call(sync_box, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_INTERRUPTIBLE)) { - /* The answerbox and the call will be freed by ipc_cleanup(). */ + /* The answerbox and the call will be freed by ipc_cleanup(). */ return EINTR; } - + /* * The answer arrived without interruption so we can remove the * answerbox from the TASK's list of synchronous answerboxes. */ - (void) interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); list_remove(&sync_box->sync_box_link); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&TASK->lock, true); + slab_free(ipc_answerbox_slab, sync_box); return EOK; @@ -211,6 +209,7 @@ /** Answer a message which was not dispatched and is not listed in any queue. * - * @param call Call structure to be answered. - * @param selflocked If true, then TASK->answebox is locked. + * @param call Call structure to be answered. + * @param selflocked If true, then TASK->answebox is locked. + * */ static void _ipc_answer_free_call(call_t *call, bool selflocked) @@ -218,15 +217,12 @@ answerbox_t *callerbox = call->callerbox; bool do_lock = ((!selflocked) || callerbox != (&TASK->answerbox)); - ipl_t ipl; - + /* Count sent answer */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); TASK->ipc_info.answer_sent++; - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&TASK->lock, true); + call->flags |= IPC_CALL_ANSWERED; - + if (call->flags & IPC_CALL_FORWARDED) { if (call->caller_phone) { @@ -235,10 +231,13 @@ } } - + if (do_lock) - spinlock_lock(&callerbox->lock); + irq_spinlock_lock(&callerbox->lock, true); + list_append(&call->link, &callerbox->answers); + if (do_lock) - spinlock_unlock(&callerbox->lock); + irq_spinlock_unlock(&callerbox->lock, true); + waitq_wakeup(&callerbox->wq, WAKEUP_FIRST); } @@ -246,13 +245,15 @@ /** Answer a message which is in a callee queue. * - * @param box Answerbox that is answering the message. - * @param call Modified request that is being sent back. + * @param box Answerbox that is answering the message. + * @param call Modified request that is being sent back. + * */ void ipc_answer(answerbox_t *box, call_t *call) { /* Remove from active box */ - spinlock_lock(&box->lock); + irq_spinlock_lock(&box->lock, true); list_remove(&call->link); - spinlock_unlock(&box->lock); + irq_spinlock_unlock(&box->lock, true); + /* Send back answer */ _ipc_answer_free_call(call, false); @@ -264,7 +265,8 @@ * message and sending it as a normal answer. * - * @param phone Phone structure the call should appear to come from. - * @param call Call structure to be answered. - * @param err Return value to be used for the answer. + * @param phone Phone structure the call should appear to come from. + * @param call Call structure to be answered. + * @param err Return value to be used for the answer. + * */ void ipc_backsend_err(phone_t *phone, call_t *call, unative_t err) @@ -278,27 +280,25 @@ /** Unsafe unchecking version of ipc_call. * - * @param phone Phone structure the call comes from. - * @param box Destination answerbox structure. - * @param call Call structure with request. + * @param phone Phone structure the call comes from. + * @param box Destination answerbox structure. + * @param call Call structure with request. + * */ static void _ipc_call(phone_t *phone, answerbox_t *box, call_t *call) { - ipl_t ipl; - /* Count sent ipc call */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); TASK->ipc_info.call_sent++; - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&TASK->lock, true); + if (!(call->flags & IPC_CALL_FORWARDED)) { atomic_inc(&phone->active_calls); call->data.phone = phone; } - - spinlock_lock(&box->lock); + + irq_spinlock_lock(&box->lock, true); list_append(&call->link, &box->calls); - spinlock_unlock(&box->lock); + irq_spinlock_unlock(&box->lock, true); + waitq_wakeup(&box->wq, WAKEUP_FIRST); } @@ -306,14 +306,13 @@ /** Send an asynchronous request using a phone to an answerbox. * - * @param phone Phone structure the call comes from and which is - * connected to the destination answerbox. - * @param call Call structure with request. - * - * @return Return 0 on success, ENOENT on error. + * @param phone Phone structure the call comes from and which is + * connected to the destination answerbox. + * @param call Call structure with request. + * + * @return Return 0 on success, ENOENT on error. + * */ int ipc_call(phone_t *phone, call_t *call) { - answerbox_t *box; - mutex_lock(&phone->lock); if (phone->state != IPC_PHONE_CONNECTED) { @@ -328,7 +327,9 @@ ipc_backsend_err(phone, call, ENOENT); } + return ENOENT; } - box = phone->callee; + + answerbox_t *box = phone->callee; _ipc_call(phone, box, call); @@ -342,14 +343,12 @@ * lazily later. * - * @param phone Phone structure to be hung up. - * - * @return Return 0 if the phone is disconnected. - * Return -1 if the phone was already disconnected. + * @param phone Phone structure to be hung up. + * + * @return 0 if the phone is disconnected. + * @return -1 if the phone was already disconnected. + * */ int ipc_phone_hangup(phone_t *phone) { - answerbox_t *box; - call_t *call; - mutex_lock(&phone->lock); if (phone->state == IPC_PHONE_FREE || @@ -359,20 +358,21 @@ return -1; } - box = phone->callee; + + answerbox_t *box = phone->callee; if (phone->state != IPC_PHONE_SLAMMED) { /* Remove myself from answerbox */ - spinlock_lock(&box->lock); + irq_spinlock_lock(&box->lock, true); list_remove(&phone->link); - spinlock_unlock(&box->lock); - - call = ipc_call_alloc(0); + irq_spinlock_unlock(&box->lock, true); + + call_t *call = ipc_call_alloc(0); IPC_SET_METHOD(call->data, IPC_M_PHONE_HUNGUP); call->flags |= IPC_CALL_DISCARD_ANSWER; _ipc_call(phone, box, call); } - + phone->state = IPC_PHONE_HUNGUP; mutex_unlock(&phone->lock); - + return 0; } @@ -380,30 +380,26 @@ /** Forwards call from one answerbox to another one. * - * @param call Call structure to be redirected. - * @param newphone Phone structure to target answerbox. - * @param oldbox Old answerbox structure. - * @param mode Flags that specify mode of the forward operation. - * - * @return Return 0 if forwarding succeeded or an error code if - * there was error. - * + * @param call Call structure to be redirected. + * @param newphone Phone structure to target answerbox. + * @param oldbox Old answerbox structure. + * @param mode Flags that specify mode of the forward operation. + * + * @return 0 if forwarding succeeded or an error code if + * there was an error. + * * The return value serves only as an information for the forwarder, * the original caller is notified automatically with EFORWARD. - */ -int ipc_forward(call_t *call, phone_t *newphone, answerbox_t *oldbox, int mode) -{ - ipl_t ipl; - + * + */ +int ipc_forward(call_t *call, phone_t *newphone, answerbox_t *oldbox, + unsigned int mode) +{ /* Count forwarded calls */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); TASK->ipc_info.forwarded++; - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - - spinlock_lock(&oldbox->lock); + irq_spinlock_pass(&TASK->lock, &oldbox->lock); list_remove(&call->link); - spinlock_unlock(&oldbox->lock); - + irq_spinlock_unlock(&oldbox->lock, true); + if (mode & IPC_FF_ROUTE_FROM_ME) { if (!call->caller_phone) @@ -411,5 +407,5 @@ call->data.phone = newphone; } - + return ipc_call(newphone, call); } @@ -418,24 +414,25 @@ /** Wait for a phone call. * - * @param box Answerbox expecting the call. - * @param usec Timeout in microseconds. See documentation for - * waitq_sleep_timeout() for decription of its special - * meaning. - * @param flags Select mode of sleep operation. See documentation for - * waitq_sleep_timeout() for description of its special - * meaning. - * @return Recived call structure or NULL. - * + * @param box Answerbox expecting the call. + * @param usec Timeout in microseconds. See documentation for + * waitq_sleep_timeout() for decription of its special + * meaning. + * @param flags Select mode of sleep operation. See documentation for + * waitq_sleep_timeout() for description of its special + * meaning. + * + * @return Recived call structure or NULL. + * * To distinguish between a call and an answer, have a look at call->flags. - */ -call_t *ipc_wait_for_call(answerbox_t *box, uint32_t usec, int flags) + * + */ +call_t *ipc_wait_for_call(answerbox_t *box, uint32_t usec, unsigned int flags) { call_t *request; - ipl_t ipl; uint64_t irq_cnt = 0; uint64_t answer_cnt = 0; uint64_t call_cnt = 0; int rc; - + restart: rc = waitq_sleep_timeout(&box->wq, usec, flags); @@ -443,21 +440,19 @@ return NULL; - spinlock_lock(&box->lock); + irq_spinlock_lock(&box->lock, true); if (!list_empty(&box->irq_notifs)) { /* Count recieved IRQ notification */ - irq_cnt++; - - ipl = interrupts_disable(); - spinlock_lock(&box->irq_lock); - + irq_cnt++; + + irq_spinlock_lock(&box->irq_lock, false); + request = list_get_instance(box->irq_notifs.next, call_t, link); list_remove(&request->link); - - spinlock_unlock(&box->irq_lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&box->irq_lock, false); } else if (!list_empty(&box->answers)) { /* Count recieved answer */ answer_cnt++; - + /* Handle asynchronous answers */ request = list_get_instance(box->answers.next, call_t, link); @@ -467,25 +462,25 @@ /* Count recieved call */ call_cnt++; - + /* Handle requests */ request = list_get_instance(box->calls.next, call_t, link); list_remove(&request->link); + /* Append request to dispatch queue */ list_append(&request->link, &box->dispatched_calls); } else { /* This can happen regularly after ipc_cleanup */ - spinlock_unlock(&box->lock); + irq_spinlock_unlock(&box->lock, true); goto restart; } - spinlock_unlock(&box->lock); - - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + + irq_spinlock_pass(&box->lock, &TASK->lock); + TASK->ipc_info.irq_notif_recieved += irq_cnt; TASK->ipc_info.answer_recieved += answer_cnt; TASK->ipc_info.call_recieved += call_cnt; - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); - + + irq_spinlock_unlock(&TASK->lock, true); + return request; } @@ -493,16 +488,16 @@ /** Answer all calls from list with EHANGUP answer. * - * @param lst Head of the list to be cleaned up. + * @param lst Head of the list to be cleaned up. + * */ void ipc_cleanup_call_list(link_t *lst) { - call_t *call; - while (!list_empty(lst)) { - call = list_get_instance(lst->next, call_t, link); + call_t *call = list_get_instance(lst->next, call_t, link); if (call->buffer) free(call->buffer); + list_remove(&call->link); - + IPC_SET_RETVAL(call->data, EHANGUP); _ipc_answer_free_call(call, true); @@ -512,7 +507,8 @@ /** Disconnects all phones connected to an answerbox. * - * @param box Answerbox to disconnect phones from. - * @param notify_box If true, the answerbox will get a hangup message for - * each disconnected phone. + * @param box Answerbox to disconnect phones from. + * @param notify_box If true, the answerbox will get a hangup message for + * each disconnected phone. + * */ void ipc_answerbox_slam_phones(answerbox_t *box, bool notify_box) @@ -520,19 +516,15 @@ phone_t *phone; DEADLOCK_PROBE_INIT(p_phonelck); - ipl_t ipl; - call_t *call; - - call = notify_box ? ipc_call_alloc(0) : NULL; - + + call_t *call = notify_box ? ipc_call_alloc(0) : NULL; + /* Disconnect all phones connected to our answerbox */ restart_phones: - ipl = interrupts_disable(); - spinlock_lock(&box->lock); + irq_spinlock_lock(&box->lock, true); while (!list_empty(&box->connected_phones)) { phone = list_get_instance(box->connected_phones.next, phone_t, link); if (SYNCH_FAILED(mutex_trylock(&phone->lock))) { - spinlock_unlock(&box->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&box->lock, true); DEADLOCK_PROBE(p_phonelck, DEADLOCK_THRESHOLD); goto restart_phones; @@ -541,13 +533,12 @@ /* Disconnect phone */ ASSERT(phone->state == IPC_PHONE_CONNECTED); - + list_remove(&phone->link); phone->state = IPC_PHONE_SLAMMED; - + if (notify_box) { mutex_unlock(&phone->lock); - spinlock_unlock(&box->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&box->lock, true); + /* * Send one message to the answerbox for each @@ -559,18 +550,17 @@ call->flags |= IPC_CALL_DISCARD_ANSWER; _ipc_call(phone, box, call); - + /* Allocate another call in advance */ call = ipc_call_alloc(0); - + /* Must start again */ goto restart_phones; } - + mutex_unlock(&phone->lock); } - - spinlock_unlock(&box->lock); - interrupts_restore(ipl); - + + irq_spinlock_unlock(&box->lock, true); + /* Free unused call */ if (call) @@ -578,47 +568,45 @@ } -/** Cleans up all IPC communication of the current task. +/** Clean up all IPC communication of the current task. * * Note: ipc_hangup sets returning answerbox to TASK->answerbox, you * have to change it as well if you want to cleanup other tasks than TASK. + * */ void ipc_cleanup(void) { - int i; - call_t *call; - ipl_t ipl; - /* Disconnect all our phones ('ipc_phone_hangup') */ + size_t i; for (i = 0; i < IPC_MAX_PHONES; i++) ipc_phone_hangup(&TASK->phones[i]); - + /* Unsubscribe from any event notifications. */ event_cleanup_answerbox(&TASK->answerbox); - + /* Disconnect all connected irqs */ ipc_irq_cleanup(&TASK->answerbox); - + /* Disconnect all phones connected to our regular answerbox */ ipc_answerbox_slam_phones(&TASK->answerbox, false); - + #ifdef CONFIG_UDEBUG /* Clean up kbox thread and communications */ ipc_kbox_cleanup(); #endif - + /* Answer all messages in 'calls' and 'dispatched_calls' queues */ - spinlock_lock(&TASK->answerbox.lock); + irq_spinlock_lock(&TASK->answerbox.lock, true); ipc_cleanup_call_list(&TASK->answerbox.dispatched_calls); ipc_cleanup_call_list(&TASK->answerbox.calls); - spinlock_unlock(&TASK->answerbox.lock); + irq_spinlock_unlock(&TASK->answerbox.lock, true); /* Wait for all answers to interrupted synchronous calls to arrive */ - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); while (!list_empty(&TASK->sync_box_head)) { answerbox_t *box = list_get_instance(TASK->sync_box_head.next, answerbox_t, sync_box_link); - + list_remove(&box->sync_box_link); - call = ipc_wait_for_call(box, SYNCH_NO_TIMEOUT, + call_t *call = ipc_wait_for_call(box, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE); ipc_call_free(call); @@ -626,10 +614,12 @@ } interrupts_restore(ipl); - + /* Wait for all answers to asynchronous calls to arrive */ - while (1) { - /* Go through all phones, until all are FREE... */ - /* Locking not needed, no one else should modify - * it, when we are in cleanup */ + while (true) { + /* + * Go through all phones, until they are all FREE + * Locking is not needed, no one else should modify + * it when we are in cleanup + */ for (i = 0; i < IPC_MAX_PHONES; i++) { if (TASK->phones[i].state == IPC_PHONE_HUNGUP && @@ -639,20 +629,26 @@ } - /* Just for sure, we might have had some - * IPC_PHONE_CONNECTING phones */ + /* + * Just for sure, we might have had some + * IPC_PHONE_CONNECTING phones + */ if (TASK->phones[i].state == IPC_PHONE_CONNECTED) ipc_phone_hangup(&TASK->phones[i]); - /* If the hangup succeeded, it has sent a HANGUP + + /* + * If the hangup succeeded, it has sent a HANGUP * message, the IPC is now in HUNGUP state, we - * wait for the reply to come */ + * wait for the reply to come + */ if (TASK->phones[i].state != IPC_PHONE_FREE) break; } - /* Voila, got into cleanup */ + + /* Got into cleanup */ if (i == IPC_MAX_PHONES) break; - call = ipc_wait_for_call(&TASK->answerbox, SYNCH_NO_TIMEOUT, + call_t *call = ipc_wait_for_call(&TASK->answerbox, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE); ASSERT((call->flags & IPC_CALL_ANSWERED) || @@ -666,10 +662,12 @@ if (!(call->flags & IPC_CALL_DISCARD_ANSWER)) atomic_dec(&TASK->active_calls); + ipc_call_free(call); } } - -/** Initilize IPC subsystem */ +/** Initilize IPC subsystem + * + */ void ipc_init(void) { @@ -680,30 +678,26 @@ } - /** List answerbox contents. * - * @param taskid Task ID. + * @param taskid Task ID. + * */ void ipc_print_task(task_id_t taskid) { - task_t *task; - int i; - call_t *call; - link_t *tmp; - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - task = task_find_by_id(taskid); - if (task) - spinlock_lock(&task->lock); - spinlock_unlock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); + task_t *task = task_find_by_id(taskid); + if (!task) { - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return; } - + + /* Hand-over-hand locking */ + irq_spinlock_exchange(&tasks_lock, &task->lock); + /* Print opened phones & details */ printf("PHONE:\n"); + + size_t i; for (i = 0; i < IPC_MAX_PHONES; i++) { if (SYNCH_FAILED(mutex_trylock(&task->phones[i].lock))) { @@ -711,6 +705,8 @@ continue; } + if (task->phones[i].state != IPC_PHONE_FREE) { - printf("%d: ", i); + printf("%" PRIs ": ", i); + switch (task->phones[i].state) { case IPC_PHONE_CONNECTING: @@ -718,31 +714,35 @@ break; case IPC_PHONE_CONNECTED: - printf("connected to: %p ", - task->phones[i].callee); + printf("connected to: %p ", + task->phones[i].callee); break; case IPC_PHONE_SLAMMED: printf("slammed by: %p ", - task->phones[i].callee); + task->phones[i].callee); break; case IPC_PHONE_HUNGUP: printf("hung up - was: %p ", - task->phones[i].callee); + task->phones[i].callee); break; default: break; } - printf("active: %ld\n", + + printf("active: %" PRIun "\n", atomic_get(&task->phones[i].active_calls)); } + mutex_unlock(&task->phones[i].lock); } - - + + irq_spinlock_lock(&task->answerbox.lock, false); + + link_t *cur; + /* Print answerbox - calls */ - spinlock_lock(&task->answerbox.lock); printf("ABOX - CALLS:\n"); - for (tmp = task->answerbox.calls.next; tmp != &task->answerbox.calls; - tmp = tmp->next) { - call = list_get_instance(tmp, call_t, link); + for (cur = task->answerbox.calls.next; cur != &task->answerbox.calls; + cur = cur->next) { + call_t *call = list_get_instance(cur, call_t, link); printf("Callid: %p Srctask:%" PRIu64 " M:%" PRIun " A1:%" PRIun " A2:%" PRIun " A3:%" PRIun @@ -754,10 +754,11 @@ call->flags); } - /* Print answerbox - calls */ + + /* Print answerbox - dispatched calls */ printf("ABOX - DISPATCHED CALLS:\n"); - for (tmp = task->answerbox.dispatched_calls.next; - tmp != &task->answerbox.dispatched_calls; - tmp = tmp->next) { - call = list_get_instance(tmp, call_t, link); + for (cur = task->answerbox.dispatched_calls.next; + cur != &task->answerbox.dispatched_calls; + cur = cur->next) { + call_t *call = list_get_instance(cur, call_t, link); printf("Callid: %p Srctask:%" PRIu64 " M:%" PRIun " A1:%" PRIun " A2:%" PRIun " A3:%" PRIun @@ -769,10 +770,11 @@ call->flags); } - /* Print answerbox - calls */ + + /* Print answerbox - answers */ printf("ABOX - ANSWERS:\n"); - for (tmp = task->answerbox.answers.next; - tmp != &task->answerbox.answers; - tmp = tmp->next) { - call = list_get_instance(tmp, call_t, link); + for (cur = task->answerbox.answers.next; + cur != &task->answerbox.answers; + cur = cur->next) { + call_t *call = list_get_instance(cur, call_t, link); printf("Callid:%p M:%" PRIun " A1:%" PRIun " A2:%" PRIun " A3:%" PRIun " A4:%" PRIun " A5:%" PRIun " Flags:%x\n", @@ -782,8 +784,7 @@ call->flags); } - - spinlock_unlock(&task->answerbox.lock); - spinlock_unlock(&task->lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&task->answerbox.lock, false); + irq_spinlock_unlock(&task->lock, true); } Index: kernel/generic/src/ipc/ipcrsc.c =================================================================== --- kernel/generic/src/ipc/ipcrsc.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/ipcrsc.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -45,5 +45,5 @@ * - hangup phone (the caller has hung up) * - hangup phone (the answerbox is exiting) - * + * * Locking strategy * @@ -85,5 +85,5 @@ * * Phone hangup - * + * * *** The caller hangs up (sys_ipc_hangup) *** * - The phone is disconnected (no more messages can be sent over this phone), @@ -99,5 +99,5 @@ * * Call forwarding - * + * * The call can be forwarded, so that the answer to call is passed directly * to the original sender. However, this poses special problems regarding @@ -114,5 +114,5 @@ * * Cleanup strategy - * + * * 1) Disconnect all our phones ('ipc_phone_hangup'). * @@ -123,5 +123,5 @@ * * 4) Wait for all async answers to arrive and dispose of them. - * + * */ @@ -137,19 +137,20 @@ * @todo Some speedup (hash table?) * - * @param callid Userspace hash of the call. Currently it is the call - * structure kernel address. - * - * @return NULL on not found, otherwise pointer to the call - * structure. + * @param callid Userspace hash of the call. Currently it is the call + * structure kernel address. + * + * @return NULL on not found, otherwise pointer to the call + * structure. + * */ call_t *get_call(unative_t callid) { link_t *lst; - call_t *call, *result = NULL; - - spinlock_lock(&TASK->answerbox.lock); + call_t *result = NULL; + + irq_spinlock_lock(&TASK->answerbox.lock, true); for (lst = TASK->answerbox.dispatched_calls.next; lst != &TASK->answerbox.dispatched_calls; lst = lst->next) { - call = list_get_instance(lst, call_t, link); + call_t *call = list_get_instance(lst, call_t, link); if ((unative_t) call == callid) { result = call; @@ -157,5 +158,6 @@ } } - spinlock_unlock(&TASK->answerbox.lock); + + irq_spinlock_unlock(&TASK->answerbox.lock, true); return result; } @@ -163,29 +165,31 @@ /** Allocate new phone slot in the specified task. * - * @param t Task for which to allocate a new phone. - * - * @return New phone handle or -1 if the phone handle limit is - * exceeded. - */ -int phone_alloc(task_t *t) -{ - int i; - - spinlock_lock(&t->lock); + * @param task Task for which to allocate a new phone. + * + * @return New phone handle or -1 if the phone handle limit is + * exceeded. + * + */ +int phone_alloc(task_t *task) +{ + irq_spinlock_lock(&task->lock, true); + + size_t i; for (i = 0; i < IPC_MAX_PHONES; i++) { - if (t->phones[i].state == IPC_PHONE_HUNGUP && - atomic_get(&t->phones[i].active_calls) == 0) - t->phones[i].state = IPC_PHONE_FREE; - - if (t->phones[i].state == IPC_PHONE_FREE) { - t->phones[i].state = IPC_PHONE_CONNECTING; + if ((task->phones[i].state == IPC_PHONE_HUNGUP) && + (atomic_get(&task->phones[i].active_calls) == 0)) + task->phones[i].state = IPC_PHONE_FREE; + + if (task->phones[i].state == IPC_PHONE_FREE) { + task->phones[i].state = IPC_PHONE_CONNECTING; break; } } - spinlock_unlock(&t->lock); - + + irq_spinlock_unlock(&task->lock, true); + if (i == IPC_MAX_PHONES) return -1; - + return i; } @@ -193,5 +197,6 @@ /** Mark a phone structure free. * - * @param phone Phone structure to be marked free. + * @param phone Phone structure to be marked free. + * */ static void phone_deallocp(phone_t *phone) @@ -199,5 +204,5 @@ ASSERT(phone->state == IPC_PHONE_CONNECTING); - /* atomic operation */ + /* Atomic operation */ phone->state = IPC_PHONE_FREE; } @@ -207,5 +212,6 @@ * All already sent messages will be correctly processed. * - * @param phoneid Phone handle of the phone to be freed. + * @param phoneid Phone handle of the phone to be freed. + * */ void phone_dealloc(int phoneid) @@ -216,10 +222,11 @@ /** Connect phone to a given answerbox. * - * @param phoneid Phone handle to be connected. - * @param box Answerbox to which to connect the phone handle. + * @param phoneid Phone handle to be connected. + * @param box Answerbox to which to connect the phone handle. * * The procedure _enforces_ that the user first marks the phone * busy (e.g. via phone_alloc) and then connects the phone, otherwise * race condition may appear. + * */ void phone_connect(int phoneid, answerbox_t *box) Index: kernel/generic/src/ipc/irq.c =================================================================== --- kernel/generic/src/ipc/irq.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/irq.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -31,4 +31,5 @@ * @{ */ + /** * @file @@ -67,4 +68,5 @@ * structure are finished. Because we hold the hash table lock, we prevent new * IRQs from taking new references to the IRQ structure. + * */ @@ -81,5 +83,6 @@ /** Free the top-half pseudocode. * - * @param code Pointer to the top-half pseudocode. + * @param code Pointer to the top-half pseudocode. + * */ static void code_free(irq_code_t *code) @@ -93,16 +96,13 @@ /** Copy the top-half pseudocode from userspace into the kernel. * - * @param ucode Userspace address of the top-half pseudocode. - * - * @return Kernel address of the copied pseudocode. + * @param ucode Userspace address of the top-half pseudocode. + * + * @return Kernel address of the copied pseudocode. + * */ static irq_code_t *code_from_uspace(irq_code_t *ucode) { - irq_code_t *code; - irq_cmd_t *ucmds; - int rc; - - code = malloc(sizeof(*code), 0); - rc = copy_from_uspace(code, ucode, sizeof(*code)); + irq_code_t *code = malloc(sizeof(*code), 0); + int rc = copy_from_uspace(code, ucode, sizeof(*code)); if (rc != 0) { free(code); @@ -114,5 +114,6 @@ return NULL; } - ucmds = code->cmds; + + irq_cmd_t *ucmds = code->cmds; code->cmds = malloc(sizeof(code->cmds[0]) * code->cmdcount, 0); rc = copy_from_uspace(code->cmds, ucmds, @@ -123,5 +124,5 @@ return NULL; } - + return code; } @@ -141,8 +142,4 @@ unative_t method, irq_code_t *ucode) { - ipl_t ipl; - irq_code_t *code; - irq_t *irq; - link_t *hlp; unative_t key[] = { (unative_t) inr, @@ -150,16 +147,17 @@ }; + irq_code_t *code; if (ucode) { code = code_from_uspace(ucode); if (!code) return EBADMEM; - } else { + } else code = NULL; - } /* * Allocate and populate the IRQ structure. */ - irq = malloc(sizeof(irq_t), 0); + irq_t *irq = malloc(sizeof(irq_t), 0); + irq_initialize(irq); irq->devno = devno; @@ -177,29 +175,30 @@ * answerbox's list. */ - ipl = interrupts_disable(); - spinlock_lock(&irq_uspace_hash_table_lock); - hlp = hash_table_find(&irq_uspace_hash_table, key); + irq_spinlock_lock(&irq_uspace_hash_table_lock, true); + + link_t *hlp = hash_table_find(&irq_uspace_hash_table, key); if (hlp) { - irq_t *hirq __attribute__((unused)) - = hash_table_get_instance(hlp, irq_t, link); + irq_t *hirq = hash_table_get_instance(hlp, irq_t, link); /* hirq is locked */ - spinlock_unlock(&hirq->lock); + irq_spinlock_unlock(&hirq->lock, false); code_free(code); - spinlock_unlock(&irq_uspace_hash_table_lock); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); + free(irq); - interrupts_restore(ipl); return EEXISTS; } - spinlock_lock(&irq->lock); /* Not really necessary, but paranoid */ - spinlock_lock(&box->irq_lock); + /* Locking is not really necessary, but paranoid */ + irq_spinlock_lock(&irq->lock, false); + irq_spinlock_lock(&box->irq_lock, false); + hash_table_insert(&irq_uspace_hash_table, key, &irq->link); list_append(&irq->notif_cfg.link, &box->irq_head); - spinlock_unlock(&box->irq_lock); - spinlock_unlock(&irq->lock); - spinlock_unlock(&irq_uspace_hash_table_lock); - - interrupts_restore(ipl); + + irq_spinlock_unlock(&box->irq_lock, false); + irq_spinlock_unlock(&irq->lock, false); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); + return EOK; } @@ -207,29 +206,27 @@ /** Unregister task from IRQ notification. * - * @param box Answerbox associated with the notification. - * @param inr IRQ number. - * @param devno Device number. + * @param box Answerbox associated with the notification. + * @param inr IRQ number. + * @param devno Device number. + * */ int ipc_irq_unregister(answerbox_t *box, inr_t inr, devno_t devno) { - ipl_t ipl; unative_t key[] = { (unative_t) inr, (unative_t) devno }; - link_t *lnk; - irq_t *irq; - - ipl = interrupts_disable(); - spinlock_lock(&irq_uspace_hash_table_lock); - lnk = hash_table_find(&irq_uspace_hash_table, key); + + irq_spinlock_lock(&irq_uspace_hash_table_lock, true); + link_t *lnk = hash_table_find(&irq_uspace_hash_table, key); if (!lnk) { - spinlock_unlock(&irq_uspace_hash_table_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); return ENOENT; } - irq = hash_table_get_instance(lnk, irq_t, link); + + irq_t *irq = hash_table_get_instance(lnk, irq_t, link); + /* irq is locked */ - spinlock_lock(&box->irq_lock); + irq_spinlock_lock(&box->irq_lock, false); ASSERT(irq->notif_cfg.answerbox == box); @@ -237,8 +234,8 @@ /* Free up the pseudo code and associated structures. */ code_free(irq->notif_cfg.code); - - /* Remove the IRQ from the answerbox's list. */ + + /* Remove the IRQ from the answerbox's list. */ list_remove(&irq->notif_cfg.link); - + /* * We need to drop the IRQ lock now because hash_table_remove() will try @@ -248,19 +245,17 @@ * the meantime. */ - spinlock_unlock(&irq->lock); - + irq_spinlock_unlock(&irq->lock, false); + /* Remove the IRQ from the uspace IRQ hash table. */ hash_table_remove(&irq_uspace_hash_table, key, 2); - spinlock_unlock(&irq_uspace_hash_table_lock); - spinlock_unlock(&box->irq_lock); + irq_spinlock_unlock(&box->irq_lock, false); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); /* Free up the IRQ structure. */ free(irq); - interrupts_restore(ipl); return EOK; } - /** Disconnect all IRQ notifications from an answerbox. @@ -270,35 +265,32 @@ * send notifications to it. * - * @param box Answerbox for which we want to carry out the cleanup. + * @param box Answerbox for which we want to carry out the cleanup. + * */ void ipc_irq_cleanup(answerbox_t *box) { - ipl_t ipl; - loop: - ipl = interrupts_disable(); - spinlock_lock(&irq_uspace_hash_table_lock); - spinlock_lock(&box->irq_lock); + irq_spinlock_lock(&irq_uspace_hash_table_lock, true); + irq_spinlock_lock(&box->irq_lock, false); while (box->irq_head.next != &box->irq_head) { - link_t *cur = box->irq_head.next; - irq_t *irq; DEADLOCK_PROBE_INIT(p_irqlock); - unative_t key[2]; - - irq = list_get_instance(cur, irq_t, notif_cfg.link); - if (!spinlock_trylock(&irq->lock)) { + + irq_t *irq = list_get_instance(box->irq_head.next, irq_t, + notif_cfg.link); + + if (!irq_spinlock_trylock(&irq->lock)) { /* * Avoid deadlock by trying again. */ - spinlock_unlock(&box->irq_lock); - spinlock_unlock(&irq_uspace_hash_table_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&box->irq_lock, false); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); DEADLOCK_PROBE(p_irqlock, DEADLOCK_THRESHOLD); goto loop; } + + unative_t key[2]; key[0] = irq->inr; key[1] = irq->devno; - ASSERT(irq->notif_cfg.answerbox == box); @@ -317,5 +309,5 @@ * didn't drop the hash table lock in the meantime. */ - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, false); /* Remove from the hash table. */ @@ -325,22 +317,22 @@ } - spinlock_unlock(&box->irq_lock); - spinlock_unlock(&irq_uspace_hash_table_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&box->irq_lock, false); + irq_spinlock_unlock(&irq_uspace_hash_table_lock, true); } /** Add a call to the proper answerbox queue. * - * Assume irq->lock is locked. - * - * @param irq IRQ structure referencing the target answerbox. - * @param call IRQ notification call. + * Assume irq->lock is locked and interrupts disabled. + * + * @param irq IRQ structure referencing the target answerbox. + * @param call IRQ notification call. + * */ static void send_call(irq_t *irq, call_t *call) { - spinlock_lock(&irq->notif_cfg.answerbox->irq_lock); + irq_spinlock_lock(&irq->notif_cfg.answerbox->irq_lock, false); list_append(&call->link, &irq->notif_cfg.answerbox->irq_notifs); - spinlock_unlock(&irq->notif_cfg.answerbox->irq_lock); - + irq_spinlock_unlock(&irq->notif_cfg.answerbox->irq_lock, false); + waitq_wakeup(&irq->notif_cfg.answerbox->wq, WAKEUP_FIRST); } @@ -348,16 +340,14 @@ /** Apply the top-half pseudo code to find out whether to accept the IRQ or not. * - * @param irq IRQ structure. - * - * @return IRQ_ACCEPT if the interrupt is accepted by the - * pseudocode. IRQ_DECLINE otherwise. + * @param irq IRQ structure. + * + * @return IRQ_ACCEPT if the interrupt is accepted by the + * pseudocode, IRQ_DECLINE otherwise. + * */ irq_ownership_t ipc_irq_top_half_claim(irq_t *irq) { - unsigned int i; - unative_t dstval; irq_code_t *code = irq->notif_cfg.code; - unative_t *scratch = irq->notif_cfg.scratch; - + uint32_t *scratch = irq->notif_cfg.scratch; if (!irq->notif_cfg.notify) @@ -367,10 +357,13 @@ return IRQ_DECLINE; + size_t i; for (i = 0; i < code->cmdcount; i++) { - unsigned int srcarg = code->cmds[i].srcarg; - unsigned int dstarg = code->cmds[i].dstarg; + uint32_t dstval; + uintptr_t srcarg = code->cmds[i].srcarg; + uintptr_t dstarg = code->cmds[i].dstarg; if (srcarg >= IPC_CALL_LEN) break; + if (dstarg >= IPC_CALL_LEN) break; @@ -405,5 +398,5 @@ break; case CMD_BTEST: - if (srcarg && dstarg) { + if ((srcarg) && (dstarg)) { dstval = scratch[srcarg] & code->cmds[i].value; scratch[dstarg] = dstval; @@ -411,5 +404,5 @@ break; case CMD_PREDICATE: - if (srcarg && !scratch[srcarg]) { + if ((srcarg) && (!scratch[srcarg])) { i += code->cmds[i].value; continue; @@ -427,10 +420,10 @@ } - /* IRQ top-half handler. * * We expect interrupts to be disabled and the irq->lock already held. * - * @param irq IRQ structure. + * @param irq IRQ structure. + * */ void ipc_irq_top_half_handler(irq_t *irq) @@ -438,8 +431,9 @@ ASSERT(irq); + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&irq->lock)); + if (irq->notif_cfg.answerbox) { - call_t *call; - - call = ipc_call_alloc(FRAME_ATOMIC); + call_t *call = ipc_call_alloc(FRAME_ATOMIC); if (!call) return; @@ -448,5 +442,5 @@ /* Put a counter to the message */ call->priv = ++irq->notif_cfg.counter; - + /* Set up args */ IPC_SET_METHOD(call->data, irq->notif_cfg.method); @@ -456,5 +450,5 @@ IPC_SET_ARG4(call->data, irq->notif_cfg.scratch[4]); IPC_SET_ARG5(call->data, irq->notif_cfg.scratch[5]); - + send_call(irq, call); } @@ -463,28 +457,28 @@ /** Send notification message. * - * @param irq IRQ structure. - * @param a1 Driver-specific payload argument. - * @param a2 Driver-specific payload argument. - * @param a3 Driver-specific payload argument. - * @param a4 Driver-specific payload argument. - * @param a5 Driver-specific payload argument. + * @param irq IRQ structure. + * @param a1 Driver-specific payload argument. + * @param a2 Driver-specific payload argument. + * @param a3 Driver-specific payload argument. + * @param a4 Driver-specific payload argument. + * @param a5 Driver-specific payload argument. + * */ void ipc_irq_send_msg(irq_t *irq, unative_t a1, unative_t a2, unative_t a3, unative_t a4, unative_t a5) { - call_t *call; - - spinlock_lock(&irq->lock); - + irq_spinlock_lock(&irq->lock, true); + if (irq->notif_cfg.answerbox) { - call = ipc_call_alloc(FRAME_ATOMIC); + call_t *call = ipc_call_alloc(FRAME_ATOMIC); if (!call) { - spinlock_unlock(&irq->lock); + irq_spinlock_unlock(&irq->lock, true); return; } + call->flags |= IPC_CALL_NOTIF; /* Put a counter to the message */ call->priv = ++irq->notif_cfg.counter; - + IPC_SET_METHOD(call->data, irq->notif_cfg.method); IPC_SET_ARG1(call->data, a1); @@ -496,5 +490,6 @@ send_call(irq, call); } - spinlock_unlock(&irq->lock); + + irq_spinlock_unlock(&irq->lock, true); } Index: kernel/generic/src/ipc/kbox.c =================================================================== --- kernel/generic/src/ipc/kbox.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/kbox.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -47,8 +47,5 @@ void ipc_kbox_cleanup(void) { - ipl_t ipl; - bool have_kb_thread; - - /* + /* * Only hold kb.cleanup_lock while setting kb.finished - * this is enough. @@ -57,12 +54,12 @@ TASK->kb.finished = true; mutex_unlock(&TASK->kb.cleanup_lock); - - have_kb_thread = (TASK->kb.thread != NULL); - + + bool have_kb_thread = (TASK->kb.thread != NULL); + /* * From now on nobody will try to connect phones or attach * kbox threads */ - + /* * Disconnect all phones connected to our kbox. Passing true for @@ -72,15 +69,13 @@ */ ipc_answerbox_slam_phones(&TASK->kb.box, have_kb_thread); - - /* + + /* * If the task was being debugged, clean up debugging session. * This is necessarry as slamming the phones won't force * kbox thread to clean it up since sender != debugger. */ - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + mutex_lock(&TASK->udebug.lock); udebug_task_cleanup(TASK); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + mutex_unlock(&TASK->udebug.lock); if (have_kb_thread) { @@ -91,42 +86,40 @@ TASK->kb.thread = NULL; } - + /* Answer all messages in 'calls' and 'dispatched_calls' queues. */ - spinlock_lock(&TASK->kb.box.lock); + irq_spinlock_lock(&TASK->kb.box.lock, true); ipc_cleanup_call_list(&TASK->kb.box.dispatched_calls); ipc_cleanup_call_list(&TASK->kb.box.calls); - spinlock_unlock(&TASK->kb.box.lock); + irq_spinlock_unlock(&TASK->kb.box.lock, true); } /** Handle hangup message in kbox. * - * @param call The IPC_M_PHONE_HUNGUP call structure. - * @param last Output, the function stores @c true here if - * this was the last phone, @c false otherwise. - **/ + * @param call The IPC_M_PHONE_HUNGUP call structure. + * @param last Output, the function stores @c true here if + * this was the last phone, @c false otherwise. + * + */ static void kbox_proc_phone_hungup(call_t *call, bool *last) { - ipl_t ipl; - /* Was it our debugger, who hung up? */ if (call->sender == TASK->udebug.debugger) { /* Terminate debugging session (if any). */ LOG("Terminate debugging session."); - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); + irq_spinlock_lock(&TASK->lock, true); udebug_task_cleanup(TASK); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&TASK->lock, true); } else { LOG("Was not debugger."); } - + LOG("Continue with hangup message."); IPC_SET_RETVAL(call->data, 0); ipc_answer(&TASK->kb.box, call); - - ipl = interrupts_disable(); - spinlock_lock(&TASK->lock); - spinlock_lock(&TASK->kb.box.lock); + + mutex_lock(&TASK->kb.cleanup_lock); + + irq_spinlock_lock(&TASK->lock, true); + irq_spinlock_lock(&TASK->kb.box.lock, false); if (list_empty(&TASK->kb.box.connected_phones)) { /* @@ -134,7 +127,6 @@ * gets freed and signal to the caller. */ - + /* Only detach kbox thread unless already terminating. */ - mutex_lock(&TASK->kb.cleanup_lock); if (TASK->kb.finished == false) { /* Detach kbox thread so it gets freed from memory. */ @@ -142,15 +134,14 @@ TASK->kb.thread = NULL; } - mutex_unlock(&TASK->kb.cleanup_lock); - + LOG("Phone list is empty."); *last = true; - } else { + } else *last = false; - } - - spinlock_unlock(&TASK->kb.box.lock); - spinlock_unlock(&TASK->lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&TASK->kb.box.lock, false); + irq_spinlock_unlock(&TASK->lock, true); + + mutex_unlock(&TASK->kb.cleanup_lock); } @@ -160,29 +151,27 @@ * when all phones are disconnected from the kbox. * - * @param arg Ignored. + * @param arg Ignored. + * */ static void kbox_thread_proc(void *arg) { - call_t *call; - bool done; - - (void)arg; + (void) arg; LOG("Starting."); - done = false; - + bool done = false; + while (!done) { - call = ipc_wait_for_call(&TASK->kb.box, SYNCH_NO_TIMEOUT, + call_t *call = ipc_wait_for_call(&TASK->kb.box, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE); - + if (call == NULL) - continue; /* Try again. */ - + continue; /* Try again. */ + switch (IPC_GET_METHOD(call->data)) { - + case IPC_M_DEBUG_ALL: /* Handle debug call. */ udebug_call_receive(call); break; - + case IPC_M_PHONE_HUNGUP: /* @@ -193,5 +182,5 @@ kbox_proc_phone_hungup(call, &done); break; - + default: /* Ignore */ @@ -199,11 +188,10 @@ } } - + LOG("Exiting."); } -/** - * Connect phone to a task kernel-box specified by id. +/** Connect phone to a task kernel-box specified by id. * * Note that this is not completely atomic. For optimisation reasons, the task @@ -212,68 +200,61 @@ * cleanup code. * - * @return Phone id on success, or negative error code. + * @return Phone id on success, or negative error code. + * */ int ipc_connect_kbox(task_id_t taskid) { - int newphid; - task_t *ta; - thread_t *kb_thread; - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - - ta = task_find_by_id(taskid); - if (ta == NULL) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_lock(&tasks_lock, true); + + task_t *task = task_find_by_id(taskid); + if (task == NULL) { + irq_spinlock_unlock(&tasks_lock, true); return ENOENT; } - - atomic_inc(&ta->refcount); - - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); - - mutex_lock(&ta->kb.cleanup_lock); - - if (atomic_predec(&ta->refcount) == 0) { - mutex_unlock(&ta->kb.cleanup_lock); - task_destroy(ta); + + atomic_inc(&task->refcount); + + irq_spinlock_unlock(&tasks_lock, true); + + mutex_lock(&task->kb.cleanup_lock); + + if (atomic_predec(&task->refcount) == 0) { + mutex_unlock(&task->kb.cleanup_lock); + task_destroy(task); return ENOENT; } - - if (ta->kb.finished != false) { - mutex_unlock(&ta->kb.cleanup_lock); + + if (task->kb.finished != false) { + mutex_unlock(&task->kb.cleanup_lock); return EINVAL; } - - newphid = phone_alloc(TASK); + + int newphid = phone_alloc(TASK); if (newphid < 0) { - mutex_unlock(&ta->kb.cleanup_lock); + mutex_unlock(&task->kb.cleanup_lock); return ELIMIT; } - + /* Connect the newly allocated phone to the kbox */ - ipc_phone_connect(&TASK->phones[newphid], &ta->kb.box); - - if (ta->kb.thread != NULL) { - mutex_unlock(&ta->kb.cleanup_lock); + ipc_phone_connect(&TASK->phones[newphid], &task->kb.box); + + if (task->kb.thread != NULL) { + mutex_unlock(&task->kb.cleanup_lock); return newphid; } - + /* Create a kbox thread */ - kb_thread = thread_create(kbox_thread_proc, NULL, ta, 0, + thread_t *kb_thread = thread_create(kbox_thread_proc, NULL, task, 0, "kbox", false); if (!kb_thread) { - mutex_unlock(&ta->kb.cleanup_lock); + mutex_unlock(&task->kb.cleanup_lock); return ENOMEM; } - - ta->kb.thread = kb_thread; + + task->kb.thread = kb_thread; thread_ready(kb_thread); - - mutex_unlock(&ta->kb.cleanup_lock); - + + mutex_unlock(&task->kb.cleanup_lock); + return newphid; } Index: kernel/generic/src/ipc/sysipc.c =================================================================== --- kernel/generic/src/ipc/sysipc.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/ipc/sysipc.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -56,11 +56,15 @@ * requests. */ -#define DATA_XFER_LIMIT (64 * 1024) +#define DATA_XFER_LIMIT (64 * 1024) + +#define STRUCT_TO_USPACE(dst, src) copy_to_uspace((dst), (src), sizeof(*(src))) /** Get phone from the current task by ID. * - * @param phoneid Phone ID. - * @param phone Place to store pointer to phone. - * @return EOK on success, EINVAL if ID is invalid. + * @param phoneid Phone ID. + * @param phone Place to store pointer to phone. + * + * @return EOK on success, EINVAL if ID is invalid. + * */ static int phone_get(unative_t phoneid, phone_t **phone) @@ -68,23 +72,22 @@ if (phoneid >= IPC_MAX_PHONES) return EINVAL; - + *phone = &TASK->phones[phoneid]; return EOK; } -#define STRUCT_TO_USPACE(dst, src) copy_to_uspace(dst, src, sizeof(*(src))) - /** Decide if the method is a system method. * - * @param method Method to be decided. - * - * @return Return 1 if the method is a system method. - * Otherwise return 0. - */ -static inline int method_is_system(unative_t method) + * @param method Method to be decided. + * + * @return true if the method is a system method. + * + */ +static inline bool method_is_system(unative_t method) { if (method <= IPC_M_LAST_SYSTEM) - return 1; - return 0; + return true; + + return false; } @@ -94,10 +97,10 @@ * it is useless * - * @param method Method to be decided. - * - * @return Return 1 if the method is forwardable. - * Otherwise return 0. - */ -static inline int method_is_forwardable(unative_t method) + * @param method Method to be decided. + * + * @return true if the method is forwardable. + * + */ +static inline bool method_is_forwardable(unative_t method) { switch (method) { @@ -106,7 +109,7 @@ case IPC_M_PHONE_HUNGUP: /* This message is meant only for the original recipient. */ - return 0; + return false; default: - return 1; + return true; } } @@ -116,10 +119,10 @@ * - some system messages may be forwarded but their content cannot be altered * - * @param method Method to be decided. - * - * @return Return 1 if the method is immutable on forward. - * Otherwise return 0. - */ -static inline int method_is_immutable(unative_t method) + * @param method Method to be decided. + * + * @return true if the method is immutable on forward. + * + */ +static inline bool method_is_immutable(unative_t method) { switch (method) { @@ -128,7 +131,7 @@ case IPC_M_DATA_WRITE: case IPC_M_DATA_READ: - return 1; + return true; default: - return 0; + return false; } } @@ -142,10 +145,10 @@ * for answer_preprocess(). * - * @param call Call structure to be decided. - * - * @return Return 1 if the old call contents should be saved. - * Return 0 otherwise. - */ -static inline int answer_need_old(call_t *call) + * @param call Call structure to be decided. + * + * @return true if the old call contents should be saved. + * + */ +static inline bool answer_need_old(call_t *call) { switch (IPC_GET_METHOD(call->data)) { @@ -158,7 +161,7 @@ case IPC_M_DATA_WRITE: case IPC_M_DATA_READ: - return 1; + return true; default: - return 0; + return false; } } @@ -168,13 +171,12 @@ * This function is called directly after sys_ipc_answer(). * - * @param answer Call structure with the answer. - * @param olddata Saved data of the request. - * - * @return Return 0 on success or an error code. + * @param answer Call structure with the answer. + * @param olddata Saved data of the request. + * + * @return Return 0 on success or an error code. + * */ static inline int answer_preprocess(call_t *answer, ipc_data_t *olddata) { - int phoneid; - if ((native_t) IPC_GET_RETVAL(answer->data) == EHANGUP) { /* In case of forward, hangup the forwared phone, @@ -182,19 +184,20 @@ */ mutex_lock(&answer->data.phone->lock); - spinlock_lock(&TASK->answerbox.lock); + irq_spinlock_lock(&TASK->answerbox.lock, true); if (answer->data.phone->state == IPC_PHONE_CONNECTED) { list_remove(&answer->data.phone->link); answer->data.phone->state = IPC_PHONE_SLAMMED; } - spinlock_unlock(&TASK->answerbox.lock); + irq_spinlock_unlock(&TASK->answerbox.lock, true); mutex_unlock(&answer->data.phone->lock); } - + if (!olddata) return 0; - + if (IPC_GET_METHOD(*olddata) == IPC_M_CONNECTION_CLONE) { - phoneid = IPC_GET_ARG1(*olddata); - phone_t *phone = &TASK->phones[phoneid]; + int phoneid = IPC_GET_ARG1(*olddata); + phone_t *phone = &TASK->phones[phoneid]; + if (IPC_GET_RETVAL(answer->data) != EOK) { /* @@ -208,13 +211,14 @@ mutex_lock(&phone->lock); if (phone->state == IPC_PHONE_CONNECTED) { - spinlock_lock(&phone->callee->lock); + irq_spinlock_lock(&phone->callee->lock, true); list_remove(&phone->link); phone->state = IPC_PHONE_SLAMMED; - spinlock_unlock(&phone->callee->lock); + irq_spinlock_unlock(&phone->callee->lock, true); } mutex_unlock(&phone->lock); } } else if (IPC_GET_METHOD(*olddata) == IPC_M_CONNECT_ME) { - phone_t *phone = (phone_t *)IPC_GET_ARG5(*olddata); + phone_t *phone = (phone_t *) IPC_GET_ARG5(*olddata); + if (IPC_GET_RETVAL(answer->data) != EOK) { /* @@ -226,13 +230,14 @@ mutex_lock(&phone->lock); if (phone->state == IPC_PHONE_CONNECTED) { - spinlock_lock(&phone->callee->lock); + irq_spinlock_lock(&phone->callee->lock, true); list_remove(&phone->link); phone->state = IPC_PHONE_SLAMMED; - spinlock_unlock(&phone->callee->lock); + irq_spinlock_unlock(&phone->callee->lock, true); } mutex_unlock(&phone->lock); } } else if (IPC_GET_METHOD(*olddata) == IPC_M_CONNECT_TO_ME) { - phoneid = IPC_GET_ARG5(*olddata); + int phoneid = IPC_GET_ARG5(*olddata); + if (IPC_GET_RETVAL(answer->data) != EOK) { /* The connection was not accepted */ @@ -254,15 +259,10 @@ if (!IPC_GET_RETVAL(answer->data)) { /* Accepted, handle as_area receipt */ - ipl_t ipl; - int rc; - as_t *as; - ipl = interrupts_disable(); - spinlock_lock(&answer->sender->lock); - as = answer->sender->as; - spinlock_unlock(&answer->sender->lock); - interrupts_restore(ipl); + irq_spinlock_lock(&answer->sender->lock, true); + as_t *as = answer->sender->as; + irq_spinlock_unlock(&answer->sender->lock, true); - rc = as_area_share(as, IPC_GET_ARG1(*olddata), + int rc = as_area_share(as, IPC_GET_ARG1(*olddata), IPC_GET_ARG2(*olddata), AS, IPC_GET_ARG1(answer->data), IPC_GET_ARG3(*olddata)); @@ -272,15 +272,9 @@ } else if (IPC_GET_METHOD(*olddata) == IPC_M_SHARE_IN) { if (!IPC_GET_RETVAL(answer->data)) { - ipl_t ipl; - as_t *as; - int rc; + irq_spinlock_lock(&answer->sender->lock, true); + as_t *as = answer->sender->as; + irq_spinlock_unlock(&answer->sender->lock, true); - ipl = interrupts_disable(); - spinlock_lock(&answer->sender->lock); - as = answer->sender->as; - spinlock_unlock(&answer->sender->lock); - interrupts_restore(ipl); - - rc = as_area_share(AS, IPC_GET_ARG1(answer->data), + int rc = as_area_share(AS, IPC_GET_ARG1(answer->data), IPC_GET_ARG2(*olddata), as, IPC_GET_ARG1(*olddata), IPC_GET_ARG2(answer->data)); @@ -301,5 +295,5 @@ */ IPC_SET_ARG1(answer->data, dst); - + answer->buffer = malloc(size, 0); int rc = copy_from_uspace(answer->buffer, @@ -320,15 +314,10 @@ if (!IPC_GET_RETVAL(answer->data)) { /* The recipient agreed to receive data. */ - int rc; - uintptr_t dst; - size_t size; - size_t max_size; - - dst = (uintptr_t)IPC_GET_ARG1(answer->data); - size = (size_t)IPC_GET_ARG2(answer->data); - max_size = (size_t)IPC_GET_ARG2(*olddata); - + uintptr_t dst = (uintptr_t)IPC_GET_ARG1(answer->data); + size_t size = (size_t)IPC_GET_ARG2(answer->data); + size_t max_size = (size_t)IPC_GET_ARG2(*olddata); + if (size <= max_size) { - rc = copy_to_uspace((void *) dst, + int rc = copy_to_uspace((void *) dst, answer->buffer, size); if (rc) @@ -341,4 +330,5 @@ answer->buffer = NULL; } + return 0; } @@ -352,7 +342,6 @@ mutex_lock(&p2->lock); mutex_lock(&p1->lock); - } else { + } else mutex_lock(&p1->lock); - } } @@ -366,24 +355,20 @@ /** Called before the request is sent. * - * @param call Call structure with the request. - * @param phone Phone that the call will be sent through. - * - * @return Return 0 on success, ELIMIT or EPERM on error. + * @param call Call structure with the request. + * @param phone Phone that the call will be sent through. + * + * @return Return 0 on success, ELIMIT or EPERM on error. + * */ static int request_preprocess(call_t *call, phone_t *phone) { - int newphid; - size_t size; - uintptr_t src; - int rc; - switch (IPC_GET_METHOD(call->data)) { case IPC_M_CONNECTION_CLONE: { phone_t *cloned_phone; - if (phone_get(IPC_GET_ARG1(call->data), &cloned_phone) != EOK) return ENOENT; + phones_lock(cloned_phone, phone); - + if ((cloned_phone->state != IPC_PHONE_CONNECTED) || phone->state != IPC_PHONE_CONNECTED) { @@ -391,4 +376,5 @@ return EINVAL; } + /* * We can be pretty sure now that both tasks exist and we are @@ -396,13 +382,16 @@ * we are effectively preventing them from finishing their * potential cleanup. + * */ - newphid = phone_alloc(phone->callee->task); + int newphid = phone_alloc(phone->callee->task); if (newphid < 0) { phones_unlock(cloned_phone, phone); return ELIMIT; } + ipc_phone_connect(&phone->callee->task->phones[newphid], cloned_phone->callee); phones_unlock(cloned_phone, phone); + /* Set the new phone for the callee. */ IPC_SET_ARG1(call->data, newphid); @@ -412,8 +401,9 @@ IPC_SET_ARG5(call->data, (unative_t) phone); break; - case IPC_M_CONNECT_ME_TO: - newphid = phone_alloc(TASK); + case IPC_M_CONNECT_ME_TO: { + int newphid = phone_alloc(TASK); if (newphid < 0) return ELIMIT; + /* Set arg5 for server */ IPC_SET_ARG5(call->data, (unative_t) &TASK->phones[newphid]); @@ -421,18 +411,23 @@ call->priv = newphid; break; - case IPC_M_SHARE_OUT: - size = as_area_get_size(IPC_GET_ARG1(call->data)); + } + case IPC_M_SHARE_OUT: { + size_t size = as_area_get_size(IPC_GET_ARG1(call->data)); if (!size) return EPERM; + IPC_SET_ARG2(call->data, size); break; - case IPC_M_DATA_READ: - size = IPC_GET_ARG2(call->data); + } + case IPC_M_DATA_READ: { + size_t size = IPC_GET_ARG2(call->data); if ((size <= 0 || (size > DATA_XFER_LIMIT))) return ELIMIT; + break; - case IPC_M_DATA_WRITE: - src = IPC_GET_ARG1(call->data); - size = IPC_GET_ARG2(call->data); + } + case IPC_M_DATA_WRITE: { + uintptr_t src = IPC_GET_ARG1(call->data); + size_t size = IPC_GET_ARG2(call->data); if (size > DATA_XFER_LIMIT) @@ -440,10 +435,12 @@ call->buffer = (uint8_t *) malloc(size, 0); - rc = copy_from_uspace(call->buffer, (void *) src, size); + int rc = copy_from_uspace(call->buffer, (void *) src, size); if (rc != 0) { free(call->buffer); return rc; } + break; + } #ifdef CONFIG_UDEBUG case IPC_M_DEBUG_ALL: @@ -453,4 +450,5 @@ break; } + return 0; } @@ -462,5 +460,6 @@ /** Do basic kernel processing of received call answer. * - * @param call Call structure with the answer. + * @param call Call structure with the answer. + * */ static void process_answer(call_t *call) @@ -469,5 +468,5 @@ (call->flags & IPC_CALL_FORWARDED)) IPC_SET_RETVAL(call->data, EFORWARD); - + if (call->flags & IPC_CALL_CONN_ME_TO) { if (IPC_GET_RETVAL(call->data)) @@ -476,8 +475,11 @@ IPC_SET_ARG5(call->data, call->priv); } - + if (call->buffer) { - /* This must be an affirmative answer to IPC_M_DATA_READ. */ - /* or IPC_M_DEBUG_ALL/UDEBUG_M_MEM_READ... */ + /* + * This must be an affirmative answer to IPC_M_DATA_READ + * or IPC_M_DEBUG_ALL/UDEBUG_M_MEM_READ... + * + */ uintptr_t dst = IPC_GET_ARG1(call->data); size_t size = IPC_GET_ARG2(call->data); @@ -492,16 +494,15 @@ /** Do basic kernel processing of received call request. * - * @param box Destination answerbox structure. - * @param call Call structure with the request. - * - * @return Return 0 if the call should be passed to userspace. - * Return -1 if the call should be ignored. + * @param box Destination answerbox structure. + * @param call Call structure with the request. + * + * @return 0 if the call should be passed to userspace. + * @return -1 if the call should be ignored. + * */ static int process_request(answerbox_t *box, call_t *call) { - int phoneid; - if (IPC_GET_METHOD(call->data) == IPC_M_CONNECT_TO_ME) { - phoneid = phone_alloc(TASK); + int phoneid = phone_alloc(TASK); if (phoneid < 0) { /* Failed to allocate phone */ IPC_SET_RETVAL(call->data, ELIMIT); @@ -509,6 +510,8 @@ return -1; } + IPC_SET_ARG5(call->data, phoneid); } + switch (IPC_GET_METHOD(call->data)) { case IPC_M_DEBUG_ALL: @@ -517,4 +520,5 @@ break; } + return 0; } @@ -525,31 +529,29 @@ * the generic function (i.e. sys_ipc_call_sync_slow()). * - * @param phoneid Phone handle for the call. - * @param method Method of the call. - * @param arg1 Service-defined payload argument. - * @param arg2 Service-defined payload argument. - * @param arg3 Service-defined payload argument. - * @param data Address of userspace structure where the reply call will - * be stored. - * - * @return Returns 0 on success. - * Return ENOENT if there is no such phone handle. + * @param phoneid Phone handle for the call. + * @param method Method of the call. + * @param arg1 Service-defined payload argument. + * @param arg2 Service-defined payload argument. + * @param arg3 Service-defined payload argument. + * @param data Address of userspace structure where the reply call will + * be stored. + * + * @return 0 on success. + * @return ENOENT if there is no such phone handle. + * */ unative_t sys_ipc_call_sync_fast(unative_t phoneid, unative_t method, unative_t arg1, unative_t arg2, unative_t arg3, ipc_data_t *data) { - call_t *call; phone_t *phone; - int res; - int rc; - if (phone_get(phoneid, &phone) != EOK) return ENOENT; - call = ipc_call_alloc(0); + call_t *call = ipc_call_alloc(0); IPC_SET_METHOD(call->data, method); IPC_SET_ARG1(call->data, arg1); IPC_SET_ARG2(call->data, arg2); IPC_SET_ARG3(call->data, arg3); + /* * To achieve deterministic behavior, zero out arguments that are beyond @@ -558,6 +560,9 @@ IPC_SET_ARG4(call->data, 0); IPC_SET_ARG5(call->data, 0); - - if (!(res = request_preprocess(call, phone))) { + + int res = request_preprocess(call, phone); + int rc; + + if (!res) { #ifdef CONFIG_UDEBUG udebug_stoppable_begin(); @@ -567,18 +572,20 @@ udebug_stoppable_end(); #endif + if (rc != EOK) { /* The call will be freed by ipc_cleanup(). */ return rc; } + process_answer(call); - - } else { + + } else IPC_SET_RETVAL(call->data, res); - } + rc = STRUCT_TO_USPACE(&data->args, &call->data.args); ipc_call_free(call); if (rc != 0) return rc; - + return 0; } @@ -586,24 +593,21 @@ /** Make a synchronous IPC call allowing to transmit the entire payload. * - * @param phoneid Phone handle for the call. - * @param question Userspace address of call data with the request. - * @param reply Userspace address of call data where to store the - * answer. - * - * @return Zero on success or an error code. + * @param phoneid Phone handle for the call. + * @param question Userspace address of call data with the request. + * @param reply Userspace address of call data where to store the + * answer. + * + * @return Zero on success or an error code. + * */ unative_t sys_ipc_call_sync_slow(unative_t phoneid, ipc_data_t *question, ipc_data_t *reply) { - call_t *call; phone_t *phone; - int res; - int rc; - if (phone_get(phoneid, &phone) != EOK) return ENOENT; - call = ipc_call_alloc(0); - rc = copy_from_uspace(&call->data.args, &question->args, + call_t *call = ipc_call_alloc(0); + int rc = copy_from_uspace(&call->data.args, &question->args, sizeof(call->data.args)); if (rc != 0) { @@ -611,7 +615,8 @@ return (unative_t) rc; } - - - if (!(res = request_preprocess(call, phone))) { + + int res = request_preprocess(call, phone); + + if (!res) { #ifdef CONFIG_UDEBUG udebug_stoppable_begin(); @@ -621,17 +626,19 @@ udebug_stoppable_end(); #endif + if (rc != EOK) { /* The call will be freed by ipc_cleanup(). */ return rc; } + process_answer(call); - } else + } else IPC_SET_RETVAL(call->data, res); - + rc = STRUCT_TO_USPACE(&reply->args, &call->data.args); ipc_call_free(call); if (rc != 0) return rc; - + return 0; } @@ -639,5 +646,6 @@ /** Check that the task did not exceed the allowed limit of asynchronous calls. * - * @return Return 0 if limit not reached or -1 if limit exceeded. + * @return 0 if limit not reached or -1 if limit exceeded. + * */ static int check_call_limit(void) @@ -647,4 +655,5 @@ return -1; } + return 0; } @@ -655,30 +664,28 @@ * the generic function sys_ipc_call_async_slow(). * - * @param phoneid Phone handle for the call. - * @param method Method of the call. - * @param arg1 Service-defined payload argument. - * @param arg2 Service-defined payload argument. - * @param arg3 Service-defined payload argument. - * @param arg4 Service-defined payload argument. - * - * @return Return call hash on success. - * Return IPC_CALLRET_FATAL in case of a fatal error and - * IPC_CALLRET_TEMPORARY if there are too many pending - * asynchronous requests; answers should be handled first. + * @param phoneid Phone handle for the call. + * @param method Method of the call. + * @param arg1 Service-defined payload argument. + * @param arg2 Service-defined payload argument. + * @param arg3 Service-defined payload argument. + * @param arg4 Service-defined payload argument. + * + * @return Call hash on success. + * @return IPC_CALLRET_FATAL in case of a fatal error. + * @return IPC_CALLRET_TEMPORARY if there are too many pending + * asynchronous requests; answers should be handled first. + * */ unative_t sys_ipc_call_async_fast(unative_t phoneid, unative_t method, unative_t arg1, unative_t arg2, unative_t arg3, unative_t arg4) { - call_t *call; - phone_t *phone; - int res; - if (check_call_limit()) return IPC_CALLRET_TEMPORARY; - + + phone_t *phone; if (phone_get(phoneid, &phone) != EOK) return IPC_CALLRET_FATAL; - - call = ipc_call_alloc(0); + + call_t *call = ipc_call_alloc(0); IPC_SET_METHOD(call->data, method); IPC_SET_ARG1(call->data, arg1); @@ -686,4 +693,5 @@ IPC_SET_ARG3(call->data, arg3); IPC_SET_ARG4(call->data, arg4); + /* * To achieve deterministic behavior, zero out arguments that are beyond @@ -691,10 +699,12 @@ */ IPC_SET_ARG5(call->data, 0); - - if (!(res = request_preprocess(call, phone))) + + int res = request_preprocess(call, phone); + + if (!res) ipc_call(phone, call); else ipc_backsend_err(phone, call, res); - + return (unative_t) call; } @@ -702,24 +712,21 @@ /** Make an asynchronous IPC call allowing to transmit the entire payload. * - * @param phoneid Phone handle for the call. - * @param data Userspace address of call data with the request. - * - * @return See sys_ipc_call_async_fast(). + * @param phoneid Phone handle for the call. + * @param data Userspace address of call data with the request. + * + * @return See sys_ipc_call_async_fast(). + * */ unative_t sys_ipc_call_async_slow(unative_t phoneid, ipc_data_t *data) { - call_t *call; - phone_t *phone; - int res; - int rc; - if (check_call_limit()) return IPC_CALLRET_TEMPORARY; - + + phone_t *phone; if (phone_get(phoneid, &phone) != EOK) return IPC_CALLRET_FATAL; - call = ipc_call_alloc(0); - rc = copy_from_uspace(&call->data.args, &data->args, + call_t *call = ipc_call_alloc(0); + int rc = copy_from_uspace(&call->data.args, &data->args, sizeof(call->data.args)); if (rc != 0) { @@ -727,45 +734,48 @@ return (unative_t) rc; } - if (!(res = request_preprocess(call, phone))) + + int res = request_preprocess(call, phone); + + if (!res) ipc_call(phone, call); else ipc_backsend_err(phone, call, res); - + return (unative_t) call; } -/** Forward a received call to another destination - common code for both the - * fast and the slow version. - * - * @param callid Hash of the call to forward. - * @param phoneid Phone handle to use for forwarding. - * @param method New method to use for the forwarded call. - * @param arg1 New value of the first argument for the forwarded call. - * @param arg2 New value of the second argument for the forwarded call. - * @param arg3 New value of the third argument for the forwarded call. - * @param arg4 New value of the fourth argument for the forwarded call. - * @param arg5 New value of the fifth argument for the forwarded call. - * @param mode Flags that specify mode of the forward operation. - * @param slow If true, arg3, arg4 and arg5 are considered. Otherwise - * the function considers only the fast version arguments: - * i.e. arg1 and arg2. - * - * @return Return 0 on succes, otherwise return an error code. - * - * Warning: Make sure that ARG5 is not rewritten for certain system IPC +/** Forward a received call to another destination + * + * Common code for both the fast and the slow version. + * + * @param callid Hash of the call to forward. + * @param phoneid Phone handle to use for forwarding. + * @param method New method to use for the forwarded call. + * @param arg1 New value of the first argument for the forwarded call. + * @param arg2 New value of the second argument for the forwarded call. + * @param arg3 New value of the third argument for the forwarded call. + * @param arg4 New value of the fourth argument for the forwarded call. + * @param arg5 New value of the fifth argument for the forwarded call. + * @param mode Flags that specify mode of the forward operation. + * @param slow If true, arg3, arg4 and arg5 are considered. Otherwise + * the function considers only the fast version arguments: + * i.e. arg1 and arg2. + * + * @return 0 on succes, otherwise an error code. + * + * Warning: Make sure that ARG5 is not rewritten for certain system IPC + * */ static unative_t sys_ipc_forward_common(unative_t callid, unative_t phoneid, unative_t method, unative_t arg1, unative_t arg2, unative_t arg3, - unative_t arg4, unative_t arg5, int mode, bool slow) -{ - call_t *call; - phone_t *phone; - - call = get_call(callid); + unative_t arg4, unative_t arg5, unsigned int mode, bool slow) +{ + call_t *call = get_call(callid); if (!call) return ENOENT; call->flags |= IPC_CALL_FORWARDED; - + + phone_t *phone; if (phone_get(phoneid, &phone) != EOK) { IPC_SET_RETVAL(call->data, EFORWARD); @@ -773,5 +783,5 @@ return ENOENT; } - + if (!method_is_forwardable(IPC_GET_METHOD(call->data))) { IPC_SET_RETVAL(call->data, EFORWARD); @@ -779,5 +789,5 @@ return EPERM; } - + /* * Userspace is not allowed to change method of system methods on @@ -790,8 +800,9 @@ if (IPC_GET_METHOD(call->data) == IPC_M_CONNECT_TO_ME) phone_dealloc(IPC_GET_ARG5(call->data)); - + IPC_SET_ARG1(call->data, method); IPC_SET_ARG2(call->data, arg1); IPC_SET_ARG3(call->data, arg2); + if (slow) { IPC_SET_ARG4(call->data, arg3); @@ -812,18 +823,9 @@ } } - + return ipc_forward(call, phone, &TASK->answerbox, mode); } /** Forward a received call to another destination - fast version. - * - * @param callid Hash of the call to forward. - * @param phoneid Phone handle to use for forwarding. - * @param method New method to use for the forwarded call. - * @param arg1 New value of the first argument for the forwarded call. - * @param arg2 New value of the second argument for the forwarded call. - * @param mode Flags that specify mode of the forward operation. - * - * @return Return 0 on succes, otherwise return an error code. * * In case the original method is a system method, ARG1, ARG2 and ARG3 are @@ -833,7 +835,17 @@ * is a set of immutable methods, for which the new method and arguments are not * set and these values are ignored. + * + * @param callid Hash of the call to forward. + * @param phoneid Phone handle to use for forwarding. + * @param method New method to use for the forwarded call. + * @param arg1 New value of the first argument for the forwarded call. + * @param arg2 New value of the second argument for the forwarded call. + * @param mode Flags that specify mode of the forward operation. + * + * @return 0 on succes, otherwise an error code. + * */ unative_t sys_ipc_forward_fast(unative_t callid, unative_t phoneid, - unative_t method, unative_t arg1, unative_t arg2, int mode) + unative_t method, unative_t arg1, unative_t arg2, unsigned int mode) { return sys_ipc_forward_common(callid, phoneid, method, arg1, arg2, 0, 0, @@ -842,11 +854,4 @@ /** Forward a received call to another destination - slow version. - * - * @param callid Hash of the call to forward. - * @param phoneid Phone handle to use for forwarding. - * @param data Userspace address of the new IPC data. - * @param mode Flags that specify mode of the forward operation. - * - * @return Return 0 on succes, otherwise return an error code. * * This function is the slow verision of the sys_ipc_forward_fast interface. @@ -856,16 +861,22 @@ * methods, it additionally stores the new value of arg3, arg4 and arg5, * respectively, to ARG3, ARG4 and ARG5, respectively. + * + * @param callid Hash of the call to forward. + * @param phoneid Phone handle to use for forwarding. + * @param data Userspace address of the new IPC data. + * @param mode Flags that specify mode of the forward operation. + * + * @return 0 on succes, otherwise an error code. + * */ unative_t sys_ipc_forward_slow(unative_t callid, unative_t phoneid, - ipc_data_t *data, int mode) + ipc_data_t *data, unsigned int mode) { ipc_data_t newdata; - int rc; - - rc = copy_from_uspace(&newdata.args, &data->args, + int rc = copy_from_uspace(&newdata.args, &data->args, sizeof(newdata.args)); - if (rc != 0) + if (rc != 0) return (unative_t) rc; - + return sys_ipc_forward_common(callid, phoneid, IPC_GET_METHOD(newdata), IPC_GET_ARG1(newdata), @@ -879,34 +890,34 @@ * than the generic sys_ipc_answer(). * - * @param callid Hash of the call to be answered. - * @param retval Return value of the answer. - * @param arg1 Service-defined return value. - * @param arg2 Service-defined return value. - * @param arg3 Service-defined return value. - * @param arg4 Service-defined return value. - * - * @return Return 0 on success, otherwise return an error code. + * @param callid Hash of the call to be answered. + * @param retval Return value of the answer. + * @param arg1 Service-defined return value. + * @param arg2 Service-defined return value. + * @param arg3 Service-defined return value. + * @param arg4 Service-defined return value. + * + * @return 0 on success, otherwise an error code. + * */ unative_t sys_ipc_answer_fast(unative_t callid, unative_t retval, unative_t arg1, unative_t arg2, unative_t arg3, unative_t arg4) { - call_t *call; - ipc_data_t saved_data; - int saveddata = 0; - int rc; - /* Do not answer notification callids */ if (callid & IPC_CALLID_NOTIFICATION) return 0; - - call = get_call(callid); + + call_t *call = get_call(callid); if (!call) return ENOENT; - + + ipc_data_t saved_data; + bool saved; + if (answer_need_old(call)) { memcpy(&saved_data, &call->data, sizeof(call->data)); - saveddata = 1; - } - + saved = true; + } else + saved = false; + IPC_SET_RETVAL(call->data, retval); IPC_SET_ARG1(call->data, arg1); @@ -914,4 +925,5 @@ IPC_SET_ARG3(call->data, arg3); IPC_SET_ARG4(call->data, arg4); + /* * To achieve deterministic behavior, zero out arguments that are beyond @@ -919,6 +931,6 @@ */ IPC_SET_ARG5(call->data, 0); - rc = answer_preprocess(call, saveddata ? &saved_data : NULL); - + int rc = answer_preprocess(call, saved ? &saved_data : NULL); + ipc_answer(&TASK->answerbox, call); return rc; @@ -927,37 +939,37 @@ /** Answer an IPC call. * - * @param callid Hash of the call to be answered. - * @param data Userspace address of call data with the answer. - * - * @return Return 0 on success, otherwise return an error code. + * @param callid Hash of the call to be answered. + * @param data Userspace address of call data with the answer. + * + * @return 0 on success, otherwise an error code. + * */ unative_t sys_ipc_answer_slow(unative_t callid, ipc_data_t *data) { - call_t *call; - ipc_data_t saved_data; - int saveddata = 0; - int rc; - /* Do not answer notification callids */ if (callid & IPC_CALLID_NOTIFICATION) return 0; - - call = get_call(callid); + + call_t *call = get_call(callid); if (!call) return ENOENT; - + + ipc_data_t saved_data; + bool saved; + if (answer_need_old(call)) { memcpy(&saved_data, &call->data, sizeof(call->data)); - saveddata = 1; - } - rc = copy_from_uspace(&call->data.args, &data->args, + saved = true; + } else + saved = false; + + int rc = copy_from_uspace(&call->data.args, &data->args, sizeof(call->data.args)); if (rc != 0) return rc; - - rc = answer_preprocess(call, saveddata ? &saved_data : NULL); + + rc = answer_preprocess(call, saved ? &saved_data : NULL); ipc_answer(&TASK->answerbox, call); - return rc; } @@ -965,18 +977,19 @@ /** Hang up a phone. * - * @param Phone handle of the phone to be hung up. - * - * @return Return 0 on success or an error code. + * @param Phone handle of the phone to be hung up. + * + * @return 0 on success or an error code. + * */ unative_t sys_ipc_hangup(unative_t phoneid) { phone_t *phone; - + if (phone_get(phoneid, &phone) != EOK) return ENOENT; - + if (ipc_phone_hangup(phone)) return -1; - + return 0; } @@ -984,32 +997,36 @@ /** Wait for an incoming IPC call or an answer. * - * @param calldata Pointer to buffer where the call/answer data is stored. - * @param usec Timeout. See waitq_sleep_timeout() for explanation. - * @param flags Select mode of sleep operation. See waitq_sleep_timeout() - * for explanation. - * - * @return Hash of the call. - * If IPC_CALLID_NOTIFICATION bit is set in the hash, the - * call is a notification. IPC_CALLID_ANSWERED denotes an - * answer. - */ -unative_t sys_ipc_wait_for_call(ipc_data_t *calldata, uint32_t usec, int flags) + * @param calldata Pointer to buffer where the call/answer data is stored. + * @param usec Timeout. See waitq_sleep_timeout() for explanation. + * @param flags Select mode of sleep operation. See waitq_sleep_timeout() + * for explanation. + * + * @return Hash of the call. + * If IPC_CALLID_NOTIFICATION bit is set in the hash, the + * call is a notification. IPC_CALLID_ANSWERED denotes an + * answer. + * + */ +unative_t sys_ipc_wait_for_call(ipc_data_t *calldata, uint32_t usec, + unsigned int flags) { call_t *call; - + restart: - + #ifdef CONFIG_UDEBUG udebug_stoppable_begin(); -#endif +#endif + call = ipc_wait_for_call(&TASK->answerbox, usec, flags | SYNCH_FLAGS_INTERRUPTIBLE); - + #ifdef CONFIG_UDEBUG udebug_stoppable_end(); #endif + if (!call) return 0; - + if (call->flags & IPC_CALL_NOTIF) { /* Set in_phone_hash to the interrupt counter */ @@ -1017,13 +1034,13 @@ STRUCT_TO_USPACE(calldata, &call->data); - + ipc_call_free(call); return ((unative_t) call) | IPC_CALLID_NOTIFICATION; } - + if (call->flags & IPC_CALL_ANSWERED) { process_answer(call); - + if (call->flags & IPC_CALL_DISCARD_ANSWER) { ipc_call_free(call); @@ -1037,14 +1054,14 @@ atomic_dec(&TASK->active_calls); } - + STRUCT_TO_USPACE(&calldata->args, &call->data.args); ipc_call_free(call); - + return ((unative_t) call) | IPC_CALLID_ANSWERED; } - + if (process_request(&TASK->answerbox, call)) goto restart; - + /* Include phone address('id') of the caller in the request, * copy whole call->data, not only call->data.args */ @@ -1055,23 +1072,27 @@ */ ipc_data_t saved_data; - int saveddata = 0; - + bool saved; + if (answer_need_old(call)) { memcpy(&saved_data, &call->data, sizeof(call->data)); - saveddata = 1; - } + saved = true; + } else + saved = false; IPC_SET_RETVAL(call->data, EPARTY); - (void) answer_preprocess(call, saveddata ? &saved_data : NULL); + (void) answer_preprocess(call, saved ? &saved_data : NULL); ipc_answer(&TASK->answerbox, call); return 0; } - return (unative_t)call; -} - -/** Interrupt one thread from sys_ipc_wait_for_call(). */ + + return (unative_t) call; +} + +/** Interrupt one thread from sys_ipc_wait_for_call(). + * + */ unative_t sys_ipc_poke(void) { - waitq_unsleep(&TASK->answerbox.wq); + waitq_unsleep(&TASK->answerbox.wq); return EOK; } @@ -1079,10 +1100,11 @@ /** Connect an IRQ handler to a task. * - * @param inr IRQ number. - * @param devno Device number. - * @param method Method to be associated with the notification. - * @param ucode Uspace pointer to the top-half pseudocode. - * - * @return EPERM or a return code returned by ipc_irq_register(). + * @param inr IRQ number. + * @param devno Device number. + * @param method Method to be associated with the notification. + * @param ucode Uspace pointer to the top-half pseudocode. + * + * @return EPERM or a return code returned by ipc_irq_register(). + * */ unative_t sys_ipc_register_irq(inr_t inr, devno_t devno, unative_t method, @@ -1091,5 +1113,5 @@ if (!(cap_get(TASK) & CAP_IRQ_REG)) return EPERM; - + return ipc_irq_register(&TASK->answerbox, inr, devno, method, ucode); } @@ -1097,8 +1119,9 @@ /** Disconnect an IRQ handler from a task. * - * @param inr IRQ number. - * @param devno Device number. - * - * @return Zero on success or EPERM on error.. + * @param inr IRQ number. + * @param devno Device number. + * + * @return Zero on success or EPERM on error. + * */ unative_t sys_ipc_unregister_irq(inr_t inr, devno_t devno) @@ -1106,7 +1129,7 @@ if (!(cap_get(TASK) & CAP_IRQ_REG)) return EPERM; - + ipc_irq_unregister(&TASK->answerbox, inr, devno); - + return 0; } @@ -1114,8 +1137,8 @@ #include -/** - * Syscall connect to a task by id. - * - * @return Phone id on success, or negative error code. +/** Syscall connect to a task by id. + * + * @return Phone id on success, or negative error code. + * */ unative_t sys_ipc_connect_kbox(sysarg64_t *uspace_taskid_arg) @@ -1123,12 +1146,10 @@ #ifdef CONFIG_UDEBUG sysarg64_t taskid_arg; - int rc; - - rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); + int rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); if (rc != 0) return (unative_t) rc; - + LOG("sys_ipc_connect_kbox(%" PRIu64 ")\n", taskid_arg.value); - + return ipc_connect_kbox(taskid_arg.value); #else Index: kernel/generic/src/lib/elf.c =================================================================== --- kernel/generic/src/lib/elf.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/lib/elf.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -28,5 +28,5 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ @@ -34,5 +34,5 @@ /** * @file - * @brief Kernel ELF loader. + * @brief Kernel ELF loader. */ @@ -57,75 +57,72 @@ }; -static int segment_header(elf_segment_header_t *entry, elf_header_t *elf, - as_t *as, int flags); -static int section_header(elf_section_header_t *entry, elf_header_t *elf, - as_t *as); -static int load_segment(elf_segment_header_t *entry, elf_header_t *elf, - as_t *as); +static int segment_header(elf_segment_header_t *, elf_header_t *, as_t *, + unsigned int); +static int section_header(elf_section_header_t *, elf_header_t *, as_t *); +static int load_segment(elf_segment_header_t *, elf_header_t *, as_t *); /** ELF loader * * @param header Pointer to ELF header in memory - * @param as Created and properly mapped address space - * @param flags A combination of ELD_F_* + * @param as Created and properly mapped address space + * @param flags A combination of ELD_F_* + * * @return EE_OK on success - */ -unsigned int elf_load(elf_header_t *header, as_t * as, int flags) -{ - int i, rc; - + * + */ +unsigned int elf_load(elf_header_t *header, as_t *as, unsigned int flags) +{ /* Identify ELF */ - if (header->e_ident[EI_MAG0] != ELFMAG0 || - header->e_ident[EI_MAG1] != ELFMAG1 || - header->e_ident[EI_MAG2] != ELFMAG2 || - header->e_ident[EI_MAG3] != ELFMAG3) { + if ((header->e_ident[EI_MAG0] != ELFMAG0) || + (header->e_ident[EI_MAG1] != ELFMAG1) || + (header->e_ident[EI_MAG2] != ELFMAG2) || + (header->e_ident[EI_MAG3] != ELFMAG3)) return EE_INVALID; - } /* Identify ELF compatibility */ - if (header->e_ident[EI_DATA] != ELF_DATA_ENCODING || - header->e_machine != ELF_MACHINE || - header->e_ident[EI_VERSION] != EV_CURRENT || - header->e_version != EV_CURRENT || - header->e_ident[EI_CLASS] != ELF_CLASS) { + if ((header->e_ident[EI_DATA] != ELF_DATA_ENCODING) || + (header->e_machine != ELF_MACHINE) || + (header->e_ident[EI_VERSION] != EV_CURRENT) || + (header->e_version != EV_CURRENT) || + (header->e_ident[EI_CLASS] != ELF_CLASS)) return EE_INCOMPATIBLE; - } - + if (header->e_phentsize != sizeof(elf_segment_header_t)) return EE_INCOMPATIBLE; - + if (header->e_shentsize != sizeof(elf_section_header_t)) return EE_INCOMPATIBLE; - + /* Check if the object type is supported. */ if (header->e_type != ET_EXEC) return EE_UNSUPPORTED; - + /* Check if the ELF image starts on a page boundary */ - if (ALIGN_UP((uintptr_t)header, PAGE_SIZE) != (uintptr_t)header) + if (ALIGN_UP((uintptr_t) header, PAGE_SIZE) != (uintptr_t) header) return EE_UNSUPPORTED; - + /* Walk through all segment headers and process them. */ + elf_half i; for (i = 0; i < header->e_phnum; i++) { - elf_segment_header_t *seghdr; - - seghdr = &((elf_segment_header_t *)(((uint8_t *) header) + + elf_segment_header_t *seghdr = + &((elf_segment_header_t *)(((uint8_t *) header) + header->e_phoff))[i]; - rc = segment_header(seghdr, header, as, flags); + + int rc = segment_header(seghdr, header, as, flags); if (rc != EE_OK) return rc; } - + /* Inspect all section headers and proccess them. */ for (i = 0; i < header->e_shnum; i++) { - elf_section_header_t *sechdr; - - sechdr = &((elf_section_header_t *)(((uint8_t *) header) + + elf_section_header_t *sechdr = + &((elf_section_header_t *)(((uint8_t *) header) + header->e_shoff))[i]; - rc = section_header(sechdr, header, as); + + int rc = section_header(sechdr, header, as); if (rc != EE_OK) return rc; } - + return EE_OK; } @@ -136,9 +133,10 @@ * * @return NULL terminated description of error. + * */ const char *elf_error(unsigned int rc) { ASSERT(rc < sizeof(error_codes) / sizeof(char *)); - + return error_codes[rc]; } @@ -147,11 +145,12 @@ * * @param entry Segment header. - * @param elf ELF header. - * @param as Address space into wich the ELF is being loaded. + * @param elf ELF header. + * @param as Address space into wich the ELF is being loaded. * * @return EE_OK on success, error code otherwise. + * */ static int segment_header(elf_segment_header_t *entry, elf_header_t *elf, - as_t *as, int flags) + as_t *as, unsigned int flags) { switch (entry->p_type) { @@ -170,7 +169,6 @@ return EE_UNSUPPORTED; } */ - if ((flags & ELD_F_LOADER) == 0) { + if ((flags & ELD_F_LOADER) == 0) return EE_LOADER; - } break; case PT_SHLIB: @@ -187,51 +185,53 @@ * * @param entry Program header entry describing segment to be loaded. - * @param elf ELF header. - * @param as Address space into wich the ELF is being loaded. + * @param elf ELF header. + * @param as Address space into wich the ELF is being loaded. * * @return EE_OK on success, error code otherwise. + * */ int load_segment(elf_segment_header_t *entry, elf_header_t *elf, as_t *as) { - as_area_t *a; - int flags = 0; mem_backend_data_t backend_data; - uintptr_t base; - size_t mem_sz; - backend_data.elf = elf; backend_data.segment = entry; - + if (entry->p_align > 1) { if ((entry->p_offset % entry->p_align) != - (entry->p_vaddr % entry->p_align)) { + (entry->p_vaddr % entry->p_align)) return EE_INVALID; - } - } - + } + + unsigned int flags = 0; + if (entry->p_flags & PF_X) flags |= AS_AREA_EXEC; + if (entry->p_flags & PF_W) flags |= AS_AREA_WRITE; + if (entry->p_flags & PF_R) flags |= AS_AREA_READ; + flags |= AS_AREA_CACHEABLE; - - /* + + /* * Align vaddr down, inserting a little "gap" at the beginning. * Adjust area size, so that its end remains in place. + * */ - base = ALIGN_DOWN(entry->p_vaddr, PAGE_SIZE); - mem_sz = entry->p_memsz + (entry->p_vaddr - base); - - a = as_area_create(as, flags, mem_sz, base, + uintptr_t base = ALIGN_DOWN(entry->p_vaddr, PAGE_SIZE); + size_t mem_sz = entry->p_memsz + (entry->p_vaddr - base); + + as_area_t *area = as_area_create(as, flags, mem_sz, base, AS_AREA_ATTR_NONE, &elf_backend, &backend_data); - if (!a) + if (!area) return EE_MEMORY; /* * The segment will be mapped on demand by elf_page_fault(). + * */ - + return EE_OK; } @@ -240,8 +240,9 @@ * * @param entry Segment header. - * @param elf ELF header. - * @param as Address space into wich the ELF is being loaded. + * @param elf ELF header. + * @param as Address space into wich the ELF is being loaded. * * @return EE_OK on success, error code otherwise. + * */ static int section_header(elf_section_header_t *entry, elf_header_t *elf, Index: kernel/generic/src/main/kinit.c =================================================================== --- kernel/generic/src/main/kinit.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/main/kinit.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -95,7 +95,5 @@ void kinit(void *arg) { -#if defined(CONFIG_SMP) || defined(CONFIG_KCONSOLE) thread_t *thread; -#endif /* @@ -117,7 +115,7 @@ thread = thread_create(kmp, NULL, TASK, THREAD_FLAG_WIRED, "kmp", true); if (thread != NULL) { - spinlock_lock(&thread->lock); + irq_spinlock_lock(&thread->lock, false); thread->cpu = &cpus[0]; - spinlock_unlock(&thread->lock); + irq_spinlock_unlock(&thread->lock, false); thread_ready(thread); } else @@ -137,7 +135,7 @@ thread = thread_create(kcpulb, NULL, TASK, THREAD_FLAG_WIRED, "kcpulb", true); if (thread != NULL) { - spinlock_lock(&thread->lock); + irq_spinlock_lock(&thread->lock, false); thread->cpu = &cpus[i]; - spinlock_unlock(&thread->lock); + irq_spinlock_unlock(&thread->lock, false); thread_ready(thread); } else @@ -201,5 +199,5 @@ str_cpy(namebuf + INIT_PREFIX_LEN, TASK_NAME_BUFLEN - INIT_PREFIX_LEN, name); - + int rc = program_create_from_image((void *) init.tasks[i].addr, namebuf, &programs[i]); @@ -224,5 +222,5 @@ } } - + /* * Run user tasks. @@ -232,5 +230,5 @@ program_ready(&programs[i]); } - + #ifdef CONFIG_KCONSOLE if (!stdin) { Index: kernel/generic/src/mm/as.c =================================================================== --- kernel/generic/src/mm/as.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/as.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Address space related functions. + * @brief Address space related functions. * * This file contains address space manipulation functions. @@ -86,4 +86,5 @@ * Each architecture decides what functions will be used to carry out * address space operations such as creating or locking page tables. + * */ as_operations_t *as_operations = NULL; @@ -91,4 +92,5 @@ /** * Slab for as_t objects. + * */ static slab_cache_t *as_slab; @@ -100,4 +102,5 @@ * - as->asid for each as of the as_t type * - asids_allocated counter + * */ SPINLOCK_INITIALIZE(asidlock); @@ -106,4 +109,5 @@ * This list contains address spaces that are not active on any * processor and that have valid ASID. + * */ LIST_INITIALIZE(inactive_as_with_asid_head); @@ -112,26 +116,24 @@ as_t *AS_KERNEL = NULL; -static int area_flags_to_page_flags(int); +static unsigned int area_flags_to_page_flags(unsigned int); static as_area_t *find_area_and_lock(as_t *, uintptr_t); static bool check_area_conflicts(as_t *, uintptr_t, size_t, as_area_t *); static void sh_info_remove_reference(share_info_t *); -static int as_constructor(void *obj, int flags) +static int as_constructor(void *obj, unsigned int flags) { as_t *as = (as_t *) obj; - int rc; - + link_initialize(&as->inactive_as_with_asid_link); mutex_initialize(&as->lock, MUTEX_PASSIVE); - rc = as_constructor_arch(as, flags); + int rc = as_constructor_arch(as, flags); return rc; } -static int as_destructor(void *obj) +static size_t as_destructor(void *obj) { as_t *as = (as_t *) obj; - return as_destructor_arch(as); } @@ -141,5 +143,5 @@ { as_arch_init(); - + as_slab = slab_cache_create("as_slab", sizeof(as_t), 0, as_constructor, as_destructor, SLAB_CACHE_MAGDEFERRED); @@ -157,12 +159,11 @@ /** Create address space. * - * @param flags Flags that influence the way in wich the address space - * is created. - */ -as_t *as_create(int flags) -{ - as_t *as; - - as = (as_t *) slab_alloc(as_slab, 0); + * @param flags Flags that influence the way in wich the address + * space is created. + * + */ +as_t *as_create(unsigned int flags) +{ + as_t *as = (as_t *) slab_alloc(as_slab, 0); (void) as_create_arch(as, 0); @@ -176,4 +177,5 @@ atomic_set(&as->refcount, 0); as->cpu_refcount = 0; + #ifdef AS_PAGE_TABLE as->genarch.page_table = page_table_create(flags); @@ -192,10 +194,9 @@ * We know that we don't hold any spinlock. * - * @param as Address space to be destroyed. + * @param as Address space to be destroyed. + * */ void as_destroy(as_t *as) { - ipl_t ipl; - bool cond; DEADLOCK_PROBE_INIT(p_asidlock); @@ -214,7 +215,9 @@ * disabled to prevent nested context switches. We also depend on the * fact that so far no spinlocks are held. + * */ preemption_disable(); - ipl = interrupts_read(); + ipl_t ipl = interrupts_read(); + retry: interrupts_disable(); @@ -224,30 +227,37 @@ goto retry; } - preemption_enable(); /* Interrupts disabled, enable preemption */ - if (as->asid != ASID_INVALID && as != AS_KERNEL) { + + /* Interrupts disabled, enable preemption */ + preemption_enable(); + + if ((as->asid != ASID_INVALID) && (as != AS_KERNEL)) { if (as->cpu_refcount == 0) list_remove(&as->inactive_as_with_asid_link); + asid_put(as->asid); } + spinlock_unlock(&asidlock); - + /* * Destroy address space areas of the address space. * The B+tree must be walked carefully because it is * also being destroyed. - */ - for (cond = true; cond; ) { - btree_node_t *node; - + * + */ + bool cond = true; + while (cond) { ASSERT(!list_empty(&as->as_area_btree.leaf_head)); - node = list_get_instance(as->as_area_btree.leaf_head.next, + + btree_node_t *node = + list_get_instance(as->as_area_btree.leaf_head.next, btree_node_t, leaf_link); - - if ((cond = node->keys)) { + + if ((cond = node->keys)) as_area_destroy(as, node->key[0]); - } - } - + } + btree_destroy(&as->as_area_btree); + #ifdef AS_PAGE_TABLE page_table_destroy(as->genarch.page_table); @@ -255,7 +265,7 @@ page_table_destroy(NULL); #endif - + interrupts_restore(ipl); - + slab_free(as_slab, as); } @@ -266,5 +276,6 @@ * space. * - * @param a Address space to be held. + * @param as Address space to be held. + * */ void as_hold(as_t *as) @@ -278,5 +289,6 @@ * space. * - * @param a Address space to be released. + * @param asAddress space to be released. + * */ void as_release(as_t *as) @@ -290,32 +302,30 @@ * The created address space area is added to the target address space. * - * @param as Target address space. - * @param flags Flags of the area memory. - * @param size Size of area. - * @param base Base address of area. - * @param attrs Attributes of the area. - * @param backend Address space area backend. NULL if no backend is used. - * @param backend_data NULL or a pointer to an array holding two void *. - * - * @return Address space area on success or NULL on failure. - */ -as_area_t * -as_area_create(as_t *as, int flags, size_t size, uintptr_t base, int attrs, - mem_backend_t *backend, mem_backend_data_t *backend_data) -{ - ipl_t ipl; - as_area_t *a; - + * @param as Target address space. + * @param flags Flags of the area memory. + * @param size Size of area. + * @param base Base address of area. + * @param attrs Attributes of the area. + * @param backend Address space area backend. NULL if no backend is used. + * @param backend_data NULL or a pointer to an array holding two void *. + * + * @return Address space area on success or NULL on failure. + * + */ +as_area_t *as_area_create(as_t *as, unsigned int flags, size_t size, + uintptr_t base, unsigned int attrs, mem_backend_t *backend, + mem_backend_data_t *backend_data) +{ if (base % PAGE_SIZE) return NULL; - + if (!size) return NULL; - + /* Writeable executable areas are not supported. */ if ((flags & AS_AREA_EXEC) && (flags & AS_AREA_WRITE)) return NULL; - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); @@ -326,54 +336,52 @@ } - a = (as_area_t *) malloc(sizeof(as_area_t), 0); - - mutex_initialize(&a->lock, MUTEX_PASSIVE); - - a->as = as; - a->flags = flags; - a->attributes = attrs; - a->pages = SIZE2FRAMES(size); - a->base = base; - a->sh_info = NULL; - a->backend = backend; + as_area_t *area = (as_area_t *) malloc(sizeof(as_area_t), 0); + + mutex_initialize(&area->lock, MUTEX_PASSIVE); + + area->as = as; + area->flags = flags; + area->attributes = attrs; + area->pages = SIZE2FRAMES(size); + area->base = base; + area->sh_info = NULL; + area->backend = backend; + if (backend_data) - a->backend_data = *backend_data; + area->backend_data = *backend_data; else - memsetb(&a->backend_data, sizeof(a->backend_data), 0); - - btree_create(&a->used_space); - - btree_insert(&as->as_area_btree, base, (void *) a, NULL); - + memsetb(&area->backend_data, sizeof(area->backend_data), 0); + + btree_create(&area->used_space); + btree_insert(&as->as_area_btree, base, (void *) area, NULL); + mutex_unlock(&as->lock); interrupts_restore(ipl); - - return a; + + return area; } /** Find address space area and change it. * - * @param as Address space. - * @param address Virtual address belonging to the area to be changed. - * Must be page-aligned. - * @param size New size of the virtual memory block starting at - * address. - * @param flags Flags influencing the remap operation. Currently unused. - * - * @return Zero on success or a value from @ref errno.h otherwise. - */ -int as_area_resize(as_t *as, uintptr_t address, size_t size, int flags) -{ - as_area_t *area; - ipl_t ipl; - size_t pages; - - ipl = interrupts_disable(); + * @param as Address space. + * @param address Virtual address belonging to the area to be changed. + * Must be page-aligned. + * @param size New size of the virtual memory block starting at + * address. + * @param flags Flags influencing the remap operation. Currently unused. + * + * @return Zero on success or a value from @ref errno.h otherwise. + * + */ +int as_area_resize(as_t *as, uintptr_t address, size_t size, unsigned int flags) +{ + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); /* * Locate the area. - */ - area = find_area_and_lock(as, address); + * + */ + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -381,9 +389,10 @@ return ENOENT; } - + if (area->backend == &phys_backend) { /* * Remapping of address space areas associated * with memory mapped devices is not supported. + * */ mutex_unlock(&area->lock); @@ -392,8 +401,10 @@ return ENOTSUP; } + if (area->sh_info) { /* - * Remapping of shared address space areas + * Remapping of shared address space areas * is not supported. + * */ mutex_unlock(&area->lock); @@ -402,9 +413,10 @@ return ENOTSUP; } - - pages = SIZE2FRAMES((address - area->base) + size); + + size_t pages = SIZE2FRAMES((address - area->base) + size); if (!pages) { /* * Zero size address space areas are not allowed. + * */ mutex_unlock(&area->lock); @@ -415,18 +427,21 @@ if (pages < area->pages) { - bool cond; uintptr_t start_free = area->base + pages * PAGE_SIZE; - + /* * Shrinking the area. * No need to check for overlaps. - */ - + * + */ + + page_table_lock(as, false); + /* * Start TLB shootdown sequence. + * */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base + pages * PAGE_SIZE, area->pages - pages); - + /* * Remove frames belonging to used space starting from @@ -435,42 +450,46 @@ * is also the right way to remove part of the used_space * B+tree leaf list. - */ - for (cond = true; cond;) { - btree_node_t *node; - + * + */ + bool cond = true; + while (cond) { ASSERT(!list_empty(&area->used_space.leaf_head)); - node = + + btree_node_t *node = list_get_instance(area->used_space.leaf_head.prev, btree_node_t, leaf_link); + if ((cond = (bool) node->keys)) { - uintptr_t b = node->key[node->keys - 1]; - size_t c = + uintptr_t ptr = node->key[node->keys - 1]; + size_t size = (size_t) node->value[node->keys - 1]; - unsigned int i = 0; - - if (overlaps(b, c * PAGE_SIZE, area->base, + size_t i = 0; + + if (overlaps(ptr, size * PAGE_SIZE, area->base, pages * PAGE_SIZE)) { - if (b + c * PAGE_SIZE <= start_free) { + if (ptr + size * PAGE_SIZE <= start_free) { /* * The whole interval fits * completely in the resized * address space area. + * */ break; } - + /* * Part of the interval corresponding * to b and c overlaps with the resized * address space area. + * */ - - cond = false; /* we are almost done */ - i = (start_free - b) >> PAGE_WIDTH; + + /* We are almost done */ + cond = false; + i = (start_free - ptr) >> PAGE_WIDTH; if (!used_space_remove(area, start_free, - c - i)) - panic("Cannot remove used " - "space."); + size - i)) + panic("Cannot remove used space."); } else { /* @@ -478,38 +497,40 @@ * completely removed. */ - if (!used_space_remove(area, b, c)) - panic("Cannot remove used " - "space."); + if (!used_space_remove(area, ptr, size)) + panic("Cannot remove used space."); } - - for (; i < c; i++) { - pte_t *pte; - - page_table_lock(as, false); - pte = page_mapping_find(as, b + + + for (; i < size; i++) { + pte_t *pte = page_mapping_find(as, ptr + i * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); - if (area->backend && - area->backend->frame_free) { + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + + if ((area->backend) && + (area->backend->frame_free)) { area->backend->frame_free(area, - b + i * PAGE_SIZE, + ptr + i * PAGE_SIZE, PTE_GET_FRAME(pte)); } - page_mapping_remove(as, b + + + page_mapping_remove(as, ptr + i * PAGE_SIZE); - page_table_unlock(as, false); } } } - + /* * Finish TLB shootdown sequence. - */ - + * + */ + tlb_invalidate_pages(as->asid, area->base + pages * PAGE_SIZE, area->pages - pages); + /* * Invalidate software translation caches (e.g. TSB on sparc64). + * */ as_invalidate_translation_cache(as, area->base + @@ -517,18 +538,20 @@ tlb_shootdown_finalize(); + page_table_unlock(as, false); } else { /* * Growing the area. * Check for overlaps with other address space areas. + * */ if (!check_area_conflicts(as, address, pages * PAGE_SIZE, area)) { mutex_unlock(&area->lock); - mutex_unlock(&as->lock); + mutex_unlock(&as->lock); interrupts_restore(ipl); return EADDRNOTAVAIL; } - } - + } + area->pages = pages; @@ -536,5 +559,5 @@ mutex_unlock(&as->lock); interrupts_restore(ipl); - + return 0; } @@ -542,20 +565,16 @@ /** Destroy address space area. * - * @param as Address space. - * @param address Address within the area to be deleted. - * - * @return Zero on success or a value from @ref errno.h on failure. + * @param as Address space. + * @param address Address within the area to be deleted. + * + * @return Zero on success or a value from @ref errno.h on failure. + * */ int as_area_destroy(as_t *as, uintptr_t address) { - as_area_t *area; - uintptr_t base; - link_t *cur; - ipl_t ipl; - - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); - - area = find_area_and_lock(as, address); + + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -563,65 +582,75 @@ return ENOENT; } - - base = area->base; - + + uintptr_t base = area->base; + + page_table_lock(as, false); + /* * Start TLB shootdown sequence. */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, area->pages); - + /* * Visit only the pages mapped by used_space B+tree. */ + link_t *cur; for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { btree_node_t *node; - unsigned int i; + btree_key_t i; node = list_get_instance(cur, btree_node_t, leaf_link); for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; - pte_t *pte; + uintptr_t ptr = node->key[i]; + size_t size; - for (j = 0; j < (size_t) node->value[i]; j++) { - page_table_lock(as, false); - pte = page_mapping_find(as, b + j * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); - if (area->backend && - area->backend->frame_free) { - area->backend->frame_free(area, b + - j * PAGE_SIZE, PTE_GET_FRAME(pte)); + for (size = 0; size < (size_t) node->value[i]; size++) { + pte_t *pte = page_mapping_find(as, ptr + size * PAGE_SIZE); + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + + if ((area->backend) && + (area->backend->frame_free)) { + area->backend->frame_free(area, + ptr + size * PAGE_SIZE, PTE_GET_FRAME(pte)); } - page_mapping_remove(as, b + j * PAGE_SIZE); - page_table_unlock(as, false); + + page_mapping_remove(as, ptr + size * PAGE_SIZE); } } } - + /* * Finish TLB shootdown sequence. - */ - + * + */ + tlb_invalidate_pages(as->asid, area->base, area->pages); + /* * Invalidate potential software translation caches (e.g. TSB on * sparc64). + * */ as_invalidate_translation_cache(as, area->base, area->pages); tlb_shootdown_finalize(); + page_table_unlock(as, false); + btree_destroy(&area->used_space); - + area->attributes |= AS_AREA_ATTR_PARTIAL; if (area->sh_info) sh_info_remove_reference(area->sh_info); - + mutex_unlock(&area->lock); - + /* * Remove the empty area from address space. + * */ btree_remove(&as->as_area_btree, base, NULL); @@ -641,36 +670,31 @@ * sh_info of the source area. The process of duplicating the * mapping is done through the backend share function. - * - * @param src_as Pointer to source address space. - * @param src_base Base address of the source address space area. - * @param acc_size Expected size of the source area. - * @param dst_as Pointer to destination address space. - * @param dst_base Target base address. + * + * @param src_as Pointer to source address space. + * @param src_base Base address of the source address space area. + * @param acc_size Expected size of the source area. + * @param dst_as Pointer to destination address space. + * @param dst_base Target base address. * @param dst_flags_mask Destination address space area flags mask. * - * @return Zero on success or ENOENT if there is no such task or if - * there is no such address space area, EPERM if there was - * a problem in accepting the area or ENOMEM if there was a - * problem in allocating destination address space area. - * ENOTSUP is returned if the address space area backend - * does not support sharing. + * @return Zero on success. + * @return ENOENT if there is no such task or such address space. + * @return EPERM if there was a problem in accepting the area. + * @return ENOMEM if there was a problem in allocating destination + * address space area. + * @return ENOTSUP if the address space area backend does not support + * sharing. + * */ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, - as_t *dst_as, uintptr_t dst_base, int dst_flags_mask) -{ - ipl_t ipl; - int src_flags; - size_t src_size; - as_area_t *src_area, *dst_area; - share_info_t *sh_info; - mem_backend_t *src_backend; - mem_backend_data_t src_backend_data; - - ipl = interrupts_disable(); + as_t *dst_as, uintptr_t dst_base, unsigned int dst_flags_mask) +{ + ipl_t ipl = interrupts_disable(); mutex_lock(&src_as->lock); - src_area = find_area_and_lock(src_as, src_base); + as_area_t *src_area = find_area_and_lock(src_as, src_base); if (!src_area) { /* * Could not find the source address space area. + * */ mutex_unlock(&src_as->lock); @@ -678,9 +702,10 @@ return ENOENT; } - - if (!src_area->backend || !src_area->backend->share) { + + if ((!src_area->backend) || (!src_area->backend->share)) { /* * There is no backend or the backend does not * know how to share the area. + * */ mutex_unlock(&src_area->lock); @@ -690,15 +715,15 @@ } - src_size = src_area->pages * PAGE_SIZE; - src_flags = src_area->flags; - src_backend = src_area->backend; - src_backend_data = src_area->backend_data; - + size_t src_size = src_area->pages * PAGE_SIZE; + unsigned int src_flags = src_area->flags; + mem_backend_t *src_backend = src_area->backend; + mem_backend_data_t src_backend_data = src_area->backend_data; + /* Share the cacheable flag from the original mapping */ if (src_flags & AS_AREA_CACHEABLE) dst_flags_mask |= AS_AREA_CACHEABLE; - - if (src_size != acc_size || - (src_flags & dst_flags_mask) != dst_flags_mask) { + + if ((src_size != acc_size) || + ((src_flags & dst_flags_mask) != dst_flags_mask)) { mutex_unlock(&src_area->lock); mutex_unlock(&src_as->lock); @@ -706,11 +731,12 @@ return EPERM; } - + /* * Now we are committed to sharing the area. * First, prepare the area for sharing. * Then it will be safe to unlock it. - */ - sh_info = src_area->sh_info; + * + */ + share_info_t *sh_info = src_area->sh_info; if (!sh_info) { sh_info = (share_info_t *) malloc(sizeof(share_info_t), 0); @@ -719,6 +745,8 @@ btree_create(&sh_info->pagemap); src_area->sh_info = sh_info; + /* * Call the backend to setup sharing. + * */ src_area->backend->share(src_area); @@ -728,8 +756,8 @@ mutex_unlock(&sh_info->lock); } - + mutex_unlock(&src_area->lock); mutex_unlock(&src_as->lock); - + /* * Create copy of the source address space area. @@ -739,7 +767,8 @@ * The flags of the source area are masked against dst_flags_mask * to support sharing in less privileged mode. - */ - dst_area = as_area_create(dst_as, dst_flags_mask, src_size, dst_base, - AS_AREA_ATTR_PARTIAL, src_backend, &src_backend_data); + * + */ + as_area_t *dst_area = as_area_create(dst_as, dst_flags_mask, src_size, + dst_base, AS_AREA_ATTR_PARTIAL, src_backend, &src_backend_data); if (!dst_area) { /* @@ -751,17 +780,18 @@ return ENOMEM; } - + /* * Now the destination address space area has been * fully initialized. Clear the AS_AREA_ATTR_PARTIAL * attribute and set the sh_info. - */ - mutex_lock(&dst_as->lock); + * + */ + mutex_lock(&dst_as->lock); mutex_lock(&dst_area->lock); dst_area->attributes &= ~AS_AREA_ATTR_PARTIAL; dst_area->sh_info = sh_info; mutex_unlock(&dst_area->lock); - mutex_unlock(&dst_as->lock); - + mutex_unlock(&dst_as->lock); + interrupts_restore(ipl); @@ -771,11 +801,10 @@ /** Check access mode for address space area. * - * The address space area must be locked prior to this call. - * - * @param area Address space area. - * @param access Access mode. - * - * @return False if access violates area's permissions, true - * otherwise. + * @param area Address space area. + * @param access Access mode. + * + * @return False if access violates area's permissions, true + * otherwise. + * */ bool as_area_check_access(as_area_t *area, pf_access_t access) @@ -787,4 +816,7 @@ }; + ASSERT(interrupts_disabled()); + ASSERT(mutex_locked(&area->lock)); + if (!(area->flags & flagmap[access])) return false; @@ -807,21 +839,13 @@ * */ -int as_area_change_flags(as_t *as, int flags, uintptr_t address) -{ - as_area_t *area; - link_t *cur; - ipl_t ipl; - int page_flags; - uintptr_t *old_frame; - size_t frame_idx; - size_t used_pages; - +int as_area_change_flags(as_t *as, unsigned int flags, uintptr_t address) +{ /* Flags for the new memory mapping */ - page_flags = area_flags_to_page_flags(flags); - - ipl = interrupts_disable(); + unsigned int page_flags = area_flags_to_page_flags(flags); + + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); - - area = find_area_and_lock(as, address); + + as_area_t *area = find_area_and_lock(as, address); if (!area) { mutex_unlock(&as->lock); @@ -829,5 +853,5 @@ return ENOENT; } - + if ((area->sh_info) || (area->backend != &anon_backend)) { /* Copying shared areas not supported yet */ @@ -838,116 +862,124 @@ return ENOTSUP; } - + /* * Compute total number of used pages in the used_space B+tree - */ - used_pages = 0; - + * + */ + size_t used_pages = 0; + link_t *cur; + for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); - for (i = 0; i < node->keys; i++) { + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + + for (i = 0; i < node->keys; i++) used_pages += (size_t) node->value[i]; - } - } - + } + /* An array for storing frame numbers */ - old_frame = malloc(used_pages * sizeof(uintptr_t), 0); - + uintptr_t *old_frame = malloc(used_pages * sizeof(uintptr_t), 0); + + page_table_lock(as, false); + /* * Start TLB shootdown sequence. + * */ tlb_shootdown_start(TLB_INVL_PAGES, as->asid, area->base, area->pages); - + /* * Remove used pages from page tables and remember their frame * numbers. - */ - frame_idx = 0; - + * + */ + size_t frame_idx = 0; + for (cur = area->used_space.leaf_head.next; cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; - pte_t *pte; + uintptr_t ptr = node->key[i]; + size_t size; - for (j = 0; j < (size_t) node->value[i]; j++) { + for (size = 0; size < (size_t) node->value[i]; size++) { + pte_t *pte = page_mapping_find(as, ptr + size * PAGE_SIZE); + + ASSERT(pte); + ASSERT(PTE_VALID(pte)); + ASSERT(PTE_PRESENT(pte)); + + old_frame[frame_idx++] = PTE_GET_FRAME(pte); + + /* Remove old mapping */ + page_mapping_remove(as, ptr + size * PAGE_SIZE); + } + } + } + + /* + * Finish TLB shootdown sequence. + * + */ + + tlb_invalidate_pages(as->asid, area->base, area->pages); + + /* + * Invalidate potential software translation caches (e.g. TSB on + * sparc64). + * + */ + as_invalidate_translation_cache(as, area->base, area->pages); + tlb_shootdown_finalize(); + + page_table_unlock(as, false); + + /* + * Set the new flags. + */ + area->flags = flags; + + /* + * Map pages back in with new flags. This step is kept separate + * so that the memory area could not be accesed with both the old and + * the new flags at once. + */ + frame_idx = 0; + + for (cur = area->used_space.leaf_head.next; + cur != &area->used_space.leaf_head; cur = cur->next) { + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + + for (i = 0; i < node->keys; i++) { + uintptr_t ptr = node->key[i]; + size_t size; + + for (size = 0; size < (size_t) node->value[i]; size++) { page_table_lock(as, false); - pte = page_mapping_find(as, b + j * PAGE_SIZE); - ASSERT(pte && PTE_VALID(pte) && - PTE_PRESENT(pte)); - old_frame[frame_idx++] = PTE_GET_FRAME(pte); - - /* Remove old mapping */ - page_mapping_remove(as, b + j * PAGE_SIZE); + + /* Insert the new mapping */ + page_mapping_insert(as, ptr + size * PAGE_SIZE, + old_frame[frame_idx++], page_flags); + page_table_unlock(as, false); } } } - - /* - * Finish TLB shootdown sequence. - */ - - tlb_invalidate_pages(as->asid, area->base, area->pages); - - /* - * Invalidate potential software translation caches (e.g. TSB on - * sparc64). - */ - as_invalidate_translation_cache(as, area->base, area->pages); - tlb_shootdown_finalize(); - - /* - * Set the new flags. - */ - area->flags = flags; - - /* - * Map pages back in with new flags. This step is kept separate - * so that the memory area could not be accesed with both the old and - * the new flags at once. - */ - frame_idx = 0; - - for (cur = area->used_space.leaf_head.next; - cur != &area->used_space.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; - - node = list_get_instance(cur, btree_node_t, leaf_link); - for (i = 0; i < node->keys; i++) { - uintptr_t b = node->key[i]; - size_t j; - - for (j = 0; j < (size_t) node->value[i]; j++) { - page_table_lock(as, false); - - /* Insert the new mapping */ - page_mapping_insert(as, b + j * PAGE_SIZE, - old_frame[frame_idx++], page_flags); - - page_table_unlock(as, false); - } - } - } - + free(old_frame); - + mutex_unlock(&area->lock); mutex_unlock(&as->lock); interrupts_restore(ipl); - + return 0; } - /** Handle page fault within the current address space. @@ -959,18 +991,17 @@ * Interrupts are assumed disabled. * - * @param page Faulting page. - * @param access Access mode that caused the page fault (i.e. - * read/write/exec). - * @param istate Pointer to the interrupted state. - * - * @return AS_PF_FAULT on page fault, AS_PF_OK on success or - * AS_PF_DEFER if the fault was caused by copy_to_uspace() - * or copy_from_uspace(). + * @param page Faulting page. + * @param access Access mode that caused the page fault (i.e. + * read/write/exec). + * @param istate Pointer to the interrupted state. + * + * @return AS_PF_FAULT on page fault. + * @return AS_PF_OK on success. + * @return AS_PF_DEFER if the fault was caused by copy_to_uspace() + * or copy_from_uspace(). + * */ int as_page_fault(uintptr_t page, pf_access_t access, istate_t *istate) { - pte_t *pte; - as_area_t *area; - if (!THREAD) return AS_PF_FAULT; @@ -980,14 +1011,15 @@ mutex_lock(&AS->lock); - area = find_area_and_lock(AS, page); + as_area_t *area = find_area_and_lock(AS, page); if (!area) { /* * No area contained mapping for 'page'. * Signal page fault to low-level handler. + * */ mutex_unlock(&AS->lock); goto page_fault; } - + if (area->attributes & AS_AREA_ATTR_PARTIAL) { /* @@ -997,17 +1029,18 @@ mutex_unlock(&area->lock); mutex_unlock(&AS->lock); - goto page_fault; - } - - if (!area->backend || !area->backend->page_fault) { + goto page_fault; + } + + if ((!area->backend) || (!area->backend->page_fault)) { /* * The address space area is not backed by any backend * or the backend cannot handle page faults. + * */ mutex_unlock(&area->lock); mutex_unlock(&AS->lock); - goto page_fault; - } - + goto page_fault; + } + page_table_lock(AS, false); @@ -1015,5 +1048,7 @@ * To avoid race condition between two page faults on the same address, * we need to make sure the mapping has not been already inserted. - */ + * + */ + pte_t *pte; if ((pte = page_mapping_find(AS, page))) { if (PTE_PRESENT(pte)) { @@ -1031,4 +1066,5 @@ /* * Resort to the backend page fault handler. + * */ if (area->backend->page_fault(area, page, access) != AS_PF_OK) { @@ -1043,5 +1079,5 @@ mutex_unlock(&AS->lock); return AS_PF_OK; - + page_fault: if (THREAD->in_copy_from_uspace) { @@ -1056,5 +1092,5 @@ return AS_PF_FAULT; } - + return AS_PF_DEFER; } @@ -1068,6 +1104,7 @@ * When this function is enetered, no spinlocks may be held. * - * @param old Old address space or NULL. - * @param new New address space. + * @param old Old address space or NULL. + * @param new New address space. + * */ void as_switch(as_t *old_as, as_t *new_as) @@ -1075,12 +1112,14 @@ DEADLOCK_PROBE_INIT(p_asidlock); preemption_disable(); + retry: (void) interrupts_disable(); if (!spinlock_trylock(&asidlock)) { - /* + /* * Avoid deadlock with TLB shootdown. * We can enable interrupts here because * preemption is disabled. We should not be * holding any other lock. + * */ (void) interrupts_enable(); @@ -1089,11 +1128,12 @@ } preemption_enable(); - + /* * First, take care of the old address space. - */ + */ if (old_as) { ASSERT(old_as->cpu_refcount); - if((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { + + if ((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { /* * The old address space is no longer active on @@ -1101,19 +1141,23 @@ * list of inactive address spaces with assigned * ASID. + * */ ASSERT(old_as->asid != ASID_INVALID); + list_append(&old_as->inactive_as_with_asid_link, &inactive_as_with_asid_head); } - + /* * Perform architecture-specific tasks when the address space * is being removed from the CPU. + * */ as_deinstall_arch(old_as); } - + /* * Second, prepare the new address space. + * */ if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { @@ -1123,4 +1167,5 @@ new_as->asid = asid_get(); } + #ifdef AS_PAGE_TABLE SET_PTL0_ADDRESS(new_as->genarch.page_table); @@ -1130,7 +1175,8 @@ * Perform architecture-specific steps. * (e.g. write ASID to hardware register etc.) + * */ as_install_arch(new_as); - + spinlock_unlock(&asidlock); @@ -1140,13 +1186,12 @@ /** Convert address space area flags to page flags. * - * @param aflags Flags of some address space area. - * - * @return Flags to be passed to page_mapping_insert(). - */ -int area_flags_to_page_flags(int aflags) -{ - int flags; - - flags = PAGE_USER | PAGE_PRESENT; + * @param aflags Flags of some address space area. + * + * @return Flags to be passed to page_mapping_insert(). + * + */ +unsigned int area_flags_to_page_flags(unsigned int aflags) +{ + unsigned int flags = PAGE_USER | PAGE_PRESENT; if (aflags & AS_AREA_READ) @@ -1161,5 +1206,5 @@ if (aflags & AS_AREA_CACHEABLE) flags |= PAGE_CACHEABLE; - + return flags; } @@ -1167,14 +1212,15 @@ /** Compute flags for virtual address translation subsytem. * - * The address space area must be locked. - * Interrupts must be disabled. - * - * @param a Address space area. - * - * @return Flags to be used in page_mapping_insert(). - */ -int as_area_get_flags(as_area_t *a) -{ - return area_flags_to_page_flags(a->flags); + * @param area Address space area. + * + * @return Flags to be used in page_mapping_insert(). + * + */ +unsigned int as_area_get_flags(as_area_t *area) +{ + ASSERT(interrupts_disabled()); + ASSERT(mutex_locked(&area->lock)); + + return area_flags_to_page_flags(area->flags); } @@ -1184,10 +1230,11 @@ * table. * - * @param flags Flags saying whether the page table is for the kernel - * address space. - * - * @return First entry of the page table. - */ -pte_t *page_table_create(int flags) + * @param flags Flags saying whether the page table is for the kernel + * address space. + * + * @return First entry of the page table. + * + */ +pte_t *page_table_create(unsigned int flags) { ASSERT(as_operations); @@ -1201,5 +1248,6 @@ * Destroy page table in architecture specific way. * - * @param page_table Physical address of PTL0. + * @param page_table Physical address of PTL0. + * */ void page_table_destroy(pte_t *page_table) @@ -1215,11 +1263,12 @@ * This function should be called before any page_mapping_insert(), * page_mapping_remove() and page_mapping_find(). - * + * * Locking order is such that address space areas must be locked * prior to this call. Address space can be locked prior to this * call in which case the lock argument is false. * - * @param as Address space. - * @param lock If false, do not attempt to lock as->lock. + * @param as Address space. + * @param lock If false, do not attempt to lock as->lock. + * */ void page_table_lock(as_t *as, bool lock) @@ -1233,6 +1282,7 @@ /** Unlock page table. * - * @param as Address space. - * @param unlock If false, do not attempt to unlock as->lock. + * @param as Address space. + * @param unlock If false, do not attempt to unlock as->lock. + * */ void page_table_unlock(as_t *as, bool unlock) @@ -1244,26 +1294,40 @@ } +/** Test whether page tables are locked. + * + * @param as Address space where the page tables belong. + * + * @return True if the page tables belonging to the address soace + * are locked, otherwise false. + */ +bool page_table_locked(as_t *as) +{ + ASSERT(as_operations); + ASSERT(as_operations->page_table_locked); + + return as_operations->page_table_locked(as); +} + /** Find address space area and lock it. * - * The address space must be locked and interrupts must be disabled. - * - * @param as Address space. - * @param va Virtual address. - * - * @return Locked address space area containing va on success or - * NULL on failure. + * @param as Address space. + * @param va Virtual address. + * + * @return Locked address space area containing va on success or + * NULL on failure. + * */ as_area_t *find_area_and_lock(as_t *as, uintptr_t va) { - as_area_t *a; - btree_node_t *leaf, *lnode; - unsigned int i; - - a = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); - if (a) { + ASSERT(interrupts_disabled()); + ASSERT(mutex_locked(&as->lock)); + + btree_node_t *leaf; + as_area_t *area = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); + if (area) { /* va is the base address of an address space area */ - mutex_lock(&a->lock); - return a; + mutex_lock(&area->lock); + return area; } @@ -1272,30 +1336,38 @@ * to find out whether this is a miss or va belongs to an address * space area found there. + * */ /* First, search the leaf node itself. */ + btree_key_t i; + for (i = 0; i < leaf->keys; i++) { - a = (as_area_t *) leaf->value[i]; - mutex_lock(&a->lock); - if ((a->base <= va) && (va < a->base + a->pages * PAGE_SIZE)) { - return a; - } - mutex_unlock(&a->lock); - } - + area = (as_area_t *) leaf->value[i]; + + mutex_lock(&area->lock); + + if ((area->base <= va) && (va < area->base + area->pages * PAGE_SIZE)) + return area; + + mutex_unlock(&area->lock); + } + /* * Second, locate the left neighbour and test its last record. * Because of its position in the B+tree, it must have base < va. - */ - lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); + * + */ + btree_node_t *lnode = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); if (lnode) { - a = (as_area_t *) lnode->value[lnode->keys - 1]; - mutex_lock(&a->lock); - if (va < a->base + a->pages * PAGE_SIZE) { - return a; - } - mutex_unlock(&a->lock); - } - + area = (as_area_t *) lnode->value[lnode->keys - 1]; + + mutex_lock(&area->lock); + + if (va < area->base + area->pages * PAGE_SIZE) + return area; + + mutex_unlock(&area->lock); + } + return NULL; } @@ -1303,22 +1375,21 @@ /** Check area conflicts with other areas. * - * The address space must be locked and interrupts must be disabled. - * - * @param as Address space. - * @param va Starting virtual address of the area being tested. - * @param size Size of the area being tested. - * @param avoid_area Do not touch this area. - * - * @return True if there is no conflict, false otherwise. - */ -bool -check_area_conflicts(as_t *as, uintptr_t va, size_t size, as_area_t *avoid_area) -{ - as_area_t *a; - btree_node_t *leaf, *node; - unsigned int i; - + * @param as Address space. + * @param va Starting virtual address of the area being tested. + * @param size Size of the area being tested. + * @param avoid_area Do not touch this area. + * + * @return True if there is no conflict, false otherwise. + * + */ +bool check_area_conflicts(as_t *as, uintptr_t va, size_t size, + as_area_t *avoid_area) +{ + ASSERT(interrupts_disabled()); + ASSERT(mutex_locked(&as->lock)); + /* * We don't want any area to have conflicts with NULL page. + * */ if (overlaps(va, size, NULL, PAGE_SIZE)) @@ -1331,57 +1402,73 @@ * record in the left neighbour, the leftmost record in the right * neighbour and all records in the leaf node itself. - */ - - if ((a = (as_area_t *) btree_search(&as->as_area_btree, va, &leaf))) { - if (a != avoid_area) + * + */ + btree_node_t *leaf; + as_area_t *area = + (as_area_t *) btree_search(&as->as_area_btree, va, &leaf); + if (area) { + if (area != avoid_area) return false; } /* First, check the two border cases. */ - if ((node = btree_leaf_node_left_neighbour(&as->as_area_btree, leaf))) { - a = (as_area_t *) node->value[node->keys - 1]; - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + btree_node_t *node = + btree_leaf_node_left_neighbour(&as->as_area_btree, leaf); + if (node) { + area = (as_area_t *) node->value[node->keys - 1]; + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); - } + + mutex_unlock(&area->lock); + } + node = btree_leaf_node_right_neighbour(&as->as_area_btree, leaf); if (node) { - a = (as_area_t *) node->value[0]; - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + area = (as_area_t *) node->value[0]; + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); + + mutex_unlock(&area->lock); } /* Second, check the leaf node. */ + btree_key_t i; for (i = 0; i < leaf->keys; i++) { - a = (as_area_t *) leaf->value[i]; - - if (a == avoid_area) + area = (as_area_t *) leaf->value[i]; + + if (area == avoid_area) continue; - - mutex_lock(&a->lock); - if (overlaps(va, size, a->base, a->pages * PAGE_SIZE)) { - mutex_unlock(&a->lock); + + mutex_lock(&area->lock); + + if (overlaps(va, size, area->base, area->pages * PAGE_SIZE)) { + mutex_unlock(&area->lock); return false; } - mutex_unlock(&a->lock); - } - + + mutex_unlock(&area->lock); + } + /* * So far, the area does not conflict with other areas. * Check if it doesn't conflict with kernel address space. - */ + * + */ if (!KERNEL_ADDRESS_SPACE_SHADOWED) { - return !overlaps(va, size, + return !overlaps(va, size, KERNEL_ADDRESS_SPACE_START, KERNEL_ADDRESS_SPACE_END - KERNEL_ADDRESS_SPACE_START); } - + return true; } @@ -1389,23 +1476,25 @@ /** Return size of the address space area with given base. * - * @param base Arbitrary address insede the address space area. - * - * @return Size of the address space area in bytes or zero if it - * does not exist. + * @param base Arbitrary address inside the address space area. + * + * @return Size of the address space area in bytes or zero if it + * does not exist. + * */ size_t as_area_get_size(uintptr_t base) { - ipl_t ipl; - as_area_t *src_area; size_t size; - - ipl = interrupts_disable(); - src_area = find_area_and_lock(AS, base); + + ipl_t ipl = interrupts_disable(); + page_table_lock(AS, true); + as_area_t *src_area = find_area_and_lock(AS, base); + if (src_area) { size = src_area->pages * PAGE_SIZE; mutex_unlock(&src_area->lock); - } else { + } else size = 0; - } + + page_table_unlock(AS, true); interrupts_restore(ipl); return size; @@ -1416,33 +1505,33 @@ * The address space area must be already locked. * - * @param a Address space area. - * @param page First page to be marked. - * @param count Number of page to be marked. - * - * @return Zero on failure and non-zero on success. - */ -int used_space_insert(as_area_t *a, uintptr_t page, size_t count) -{ - btree_node_t *leaf, *node; - size_t pages; - unsigned int i; - + * @param area Address space area. + * @param page First page to be marked. + * @param count Number of page to be marked. + * + * @return Zero on failure and non-zero on success. + * + */ +int used_space_insert(as_area_t *area, uintptr_t page, size_t count) +{ + ASSERT(mutex_locked(&area->lock)); ASSERT(page == ALIGN_DOWN(page, PAGE_SIZE)); ASSERT(count); - - pages = (size_t) btree_search(&a->used_space, page, &leaf); + + btree_node_t *leaf; + size_t pages = (size_t) btree_search(&area->used_space, page, &leaf); if (pages) { /* * We hit the beginning of some used space. + * */ return 0; } - + if (!leaf->keys) { - btree_insert(&a->used_space, page, (void *) count, leaf); + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; } - - node = btree_leaf_node_left_neighbour(&a->used_space, leaf); + + btree_node_t *node = btree_leaf_node_left_neighbour(&area->used_space, leaf); if (node) { uintptr_t left_pg = node->key[node->keys - 1]; @@ -1455,6 +1544,7 @@ * somewhere between the rightmost interval of * the left neigbour and the first interval of the leaf. - */ - + * + */ + if (page >= right_pg) { /* Do nothing. */ @@ -1466,5 +1556,5 @@ right_cnt * PAGE_SIZE)) { /* The interval intersects with the right interval. */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1472,12 +1562,14 @@ * The interval can be added by merging the two already * present intervals. + * */ node->value[node->keys - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, leaf); - return 1; + btree_remove(&area->used_space, right_pg, leaf); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { - /* + /* * The interval can be added by simply growing the left * interval. + * */ node->value[node->keys - 1] += count; @@ -1488,4 +1580,5 @@ * the right interval down and increasing its size * accordingly. + * */ leaf->value[0] += count; @@ -1496,6 +1589,7 @@ * The interval is between both neigbouring intervals, * but cannot be merged with any of them. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1504,10 +1598,11 @@ uintptr_t right_pg = leaf->key[0]; size_t right_cnt = (size_t) leaf->value[0]; - + /* * Investigate the border case in which the left neighbour does * not exist but the interval fits from the left. - */ - + * + */ + if (overlaps(page, count * PAGE_SIZE, right_pg, right_cnt * PAGE_SIZE)) { @@ -1519,4 +1614,5 @@ * right interval down and increasing its size * accordingly. + * */ leaf->key[0] = page; @@ -1527,12 +1623,13 @@ * The interval doesn't adjoin with the right interval. * It must be added individually. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; } } - - node = btree_leaf_node_right_neighbour(&a->used_space, leaf); + + node = btree_leaf_node_right_neighbour(&area->used_space, leaf); if (node) { uintptr_t left_pg = leaf->key[leaf->keys - 1]; @@ -1545,6 +1642,7 @@ * somewhere between the leftmost interval of * the right neigbour and the last interval of the leaf. - */ - + * + */ + if (page < left_pg) { /* Do nothing. */ @@ -1556,5 +1654,5 @@ right_cnt * PAGE_SIZE)) { /* The interval intersects with the right interval. */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1562,13 +1660,15 @@ * The interval can be added by merging the two already * present intervals. - * */ + * + */ leaf->value[leaf->keys - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, node); - return 1; + btree_remove(&area->used_space, right_pg, node); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { /* * The interval can be added by simply growing the left * interval. - * */ + * + */ leaf->value[leaf->keys - 1] += count; return 1; @@ -1578,4 +1678,5 @@ * the right interval down and increasing its size * accordingly. + * */ node->value[0] += count; @@ -1586,6 +1687,7 @@ * The interval is between both neigbouring intervals, * but cannot be merged with any of them. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1594,10 +1696,11 @@ uintptr_t left_pg = leaf->key[leaf->keys - 1]; size_t left_cnt = (size_t) leaf->value[leaf->keys - 1]; - + /* * Investigate the border case in which the right neighbour * does not exist but the interval fits from the right. - */ - + * + */ + if (overlaps(page, count * PAGE_SIZE, left_pg, left_cnt * PAGE_SIZE)) { @@ -1608,4 +1711,5 @@ * The interval can be added by growing the left * interval. + * */ leaf->value[leaf->keys - 1] += count; @@ -1615,6 +1719,7 @@ * The interval doesn't adjoin with the left interval. * It must be added individually. + * */ - btree_insert(&a->used_space, page, (void *) count, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1626,5 +1731,7 @@ * only between two other intervals of the leaf. The two border cases * were already resolved. - */ + * + */ + btree_key_t i; for (i = 1; i < leaf->keys; i++) { if (page < leaf->key[i]) { @@ -1633,9 +1740,10 @@ size_t left_cnt = (size_t) leaf->value[i - 1]; size_t right_cnt = (size_t) leaf->value[i]; - + /* * The interval fits between left_pg and right_pg. + * */ - + if (overlaps(page, count * PAGE_SIZE, left_pg, left_cnt * PAGE_SIZE)) { @@ -1643,4 +1751,5 @@ * The interval intersects with the left * interval. + * */ return 0; @@ -1650,6 +1759,7 @@ * The interval intersects with the right * interval. + * */ - return 0; + return 0; } else if ((page == left_pg + left_cnt * PAGE_SIZE) && (page + count * PAGE_SIZE == right_pg)) { @@ -1657,12 +1767,14 @@ * The interval can be added by merging the two * already present intervals. + * */ leaf->value[i - 1] += count + right_cnt; - btree_remove(&a->used_space, right_pg, leaf); - return 1; + btree_remove(&area->used_space, right_pg, leaf); + return 1; } else if (page == left_pg + left_cnt * PAGE_SIZE) { /* * The interval can be added by simply growing * the left interval. + * */ leaf->value[i - 1] += count; @@ -1670,8 +1782,9 @@ } else if (page + count * PAGE_SIZE == right_pg) { /* - * The interval can be addded by simply moving + * The interval can be addded by simply moving * base of the right interval down and * increasing its size accordingly. - */ + * + */ leaf->value[i] += count; leaf->key[i] = page; @@ -1682,6 +1795,7 @@ * intervals, but cannot be merged with any of * them. + * */ - btree_insert(&a->used_space, page, + btree_insert(&area->used_space, page, (void *) count, leaf); return 1; @@ -1689,5 +1803,5 @@ } } - + panic("Inconsistency detected while adding %" PRIs " pages of used " "space at %p.", count, page); @@ -1698,28 +1812,28 @@ * The address space area must be already locked. * - * @param a Address space area. - * @param page First page to be marked. - * @param count Number of page to be marked. - * - * @return Zero on failure and non-zero on success. - */ -int used_space_remove(as_area_t *a, uintptr_t page, size_t count) -{ - btree_node_t *leaf, *node; - size_t pages; - unsigned int i; - + * @param area Address space area. + * @param page First page to be marked. + * @param count Number of page to be marked. + * + * @return Zero on failure and non-zero on success. + * + */ +int used_space_remove(as_area_t *area, uintptr_t page, size_t count) +{ + ASSERT(mutex_locked(&area->lock)); ASSERT(page == ALIGN_DOWN(page, PAGE_SIZE)); ASSERT(count); - - pages = (size_t) btree_search(&a->used_space, page, &leaf); + + btree_node_t *leaf; + size_t pages = (size_t) btree_search(&area->used_space, page, &leaf); if (pages) { /* * We are lucky, page is the beginning of some interval. + * */ if (count > pages) { return 0; } else if (count == pages) { - btree_remove(&a->used_space, page, leaf); + btree_remove(&area->used_space, page, leaf); return 1; } else { @@ -1727,5 +1841,7 @@ * Find the respective interval. * Decrease its size and relocate its start address. + * */ + btree_key_t i; for (i = 0; i < leaf->keys; i++) { if (leaf->key[i] == page) { @@ -1738,10 +1854,10 @@ } } - - node = btree_leaf_node_left_neighbour(&a->used_space, leaf); - if (node && page < leaf->key[0]) { + + btree_node_t *node = btree_leaf_node_left_neighbour(&area->used_space, leaf); + if ((node) && (page < leaf->key[0])) { uintptr_t left_pg = node->key[node->keys - 1]; size_t left_cnt = (size_t) node->value[node->keys - 1]; - + if (overlaps(left_pg, left_cnt * PAGE_SIZE, page, count * PAGE_SIZE)) { @@ -1753,4 +1869,5 @@ * removed by updating the size of the bigger * interval. + * */ node->value[node->keys - 1] -= count; @@ -1758,6 +1875,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt*PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the rightmost @@ -1766,9 +1881,10 @@ * the original interval and also inserting a * new interval. + * */ - new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count*PAGE_SIZE)) >> PAGE_WIDTH; node->value[node->keys - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); return 1; @@ -1776,15 +1892,14 @@ } return 0; - } else if (page < leaf->key[0]) { + } else if (page < leaf->key[0]) return 0; - } if (page > leaf->key[leaf->keys - 1]) { uintptr_t left_pg = leaf->key[leaf->keys - 1]; size_t left_cnt = (size_t) leaf->value[leaf->keys - 1]; - + if (overlaps(left_pg, left_cnt * PAGE_SIZE, page, count * PAGE_SIZE)) { - if (page + count * PAGE_SIZE == + if (page + count * PAGE_SIZE == left_pg + left_cnt * PAGE_SIZE) { /* @@ -1792,4 +1907,5 @@ * interval of the leaf and can be removed by * updating the size of the bigger interval. + * */ leaf->value[leaf->keys - 1] -= count; @@ -1797,6 +1913,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt * PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the rightmost @@ -1805,9 +1919,10 @@ * original interval and also inserting a new * interval. + * */ - new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count * PAGE_SIZE)) >> PAGE_WIDTH; leaf->value[leaf->keys - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); return 1; @@ -1815,5 +1930,5 @@ } return 0; - } + } /* @@ -1821,9 +1936,10 @@ * Now the interval can be only between intervals of the leaf. */ + btree_key_t i; for (i = 1; i < leaf->keys - 1; i++) { if (page < leaf->key[i]) { uintptr_t left_pg = leaf->key[i - 1]; size_t left_cnt = (size_t) leaf->value[i - 1]; - + /* * Now the interval is between intervals corresponding @@ -1839,4 +1955,5 @@ * be removed by updating the size of * the bigger interval. + * */ leaf->value[i - 1] -= count; @@ -1844,6 +1961,4 @@ } else if (page + count * PAGE_SIZE < left_pg + left_cnt * PAGE_SIZE) { - size_t new_cnt; - /* * The interval is contained in the @@ -1853,10 +1968,10 @@ * also inserting a new interval. */ - new_cnt = ((left_pg + + size_t new_cnt = ((left_pg + left_cnt * PAGE_SIZE) - (page + count * PAGE_SIZE)) >> PAGE_WIDTH; leaf->value[i - 1] -= count + new_cnt; - btree_insert(&a->used_space, page + + btree_insert(&area->used_space, page + count * PAGE_SIZE, (void *) new_cnt, leaf); @@ -1867,5 +1982,5 @@ } } - + error: panic("Inconsistency detected while removing %" PRIs " pages of used " @@ -1877,12 +1992,14 @@ * If the reference count drops to 0, the sh_info is deallocated. * - * @param sh_info Pointer to address space area share info. + * @param sh_info Pointer to address space area share info. + * */ void sh_info_remove_reference(share_info_t *sh_info) { bool dealloc = false; - + mutex_lock(&sh_info->lock); ASSERT(sh_info->refcount); + if (--sh_info->refcount == 0) { dealloc = true; @@ -1895,9 +2012,9 @@ for (cur = sh_info->pagemap.leaf_head.next; cur != &sh_info->pagemap.leaf_head; cur = cur->next) { - btree_node_t *node; - unsigned int i; + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; - node = list_get_instance(cur, btree_node_t, leaf_link); - for (i = 0; i < node->keys; i++) + for (i = 0; i < node->keys; i++) frame_free((uintptr_t) node->value[i]); } @@ -1917,5 +2034,5 @@ /** Wrapper for as_area_create(). */ -unative_t sys_as_area_create(uintptr_t address, size_t size, int flags) +unative_t sys_as_area_create(uintptr_t address, size_t size, unsigned int flags) { if (as_area_create(AS, flags | AS_AREA_CACHEABLE, size, address, @@ -1927,5 +2044,5 @@ /** Wrapper for as_area_resize(). */ -unative_t sys_as_area_resize(uintptr_t address, size_t size, int flags) +unative_t sys_as_area_resize(uintptr_t address, size_t size, unsigned int flags) { return (unative_t) as_area_resize(AS, address, size, 0); @@ -1933,5 +2050,5 @@ /** Wrapper for as_area_change_flags(). */ -unative_t sys_as_area_change_flags(uintptr_t address, int flags) +unative_t sys_as_area_change_flags(uintptr_t address, unsigned int flags) { return (unative_t) as_area_change_flags(AS, flags, address); @@ -1946,77 +2063,69 @@ /** Get list of adress space areas. * - * @param as Address space. - * @param obuf Place to save pointer to returned buffer. - * @param osize Place to save size of returned buffer. + * @param as Address space. + * @param obuf Place to save pointer to returned buffer. + * @param osize Place to save size of returned buffer. + * */ void as_get_area_info(as_t *as, as_area_info_t **obuf, size_t *osize) { - ipl_t ipl; - size_t area_cnt, area_idx, i; + ipl_t ipl = interrupts_disable(); + mutex_lock(&as->lock); + + /* First pass, count number of areas. */ + + size_t area_cnt = 0; link_t *cur; - - as_area_info_t *info; - size_t isize; - - ipl = interrupts_disable(); - mutex_lock(&as->lock); - - /* First pass, count number of areas. */ - - area_cnt = 0; - + for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); + btree_node_t *node = + list_get_instance(cur, btree_node_t, leaf_link); area_cnt += node->keys; } - - isize = area_cnt * sizeof(as_area_info_t); - info = malloc(isize, 0); - + + size_t isize = area_cnt * sizeof(as_area_info_t); + as_area_info_t *info = malloc(isize, 0); + /* Second pass, record data. */ - - area_idx = 0; - + + size_t area_idx = 0; + for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); - + btree_node_t *node = + list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { as_area_t *area = node->value[i]; - + ASSERT(area_idx < area_cnt); mutex_lock(&area->lock); - + info[area_idx].start_addr = area->base; info[area_idx].size = FRAMES2SIZE(area->pages); info[area_idx].flags = area->flags; ++area_idx; - + mutex_unlock(&area->lock); } } - + mutex_unlock(&as->lock); interrupts_restore(ipl); - + *obuf = info; *osize = isize; } - /** Print out information about address space. * - * @param as Address space. + * @param as Address space. + * */ void as_print(as_t *as) { - ipl_t ipl; - - ipl = interrupts_disable(); + ipl_t ipl = interrupts_disable(); mutex_lock(&as->lock); @@ -2025,12 +2134,11 @@ for (cur = as->as_area_btree.leaf_head.next; cur != &as->as_area_btree.leaf_head; cur = cur->next) { - btree_node_t *node; - - node = list_get_instance(cur, btree_node_t, leaf_link); - - unsigned int i; + btree_node_t *node + = list_get_instance(cur, btree_node_t, leaf_link); + btree_key_t i; + for (i = 0; i < node->keys; i++) { as_area_t *area = node->value[i]; - + mutex_lock(&area->lock); printf("as_area: %p, base=%p, pages=%" PRIs Index: kernel/generic/src/mm/backend_anon.c =================================================================== --- kernel/generic/src/mm/backend_anon.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/backend_anon.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -79,4 +79,7 @@ { uintptr_t frame; + + ASSERT(page_table_locked(AS)); + ASSERT(mutex_locked(&area->lock)); if (!as_area_check_access(area, access)) @@ -168,4 +171,7 @@ void anon_frame_free(as_area_t *area, uintptr_t page, uintptr_t frame) { + ASSERT(page_table_locked(area->as)); + ASSERT(mutex_locked(&area->lock)); + frame_free(frame); } @@ -183,4 +189,7 @@ { link_t *cur; + + ASSERT(mutex_locked(&area->as->lock)); + ASSERT(mutex_locked(&area->lock)); /* Index: kernel/generic/src/mm/backend_elf.c =================================================================== --- kernel/generic/src/mm/backend_elf.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/backend_elf.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -86,4 +86,7 @@ bool dirty = false; + ASSERT(page_table_locked(AS)); + ASSERT(mutex_locked(&area->lock)); + if (!as_area_check_access(area, access)) return AS_PF_FAULT; @@ -235,6 +238,10 @@ uintptr_t start_anon; - ASSERT((page >= ALIGN_DOWN(entry->p_vaddr, PAGE_SIZE)) && - (page < entry->p_vaddr + entry->p_memsz)); + ASSERT(page_table_locked(area->as)); + ASSERT(mutex_locked(&area->lock)); + + ASSERT(page >= ALIGN_DOWN(entry->p_vaddr, PAGE_SIZE)); + ASSERT(page < entry->p_vaddr + entry->p_memsz); + start_anon = entry->p_vaddr + entry->p_filesz; @@ -263,6 +270,4 @@ * are put into the pagemap. * - * The address space and address space area must be locked prior to the call. - * * @param area Address space area. */ @@ -273,4 +278,7 @@ btree_node_t *leaf, *node; uintptr_t start_anon = entry->p_vaddr + entry->p_filesz; + + ASSERT(mutex_locked(&area->as->lock)); + ASSERT(mutex_locked(&area->lock)); /* Index: kernel/generic/src/mm/backend_phys.c =================================================================== --- kernel/generic/src/mm/backend_phys.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/backend_phys.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -72,4 +72,7 @@ uintptr_t base = area->backend_data.base; + ASSERT(page_table_locked(AS)); + ASSERT(mutex_locked(&area->lock)); + if (!as_area_check_access(area, access)) return AS_PF_FAULT; @@ -93,4 +96,6 @@ void phys_share(as_area_t *area) { + ASSERT(mutex_locked(&area->as->lock)); + ASSERT(mutex_locked(&area->lock)); } Index: kernel/generic/src/mm/frame.c =================================================================== --- kernel/generic/src/mm/frame.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/frame.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -66,8 +66,8 @@ * available. */ -mutex_t mem_avail_mtx; -condvar_t mem_avail_cv; -size_t mem_avail_req = 0; /**< Number of frames requested. */ -size_t mem_avail_gen = 0; /**< Generation counter. */ +static mutex_t mem_avail_mtx; +static condvar_t mem_avail_cv; +static size_t mem_avail_req = 0; /**< Number of frames requested. */ +static size_t mem_avail_gen = 0; /**< Generation counter. */ /********************/ @@ -171,5 +171,5 @@ return total; } -#endif +#endif /* CONFIG_DEBUG */ /** Find a zone with a given frames. @@ -199,4 +199,5 @@ if (i >= zones.count) i = 0; + } while (i != hint); @@ -242,4 +243,5 @@ if (i >= zones.count) i = 0; + } while (i != hint); @@ -296,5 +298,5 @@ index = (frame_index(zone, frame)) + (1 << frame->buddy_order); - } else { /* is_right */ + } else { /* is_right */ index = (frame_index(zone, frame)) - (1 << frame->buddy_order); @@ -673,6 +675,5 @@ bool zone_merge(size_t z1, size_t z2) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); bool ret = true; @@ -744,6 +745,5 @@ errout: - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return ret; @@ -777,5 +777,6 @@ * */ -static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, size_t count, zone_flags_t flags) +static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, + size_t count, zone_flags_t flags) { zone->base = start; @@ -841,8 +842,8 @@ * */ -size_t zone_create(pfn_t start, size_t count, pfn_t confframe, zone_flags_t flags) -{ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); +size_t zone_create(pfn_t start, size_t count, pfn_t confframe, + zone_flags_t flags) +{ + irq_spinlock_lock(&zones.lock, true); if (zone_flags_available(flags)) { /* Create available zone */ @@ -889,6 +890,5 @@ size_t znum = zones_insert_zone(start, count); if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return (size_t) -1; } @@ -905,6 +905,5 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return znum; @@ -914,12 +913,10 @@ size_t znum = zones_insert_zone(start, count); if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return (size_t) -1; } zone_construct(&zones.info[znum], NULL, start, count, flags); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return znum; @@ -933,6 +930,5 @@ void frame_set_parent(pfn_t pfn, void *data, size_t hint) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = find_zone(pfn, 1, hint); @@ -943,12 +939,10 @@ pfn - zones.info[znum].base)->parent = data; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); } void *frame_get_parent(pfn_t pfn, size_t hint) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = find_zone(pfn, 1, hint); @@ -959,6 +953,5 @@ pfn - zones.info[znum].base)->parent; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return res; @@ -977,10 +970,8 @@ { size_t size = ((size_t) 1) << order; - ipl_t ipl; size_t hint = pzone ? (*pzone) : 0; loop: - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -993,11 +984,7 @@ if it does not help, reclaim all */ if ((znum == (size_t) -1) && (!(flags & FRAME_NO_RECLAIM))) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); size_t freed = slab_reclaim(0); - - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (freed > 0) @@ -1006,11 +993,7 @@ if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); freed = slab_reclaim(SLAB_RECLAIM_ALL); - - ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (freed > 0) @@ -1022,6 +1005,5 @@ if (znum == (size_t) -1) { if (flags & FRAME_ATOMIC) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return NULL; } @@ -1031,7 +1013,6 @@ #endif - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&zones.lock, true); + if (!THREAD) panic("Cannot wait for memory to become available."); @@ -1069,6 +1050,5 @@ + zones.info[znum].base; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); if (pzone) @@ -1092,6 +1072,5 @@ void frame_free(uintptr_t frame) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -1105,6 +1084,5 @@ zone_frame_free(&zones.info[znum], pfn - zones.info[znum].base); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); /* @@ -1132,6 +1110,5 @@ void frame_reference_add(pfn_t pfn) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); /* @@ -1144,13 +1121,13 @@ zones.info[znum].frames[pfn - zones.info[znum].base].refcount++; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Mark given range unavailable in frame zones. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Mark given range unavailable in frame zones. + * + */ void frame_mark_unavailable(pfn_t start, size_t count) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t i; @@ -1164,14 +1141,15 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Initialize physical memory management. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Initialize physical memory management. + * + */ void frame_init(void) { if (config.cpu_active == 1) { zones.count = 0; - spinlock_initialize(&zones.lock, "zones.lock"); + irq_spinlock_initialize(&zones.lock, "frame.zones.lock"); mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE); condvar_initialize(&mem_avail_cv); @@ -1204,9 +1182,10 @@ } -/** Return total size of all zones. */ +/** Return total size of all zones. + * + */ uint64_t zones_total_size(void) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); uint64_t total = 0; @@ -1215,6 +1194,5 @@ total += (uint64_t) FRAMES2SIZE(zones.info[i].count); - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); return total; @@ -1229,6 +1207,5 @@ ASSERT(free != NULL); - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); *total = 0; @@ -1248,9 +1225,10 @@ } - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); -} - -/** Prints list of zones. */ + irq_spinlock_unlock(&zones.lock, true); +} + +/** Prints list of zones. + * + */ void zones_print_list(void) { @@ -1278,10 +1256,8 @@ size_t i; for (i = 0;; i++) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); if (i >= zones.count) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); break; } @@ -1293,6 +1269,5 @@ size_t busy_count = zones.info[i].busy_count; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); bool available = zone_flags_available(flags); @@ -1328,6 +1303,5 @@ void zone_print_one(size_t num) { - ipl_t ipl = interrupts_disable(); - spinlock_lock(&zones.lock); + irq_spinlock_lock(&zones.lock, true); size_t znum = (size_t) -1; @@ -1341,6 +1315,5 @@ if (znum == (size_t) -1) { - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); printf("Zone not found.\n"); return; @@ -1353,6 +1326,5 @@ size_t busy_count = zones.info[i].busy_count; - spinlock_unlock(&zones.lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&zones.lock, true); bool available = zone_flags_available(flags); Index: kernel/generic/src/mm/page.c =================================================================== --- kernel/generic/src/mm/page.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/page.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Virtual Address Translation subsystem. + * @brief Virtual Address Translation subsystem. * * This file contains code for creating, destroying and searching @@ -39,4 +39,5 @@ * Functions here are mere wrappers that call the real implementation. * They however, define the single interface. + * */ @@ -55,4 +56,5 @@ * will do an implicit serialization by virtue of running the TLB shootdown * interrupt handler. + * */ @@ -83,18 +85,18 @@ * of page boundaries. * - * @param s Address of the structure. - * @param size Size of the structure. + * @param addr Address of the structure. + * @param size Size of the structure. + * */ -void map_structure(uintptr_t s, size_t size) +void map_structure(uintptr_t addr, size_t size) { - int i, cnt, length; - - length = size + (s - (s & ~(PAGE_SIZE - 1))); - cnt = length / PAGE_SIZE + (length % PAGE_SIZE > 0); - + size_t length = size + (addr - (addr & ~(PAGE_SIZE - 1))); + size_t cnt = length / PAGE_SIZE + (length % PAGE_SIZE > 0); + + size_t i; for (i = 0; i < cnt; i++) - page_mapping_insert(AS_KERNEL, s + i * PAGE_SIZE, - s + i * PAGE_SIZE, PAGE_NOT_CACHEABLE | PAGE_WRITE); - + page_mapping_insert(AS_KERNEL, addr + i * PAGE_SIZE, + addr + i * PAGE_SIZE, PAGE_NOT_CACHEABLE | PAGE_WRITE); + /* Repel prefetched accesses to the old mapping. */ memory_barrier(); @@ -106,17 +108,20 @@ * using flags. Allocate and setup any missing page tables. * - * The page table must be locked and interrupts must be disabled. + * @param as Address space to wich page belongs. + * @param page Virtual address of the page to be mapped. + * @param frame Physical address of memory frame to which the mapping is + * done. + * @param flags Flags to be used for mapping. * - * @param as Address space to wich page belongs. - * @param page Virtual address of the page to be mapped. - * @param frame Physical address of memory frame to which the mapping is - * done. - * @param flags Flags to be used for mapping. */ -void page_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, int flags) +void page_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame, + unsigned int flags) { + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + ASSERT(page_mapping_operations); ASSERT(page_mapping_operations->mapping_insert); - + page_mapping_operations->mapping_insert(as, page, frame, flags); @@ -131,16 +136,18 @@ * this call visible. * - * The page table must be locked and interrupts must be disabled. + * @param as Address space to wich page belongs. + * @param page Virtual address of the page to be demapped. * - * @param as Address space to wich page belongs. - * @param page Virtual address of the page to be demapped. */ void page_mapping_remove(as_t *as, uintptr_t page) { + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + ASSERT(page_mapping_operations); ASSERT(page_mapping_operations->mapping_remove); page_mapping_operations->mapping_remove(as, page); - + /* Repel prefetched accesses to the old mapping. */ memory_barrier(); @@ -151,17 +158,19 @@ * Find mapping for virtual page. * - * The page table must be locked and interrupts must be disabled. + * @param as Address space to wich page belongs. + * @param page Virtual page. * - * @param as Address space to wich page belongs. - * @param page Virtual page. + * @return NULL if there is no such mapping; requested mapping + * otherwise. * - * @return NULL if there is no such mapping; requested mapping - * otherwise. */ pte_t *page_mapping_find(as_t *as, uintptr_t page) { + ASSERT(interrupts_disabled()); + ASSERT(page_table_locked(as)); + ASSERT(page_mapping_operations); ASSERT(page_mapping_operations->mapping_find); - + return page_mapping_operations->mapping_find(as, page); } Index: kernel/generic/src/mm/slab.c =================================================================== --- kernel/generic/src/mm/slab.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/slab.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Slab allocator. + * @brief Slab allocator. * * The slab allocator is closely modelled after OpenSolaris slab allocator. @@ -50,5 +50,5 @@ * * The slab allocator supports per-CPU caches ('magazines') to facilitate - * good SMP scaling. + * good SMP scaling. * * When a new object is being allocated, it is first checked, if it is @@ -65,9 +65,9 @@ * thrashing when somebody is allocating/deallocating 1 item at the magazine * size boundary. LIFO order is enforced, which should avoid fragmentation - * as much as possible. - * + * as much as possible. + * * Every cache contains list of full slabs and list of partially full slabs. * Empty slabs are immediately freed (thrashing will be avoided because - * of magazines). + * of magazines). * * The slab information structure is kept inside the data area, if possible. @@ -95,7 +95,8 @@ * * @todo - * it might be good to add granularity of locks even to slab level, + * It might be good to add granularity of locks even to slab level, * we could then try_spinlock over all partial slabs and thus improve - * scalability even on slab level + * scalability even on slab level. + * */ @@ -114,11 +115,13 @@ #include -SPINLOCK_INITIALIZE(slab_cache_lock); +IRQ_SPINLOCK_STATIC_INITIALIZE(slab_cache_lock); static LIST_INITIALIZE(slab_cache_list); /** Magazine cache */ static slab_cache_t mag_cache; + /** Cache for cache descriptors */ static slab_cache_t slab_cache_cache; + /** Cache for external slab descriptors * This time we want per-cpu cache, so do not make it static @@ -128,6 +131,8 @@ */ static slab_cache_t *slab_extern_cache; + /** Caches for malloc */ static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; + static const char *malloc_names[] = { "malloc-16", @@ -154,34 +159,36 @@ /** Slab descriptor */ typedef struct { - slab_cache_t *cache; /**< Pointer to parent cache. */ - link_t link; /**< List of full/partial slabs. */ - void *start; /**< Start address of first available item. */ - size_t available; /**< Count of available items in this slab. */ - size_t nextavail; /**< The index of next available item. */ + slab_cache_t *cache; /**< Pointer to parent cache. */ + link_t link; /**< List of full/partial slabs. */ + void *start; /**< Start address of first available item. */ + size_t available; /**< Count of available items in this slab. */ + size_t nextavail; /**< The index of next available item. */ } slab_t; #ifdef CONFIG_DEBUG -static int _slab_initialized = 0; +static unsigned int _slab_initialized = 0; #endif /**************************************/ /* Slab allocation functions */ - -/** - * Allocate frames for slab space and initialize - * - */ -static slab_t *slab_space_alloc(slab_cache_t *cache, int flags) -{ - void *data; +/**************************************/ + +/** Allocate frames for slab space and initialize + * + */ +static slab_t *slab_space_alloc(slab_cache_t *cache, unsigned int flags) +{ + + + size_t zone = 0; + + void *data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); + if (!data) { + return NULL; + } + slab_t *slab; size_t fsize; - unsigned int i; - size_t zone = 0; - - data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); - if (!data) { - return NULL; - } + if (!(cache->flags & SLAB_CACHE_SLINSIDE)) { slab = slab_alloc(slab_extern_cache, flags); @@ -196,61 +203,62 @@ /* Fill in slab structures */ - for (i = 0; i < ((unsigned int) 1 << cache->order); i++) + size_t i; + for (i = 0; i < ((size_t) 1 << cache->order); i++) frame_set_parent(ADDR2PFN(KA2PA(data)) + i, slab, zone); - + slab->start = data; slab->available = cache->objects; slab->nextavail = 0; slab->cache = cache; - + for (i = 0; i < cache->objects; i++) - *((int *) (slab->start + i*cache->size)) = i + 1; - + *((size_t *) (slab->start + i * cache->size)) = i + 1; + atomic_inc(&cache->allocated_slabs); return slab; } -/** - * Deallocate space associated with slab +/** Deallocate space associated with slab * * @return number of freed frames + * */ static size_t slab_space_free(slab_cache_t *cache, slab_t *slab) { frame_free(KA2PA(slab->start)); - if (! (cache->flags & SLAB_CACHE_SLINSIDE)) + if (!(cache->flags & SLAB_CACHE_SLINSIDE)) slab_free(slab_extern_cache, slab); - + atomic_dec(&cache->allocated_slabs); - return 1 << cache->order; + return (1 << cache->order); } /** Map object to slab structure */ -static slab_t * obj2slab(void *obj) +static slab_t *obj2slab(void *obj) { return (slab_t *) frame_get_parent(ADDR2PFN(KA2PA(obj)), 0); } -/**************************************/ +/******************/ /* Slab functions */ - - -/** - * Return object to slab and call a destructor +/******************/ + +/** Return object to slab and call a destructor * * @param slab If the caller knows directly slab of the object, otherwise NULL * * @return Number of freed pages + * */ static size_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) { - int freed = 0; - if (!slab) slab = obj2slab(obj); - + ASSERT(slab->cache == cache); - + + size_t freed = 0; + if (cache->destructor) freed = cache->destructor(obj); @@ -258,9 +266,9 @@ spinlock_lock(&cache->slablock); ASSERT(slab->available < cache->objects); - - *((int *)obj) = slab->nextavail; + + *((size_t *) obj) = slab->nextavail; slab->nextavail = (obj - slab->start) / cache->size; slab->available++; - + /* Move it to correct list */ if (slab->available == cache->objects) { @@ -268,7 +276,6 @@ list_remove(&slab->link); spinlock_unlock(&cache->slablock); - + return freed + slab_space_free(cache, slab); - } else if (slab->available == 1) { /* It was in full, move to partial */ @@ -276,26 +283,28 @@ list_prepend(&slab->link, &cache->partial_slabs); } + spinlock_unlock(&cache->slablock); return freed; } -/** - * Take new object from slab or create new if needed +/** Take new object from slab or create new if needed * * @return Object address or null + * */ static void *slab_obj_create(slab_cache_t *cache, int flags) { + spinlock_lock(&cache->slablock); + slab_t *slab; - void *obj; - - spinlock_lock(&cache->slablock); - + if (list_empty(&cache->partial_slabs)) { - /* Allow recursion and reclaiming + /* + * Allow recursion and reclaiming * - this should work, as the slab control structures * are small and do not need to allocate with anything * other than frame_alloc when they are allocating, * that's why we should get recursion at most 1-level deep + * */ spinlock_unlock(&cache->slablock); @@ -303,4 +312,5 @@ if (!slab) return NULL; + spinlock_lock(&cache->slablock); } else { @@ -309,37 +319,39 @@ list_remove(&slab->link); } - obj = slab->start + slab->nextavail * cache->size; - slab->nextavail = *((int *)obj); + + void *obj = slab->start + slab->nextavail * cache->size; + slab->nextavail = *((size_t *) obj); slab->available--; - + if (!slab->available) list_prepend(&slab->link, &cache->full_slabs); else list_prepend(&slab->link, &cache->partial_slabs); - + spinlock_unlock(&cache->slablock); - - if (cache->constructor && cache->constructor(obj, flags)) { + + if ((cache->constructor) && (cache->constructor(obj, flags))) { /* Bad, bad, construction failed */ slab_obj_destroy(cache, obj, slab); return NULL; } + return obj; } -/**************************************/ +/****************************/ /* CPU-Cache slab functions */ - -/** - * Finds a full magazine in cache, takes it from list - * and returns it - * - * @param first If true, return first, else last mag - */ -static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) +/****************************/ + +/** Find a full magazine in cache, take it from list and return it + * + * @param first If true, return first, else last mag. + * + */ +static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, bool first) { slab_magazine_t *mag = NULL; link_t *cur; - + spinlock_lock(&cache->maglock); if (!list_empty(&cache->magazines)) { @@ -348,17 +360,21 @@ else cur = cache->magazines.prev; + mag = list_get_instance(cur, slab_magazine_t, link); list_remove(&mag->link); atomic_dec(&cache->magazine_counter); } + spinlock_unlock(&cache->maglock); return mag; } -/** Prepend magazine to magazine list in cache */ +/** Prepend magazine to magazine list in cache + * + */ static void put_mag_to_cache(slab_cache_t *cache, slab_magazine_t *mag) { spinlock_lock(&cache->maglock); - + list_prepend(&mag->link, &cache->magazines); atomic_inc(&cache->magazine_counter); @@ -367,14 +383,14 @@ } -/** - * Free all objects in magazine and free memory associated with magazine +/** Free all objects in magazine and free memory associated with magazine * * @return Number of freed pages + * */ static size_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) { - unsigned int i; + size_t i; size_t frames = 0; - + for (i = 0; i < mag->busy; i++) { frames += slab_obj_destroy(cache, mag->objs[i], NULL); @@ -383,24 +399,23 @@ slab_free(&mag_cache, mag); - + return frames; } -/** - * Find full magazine, set it as current and return it - * - * Assume cpu_magazine lock is held +/** Find full magazine, set it as current and return it + * */ static slab_magazine_t *get_full_current_mag(slab_cache_t *cache) { - slab_magazine_t *cmag, *lastmag, *newmag; - - cmag = cache->mag_cache[CPU->id].current; - lastmag = cache->mag_cache[CPU->id].last; + slab_magazine_t *cmag = cache->mag_cache[CPU->id].current; + slab_magazine_t *lastmag = cache->mag_cache[CPU->id].last; + + ASSERT(spinlock_locked(&cache->mag_cache[CPU->id].lock)); + if (cmag) { /* First try local CPU magazines */ if (cmag->busy) return cmag; - - if (lastmag && lastmag->busy) { + + if ((lastmag) && (lastmag->busy)) { cache->mag_cache[CPU->id].current = lastmag; cache->mag_cache[CPU->id].last = cmag; @@ -408,39 +423,40 @@ } } + /* Local magazines are empty, import one from magazine list */ - newmag = get_mag_from_cache(cache, 1); + slab_magazine_t *newmag = get_mag_from_cache(cache, 1); if (!newmag) return NULL; - + if (lastmag) magazine_destroy(cache, lastmag); - + cache->mag_cache[CPU->id].last = cmag; cache->mag_cache[CPU->id].current = newmag; + return newmag; } -/** - * Try to find object in CPU-cache magazines +/** Try to find object in CPU-cache magazines * * @return Pointer to object or NULL if not available + * */ static void *magazine_obj_get(slab_cache_t *cache) { - slab_magazine_t *mag; - void *obj; - if (!CPU) return NULL; - + spinlock_lock(&cache->mag_cache[CPU->id].lock); - - mag = get_full_current_mag(cache); + + slab_magazine_t *mag = get_full_current_mag(cache); if (!mag) { spinlock_unlock(&cache->mag_cache[CPU->id].lock); return NULL; } - obj = mag->objs[--mag->busy]; + + void *obj = mag->objs[--mag->busy]; spinlock_unlock(&cache->mag_cache[CPU->id].lock); + atomic_dec(&cache->cached_objs); @@ -448,28 +464,25 @@ } -/** - * Assure that the current magazine is empty, return pointer to it, or NULL if - * no empty magazine is available and cannot be allocated - * - * Assume mag_cache[CPU->id].lock is held - * - * We have 2 magazines bound to processor. - * First try the current. - * If full, try the last. - * If full, put to magazines list. - * allocate new, exchange last & current +/** Assure that the current magazine is empty, return pointer to it, + * or NULL if no empty magazine is available and cannot be allocated + * + * We have 2 magazines bound to processor. + * First try the current. + * If full, try the last. + * If full, put to magazines list. * */ static slab_magazine_t *make_empty_current_mag(slab_cache_t *cache) { - slab_magazine_t *cmag,*lastmag,*newmag; - - cmag = cache->mag_cache[CPU->id].current; - lastmag = cache->mag_cache[CPU->id].last; + slab_magazine_t *cmag = cache->mag_cache[CPU->id].current; + slab_magazine_t *lastmag = cache->mag_cache[CPU->id].last; + + ASSERT(spinlock_locked(&cache->mag_cache[CPU->id].lock)); if (cmag) { if (cmag->busy < cmag->size) return cmag; - if (lastmag && lastmag->busy < lastmag->size) { + + if ((lastmag) && (lastmag->busy < lastmag->size)) { cache->mag_cache[CPU->id].last = cmag; cache->mag_cache[CPU->id].current = lastmag; @@ -477,40 +490,45 @@ } } + /* current | last are full | nonexistent, allocate new */ - /* We do not want to sleep just because of caching */ - /* Especially we do not want reclaiming to start, as - * this would deadlock */ - newmag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM); + + /* + * We do not want to sleep just because of caching, + * especially we do not want reclaiming to start, as + * this would deadlock. + * + */ + slab_magazine_t *newmag = slab_alloc(&mag_cache, + FRAME_ATOMIC | FRAME_NO_RECLAIM); if (!newmag) return NULL; + newmag->size = SLAB_MAG_SIZE; newmag->busy = 0; - + /* Flush last to magazine list */ if (lastmag) put_mag_to_cache(cache, lastmag); - + /* Move current as last, save new as current */ - cache->mag_cache[CPU->id].last = cmag; - cache->mag_cache[CPU->id].current = newmag; - + cache->mag_cache[CPU->id].last = cmag; + cache->mag_cache[CPU->id].current = newmag; + return newmag; } -/** - * Put object into CPU-cache magazine - * - * @return 0 - success, -1 - could not get memory +/** Put object into CPU-cache magazine + * + * @return 0 on success, -1 on no memory + * */ static int magazine_obj_put(slab_cache_t *cache, void *obj) { - slab_magazine_t *mag; - if (!CPU) return -1; - + spinlock_lock(&cache->mag_cache[CPU->id].lock); - - mag = make_empty_current_mag(cache); + + slab_magazine_t *mag = make_empty_current_mag(cache); if (!mag) { spinlock_unlock(&cache->mag_cache[CPU->id].lock); @@ -519,94 +537,101 @@ mag->objs[mag->busy++] = obj; - + spinlock_unlock(&cache->mag_cache[CPU->id].lock); + atomic_inc(&cache->cached_objs); + return 0; } - -/**************************************/ +/************************/ /* Slab cache functions */ - -/** Return number of objects that fit in certain cache size */ -static unsigned int comp_objects(slab_cache_t *cache) +/************************/ + +/** Return number of objects that fit in certain cache size + * + */ +static size_t comp_objects(slab_cache_t *cache) { if (cache->flags & SLAB_CACHE_SLINSIDE) - return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / - cache->size; - else + return ((PAGE_SIZE << cache->order) + - sizeof(slab_t)) / cache->size; + else return (PAGE_SIZE << cache->order) / cache->size; } -/** Return wasted space in slab */ -static unsigned int badness(slab_cache_t *cache) -{ - unsigned int objects; - unsigned int ssize; - - objects = comp_objects(cache); - ssize = PAGE_SIZE << cache->order; +/** Return wasted space in slab + * + */ +static size_t badness(slab_cache_t *cache) +{ + size_t objects = comp_objects(cache); + size_t ssize = PAGE_SIZE << cache->order; + if (cache->flags & SLAB_CACHE_SLINSIDE) ssize -= sizeof(slab_t); + return ssize - objects * cache->size; } -/** - * Initialize mag_cache structure in slab cache +/** Initialize mag_cache structure in slab cache + * */ static bool make_magcache(slab_cache_t *cache) { - unsigned int i; - ASSERT(_slab_initialized >= 2); - + cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, FRAME_ATOMIC); if (!cache->mag_cache) return false; - + + size_t i; for (i = 0; i < config.cpu_count; i++) { memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); spinlock_initialize(&cache->mag_cache[i].lock, - "slab_maglock_cpu"); - } + "slab.cache.mag_cache[].lock"); + } + return true; } -/** Initialize allocated memory as a slab cache */ +/** Initialize allocated memory as a slab cache + * + */ static void _slab_cache_create(slab_cache_t *cache, const char *name, - size_t size, size_t align, int (*constructor)(void *obj, int kmflag), - int (*destructor)(void *obj), int flags) -{ - int pages; - ipl_t ipl; - + size_t size, size_t align, int (*constructor)(void *obj, + unsigned int kmflag), size_t (*destructor)(void *obj), unsigned int flags) +{ memsetb(cache, sizeof(*cache), 0); cache->name = name; - + if (align < sizeof(unative_t)) align = sizeof(unative_t); + size = ALIGN_UP(size, align); - + cache->size = size; - cache->constructor = constructor; cache->destructor = destructor; cache->flags = flags; - + list_initialize(&cache->full_slabs); list_initialize(&cache->partial_slabs); list_initialize(&cache->magazines); - spinlock_initialize(&cache->slablock, "slab_lock"); - spinlock_initialize(&cache->maglock, "slab_maglock"); + + spinlock_initialize(&cache->slablock, "slab.cache.slablock"); + spinlock_initialize(&cache->maglock, "slab.cache.maglock"); + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) (void) make_magcache(cache); - + /* Compute slab sizes, object counts in slabs etc. */ if (cache->size < SLAB_INSIDE_SIZE) cache->flags |= SLAB_CACHE_SLINSIDE; - + /* Minimum slab order */ - pages = SIZE2FRAMES(cache->size); + size_t pages = SIZE2FRAMES(cache->size); + /* We need the 2^order >= pages */ if (pages == 1) @@ -614,59 +639,58 @@ else cache->order = fnzb(pages - 1) + 1; - - while (badness(cache) > SLAB_MAX_BADNESS(cache)) { + + while (badness(cache) > SLAB_MAX_BADNESS(cache)) cache->order += 1; - } + cache->objects = comp_objects(cache); + /* If info fits in, put it inside */ if (badness(cache) > sizeof(slab_t)) cache->flags |= SLAB_CACHE_SLINSIDE; - + /* Add cache to cache list */ - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + irq_spinlock_lock(&slab_cache_lock, true); list_append(&cache->link, &slab_cache_list); - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); -} - -/** Create slab cache */ + irq_spinlock_unlock(&slab_cache_lock, true); +} + +/** Create slab cache + * + */ slab_cache_t *slab_cache_create(const char *name, size_t size, size_t align, - int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), - int flags) -{ - slab_cache_t *cache; - - cache = slab_alloc(&slab_cache_cache, 0); + int (*constructor)(void *obj, unsigned int kmflag), + size_t (*destructor)(void *obj), unsigned int flags) +{ + slab_cache_t *cache = slab_alloc(&slab_cache_cache, 0); _slab_cache_create(cache, name, size, align, constructor, destructor, flags); + return cache; } -/** - * Reclaim space occupied by objects that are already free +/** Reclaim space occupied by objects that are already free * * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing + * * @return Number of freed pages - */ -static size_t _slab_reclaim(slab_cache_t *cache, int flags) -{ - unsigned int i; + * + */ +static size_t _slab_reclaim(slab_cache_t *cache, unsigned int flags) +{ + if (cache->flags & SLAB_CACHE_NOMAGAZINE) + return 0; /* Nothing to do */ + + /* + * We count up to original magazine count to avoid + * endless loop + */ + atomic_count_t magcount = atomic_get(&cache->magazine_counter); + slab_magazine_t *mag; size_t frames = 0; - int magcount; - - if (cache->flags & SLAB_CACHE_NOMAGAZINE) - return 0; /* Nothing to do */ - - /* We count up to original magazine count to avoid - * endless loop - */ - magcount = atomic_get(&cache->magazine_counter); - while (magcount-- && (mag=get_mag_from_cache(cache, 0))) { - frames += magazine_destroy(cache,mag); - if (!(flags & SLAB_RECLAIM_ALL) && frames) + + while ((magcount--) && (mag = get_mag_from_cache(cache, 0))) { + frames += magazine_destroy(cache, mag); + if ((!(flags & SLAB_RECLAIM_ALL)) && (frames)) break; } @@ -675,7 +699,8 @@ /* Free cpu-bound magazines */ /* Destroy CPU magazines */ + size_t i; for (i = 0; i < config.cpu_count; i++) { spinlock_lock(&cache->mag_cache[i].lock); - + mag = cache->mag_cache[i].current; if (mag) @@ -687,85 +712,88 @@ frames += magazine_destroy(cache, mag); cache->mag_cache[i].last = NULL; - + spinlock_unlock(&cache->mag_cache[i].lock); } } - + return frames; } -/** Check that there are no slabs and remove cache from system */ +/** Check that there are no slabs and remove cache from system + * + */ void slab_cache_destroy(slab_cache_t *cache) { - ipl_t ipl; - - /* First remove cache from link, so that we don't need + /* + * First remove cache from link, so that we don't need * to disable interrupts later + * */ - - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + irq_spinlock_lock(&slab_cache_lock, true); list_remove(&cache->link); - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); - - /* Do not lock anything, we assume the software is correct and - * does not touch the cache when it decides to destroy it */ + irq_spinlock_unlock(&slab_cache_lock, true); + + /* + * Do not lock anything, we assume the software is correct and + * does not touch the cache when it decides to destroy it + * + */ /* Destroy all magazines */ _slab_reclaim(cache, SLAB_RECLAIM_ALL); - + /* All slabs must be empty */ - if (!list_empty(&cache->full_slabs) || - !list_empty(&cache->partial_slabs)) + if ((!list_empty(&cache->full_slabs)) || + (!list_empty(&cache->partial_slabs))) panic("Destroying cache that is not empty."); - + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) free(cache->mag_cache); + slab_free(&slab_cache_cache, cache); } -/** Allocate new object from cache - if no flags given, always returns memory */ -void *slab_alloc(slab_cache_t *cache, int flags) -{ - ipl_t ipl; +/** Allocate new object from cache - if no flags given, always returns memory + * + */ +void *slab_alloc(slab_cache_t *cache, unsigned int flags) +{ + /* Disable interrupts to avoid deadlocks with interrupt handlers */ + ipl_t ipl = interrupts_disable(); + void *result = NULL; - /* Disable interrupts to avoid deadlocks with interrupt handlers */ - ipl = interrupts_disable(); - - if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) { + if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) result = magazine_obj_get(cache); - } + if (!result) result = slab_obj_create(cache, flags); - + interrupts_restore(ipl); - + if (result) atomic_inc(&cache->allocated_objs); - + return result; } -/** Return object to cache, use slab if known */ +/** Return object to cache, use slab if known + * + */ static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab) { - ipl_t ipl; - - ipl = interrupts_disable(); - + ipl_t ipl = interrupts_disable(); + if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || - magazine_obj_put(cache, obj)) { + (magazine_obj_put(cache, obj))) slab_obj_destroy(cache, obj, slab); - - } + interrupts_restore(ipl); atomic_dec(&cache->allocated_objs); } -/** Return slab object to cache */ +/** Return slab object to cache + * + */ void slab_free(slab_cache_t *cache, void *obj) { @@ -773,45 +801,39 @@ } -/* Go through all caches and reclaim what is possible */ -size_t slab_reclaim(int flags) -{ - slab_cache_t *cache; +/** Go through all caches and reclaim what is possible + * + * Interrupts must be disabled before calling this function, + * otherwise memory allocation from interrupts can deadlock. + * + */ +size_t slab_reclaim(unsigned int flags) +{ + irq_spinlock_lock(&slab_cache_lock, false); + + size_t frames = 0; link_t *cur; - size_t frames = 0; - - spinlock_lock(&slab_cache_lock); - - /* TODO: Add assert, that interrupts are disabled, otherwise - * memory allocation from interrupts can deadlock. - */ - for (cur = slab_cache_list.next; cur != &slab_cache_list; cur = cur->next) { - cache = list_get_instance(cur, slab_cache_t, link); + slab_cache_t *cache = list_get_instance(cur, slab_cache_t, link); frames += _slab_reclaim(cache, flags); } - - spinlock_unlock(&slab_cache_lock); - + + irq_spinlock_unlock(&slab_cache_lock, false); + return frames; } - -/* Print list of slabs */ +/* Print list of slabs + * + */ void slab_print_list(void) { - int skip = 0; - - printf("slab name size pages obj/pg slabs cached allocated" + printf("slab name size pages obj/pg slabs cached allocated" " ctl\n"); - printf("---------------- -------- ------ ------ ------ ------ ---------" + printf("---------------- -------- ------ -------- ------ ------ ---------" " ---\n"); - + + size_t skip = 0; while (true) { - slab_cache_t *cache; - link_t *cur; - ipl_t ipl; - int i; - /* * We must not hold the slab_cache_lock spinlock when printing @@ -836,36 +858,34 @@ * statistics. */ - - ipl = interrupts_disable(); - spinlock_lock(&slab_cache_lock); - + + irq_spinlock_lock(&slab_cache_lock, true); + + link_t *cur; + size_t i; for (i = 0, cur = slab_cache_list.next; - i < skip && cur != &slab_cache_list; - i++, cur = cur->next) - ; - + (i < skip) && (cur != &slab_cache_list); + i++, cur = cur->next); + if (cur == &slab_cache_list) { - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&slab_cache_lock, true); break; } - + skip++; - - cache = list_get_instance(cur, slab_cache_t, link); - + + slab_cache_t *cache = list_get_instance(cur, slab_cache_t, link); + const char *name = cache->name; uint8_t order = cache->order; size_t size = cache->size; - unsigned int objects = cache->objects; + size_t objects = cache->objects; long allocated_slabs = atomic_get(&cache->allocated_slabs); long cached_objs = atomic_get(&cache->cached_objs); long allocated_objs = atomic_get(&cache->allocated_objs); - int flags = cache->flags; - - spinlock_unlock(&slab_cache_lock); - interrupts_restore(ipl); - - printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", + unsigned int flags = cache->flags; + + irq_spinlock_unlock(&slab_cache_lock, true); + + printf("%-16s %8" PRIs " %6u %8" PRIs " %6ld %6ld %9ld %-3s\n", name, size, (1 << order), objects, allocated_slabs, cached_objs, allocated_objs, @@ -876,6 +896,4 @@ void slab_cache_init(void) { - int i, size; - /* Initialize magazine cache */ _slab_cache_create(&mag_cache, "slab_magazine", @@ -883,13 +901,18 @@ sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); + /* Initialize slab_cache cache */ _slab_cache_create(&slab_cache_cache, "slab_cache", sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); + /* Initialize external slab cache */ slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); - + /* Initialize structures for malloc */ + size_t i; + size_t size; + for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); @@ -898,4 +921,5 @@ NULL, NULL, SLAB_CACHE_MAGDEFERRED); } + #ifdef CONFIG_DEBUG _slab_initialized = 1; @@ -906,35 +930,32 @@ * * Kernel calls this function, when it knows the real number of - * processors. - * Allocate slab for cpucache and enable it on all existing - * slabs that are SLAB_CACHE_MAGDEFERRED + * processors. Allocate slab for cpucache and enable it on all + * existing slabs that are SLAB_CACHE_MAGDEFERRED + * */ void slab_enable_cpucache(void) { - link_t *cur; - slab_cache_t *s; - #ifdef CONFIG_DEBUG _slab_initialized = 2; #endif - - spinlock_lock(&slab_cache_lock); - + + irq_spinlock_lock(&slab_cache_lock, false); + + link_t *cur; for (cur = slab_cache_list.next; cur != &slab_cache_list; - cur = cur->next){ - s = list_get_instance(cur, slab_cache_t, link); - if ((s->flags & SLAB_CACHE_MAGDEFERRED) != + cur = cur->next) { + slab_cache_t *slab = list_get_instance(cur, slab_cache_t, link); + if ((slab->flags & SLAB_CACHE_MAGDEFERRED) != SLAB_CACHE_MAGDEFERRED) continue; - (void) make_magcache(s); - s->flags &= ~SLAB_CACHE_MAGDEFERRED; - } - - spinlock_unlock(&slab_cache_lock); -} - -/**************************************/ -/* kalloc/kfree functions */ -void *malloc(unsigned int size, int flags) + + (void) make_magcache(slab); + slab->flags &= ~SLAB_CACHE_MAGDEFERRED; + } + + irq_spinlock_unlock(&slab_cache_lock, false); +} + +void *malloc(size_t size, unsigned int flags) { ASSERT(_slab_initialized); @@ -943,11 +964,11 @@ if (size < (1 << SLAB_MIN_MALLOC_W)) size = (1 << SLAB_MIN_MALLOC_W); - - int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; - + + uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; + return slab_alloc(malloc_caches[idx], flags); } -void *realloc(void *ptr, unsigned int size, int flags) +void *realloc(void *ptr, size_t size, unsigned int flags) { ASSERT(_slab_initialized); @@ -959,5 +980,5 @@ if (size < (1 << SLAB_MIN_MALLOC_W)) size = (1 << SLAB_MIN_MALLOC_W); - int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; + uint8_t idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; new_ptr = slab_alloc(malloc_caches[idx], flags); @@ -980,5 +1001,5 @@ if (!ptr) return; - + slab_t *slab = obj2slab(ptr); _slab_free(slab->cache, ptr, slab); Index: kernel/generic/src/mm/tlb.c =================================================================== --- kernel/generic/src/mm/tlb.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/mm/tlb.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Generic TLB shootdown algorithm. + * @brief Generic TLB shootdown algorithm. * * The algorithm implemented here is based on the CMU TLB shootdown @@ -53,11 +53,4 @@ #include -/** - * This lock is used for synchronisation between sender and - * recipients of TLB shootdown message. It must be acquired - * before CPU structure lock. - */ -SPINLOCK_INITIALIZE(tlblock); - void tlb_init(void) { @@ -66,4 +59,12 @@ #ifdef CONFIG_SMP + +/** + * This lock is used for synchronisation between sender and + * recipients of TLB shootdown message. It must be acquired + * before CPU structure lock. + * + */ +IRQ_SPINLOCK_STATIC_INITIALIZE(tlblock); /** Send TLB shootdown message. @@ -78,13 +79,13 @@ * @param page Virtual page address, if required by type. * @param count Number of pages, if required by type. + * */ void tlb_shootdown_start(tlb_invalidate_type_t type, asid_t asid, uintptr_t page, size_t count) { - unsigned int i; - - CPU->tlb_active = 0; - spinlock_lock(&tlblock); + CPU->tlb_active = false; + irq_spinlock_lock(&tlblock, false); + size_t i; for (i = 0; i < config.cpu_count; i++) { cpu_t *cpu; @@ -92,7 +93,7 @@ if (i == CPU->id) continue; - + cpu = &cpus[i]; - spinlock_lock(&cpu->lock); + irq_spinlock_lock(&cpu->lock, false); if (cpu->tlb_messages_count == TLB_MESSAGE_QUEUE_LEN) { /* @@ -115,10 +116,10 @@ cpu->tlb_messages[idx].count = count; } - spinlock_unlock(&cpu->lock); + irq_spinlock_unlock(&cpu->lock, false); } tlb_shootdown_ipi_send(); - -busy_wait: + +busy_wait: for (i = 0; i < config.cpu_count; i++) if (cpus[i].tlb_active) @@ -126,9 +127,11 @@ } -/** Finish TLB shootdown sequence. */ +/** Finish TLB shootdown sequence. + * + */ void tlb_shootdown_finalize(void) { - spinlock_unlock(&tlblock); - CPU->tlb_active = 1; + irq_spinlock_unlock(&tlblock, false); + CPU->tlb_active = true; } @@ -138,28 +141,25 @@ } -/** Receive TLB shootdown message. */ +/** Receive TLB shootdown message. + * + */ void tlb_shootdown_ipi_recv(void) { - tlb_invalidate_type_t type; - asid_t asid; - uintptr_t page; - size_t count; - unsigned int i; - ASSERT(CPU); - CPU->tlb_active = 0; - spinlock_lock(&tlblock); - spinlock_unlock(&tlblock); + CPU->tlb_active = false; + irq_spinlock_lock(&tlblock, false); + irq_spinlock_unlock(&tlblock, false); - spinlock_lock(&CPU->lock); + irq_spinlock_lock(&CPU->lock, false); ASSERT(CPU->tlb_messages_count <= TLB_MESSAGE_QUEUE_LEN); - + + size_t i; for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) { - type = CPU->tlb_messages[i].type; - asid = CPU->tlb_messages[i].asid; - page = CPU->tlb_messages[i].page; - count = CPU->tlb_messages[i].count; - + tlb_invalidate_type_t type = CPU->tlb_messages[i].type; + asid_t asid = CPU->tlb_messages[i].asid; + uintptr_t page = CPU->tlb_messages[i].page; + size_t count = CPU->tlb_messages[i].count; + switch (type) { case TLB_INVL_ALL: @@ -170,5 +170,5 @@ break; case TLB_INVL_PAGES: - ASSERT(count); + ASSERT(count); tlb_invalidate_pages(asid, page, count); break; @@ -177,10 +177,11 @@ break; } + if (type == TLB_INVL_ALL) break; } - spinlock_unlock(&CPU->lock); - CPU->tlb_active = 1; + irq_spinlock_unlock(&CPU->lock, false); + CPU->tlb_active = true; } Index: kernel/generic/src/preempt/preemption.c =================================================================== --- kernel/generic/src/preempt/preemption.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/preempt/preemption.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,13 +27,13 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ /** - * @file preemption.c - * @brief Preemption control. + * @file preemption.c + * @brief Preemption control. */ - + #include #include @@ -52,5 +52,5 @@ void preemption_enable(void) { - ASSERT(THE->preemption_disabled); + ASSERT(PREEMPTION_DISABLED); memory_barrier(); THE->preemption_disabled--; Index: kernel/generic/src/printf/vprintf.c =================================================================== --- kernel/generic/src/printf/vprintf.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/printf/vprintf.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -42,5 +42,5 @@ #include -SPINLOCK_STATIC_INITIALIZE_NAME(printf_lock, "*printf_lock"); +IRQ_SPINLOCK_STATIC_INITIALIZE_NAME(printf_lock, "*printf_lock"); static int vprintf_str_write(const char *str, size_t size, void *data) @@ -93,11 +93,7 @@ }; - ipl_t ipl = interrupts_disable(); - spinlock_lock(&printf_lock); - + irq_spinlock_lock(&printf_lock, true); int ret = printf_core(fmt, &ps, ap); - - spinlock_unlock(&printf_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&printf_lock, true); return ret; Index: kernel/generic/src/proc/scheduler.c =================================================================== --- kernel/generic/src/proc/scheduler.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/proc/scheduler.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Scheduler and load balancing. + * @brief Scheduler and load balancing. * * This file contains the scheduler and kcpulb kernel thread which @@ -68,5 +68,5 @@ static void scheduler_separated_stack(void); -atomic_t nrdy; /**< Number of ready threads in the system. */ +atomic_t nrdy; /**< Number of ready threads in the system. */ /** Carry out actions before new task runs. */ @@ -89,8 +89,8 @@ before_thread_runs_arch(); #ifdef CONFIG_FPU_LAZY - if(THREAD == CPU->fpu_owner) + if(THREAD == CPU->fpu_owner) fpu_enable(); else - fpu_disable(); + fpu_disable(); #else fpu_enable(); @@ -123,17 +123,18 @@ restart: fpu_enable(); - spinlock_lock(&CPU->lock); - + irq_spinlock_lock(&CPU->lock, false); + /* Save old context */ - if (CPU->fpu_owner != NULL) { - spinlock_lock(&CPU->fpu_owner->lock); + if (CPU->fpu_owner != NULL) { + irq_spinlock_lock(&CPU->fpu_owner->lock, false); fpu_context_save(CPU->fpu_owner->saved_fpu_context); - /* don't prevent migration */ + + /* Don't prevent migration */ CPU->fpu_owner->fpu_context_engaged = 0; - spinlock_unlock(&CPU->fpu_owner->lock); + irq_spinlock_unlock(&CPU->fpu_owner->lock, false); CPU->fpu_owner = NULL; } - - spinlock_lock(&THREAD->lock); + + irq_spinlock_lock(&THREAD->lock, false); if (THREAD->fpu_context_exists) { fpu_context_restore(THREAD->saved_fpu_context); @@ -142,21 +143,23 @@ if (!THREAD->saved_fpu_context) { /* Might sleep */ - spinlock_unlock(&THREAD->lock); - spinlock_unlock(&CPU->lock); + irq_spinlock_unlock(&THREAD->lock, false); + irq_spinlock_unlock(&CPU->lock, false); THREAD->saved_fpu_context = (fpu_context_t *) slab_alloc(fpu_context_slab, 0); + /* We may have switched CPUs during slab_alloc */ - goto restart; + goto restart; } fpu_init(); THREAD->fpu_context_exists = 1; } + CPU->fpu_owner = THREAD; THREAD->fpu_context_engaged = 1; - spinlock_unlock(&THREAD->lock); - - spinlock_unlock(&CPU->lock); -} -#endif + irq_spinlock_unlock(&THREAD->lock, false); + + irq_spinlock_unlock(&CPU->lock, false); +} +#endif /* CONFIG_FPU_LAZY */ /** Initialize scheduler @@ -180,10 +183,6 @@ static thread_t *find_best_thread(void) { - thread_t *t; - runq_t *r; - int i; - ASSERT(CPU != NULL); - + loop: @@ -194,61 +193,60 @@ * This improves energy saving and hyperthreading. */ - + /* Mark CPU as it was idle this clock tick */ - spinlock_lock(&CPU->lock); - CPU->idle = true; - spinlock_unlock(&CPU->lock); - - interrupts_enable(); - /* + irq_spinlock_lock(&CPU->lock, false); + CPU->idle = true; + irq_spinlock_unlock(&CPU->lock, false); + + interrupts_enable(); + /* * An interrupt might occur right now and wake up a thread. * In such case, the CPU will continue to go to sleep * even though there is a runnable thread. */ - cpu_sleep(); - interrupts_disable(); - goto loop; - } - + cpu_sleep(); + interrupts_disable(); + goto loop; + } + + unsigned int i; for (i = 0; i < RQ_COUNT; i++) { - r = &CPU->rq[i]; - spinlock_lock(&r->lock); - if (r->n == 0) { + irq_spinlock_lock(&(CPU->rq[i].lock), false); + if (CPU->rq[i].n == 0) { /* * If this queue is empty, try a lower-priority queue. */ - spinlock_unlock(&r->lock); + irq_spinlock_unlock(&(CPU->rq[i].lock), false); continue; } - + atomic_dec(&CPU->nrdy); atomic_dec(&nrdy); - r->n--; - + CPU->rq[i].n--; + /* * Take the first thread from the queue. */ - t = list_get_instance(r->rq_head.next, thread_t, rq_link); - list_remove(&t->rq_link); - - spinlock_unlock(&r->lock); - - spinlock_lock(&t->lock); - t->cpu = CPU; - - t->ticks = us2ticks((i + 1) * 10000); - t->priority = i; /* correct rq index */ - + thread_t *thread = + list_get_instance(CPU->rq[i].rq_head.next, thread_t, rq_link); + list_remove(&thread->rq_link); + + irq_spinlock_pass(&(CPU->rq[i].lock), &thread->lock); + + thread->cpu = CPU; + thread->ticks = us2ticks((i + 1) * 10000); + thread->priority = i; /* Correct rq index */ + /* * Clear the THREAD_FLAG_STOLEN flag so that t can be migrated * when load balancing needs emerge. */ - t->flags &= ~THREAD_FLAG_STOLEN; - spinlock_unlock(&t->lock); - - return t; - } + thread->flags &= ~THREAD_FLAG_STOLEN; + irq_spinlock_unlock(&thread->lock, false); + + return thread; + } + goto loop; - } @@ -267,30 +265,31 @@ { link_t head; - runq_t *r; - int i, n; - + list_initialize(&head); - spinlock_lock(&CPU->lock); + irq_spinlock_lock(&CPU->lock, false); + if (CPU->needs_relink > NEEDS_RELINK_MAX) { + int i; for (i = start; i < RQ_COUNT - 1; i++) { - /* remember and empty rq[i + 1] */ - r = &CPU->rq[i + 1]; - spinlock_lock(&r->lock); - list_concat(&head, &r->rq_head); - n = r->n; - r->n = 0; - spinlock_unlock(&r->lock); - - /* append rq[i + 1] to rq[i] */ - r = &CPU->rq[i]; - spinlock_lock(&r->lock); - list_concat(&r->rq_head, &head); - r->n += n; - spinlock_unlock(&r->lock); + /* Remember and empty rq[i + 1] */ + + irq_spinlock_lock(&CPU->rq[i + 1].lock, false); + list_concat(&head, &CPU->rq[i + 1].rq_head); + size_t n = CPU->rq[i + 1].n; + CPU->rq[i + 1].n = 0; + irq_spinlock_unlock(&CPU->rq[i + 1].lock, false); + + /* Append rq[i + 1] to rq[i] */ + + irq_spinlock_lock(&CPU->rq[i].lock, false); + list_concat(&CPU->rq[i].rq_head, &head); + CPU->rq[i].n += n; + irq_spinlock_unlock(&CPU->rq[i].lock, false); } + CPU->needs_relink = 0; } - spinlock_unlock(&CPU->lock); - + + irq_spinlock_unlock(&CPU->lock, false); } @@ -305,14 +304,14 @@ { volatile ipl_t ipl; - + ASSERT(CPU != NULL); - + ipl = interrupts_disable(); - + if (atomic_get(&haltstate)) halt(); if (THREAD) { - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, false); /* Update thread kernel accounting */ @@ -330,25 +329,27 @@ THREAD->last_cycle = get_cycle(); - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); interrupts_restore(THREAD->saved_context.ipl); return; } - + /* * Interrupt priority level of preempted thread is recorded * here to facilitate scheduler() invocations from - * interrupts_disable()'d code (e.g. waitq_sleep_timeout()). + * interrupts_disable()'d code (e.g. waitq_sleep_timeout()). + * */ THREAD->saved_context.ipl = ipl; } - + /* * Through the 'THE' structure, we keep track of THREAD, TASK, CPU, VM * and preemption counter. At this point THE could be coming either * from THREAD's or CPU's stack. + * */ the_copy(THE, (the_t *) CPU->stack); - + /* * We may not keep the old stack. @@ -362,4 +363,5 @@ * Therefore the scheduler() function continues in * scheduler_separated_stack(). + * */ context_save(&CPU->saved_context); @@ -367,5 +369,6 @@ (uintptr_t) CPU->stack, CPU_STACK_SIZE); context_restore(&CPU->saved_context); - /* not reached */ + + /* Not reached */ } @@ -376,13 +379,12 @@ * switch to a new thread. * - * Assume THREAD->lock is held. */ void scheduler_separated_stack(void) { - int priority; DEADLOCK_PROBE_INIT(p_joinwq); task_t *old_task = TASK; as_t *old_as = AS; - + + ASSERT(!THREAD || irq_spinlock_locked(&THREAD->lock)); ASSERT(CPU != NULL); @@ -391,36 +393,40 @@ * possible destruction should thread_destroy() be called on this or any * other processor while the scheduler is still using them. + * */ if (old_task) task_hold(old_task); + if (old_as) as_hold(old_as); - + if (THREAD) { - /* must be run after the switch to scheduler stack */ + /* Must be run after the switch to scheduler stack */ after_thread_ran(); - + switch (THREAD->state) { case Running: - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); thread_ready(THREAD); break; - + case Exiting: repeat: if (THREAD->detached) { - thread_destroy(THREAD); + thread_destroy(THREAD, false); } else { /* * The thread structure is kept allocated until * somebody calls thread_detach() on it. + * */ - if (!spinlock_trylock(&THREAD->join_wq.lock)) { + if (!irq_spinlock_trylock(&THREAD->join_wq.lock)) { /* * Avoid deadlock. + * */ - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); delay(HZ); - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, false); DEADLOCK_PROBE(p_joinwq, DEADLOCK_THRESHOLD); @@ -429,8 +435,8 @@ _waitq_wakeup_unsafe(&THREAD->join_wq, WAKEUP_FIRST); - spinlock_unlock(&THREAD->join_wq.lock); + irq_spinlock_unlock(&THREAD->join_wq.lock, false); THREAD->state = Lingering; - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); } break; @@ -439,17 +445,20 @@ /* * Prefer the thread after it's woken up. + * */ THREAD->priority = -1; - + /* * We need to release wq->lock which we locked in * waitq_sleep(). Address of wq->lock is kept in * THREAD->sleep_queue. + * */ - spinlock_unlock(&THREAD->sleep_queue->lock); - + irq_spinlock_unlock(&THREAD->sleep_queue->lock, false); + /* * Check for possible requests for out-of-context * invocation. + * */ if (THREAD->call_me) { @@ -458,12 +467,13 @@ THREAD->call_me_with = NULL; } - - spinlock_unlock(&THREAD->lock); - + + irq_spinlock_unlock(&THREAD->lock, false); + break; - + default: /* * Entering state is unexpected. + * */ panic("tid%" PRIu64 ": unexpected state %s.", @@ -471,19 +481,20 @@ break; } - + THREAD = NULL; } - + THREAD = find_best_thread(); - spinlock_lock(&THREAD->lock); - priority = THREAD->priority; - spinlock_unlock(&THREAD->lock); - - relink_rq(priority); - + irq_spinlock_lock(&THREAD->lock, false); + int priority = THREAD->priority; + irq_spinlock_unlock(&THREAD->lock, false); + + relink_rq(priority); + /* * If both the old and the new task are the same, lots of work is * avoided. + * */ if (TASK != THREAD->task) { @@ -493,4 +504,5 @@ * Note that it is possible for two tasks to share one address * space. + ( */ if (old_as != new_as) { @@ -498,26 +510,28 @@ * Both tasks and address spaces are different. * Replace the old one with the new one. + * */ as_switch(old_as, new_as); } - + TASK = THREAD->task; before_task_runs(); } - + if (old_task) task_release(old_task); + if (old_as) as_release(old_as); - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, false); THREAD->state = Running; - + #ifdef SCHEDULER_VERBOSE printf("cpu%u: tid %" PRIu64 " (priority=%d, ticks=%" PRIu64 ", nrdy=%ld)\n", CPU->id, THREAD->tid, THREAD->priority, THREAD->ticks, atomic_get(&CPU->nrdy)); -#endif - +#endif + /* * Some architectures provide late kernel PA2KA(identity) @@ -527,15 +541,18 @@ * necessary, is to be mapped in before_thread_runs(). This * function must be executed before the switch to the new stack. + * */ before_thread_runs(); - + /* * Copy the knowledge of CPU, TASK, THREAD and preemption counter to * thread's stack. + * */ the_copy(THE, (the_t *) THREAD->kstack); context_restore(&THREAD->saved_context); - /* not reached */ + + /* Not reached */ } @@ -551,12 +568,7 @@ void kcpulb(void *arg) { - thread_t *t; - int count; atomic_count_t average; - unsigned int i; - int j; - int k = 0; - ipl_t ipl; - + atomic_count_t rdy; + /* * Detach kcpulb as nobody will call thread_join_timeout() on it. @@ -569,5 +581,5 @@ */ thread_sleep(1); - + not_satisfied: /* @@ -575,46 +587,53 @@ * other CPU's. Note that situation can have changed between two * passes. Each time get the most up to date counts. + * */ average = atomic_get(&nrdy) / config.cpu_active + 1; - count = average - atomic_get(&CPU->nrdy); - - if (count <= 0) + rdy = atomic_get(&CPU->nrdy); + + if (average <= rdy) goto satisfied; - + + atomic_count_t count = average - rdy; + /* * Searching least priority queues on all CPU's first and most priority * queues on all CPU's last. - */ - for (j = RQ_COUNT - 1; j >= 0; j--) { - for (i = 0; i < config.cpu_active; i++) { - link_t *l; - runq_t *r; - cpu_t *cpu; - - cpu = &cpus[(i + k) % config.cpu_active]; - + * + */ + size_t acpu; + size_t acpu_bias = 0; + int rq; + + for (rq = RQ_COUNT - 1; rq >= 0; rq--) { + for (acpu = 0; acpu < config.cpu_active; acpu++) { + cpu_t *cpu = &cpus[(acpu + acpu_bias) % config.cpu_active]; + /* * Not interested in ourselves. * Doesn't require interrupt disabling for kcpulb has * THREAD_FLAG_WIRED. + * */ if (CPU == cpu) continue; + if (atomic_get(&cpu->nrdy) <= average) continue; - - ipl = interrupts_disable(); - r = &cpu->rq[j]; - spinlock_lock(&r->lock); - if (r->n == 0) { - spinlock_unlock(&r->lock); - interrupts_restore(ipl); + + irq_spinlock_lock(&(cpu->rq[rq].lock), true); + if (cpu->rq[rq].n == 0) { + irq_spinlock_unlock(&(cpu->rq[rq].lock), true); continue; } - - t = NULL; - l = r->rq_head.prev; /* search rq from the back */ - while (l != &r->rq_head) { - t = list_get_instance(l, thread_t, rq_link); + + thread_t *thread = NULL; + + /* Search rq from the back */ + link_t *link = cpu->rq[rq].rq_head.prev; + + while (link != &(cpu->rq[rq].rq_head)) { + thread = (thread_t *) list_get_instance(link, thread_t, rq_link); + /* * We don't want to steal CPU-wired threads @@ -624,33 +643,38 @@ * steal threads whose FPU context is still in * CPU. + * */ - spinlock_lock(&t->lock); - if ((!(t->flags & (THREAD_FLAG_WIRED | - THREAD_FLAG_STOLEN))) && - (!(t->fpu_context_engaged))) { + irq_spinlock_lock(&thread->lock, false); + + if ((!(thread->flags & (THREAD_FLAG_WIRED | THREAD_FLAG_STOLEN))) + && (!(thread->fpu_context_engaged))) { /* - * Remove t from r. + * Remove thread from ready queue. */ - spinlock_unlock(&t->lock); + irq_spinlock_unlock(&thread->lock, false); atomic_dec(&cpu->nrdy); atomic_dec(&nrdy); - - r->n--; - list_remove(&t->rq_link); - + + cpu->rq[rq].n--; + list_remove(&thread->rq_link); + break; } - spinlock_unlock(&t->lock); - l = l->prev; - t = NULL; + + irq_spinlock_unlock(&thread->lock, false); + + link = link->prev; + thread = NULL; } - spinlock_unlock(&r->lock); - - if (t) { + + if (thread) { /* - * Ready t on local CPU + * Ready thread on local CPU + * */ - spinlock_lock(&t->lock); + + irq_spinlock_pass(&(cpu->rq[rq].lock), &thread->lock); + #ifdef KCPULB_VERBOSE printf("kcpulb%u: TID %" PRIu64 " -> cpu%u, " @@ -659,30 +683,32 @@ atomic_get(&nrdy) / config.cpu_active); #endif - t->flags |= THREAD_FLAG_STOLEN; - t->state = Entering; - spinlock_unlock(&t->lock); - - thread_ready(t); - - interrupts_restore(ipl); - + + thread->flags |= THREAD_FLAG_STOLEN; + thread->state = Entering; + + irq_spinlock_unlock(&thread->lock, true); + thread_ready(thread); + if (--count == 0) goto satisfied; - + /* * We are not satisfied yet, focus on another * CPU next time. + * */ - k++; + acpu_bias++; continue; - } - interrupts_restore(ipl); + } else + irq_spinlock_unlock(&(cpu->rq[rq].lock), true); + } } - + if (atomic_get(&CPU->nrdy)) { /* * Be a little bit light-weight and let migrated threads run. + * */ scheduler(); @@ -691,60 +717,56 @@ * We failed to migrate a single thread. * Give up this turn. + * */ goto loop; } - + goto not_satisfied; - + satisfied: goto loop; } - #endif /* CONFIG_SMP */ - -/** Print information about threads & scheduler queues */ +/** Print information about threads & scheduler queues + * + */ void sched_print_list(void) { - ipl_t ipl; - unsigned int cpu, i; - runq_t *r; - thread_t *t; - link_t *cur; - - /* We are going to mess with scheduler structures, - * let's not be interrupted */ - ipl = interrupts_disable(); + size_t cpu; for (cpu = 0; cpu < config.cpu_count; cpu++) { - if (!cpus[cpu].active) continue; - - spinlock_lock(&cpus[cpu].lock); + + irq_spinlock_lock(&cpus[cpu].lock, true); + printf("cpu%u: address=%p, nrdy=%ld, needs_relink=%" PRIs "\n", cpus[cpu].id, &cpus[cpu], atomic_get(&cpus[cpu].nrdy), cpus[cpu].needs_relink); + unsigned int i; for (i = 0; i < RQ_COUNT; i++) { - r = &cpus[cpu].rq[i]; - spinlock_lock(&r->lock); - if (!r->n) { - spinlock_unlock(&r->lock); + irq_spinlock_lock(&(cpus[cpu].rq[i].lock), false); + if (cpus[cpu].rq[i].n == 0) { + irq_spinlock_unlock(&(cpus[cpu].rq[i].lock), false); continue; } + printf("\trq[%u]: ", i); - for (cur = r->rq_head.next; cur != &r->rq_head; - cur = cur->next) { - t = list_get_instance(cur, thread_t, rq_link); - printf("%" PRIu64 "(%s) ", t->tid, - thread_states[t->state]); + link_t *cur; + for (cur = cpus[cpu].rq[i].rq_head.next; + cur != &(cpus[cpu].rq[i].rq_head); + cur = cur->next) { + thread_t *thread = list_get_instance(cur, thread_t, rq_link); + printf("%" PRIu64 "(%s) ", thread->tid, + thread_states[thread->state]); } printf("\n"); - spinlock_unlock(&r->lock); + + irq_spinlock_unlock(&(cpus[cpu].rq[i].lock), false); } - spinlock_unlock(&cpus[cpu].lock); - } - - interrupts_restore(ipl); + + irq_spinlock_unlock(&cpus[cpu].lock, true); + } } Index: kernel/generic/src/proc/task.c =================================================================== --- kernel/generic/src/proc/task.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/proc/task.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -60,5 +60,5 @@ /** Spinlock protecting the tasks_tree AVL tree. */ -SPINLOCK_INITIALIZE(tasks_lock); +IRQ_SPINLOCK_INITIALIZE(tasks_lock); /** AVL tree of active tasks. @@ -81,7 +81,9 @@ /* Forward declarations. */ static void task_kill_internal(task_t *); -static int tsk_constructor(void *, int); - -/** Initialize kernel tasks support. */ +static int tsk_constructor(void *, unsigned int); + +/** Initialize kernel tasks support. + * + */ void task_init(void) { @@ -92,19 +94,23 @@ } -/* +/** Task finish walker. + * * The idea behind this walker is to kill and count all tasks different from * TASK. + * */ static bool task_done_walker(avltree_node_t *node, void *arg) { - task_t *t = avltree_get_instance(node, task_t, tasks_tree_node); - unsigned *cnt = (unsigned *) arg; - - if (t != TASK) { + task_t *task = avltree_get_instance(node, task_t, tasks_tree_node); + size_t *cnt = (size_t *) arg; + + if (task != TASK) { (*cnt)++; + #ifdef CONFIG_DEBUG - printf("[%"PRIu64"] ", t->taskid); -#endif - task_kill_internal(t); + printf("[%"PRIu64"] ", task->taskid); +#endif + + task_kill_internal(task); } @@ -113,51 +119,55 @@ } -/** Kill all tasks except the current task. */ +/** Kill all tasks except the current task. + * + */ void task_done(void) { - unsigned tasks_left; - - do { /* Repeat until there are any tasks except TASK */ - /* Messing with task structures, avoid deadlock */ + size_t tasks_left; + + /* Repeat until there are any tasks except TASK */ + do { #ifdef CONFIG_DEBUG printf("Killing tasks... "); #endif - ipl_t ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); + + irq_spinlock_lock(&tasks_lock, true); tasks_left = 0; avltree_walk(&tasks_tree, task_done_walker, &tasks_left); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); + thread_sleep(1); + #ifdef CONFIG_DEBUG printf("\n"); #endif - } while (tasks_left); -} - -int tsk_constructor(void *obj, int kmflags) -{ - task_t *ta = obj; - int i; - - atomic_set(&ta->refcount, 0); - atomic_set(&ta->lifecount, 0); - atomic_set(&ta->active_calls, 0); - - spinlock_initialize(&ta->lock, "task_ta_lock"); - mutex_initialize(&ta->futexes_lock, MUTEX_PASSIVE); - - list_initialize(&ta->th_head); - list_initialize(&ta->sync_box_head); - - ipc_answerbox_init(&ta->answerbox, ta); + } while (tasks_left > 0); +} + +int tsk_constructor(void *obj, unsigned int kmflags) +{ + task_t *task = (task_t *) obj; + + atomic_set(&task->refcount, 0); + atomic_set(&task->lifecount, 0); + atomic_set(&task->active_calls, 0); + + irq_spinlock_initialize(&task->lock, "task_t_lock"); + mutex_initialize(&task->futexes_lock, MUTEX_PASSIVE); + + list_initialize(&task->th_head); + list_initialize(&task->sync_box_head); + + ipc_answerbox_init(&task->answerbox, task); + + size_t i; for (i = 0; i < IPC_MAX_PHONES; i++) - ipc_phone_init(&ta->phones[i]); + ipc_phone_init(&task->phones[i]); #ifdef CONFIG_UDEBUG /* Init kbox stuff */ - ta->kb.thread = NULL; - ipc_answerbox_init(&ta->kb.box, ta); - mutex_initialize(&ta->kb.cleanup_lock, MUTEX_PASSIVE); + task->kb.thread = NULL; + ipc_answerbox_init(&task->kb.box, task); + mutex_initialize(&task->kb.cleanup_lock, MUTEX_PASSIVE); #endif @@ -175,86 +185,83 @@ task_t *task_create(as_t *as, const char *name) { - ipl_t ipl; - task_t *ta; - - ta = (task_t *) slab_alloc(task_slab, 0); - task_create_arch(ta); - ta->as = as; - memcpy(ta->name, name, TASK_NAME_BUFLEN); - ta->name[TASK_NAME_BUFLEN - 1] = 0; - - ta->context = CONTEXT; - ta->capabilities = 0; - ta->ucycles = 0; - ta->kcycles = 0; - - ta->ipc_info.call_sent = 0; - ta->ipc_info.call_recieved = 0; - ta->ipc_info.answer_sent = 0; - ta->ipc_info.answer_recieved = 0; - ta->ipc_info.irq_notif_recieved = 0; - ta->ipc_info.forwarded = 0; - + task_t *task = (task_t *) slab_alloc(task_slab, 0); + task_create_arch(task); + + task->as = as; + str_cpy(task->name, TASK_NAME_BUFLEN, name); + + task->context = CONTEXT; + task->capabilities = 0; + task->ucycles = 0; + task->kcycles = 0; + + task->ipc_info.call_sent = 0; + task->ipc_info.call_recieved = 0; + task->ipc_info.answer_sent = 0; + task->ipc_info.answer_recieved = 0; + task->ipc_info.irq_notif_recieved = 0; + task->ipc_info.forwarded = 0; + #ifdef CONFIG_UDEBUG /* Init debugging stuff */ - udebug_task_init(&ta->udebug); + udebug_task_init(&task->udebug); /* Init kbox stuff */ - ta->kb.finished = false; + task->kb.finished = false; #endif if ((ipc_phone_0) && - (context_check(ipc_phone_0->task->context, ta->context))) - ipc_phone_connect(&ta->phones[0], ipc_phone_0); - - btree_create(&ta->futexes); + (context_check(ipc_phone_0->task->context, task->context))) + ipc_phone_connect(&task->phones[0], ipc_phone_0); + + btree_create(&task->futexes); /* * Get a reference to the address space. */ - as_hold(ta->as); - - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - ta->taskid = ++task_counter; - avltree_node_initialize(&ta->tasks_tree_node); - ta->tasks_tree_node.key = ta->taskid; - avltree_insert(&tasks_tree, &ta->tasks_tree_node); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); - - return ta; + as_hold(task->as); + + irq_spinlock_lock(&tasks_lock, true); + + task->taskid = ++task_counter; + avltree_node_initialize(&task->tasks_tree_node); + task->tasks_tree_node.key = task->taskid; + avltree_insert(&tasks_tree, &task->tasks_tree_node); + + irq_spinlock_unlock(&tasks_lock, true); + + return task; } /** Destroy task. * - * @param t Task to be destroyed. - * - */ -void task_destroy(task_t *t) + * @param task Task to be destroyed. + * + */ +void task_destroy(task_t *task) { /* * Remove the task from the task B+tree. */ - spinlock_lock(&tasks_lock); - avltree_delete(&tasks_tree, &t->tasks_tree_node); - spinlock_unlock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); + avltree_delete(&tasks_tree, &task->tasks_tree_node); + irq_spinlock_unlock(&tasks_lock, true); /* * Perform architecture specific task destruction. */ - task_destroy_arch(t); + task_destroy_arch(task); /* * Free up dynamically allocated state. */ - btree_destroy(&t->futexes); + btree_destroy(&task->futexes); /* * Drop our reference to the address space. */ - as_release(t->as); - - slab_free(task_slab, t); + as_release(task->as); + + slab_free(task_slab, task); } @@ -263,9 +270,10 @@ * Holding a reference to a task prevents destruction of that task. * - * @param t Task to be held. - */ -void task_hold(task_t *t) -{ - atomic_inc(&t->refcount); + * @param task Task to be held. + * + */ +void task_hold(task_t *task) +{ + atomic_inc(&task->refcount); } @@ -274,10 +282,11 @@ * The last one to release a reference to a task destroys the task. * - * @param t Task to be released. - */ -void task_release(task_t *t) -{ - if ((atomic_predec(&t->refcount)) == 0) - task_destroy(t); + * @param task Task to be released. + * + */ +void task_release(task_t *task) +{ + if ((atomic_predec(&task->refcount)) == 0) + task_destroy(task); } @@ -342,9 +351,12 @@ task_t *task_find_by_id(task_id_t id) { + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&tasks_lock)); + avltree_node_t *node = avltree_search(&tasks_tree, (avltree_key_t) id); if (node) - return avltree_get_instance(node, task_t, tasks_tree_node); + return avltree_get_instance(node, task_t, tasks_tree_node); return NULL; @@ -353,34 +365,40 @@ /** Get accounting data of given task. * - * Note that task lock of 't' must be already held and interrupts must be + * Note that task lock of 'task' must be already held and interrupts must be * already disabled. * - * @param t Pointer to thread. + * @param task Pointer to the task. * @param ucycles Out pointer to sum of all user cycles. * @param kcycles Out pointer to sum of all kernel cycles. * */ -void task_get_accounting(task_t *t, uint64_t *ucycles, uint64_t *kcycles) -{ +void task_get_accounting(task_t *task, uint64_t *ucycles, uint64_t *kcycles) +{ + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&task->lock)); + /* Accumulated values of task */ - uint64_t uret = t->ucycles; - uint64_t kret = t->kcycles; + uint64_t uret = task->ucycles; + uint64_t kret = task->kcycles; /* Current values of threads */ link_t *cur; - for (cur = t->th_head.next; cur != &t->th_head; cur = cur->next) { - thread_t *thr = list_get_instance(cur, thread_t, th_link); - - spinlock_lock(&thr->lock); + for (cur = task->th_head.next; cur != &task->th_head; cur = cur->next) { + thread_t *thread = list_get_instance(cur, thread_t, th_link); + + irq_spinlock_lock(&thread->lock, false); + /* Process only counted threads */ - if (!thr->uncounted) { - if (thr == THREAD) { + if (!thread->uncounted) { + if (thread == THREAD) { /* Update accounting of current thread */ thread_update_accounting(false); - } - uret += thr->ucycles; - kret += thr->kcycles; + } + + uret += thread->ucycles; + kret += thread->kcycles; } - spinlock_unlock(&thr->lock); + + irq_spinlock_unlock(&thread->lock, false); } @@ -389,5 +407,5 @@ } -static void task_kill_internal(task_t *ta) +static void task_kill_internal(task_t *task) { link_t *cur; @@ -396,21 +414,22 @@ * Interrupt all threads. */ - spinlock_lock(&ta->lock); - for (cur = ta->th_head.next; cur != &ta->th_head; cur = cur->next) { - thread_t *thr; + irq_spinlock_lock(&task->lock, false); + for (cur = task->th_head.next; cur != &task->th_head; cur = cur->next) { + thread_t *thread = list_get_instance(cur, thread_t, th_link); bool sleeping = false; - thr = list_get_instance(cur, thread_t, th_link); - - spinlock_lock(&thr->lock); - thr->interrupted = true; - if (thr->state == Sleeping) + irq_spinlock_lock(&thread->lock, false); + + thread->interrupted = true; + if (thread->state == Sleeping) sleeping = true; - spinlock_unlock(&thr->lock); + + irq_spinlock_unlock(&thread->lock, false); if (sleeping) - waitq_interrupt_sleep(thr); + waitq_interrupt_sleep(thread); } - spinlock_unlock(&ta->lock); + + irq_spinlock_unlock(&task->lock, false); } @@ -427,58 +446,55 @@ int task_kill(task_id_t id) { - ipl_t ipl; - task_t *ta; - if (id == 1) return EPERM; - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - if (!(ta = task_find_by_id(id))) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_lock(&tasks_lock, true); + + task_t *task = task_find_by_id(id); + if (!task) { + irq_spinlock_unlock(&tasks_lock, true); return ENOENT; } - task_kill_internal(ta); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); - return 0; + + task_kill_internal(task); + irq_spinlock_unlock(&tasks_lock, true); + + return EOK; } static bool task_print_walker(avltree_node_t *node, void *arg) { - task_t *t = avltree_get_instance(node, task_t, tasks_tree_node); - int j; - - spinlock_lock(&t->lock); + task_t *task = avltree_get_instance(node, task_t, tasks_tree_node); + irq_spinlock_lock(&task->lock, false); uint64_t ucycles; uint64_t kcycles; char usuffix, ksuffix; - task_get_accounting(t, &ucycles, &kcycles); + task_get_accounting(task, &ucycles, &kcycles); order_suffix(ucycles, &ucycles, &usuffix); order_suffix(kcycles, &kcycles, &ksuffix); -#ifdef __32_BITS__ +#ifdef __32_BITS__ printf("%-6" PRIu64 " %-12s %-3" PRIu32 " %10p %10p %9" PRIu64 "%c %9" - PRIu64 "%c %7ld %6ld", t->taskid, t->name, t->context, t, t->as, - ucycles, usuffix, kcycles, ksuffix, atomic_get(&t->refcount), - atomic_get(&t->active_calls)); + PRIu64 "%c %7ld %6ld", task->taskid, task->name, task->context, + task, task->as, ucycles, usuffix, kcycles, ksuffix, + atomic_get(&task->refcount), atomic_get(&task->active_calls)); #endif #ifdef __64_BITS__ printf("%-6" PRIu64 " %-12s %-3" PRIu32 " %18p %18p %9" PRIu64 "%c %9" - PRIu64 "%c %7ld %6ld", t->taskid, t->name, t->context, t, t->as, - ucycles, usuffix, kcycles, ksuffix, atomic_get(&t->refcount), - atomic_get(&t->active_calls)); -#endif - - for (j = 0; j < IPC_MAX_PHONES; j++) { - if (t->phones[j].callee) - printf(" %d:%p", j, t->phones[j].callee); + PRIu64 "%c %7ld %6ld", task->taskid, task->name, task->context, + task, task->as, ucycles, usuffix, kcycles, ksuffix, + atomic_get(&task->refcount), atomic_get(&task->active_calls)); +#endif + + size_t i; + for (i = 0; i < IPC_MAX_PHONES; i++) { + if (task->phones[i].callee) + printf(" %" PRIs ":%p", i, task->phones[i].callee); } printf("\n"); - spinlock_unlock(&t->lock); + irq_spinlock_unlock(&task->lock, false); return true; } @@ -487,9 +503,6 @@ void task_print_list(void) { - ipl_t ipl; - /* Messing with task structures, avoid deadlock */ - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); #ifdef __32_BITS__ @@ -509,6 +522,5 @@ avltree_walk(&tasks_tree, task_print_walker, NULL); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); } Index: kernel/generic/src/proc/thread.c =================================================================== --- kernel/generic/src/proc/thread.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/proc/thread.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Thread management functions. + * @brief Thread management functions. */ @@ -94,6 +94,7 @@ * * For locking rules, see declaration thereof. - */ -SPINLOCK_INITIALIZE(threads_lock); + * + */ +IRQ_SPINLOCK_INITIALIZE(threads_lock); /** AVL tree of all threads. @@ -101,11 +102,13 @@ * When a thread is found in the threads_tree AVL tree, it is guaranteed to * exist as long as the threads_lock is held. - */ -avltree_t threads_tree; - -SPINLOCK_INITIALIZE(tidlock); -thread_id_t last_tid = 0; + * + */ +avltree_t threads_tree; + +IRQ_SPINLOCK_STATIC_INITIALIZE(tidlock); +static thread_id_t last_tid = 0; static slab_cache_t *thread_slab; + #ifdef CONFIG_FPU slab_cache_t *fpu_context_slab; @@ -125,15 +128,13 @@ void *arg = THREAD->thread_arg; THREAD->last_cycle = get_cycle(); - + /* This is where each thread wakes up after its creation */ - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); interrupts_enable(); - + f(arg); /* Accumulate accounting to the task */ - ipl_t ipl = interrupts_disable(); - - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, true); if (!THREAD->uncounted) { thread_update_accounting(true); @@ -142,73 +143,74 @@ uint64_t kcycles = THREAD->kcycles; THREAD->kcycles = 0; - - spinlock_unlock(&THREAD->lock); - spinlock_lock(&TASK->lock); + irq_spinlock_pass(&THREAD->lock, &TASK->lock); TASK->ucycles += ucycles; TASK->kcycles += kcycles; - spinlock_unlock(&TASK->lock); + irq_spinlock_unlock(&TASK->lock, true); } else - spinlock_unlock(&THREAD->lock); - - interrupts_restore(ipl); + irq_spinlock_unlock(&THREAD->lock, true); thread_exit(); - /* not reached */ -} - -/** Initialization and allocation for thread_t structure */ -static int thr_constructor(void *obj, int kmflags) -{ - thread_t *t = (thread_t *) obj; - - spinlock_initialize(&t->lock, "thread_t_lock"); - link_initialize(&t->rq_link); - link_initialize(&t->wq_link); - link_initialize(&t->th_link); - + + /* Not reached */ +} + +/** Initialization and allocation for thread_t structure + * + */ +static int thr_constructor(void *obj, unsigned int kmflags) +{ + thread_t *thread = (thread_t *) obj; + + irq_spinlock_initialize(&thread->lock, "thread_t_lock"); + link_initialize(&thread->rq_link); + link_initialize(&thread->wq_link); + link_initialize(&thread->th_link); + /* call the architecture-specific part of the constructor */ - thr_constructor_arch(t); + thr_constructor_arch(thread); #ifdef CONFIG_FPU #ifdef CONFIG_FPU_LAZY - t->saved_fpu_context = NULL; -#else - t->saved_fpu_context = slab_alloc(fpu_context_slab, kmflags); - if (!t->saved_fpu_context) + thread->saved_fpu_context = NULL; +#else /* CONFIG_FPU_LAZY */ + thread->saved_fpu_context = slab_alloc(fpu_context_slab, kmflags); + if (!thread->saved_fpu_context) return -1; -#endif -#endif - - t->kstack = (uint8_t *) frame_alloc(STACK_FRAMES, FRAME_KA | kmflags); - if (!t->kstack) { +#endif /* CONFIG_FPU_LAZY */ +#endif /* CONFIG_FPU */ + + thread->kstack = (uint8_t *) frame_alloc(STACK_FRAMES, FRAME_KA | kmflags); + if (!thread->kstack) { #ifdef CONFIG_FPU - if (t->saved_fpu_context) - slab_free(fpu_context_slab, t->saved_fpu_context); + if (thread->saved_fpu_context) + slab_free(fpu_context_slab, thread->saved_fpu_context); #endif return -1; } - + #ifdef CONFIG_UDEBUG - mutex_initialize(&t->udebug.lock, MUTEX_PASSIVE); -#endif - + mutex_initialize(&thread->udebug.lock, MUTEX_PASSIVE); +#endif + return 0; } /** Destruction of thread_t object */ -static int thr_destructor(void *obj) -{ - thread_t *t = (thread_t *) obj; - +static size_t thr_destructor(void *obj) +{ + thread_t *thread = (thread_t *) obj; + /* call the architecture-specific part of the destructor */ - thr_destructor_arch(t); - - frame_free(KA2PA(t->kstack)); + thr_destructor_arch(thread); + + frame_free(KA2PA(thread->kstack)); + #ifdef CONFIG_FPU - if (t->saved_fpu_context) - slab_free(fpu_context_slab, t->saved_fpu_context); -#endif - return 1; /* One page freed */ + if (thread->saved_fpu_context) + slab_free(fpu_context_slab, thread->saved_fpu_context); +#endif + + return 1; /* One page freed */ } @@ -221,13 +223,14 @@ { THREAD = NULL; + atomic_set(&nrdy, 0); thread_slab = slab_cache_create("thread_slab", sizeof(thread_t), 0, thr_constructor, thr_destructor, 0); - + #ifdef CONFIG_FPU fpu_context_slab = slab_cache_create("fpu_slab", sizeof(fpu_context_t), FPU_CONTEXT_ALIGN, NULL, NULL, 0); #endif - + avltree_create(&threads_tree); } @@ -235,145 +238,132 @@ /** Make thread ready * - * Switch thread t to the ready state. + * Switch thread to the ready state. * * @param t Thread to make ready. * */ -void thread_ready(thread_t *t) -{ - cpu_t *cpu; - runq_t *r; - ipl_t ipl; - int i, avg; - - ipl = interrupts_disable(); - - spinlock_lock(&t->lock); - - ASSERT(!(t->state == Ready)); - - i = (t->priority < RQ_COUNT - 1) ? ++t->priority : t->priority; - - cpu = CPU; - if (t->flags & THREAD_FLAG_WIRED) { - ASSERT(t->cpu != NULL); - cpu = t->cpu; +void thread_ready(thread_t *thread) +{ + irq_spinlock_lock(&thread->lock, true); + + ASSERT(!(thread->state == Ready)); + + int i = (thread->priority < RQ_COUNT - 1) + ? ++thread->priority : thread->priority; + + cpu_t *cpu = CPU; + if (thread->flags & THREAD_FLAG_WIRED) { + ASSERT(thread->cpu != NULL); + cpu = thread->cpu; } - t->state = Ready; - spinlock_unlock(&t->lock); + thread->state = Ready; + + irq_spinlock_pass(&thread->lock, &(cpu->rq[i].lock)); /* - * Append t to respective ready queue on respective processor. + * Append thread to respective ready queue + * on respective processor. */ - r = &cpu->rq[i]; - spinlock_lock(&r->lock); - list_append(&t->rq_link, &r->rq_head); - r->n++; - spinlock_unlock(&r->lock); - + + list_append(&thread->rq_link, &cpu->rq[i].rq_head); + cpu->rq[i].n++; + irq_spinlock_unlock(&(cpu->rq[i].lock), true); + atomic_inc(&nrdy); - // FIXME: Why is the avg value never read? - avg = atomic_get(&nrdy) / config.cpu_active; + // FIXME: Why is the avg value not used + // avg = atomic_get(&nrdy) / config.cpu_active; atomic_inc(&cpu->nrdy); - +} + +/** Create new thread + * + * Create a new thread. + * + * @param func Thread's implementing function. + * @param arg Thread's implementing function argument. + * @param task Task to which the thread belongs. The caller must + * guarantee that the task won't cease to exist during the + * call. The task's lock may not be held. + * @param flags Thread flags. + * @param name Symbolic name (a copy is made). + * @param uncounted Thread's accounting doesn't affect accumulated task + * accounting. + * + * @return New thread's structure on success, NULL on failure. + * + */ +thread_t *thread_create(void (* func)(void *), void *arg, task_t *task, + unsigned int flags, const char *name, bool uncounted) +{ + thread_t *thread = (thread_t *) slab_alloc(thread_slab, 0); + if (!thread) + return NULL; + + /* Not needed, but good for debugging */ + memsetb(thread->kstack, THREAD_STACK_SIZE * 1 << STACK_FRAMES, 0); + + irq_spinlock_lock(&tidlock, true); + thread->tid = ++last_tid; + irq_spinlock_unlock(&tidlock, true); + + context_save(&thread->saved_context); + context_set(&thread->saved_context, FADDR(cushion), + (uintptr_t) thread->kstack, THREAD_STACK_SIZE); + + the_initialize((the_t *) thread->kstack); + + ipl_t ipl = interrupts_disable(); + thread->saved_context.ipl = interrupts_read(); interrupts_restore(ipl); -} - -/** Create new thread - * - * Create a new thread. - * - * @param func Thread's implementing function. - * @param arg Thread's implementing function argument. - * @param task Task to which the thread belongs. The caller must - * guarantee that the task won't cease to exist during the - * call. The task's lock may not be held. - * @param flags Thread flags. - * @param name Symbolic name (a copy is made). - * @param uncounted Thread's accounting doesn't affect accumulated task - * accounting. - * - * @return New thread's structure on success, NULL on failure. - * - */ -thread_t *thread_create(void (* func)(void *), void *arg, task_t *task, - int flags, const char *name, bool uncounted) -{ - thread_t *t; - ipl_t ipl; - - t = (thread_t *) slab_alloc(thread_slab, 0); - if (!t) - return NULL; - - /* Not needed, but good for debugging */ - memsetb(t->kstack, THREAD_STACK_SIZE * 1 << STACK_FRAMES, 0); - - ipl = interrupts_disable(); - spinlock_lock(&tidlock); - t->tid = ++last_tid; - spinlock_unlock(&tidlock); - interrupts_restore(ipl); - - context_save(&t->saved_context); - context_set(&t->saved_context, FADDR(cushion), (uintptr_t) t->kstack, - THREAD_STACK_SIZE); - - the_initialize((the_t *) t->kstack); - - ipl = interrupts_disable(); - t->saved_context.ipl = interrupts_read(); - interrupts_restore(ipl); - - memcpy(t->name, name, THREAD_NAME_BUFLEN); - t->name[THREAD_NAME_BUFLEN - 1] = 0; - - t->thread_code = func; - t->thread_arg = arg; - t->ticks = -1; - t->ucycles = 0; - t->kcycles = 0; - t->uncounted = uncounted; - t->priority = -1; /* start in rq[0] */ - t->cpu = NULL; - t->flags = flags; - t->state = Entering; - t->call_me = NULL; - t->call_me_with = NULL; - - timeout_initialize(&t->sleep_timeout); - t->sleep_interruptible = false; - t->sleep_queue = NULL; - t->timeout_pending = 0; - - t->in_copy_from_uspace = false; - t->in_copy_to_uspace = false; - - t->interrupted = false; - t->detached = false; - waitq_initialize(&t->join_wq); - - t->rwlock_holder_type = RWLOCK_NONE; - - t->task = task; - - t->fpu_context_exists = 0; - t->fpu_context_engaged = 0; - - avltree_node_initialize(&t->threads_tree_node); - t->threads_tree_node.key = (uintptr_t) t; - + + str_cpy(thread->name, THREAD_NAME_BUFLEN, name); + + thread->thread_code = func; + thread->thread_arg = arg; + thread->ticks = -1; + thread->ucycles = 0; + thread->kcycles = 0; + thread->uncounted = uncounted; + thread->priority = -1; /* Start in rq[0] */ + thread->cpu = NULL; + thread->flags = flags; + thread->state = Entering; + thread->call_me = NULL; + thread->call_me_with = NULL; + + timeout_initialize(&thread->sleep_timeout); + thread->sleep_interruptible = false; + thread->sleep_queue = NULL; + thread->timeout_pending = false; + + thread->in_copy_from_uspace = false; + thread->in_copy_to_uspace = false; + + thread->interrupted = false; + thread->detached = false; + waitq_initialize(&thread->join_wq); + + thread->rwlock_holder_type = RWLOCK_NONE; + + thread->task = task; + + thread->fpu_context_exists = 0; + thread->fpu_context_engaged = 0; + + avltree_node_initialize(&thread->threads_tree_node); + thread->threads_tree_node.key = (uintptr_t) thread; + #ifdef CONFIG_UDEBUG /* Init debugging stuff */ - udebug_thread_initialize(&t->udebug); -#endif - - /* might depend on previous initialization */ - thread_create_arch(t); - + udebug_thread_initialize(&thread->udebug); +#endif + + /* Might depend on previous initialization */ + thread_create_arch(thread); + if (!(flags & THREAD_FLAG_NOATTACH)) - thread_attach(t, task); - - return t; + thread_attach(thread, task); + + return thread; } @@ -382,36 +372,38 @@ * Detach thread from all queues, cpus etc. and destroy it. * - * Assume thread->lock is held!! - */ -void thread_destroy(thread_t *t) -{ - ASSERT(t->state == Exiting || t->state == Lingering); - ASSERT(t->task); - ASSERT(t->cpu); - - spinlock_lock(&t->cpu->lock); - if (t->cpu->fpu_owner == t) - t->cpu->fpu_owner = NULL; - spinlock_unlock(&t->cpu->lock); - - spinlock_unlock(&t->lock); - - spinlock_lock(&threads_lock); - avltree_delete(&threads_tree, &t->threads_tree_node); - spinlock_unlock(&threads_lock); - + * @param thread Thread to be destroyed. + * @param irq_res Indicate whether it should unlock thread->lock + * in interrupts-restore mode. + * + */ +void thread_destroy(thread_t *thread, bool irq_res) +{ + ASSERT(irq_spinlock_locked(&thread->lock)); + ASSERT((thread->state == Exiting) || (thread->state == Lingering)); + ASSERT(thread->task); + ASSERT(thread->cpu); + + irq_spinlock_lock(&thread->cpu->lock, false); + if (thread->cpu->fpu_owner == thread) + thread->cpu->fpu_owner = NULL; + irq_spinlock_unlock(&thread->cpu->lock, false); + + irq_spinlock_pass(&thread->lock, &threads_lock); + + avltree_delete(&threads_tree, &thread->threads_tree_node); + + irq_spinlock_pass(&threads_lock, &thread->task->lock); + /* * Detach from the containing task. */ - spinlock_lock(&t->task->lock); - list_remove(&t->th_link); - spinlock_unlock(&t->task->lock); - + list_remove(&thread->th_link); + irq_spinlock_unlock(&thread->task->lock, irq_res); + /* * Drop the reference to the containing task. */ - task_release(t->task); - - slab_free(thread_slab, t); + task_release(thread->task); + slab_free(thread_slab, thread); } @@ -421,50 +413,51 @@ * threads_tree. * - * @param t Thread to be attached to the task. - * @param task Task to which the thread is to be attached. - */ -void thread_attach(thread_t *t, task_t *task) -{ - ipl_t ipl; - + * @param t Thread to be attached to the task. + * @param task Task to which the thread is to be attached. + * + */ +void thread_attach(thread_t *thread, task_t *task) +{ /* * Attach to the specified task. */ - ipl = interrupts_disable(); - spinlock_lock(&task->lock); - + irq_spinlock_lock(&task->lock, true); + /* Hold a reference to the task. */ task_hold(task); - + /* Must not count kbox thread into lifecount */ - if (t->flags & THREAD_FLAG_USPACE) + if (thread->flags & THREAD_FLAG_USPACE) atomic_inc(&task->lifecount); - - list_append(&t->th_link, &task->th_head); - spinlock_unlock(&task->lock); - + + list_append(&thread->th_link, &task->th_head); + + irq_spinlock_pass(&task->lock, &threads_lock); + /* * Register this thread in the system-wide list. */ - spinlock_lock(&threads_lock); - avltree_insert(&threads_tree, &t->threads_tree_node); - spinlock_unlock(&threads_lock); - - interrupts_restore(ipl); + avltree_insert(&threads_tree, &thread->threads_tree_node); + irq_spinlock_unlock(&threads_lock, true); } /** Terminate thread. * - * End current thread execution and switch it to the exiting state. All pending - * timeouts are executed. + * End current thread execution and switch it to the exiting state. + * All pending timeouts are executed. + * */ void thread_exit(void) { - ipl_t ipl; - if (THREAD->flags & THREAD_FLAG_USPACE) { #ifdef CONFIG_UDEBUG /* Generate udebug THREAD_E event */ udebug_thread_e_event(); + + /* + * This thread will not execute any code or system calls from + * now on. + */ + udebug_stoppable_begin(); #endif if (atomic_predec(&TASK->lifecount) == 0) { @@ -475,4 +468,5 @@ * can only be created by threads of the same task. * We are safe to perform cleanup. + * */ ipc_cleanup(); @@ -481,24 +475,21 @@ } } - + restart: - ipl = interrupts_disable(); - spinlock_lock(&THREAD->lock); - if (THREAD->timeout_pending) { - /* busy waiting for timeouts in progress */ - spinlock_unlock(&THREAD->lock); - interrupts_restore(ipl); + irq_spinlock_lock(&THREAD->lock, true); + if (THREAD->timeout_pending) { + /* Busy waiting for timeouts in progress */ + irq_spinlock_unlock(&THREAD->lock, true); goto restart; } THREAD->state = Exiting; - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, true); + scheduler(); - + /* Not reached */ - while (1) - ; -} - + while (true); +} /** Thread sleep @@ -515,5 +506,5 @@ while (sec > 0) { uint32_t period = (sec > 1000) ? 1000 : sec; - + thread_usleep(period * 1000000); sec -= period; @@ -523,18 +514,16 @@ /** Wait for another thread to exit. * - * @param t Thread to join on exit. - * @param usec Timeout in microseconds. - * @param flags Mode of operation. + * @param thread Thread to join on exit. + * @param usec Timeout in microseconds. + * @param flags Mode of operation. * * @return An error code from errno.h or an error code from synch.h. - */ -int thread_join_timeout(thread_t *t, uint32_t usec, int flags) -{ - ipl_t ipl; - int rc; - - if (t == THREAD) + * + */ +int thread_join_timeout(thread_t *thread, uint32_t usec, unsigned int flags) +{ + if (thread == THREAD) return EINVAL; - + /* * Since thread join can only be called once on an undetached thread, @@ -542,13 +531,9 @@ */ - ipl = interrupts_disable(); - spinlock_lock(&t->lock); - ASSERT(!t->detached); - spinlock_unlock(&t->lock); - interrupts_restore(ipl); - - rc = waitq_sleep_timeout(&t->join_wq, usec, flags); - - return rc; + irq_spinlock_lock(&thread->lock, true); + ASSERT(!thread->detached); + irq_spinlock_unlock(&thread->lock, true); + + return waitq_sleep_timeout(&thread->join_wq, usec, flags); } @@ -558,26 +543,28 @@ * state, deallocate its resources. * - * @param t Thread to be detached. - */ -void thread_detach(thread_t *t) -{ - ipl_t ipl; - + * @param thread Thread to be detached. + * + */ +void thread_detach(thread_t *thread) +{ /* * Since the thread is expected not to be already detached, * pointer to it must be still valid. */ - ipl = interrupts_disable(); - spinlock_lock(&t->lock); - ASSERT(!t->detached); - if (t->state == Lingering) { - thread_destroy(t); /* unlocks &t->lock */ - interrupts_restore(ipl); + irq_spinlock_lock(&thread->lock, true); + ASSERT(!thread->detached); + + if (thread->state == Lingering) { + /* + * Unlock &thread->lock and restore + * interrupts in thread_destroy(). + */ + thread_destroy(thread, true); return; } else { - t->detached = true; + thread->detached = true; } - spinlock_unlock(&t->lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&thread->lock, true); } @@ -601,5 +588,6 @@ * * Register a function and its argument to be executed - * on next context switch to the current thread. + * on next context switch to the current thread. Must + * be called with interrupts disabled. * * @param call_me Out-of-context function. @@ -609,67 +597,62 @@ void thread_register_call_me(void (* call_me)(void *), void *call_me_with) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, false); THREAD->call_me = call_me; THREAD->call_me_with = call_me_with; - spinlock_unlock(&THREAD->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&THREAD->lock, false); } static bool thread_walker(avltree_node_t *node, void *arg) { - thread_t *t = avltree_get_instance(node, thread_t, threads_tree_node); + thread_t *thread = avltree_get_instance(node, thread_t, threads_tree_node); uint64_t ucycles, kcycles; char usuffix, ksuffix; - order_suffix(t->ucycles, &ucycles, &usuffix); - order_suffix(t->kcycles, &kcycles, &ksuffix); - + order_suffix(thread->ucycles, &ucycles, &usuffix); + order_suffix(thread->kcycles, &kcycles, &ksuffix); + #ifdef __32_BITS__ printf("%-6" PRIu64" %-10s %10p %-8s %10p %-3" PRIu32 " %10p %10p %9" - PRIu64 "%c %9" PRIu64 "%c ", t->tid, t->name, t, - thread_states[t->state], t->task, t->task->context, t->thread_code, - t->kstack, ucycles, usuffix, kcycles, ksuffix); -#endif - + PRIu64 "%c %9" PRIu64 "%c ", thread->tid, thread->name, thread, + thread_states[thread->state], thread->task, thread->task->context, + thread->thread_code, thread->kstack, ucycles, usuffix, kcycles, ksuffix); +#endif + #ifdef __64_BITS__ printf("%-6" PRIu64" %-10s %18p %-8s %18p %-3" PRIu32 " %18p %18p %9" - PRIu64 "%c %9" PRIu64 "%c ", t->tid, t->name, t, - thread_states[t->state], t->task, t->task->context, t->thread_code, - t->kstack, ucycles, usuffix, kcycles, ksuffix); -#endif - - if (t->cpu) - printf("%-4u", t->cpu->id); + PRIu64 "%c %9" PRIu64 "%c ", thread->tid, thread->name, thread, + thread_states[thread->state], thread->task, thread->task->context, + thread->thread_code, thread->kstack, ucycles, usuffix, kcycles, ksuffix); +#endif + + if (thread->cpu) + printf("%-4u", thread->cpu->id); else printf("none"); - - if (t->state == Sleeping) { + + if (thread->state == Sleeping) { #ifdef __32_BITS__ - printf(" %10p", t->sleep_queue); -#endif - + printf(" %10p", thread->sleep_queue); +#endif + #ifdef __64_BITS__ - printf(" %18p", t->sleep_queue); + printf(" %18p", thread->sleep_queue); #endif } - + printf("\n"); - + return true; } -/** Print list of threads debug info */ +/** Print list of threads debug info + * + */ void thread_print_list(void) { - ipl_t ipl; - /* Messing with thread structures, avoid deadlock */ - ipl = interrupts_disable(); - spinlock_lock(&threads_lock); - -#ifdef __32_BITS__ + irq_spinlock_lock(&threads_lock, true); + +#ifdef __32_BITS__ printf("tid name address state task " "ctx code stack ucycles kcycles cpu " @@ -679,5 +662,5 @@ "----------\n"); #endif - + #ifdef __64_BITS__ printf("tid name address state task " @@ -688,9 +671,8 @@ "------------------\n"); #endif - + avltree_walk(&threads_tree, thread_walker, NULL); - - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&threads_lock, true); } @@ -700,13 +682,16 @@ * interrupts must be already disabled. * - * @param t Pointer to thread. + * @param thread Pointer to thread. * * @return True if thread t is known to the system, false otherwise. - */ -bool thread_exists(thread_t *t) -{ - avltree_node_t *node; - - node = avltree_search(&threads_tree, (avltree_key_t) ((uintptr_t) t)); + * + */ +bool thread_exists(thread_t *thread) +{ + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&threads_lock)); + + avltree_node_t *node = + avltree_search(&threads_tree, (avltree_key_t) ((uintptr_t) thread)); return node != NULL; @@ -718,14 +703,19 @@ * interrupts must be already disabled. * - * @param user True to update user accounting, false for kernel. + * @param user True to update user accounting, false for kernel. + * */ void thread_update_accounting(bool user) { uint64_t time = get_cycle(); - if (user) { + + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&THREAD->lock)); + + if (user) THREAD->ucycles += time - THREAD->last_cycle; - } else { + else THREAD->kcycles += time - THREAD->last_cycle; - } + THREAD->last_cycle = time; } @@ -757,4 +747,7 @@ thread_t *thread_find_by_id(thread_id_t thread_id) { + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&threads_lock)); + thread_iterator_t iterator; @@ -774,23 +767,21 @@ size_t name_len, thread_id_t *uspace_thread_id) { - thread_t *t; - char namebuf[THREAD_NAME_BUFLEN]; - uspace_arg_t *kernel_uarg; - int rc; - if (name_len > THREAD_NAME_BUFLEN - 1) name_len = THREAD_NAME_BUFLEN - 1; - - rc = copy_from_uspace(namebuf, uspace_name, name_len); + + char namebuf[THREAD_NAME_BUFLEN]; + int rc = copy_from_uspace(namebuf, uspace_name, name_len); if (rc != 0) return (unative_t) rc; - + namebuf[name_len] = 0; - + /* * In case of failure, kernel_uarg will be deallocated in this function. * In case of success, kernel_uarg will be freed in uinit(). + * */ - kernel_uarg = (uspace_arg_t *) malloc(sizeof(uspace_arg_t), 0); + uspace_arg_t *kernel_uarg = + (uspace_arg_t *) malloc(sizeof(uspace_arg_t), 0); rc = copy_from_uspace(kernel_uarg, uspace_uarg, sizeof(uspace_arg_t)); @@ -799,13 +790,11 @@ return (unative_t) rc; } - - t = thread_create(uinit, kernel_uarg, TASK, + + thread_t *thread = thread_create(uinit, kernel_uarg, TASK, THREAD_FLAG_USPACE | THREAD_FLAG_NOATTACH, namebuf, false); - if (t) { + if (thread) { if (uspace_thread_id != NULL) { - int rc; - - rc = copy_to_uspace(uspace_thread_id, &t->tid, - sizeof(t->tid)); + rc = copy_to_uspace(uspace_thread_id, &thread->tid, + sizeof(thread->tid)); if (rc != 0) { /* @@ -813,6 +802,7 @@ * has already been created. We need to undo its * creation now. + * */ - + /* * The new thread structure is initialized, but @@ -820,10 +810,11 @@ * We can safely deallocate it. */ - slab_free(thread_slab, t); - free(kernel_uarg); - + slab_free(thread_slab, thread); + free(kernel_uarg); + return (unative_t) rc; } } + #ifdef CONFIG_UDEBUG /* @@ -833,15 +824,16 @@ * THREAD_B events for threads that already existed * and could be detected with THREAD_READ before. + * */ - udebug_thread_b_event_attach(t, TASK); + udebug_thread_b_event_attach(thread, TASK); #else - thread_attach(t, TASK); -#endif - thread_ready(t); - + thread_attach(thread, TASK); +#endif + thread_ready(thread); + return 0; } else free(kernel_uarg); - + return (unative_t) ENOMEM; } @@ -853,4 +845,5 @@ { thread_exit(); + /* Unreachable */ return 0; @@ -863,4 +856,5 @@ * * @return 0 on success or an error code from @ref errno.h. + * */ unative_t sys_thread_get_id(thread_id_t *uspace_thread_id) @@ -869,4 +863,5 @@ * No need to acquire lock on THREAD because tid * remains constant for the lifespan of the thread. + * */ return (unative_t) copy_to_uspace(uspace_thread_id, &THREAD->tid, Index: kernel/generic/src/security/cap.c =================================================================== --- kernel/generic/src/security/cap.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/security/cap.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -27,15 +27,15 @@ */ -/** @addtogroup generic +/** @addtogroup generic * @{ */ /** - * @file cap.c - * @brief Capabilities control. + * @file cap.c + * @brief Capabilities control. * * @see cap.h */ - + #include #include @@ -48,37 +48,27 @@ /** Set capabilities. * - * @param t Task whose capabilities are to be changed. + * @param task Task whose capabilities are to be changed. * @param caps New set of capabilities. + * */ -void cap_set(task_t *t, cap_t caps) +void cap_set(task_t *task, cap_t caps) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&t->lock); - - t->capabilities = caps; - - spinlock_unlock(&t->lock); - interrupts_restore(ipl); + irq_spinlock_lock(&task->lock, true); + task->capabilities = caps; + irq_spinlock_unlock(&task->lock, true); } /** Get capabilities. * - * @param t Task whose capabilities are to be returned. + * @param task Task whose capabilities are to be returned. + * * @return Task's capabilities. + * */ -cap_t cap_get(task_t *t) +cap_t cap_get(task_t *task) { - ipl_t ipl; - cap_t caps; - - ipl = interrupts_disable(); - spinlock_lock(&t->lock); - - caps = t->capabilities; - - spinlock_unlock(&t->lock); - interrupts_restore(ipl); + irq_spinlock_lock(&task->lock, true); + cap_t caps = task->capabilities; + irq_spinlock_unlock(&task->lock, true); return caps; @@ -93,34 +83,29 @@ * * @return Zero on success or an error code from @ref errno.h. + * */ unative_t sys_cap_grant(sysarg64_t *uspace_taskid_arg, cap_t caps) { - sysarg64_t taskid_arg; - task_t *t; - ipl_t ipl; - int rc; - if (!(cap_get(TASK) & CAP_CAP)) return (unative_t) EPERM; - rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); + sysarg64_t taskid_arg; + int rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); if (rc != 0) return (unative_t) rc; - - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - t = task_find_by_id((task_id_t) taskid_arg.value); - if ((!t) || (!context_check(CONTEXT, t->context))) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + + irq_spinlock_lock(&tasks_lock, true); + task_t *task = task_find_by_id((task_id_t) taskid_arg.value); + + if ((!task) || (!context_check(CONTEXT, task->context))) { + irq_spinlock_unlock(&tasks_lock, true); return (unative_t) ENOENT; } - spinlock_lock(&t->lock); - cap_set(t, cap_get(t) | caps); - spinlock_unlock(&t->lock); + irq_spinlock_lock(&task->lock, false); + task->capabilities |= caps; + irq_spinlock_unlock(&task->lock, false); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return 0; } @@ -135,25 +120,21 @@ * * @return Zero on success or an error code from @ref errno.h. + * */ unative_t sys_cap_revoke(sysarg64_t *uspace_taskid_arg, cap_t caps) { sysarg64_t taskid_arg; - task_t *t; - ipl_t ipl; - int rc; - - rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); + int rc = copy_from_uspace(&taskid_arg, uspace_taskid_arg, sizeof(sysarg64_t)); if (rc != 0) return (unative_t) rc; - - ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); - t = task_find_by_id((task_id_t) taskid_arg.value); - if ((!t) || (!context_check(CONTEXT, t->context))) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + + irq_spinlock_lock(&tasks_lock, true); + + task_t *task = task_find_by_id((task_id_t) taskid_arg.value); + if ((!task) || (!context_check(CONTEXT, task->context))) { + irq_spinlock_unlock(&tasks_lock, true); return (unative_t) ENOENT; } - + /* * Revoking capabilities is different from granting them in that @@ -161,17 +142,16 @@ * doesn't have CAP_CAP. */ - if (!(cap_get(TASK) & CAP_CAP) || !(t == TASK)) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&TASK->lock, false); + + if ((!(TASK->capabilities & CAP_CAP)) || (task != TASK)) { + irq_spinlock_unlock(&TASK->lock, false); + irq_spinlock_unlock(&tasks_lock, true); return (unative_t) EPERM; } - spinlock_lock(&t->lock); - cap_set(t, cap_get(t) & ~caps); - spinlock_unlock(&t->lock); - - spinlock_unlock(&tasks_lock); - - interrupts_restore(ipl); + task->capabilities &= ~caps; + irq_spinlock_unlock(&TASK->lock, false); + + irq_spinlock_unlock(&tasks_lock, true); return 0; } @@ -179,3 +159,2 @@ /** @} */ - Index: kernel/generic/src/synch/mutex.c =================================================================== --- kernel/generic/src/synch/mutex.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/synch/mutex.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,7 +33,7 @@ /** * @file - * @brief Mutexes. + * @brief Mutexes. */ - + #include #include @@ -44,6 +44,6 @@ /** Initialize mutex. * - * @param mtx Mutex. - * @param type Type of the mutex. + * @param mtx Mutex. + * @param type Type of the mutex. */ void mutex_initialize(mutex_t *mtx, mutex_type_t type) @@ -53,27 +53,39 @@ } +/** Find out whether the mutex is currently locked. + * + * @param mtx Mutex. + * @return True if the mutex is locked, false otherwise. + */ +bool mutex_locked(mutex_t *mtx) +{ + return semaphore_count_get(&mtx->sem) <= 0; +} + /** Acquire mutex. * * Timeout mode and non-blocking mode can be requested. * - * @param mtx Mutex. - * @param usec Timeout in microseconds. - * @param flags Specify mode of operation. + * @param mtx Mutex. + * @param usec Timeout in microseconds. + * @param flags Specify mode of operation. * * For exact description of possible combinations of * usec and flags, see comment for waitq_sleep_timeout(). * - * @return See comment for waitq_sleep_timeout(). + * @return See comment for waitq_sleep_timeout(). + * */ -int _mutex_lock_timeout(mutex_t *mtx, uint32_t usec, int flags) +int _mutex_lock_timeout(mutex_t *mtx, uint32_t usec, unsigned int flags) { int rc; - if (mtx->type == MUTEX_PASSIVE && THREAD) { + if ((mtx->type == MUTEX_PASSIVE) && (THREAD)) { rc = _semaphore_down_timeout(&mtx->sem, usec, flags); } else { - ASSERT(mtx->type == MUTEX_ACTIVE || !THREAD); + ASSERT((mtx->type == MUTEX_ACTIVE) || (!THREAD)); ASSERT(usec == SYNCH_NO_TIMEOUT); ASSERT(!(flags & SYNCH_FLAGS_INTERRUPTIBLE)); + do { rc = semaphore_trydown(&mtx->sem); @@ -87,5 +99,5 @@ /** Release mutex. * - * @param mtx Mutex. + * @param mtx Mutex. */ void mutex_unlock(mutex_t *mtx) Index: kernel/generic/src/synch/rwlock.c =================================================================== --- kernel/generic/src/synch/rwlock.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/synch/rwlock.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Reader/Writer locks. + * @brief Reader/Writer locks. * * A reader/writer lock can be held by multiple readers at a time. @@ -57,5 +57,5 @@ * each thread can block on only one rwlock at a time. */ - + #include #include @@ -69,9 +69,6 @@ #include -#define ALLOW_ALL 0 -#define ALLOW_READERS_ONLY 1 - -static void let_others_in(rwlock_t *rwl, int readers_only); -static void release_spinlock(void *arg); +#define ALLOW_ALL 0 +#define ALLOW_READERS_ONLY 1 /** Initialize reader/writer lock @@ -80,11 +77,101 @@ * * @param rwl Reader/Writer lock. + * */ void rwlock_initialize(rwlock_t *rwl) { - spinlock_initialize(&rwl->lock, "rwlock_t"); + irq_spinlock_initialize(&rwl->lock, "rwl.lock"); mutex_initialize(&rwl->exclusive, MUTEX_PASSIVE); rwl->readers_in = 0; } +/** Direct handoff of reader/writer lock ownership. + * + * Direct handoff of reader/writer lock ownership + * to waiting readers or a writer. + * + * Must be called with rwl->lock locked. + * Must be called with interrupts_disable()'d. + * + * @param rwl Reader/Writer lock. + * @param readers_only See the description below. + * + * If readers_only is false: (unlock scenario) + * Let the first sleeper on 'exclusive' mutex in, no matter + * whether it is a reader or a writer. If there are more leading + * readers in line, let each of them in. + * + * Otherwise: (timeout scenario) + * Let all leading readers in. + * + */ +static void let_others_in(rwlock_t *rwl, int readers_only) +{ + rwlock_type_t type = RWLOCK_NONE; + thread_t *thread = NULL; + bool one_more = true; + + irq_spinlock_lock(&rwl->exclusive.sem.wq.lock, false); + + if (!list_empty(&rwl->exclusive.sem.wq.head)) + thread = list_get_instance(rwl->exclusive.sem.wq.head.next, + thread_t, wq_link); + + do { + if (thread) { + irq_spinlock_lock(&thread->lock, false); + type = thread->rwlock_holder_type; + irq_spinlock_unlock(&thread->lock, false); + } + + /* + * If readers_only is true, we wake all leading readers + * if and only if rwl is locked by another reader. + * Assumption: readers_only ==> rwl->readers_in + * + */ + if ((readers_only) && (type != RWLOCK_READER)) + break; + + if (type == RWLOCK_READER) { + /* + * Waking up a reader. + * We are responsible for incrementing rwl->readers_in + * for it. + * + */ + rwl->readers_in++; + } + + /* + * Only the last iteration through this loop can increment + * rwl->exclusive.sem.wq.missed_wakeup's. All preceeding + * iterations will wake up a thread. + * + */ + + /* + * We call the internal version of waitq_wakeup, which + * relies on the fact that the waitq is already locked. + * + */ + _waitq_wakeup_unsafe(&rwl->exclusive.sem.wq, WAKEUP_FIRST); + + thread = NULL; + if (!list_empty(&rwl->exclusive.sem.wq.head)) { + thread = list_get_instance(rwl->exclusive.sem.wq.head.next, + thread_t, wq_link); + + if (thread) { + irq_spinlock_lock(&thread->lock, false); + if (thread->rwlock_holder_type != RWLOCK_READER) + one_more = false; + irq_spinlock_unlock(&thread->lock, false); + } + } + } while ((type == RWLOCK_READER) && (thread) && (one_more)); + + irq_spinlock_unlock(&rwl->exclusive.sem.wq.lock, false); +} + /** Acquire reader/writer lock for reading * @@ -92,6 +179,6 @@ * Timeout and willingness to block may be specified. * - * @param rwl Reader/Writer lock. - * @param usec Timeout in microseconds. + * @param rwl Reader/Writer lock. + * @param usec Timeout in microseconds. * @param flags Specify mode of operation. * @@ -100,43 +187,54 @@ * * @return See comment for waitq_sleep_timeout(). - */ -int _rwlock_write_lock_timeout(rwlock_t *rwl, uint32_t usec, int flags) -{ - ipl_t ipl; - int rc; - - ipl = interrupts_disable(); - spinlock_lock(&THREAD->lock); + * + */ +int _rwlock_write_lock_timeout(rwlock_t *rwl, uint32_t usec, unsigned int flags) +{ + irq_spinlock_lock(&THREAD->lock, true); THREAD->rwlock_holder_type = RWLOCK_WRITER; - spinlock_unlock(&THREAD->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&THREAD->lock, true); + /* * Writers take the easy part. * They just need to acquire the exclusive mutex. + * */ - rc = _mutex_lock_timeout(&rwl->exclusive, usec, flags); + int rc = _mutex_lock_timeout(&rwl->exclusive, usec, flags); if (SYNCH_FAILED(rc)) { - /* * Lock operation timed out or was interrupted. * The state of rwl is UNKNOWN at this point. * No claims about its holder can be made. - */ - - ipl = interrupts_disable(); - spinlock_lock(&rwl->lock); + * + */ + irq_spinlock_lock(&rwl->lock, true); + /* * Now when rwl is locked, we can inspect it again. * If it is held by some readers already, we can let * readers from the head of the wait queue in. + * */ if (rwl->readers_in) let_others_in(rwl, ALLOW_READERS_ONLY); - spinlock_unlock(&rwl->lock); - interrupts_restore(ipl); + + irq_spinlock_unlock(&rwl->lock, true); } return rc; +} + +/** Release spinlock callback + * + * This is a callback function invoked from the scheduler. + * The callback is registered in _rwlock_read_lock_timeout(). + * + * @param arg Spinlock. + * + */ +static void release_spinlock(void *arg) +{ + if (arg != NULL) + irq_spinlock_unlock((irq_spinlock_t *) arg, false); } @@ -146,6 +244,6 @@ * Timeout and willingness to block may be specified. * - * @param rwl Reader/Writer lock. - * @param usec Timeout in microseconds. + * @param rwl Reader/Writer lock. + * @param usec Timeout in microseconds. * @param flags Select mode of operation. * @@ -154,23 +252,27 @@ * * @return See comment for waitq_sleep_timeout(). - */ -int _rwlock_read_lock_timeout(rwlock_t *rwl, uint32_t usec, int flags) -{ - int rc; - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&THREAD->lock); + * + */ +int _rwlock_read_lock_timeout(rwlock_t *rwl, uint32_t usec, unsigned int flags) +{ + /* + * Since the locking scenarios get a little bit too + * complicated, we do not rely on internal irq_spinlock_t + * interrupt disabling logic here and control interrupts + * manually. + * + */ + ipl_t ipl = interrupts_disable(); + + irq_spinlock_lock(&THREAD->lock, false); THREAD->rwlock_holder_type = RWLOCK_READER; - spinlock_unlock(&THREAD->lock); - - spinlock_lock(&rwl->lock); - + irq_spinlock_pass(&THREAD->lock, &rwl->lock); + /* * Find out whether we can get what we want without blocking. + * */ - rc = mutex_trylock(&rwl->exclusive); + int rc = mutex_trylock(&rwl->exclusive); if (SYNCH_FAILED(rc)) { - /* * 'exclusive' mutex is being held by someone else. @@ -178,18 +280,19 @@ * else waiting for it, we can enter the critical * section. - */ - + * + */ + if (rwl->readers_in) { - spinlock_lock(&rwl->exclusive.sem.wq.lock); + irq_spinlock_lock(&rwl->exclusive.sem.wq.lock, false); if (list_empty(&rwl->exclusive.sem.wq.head)) { /* * We can enter. */ - spinlock_unlock(&rwl->exclusive.sem.wq.lock); + irq_spinlock_unlock(&rwl->exclusive.sem.wq.lock, false); goto shortcut; } - spinlock_unlock(&rwl->exclusive.sem.wq.lock); + irq_spinlock_unlock(&rwl->exclusive.sem.wq.lock, false); } - + /* * In order to prevent a race condition when a reader @@ -197,11 +300,12 @@ * we register a function to unlock rwl->lock * after this thread is put asleep. - */ - #ifdef CONFIG_SMP + * + */ +#ifdef CONFIG_SMP thread_register_call_me(release_spinlock, &rwl->lock); - #else +#else thread_register_call_me(release_spinlock, NULL); - #endif - +#endif + rc = _mutex_lock_timeout(&rwl->exclusive, usec, flags); switch (rc) { @@ -209,7 +313,8 @@ /* * release_spinlock() wasn't called + * */ thread_register_call_me(NULL, NULL); - spinlock_unlock(&rwl->lock); + irq_spinlock_unlock(&rwl->lock, false); case ESYNCH_TIMEOUT: case ESYNCH_INTERRUPTED: @@ -217,6 +322,7 @@ * The sleep timed out. * We just restore interrupt priority level. + * */ - case ESYNCH_OK_BLOCKED: + case ESYNCH_OK_BLOCKED: /* * We were woken with rwl->readers_in already @@ -228,4 +334,5 @@ * 'readers_in' is incremented. Same time means both * events happen atomically when rwl->lock is held.) + * */ interrupts_restore(ipl); @@ -240,16 +347,15 @@ return rc; } - + shortcut: - /* * We can increment readers_in only if we didn't go to sleep. * For sleepers, rwlock_let_others_in() will do the job. + * */ rwl->readers_in++; - - spinlock_unlock(&rwl->lock); + irq_spinlock_unlock(&rwl->lock, false); interrupts_restore(ipl); - + return ESYNCH_OK_ATOMIC; } @@ -262,15 +368,11 @@ * * @param rwl Reader/Writer lock. + * */ void rwlock_write_unlock(rwlock_t *rwl) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&rwl->lock); + irq_spinlock_lock(&rwl->lock, true); let_others_in(rwl, ALLOW_ALL); - spinlock_unlock(&rwl->lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&rwl->lock, true); } @@ -283,109 +385,14 @@ * * @param rwl Reader/Writer lock. + * */ void rwlock_read_unlock(rwlock_t *rwl) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&rwl->lock); + irq_spinlock_lock(&rwl->lock, true); + if (!--rwl->readers_in) let_others_in(rwl, ALLOW_ALL); - spinlock_unlock(&rwl->lock); - interrupts_restore(ipl); -} - - -/** Direct handoff of reader/writer lock ownership. - * - * Direct handoff of reader/writer lock ownership - * to waiting readers or a writer. - * - * Must be called with rwl->lock locked. - * Must be called with interrupts_disable()'d. - * - * @param rwl Reader/Writer lock. - * @param readers_only See the description below. - * - * If readers_only is false: (unlock scenario) - * Let the first sleeper on 'exclusive' mutex in, no matter - * whether it is a reader or a writer. If there are more leading - * readers in line, let each of them in. - * - * Otherwise: (timeout scenario) - * Let all leading readers in. - */ -void let_others_in(rwlock_t *rwl, int readers_only) -{ - rwlock_type_t type = RWLOCK_NONE; - thread_t *t = NULL; - bool one_more = true; - - spinlock_lock(&rwl->exclusive.sem.wq.lock); - - if (!list_empty(&rwl->exclusive.sem.wq.head)) - t = list_get_instance(rwl->exclusive.sem.wq.head.next, thread_t, - wq_link); - do { - if (t) { - spinlock_lock(&t->lock); - type = t->rwlock_holder_type; - spinlock_unlock(&t->lock); - } - - /* - * If readers_only is true, we wake all leading readers - * if and only if rwl is locked by another reader. - * Assumption: readers_only ==> rwl->readers_in - */ - if (readers_only && (type != RWLOCK_READER)) - break; - - - if (type == RWLOCK_READER) { - /* - * Waking up a reader. - * We are responsible for incrementing rwl->readers_in - * for it. - */ - rwl->readers_in++; - } - - /* - * Only the last iteration through this loop can increment - * rwl->exclusive.sem.wq.missed_wakeup's. All preceeding - * iterations will wake up a thread. - */ - /* We call the internal version of waitq_wakeup, which - * relies on the fact that the waitq is already locked. - */ - _waitq_wakeup_unsafe(&rwl->exclusive.sem.wq, WAKEUP_FIRST); - - t = NULL; - if (!list_empty(&rwl->exclusive.sem.wq.head)) { - t = list_get_instance(rwl->exclusive.sem.wq.head.next, - thread_t, wq_link); - if (t) { - spinlock_lock(&t->lock); - if (t->rwlock_holder_type != RWLOCK_READER) - one_more = false; - spinlock_unlock(&t->lock); - } - } - } while ((type == RWLOCK_READER) && t && one_more); - - spinlock_unlock(&rwl->exclusive.sem.wq.lock); -} - -/** Release spinlock callback - * - * This is a callback function invoked from the scheduler. - * The callback is registered in _rwlock_read_lock_timeout(). - * - * @param arg Spinlock. - */ -void release_spinlock(void *arg) -{ - spinlock_unlock((spinlock_t *) arg); + + irq_spinlock_unlock(&rwl->lock, true); } Index: kernel/generic/src/synch/semaphore.c =================================================================== --- kernel/generic/src/synch/semaphore.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/synch/semaphore.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Semaphores. + * @brief Semaphores. */ @@ -47,20 +47,12 @@ * Initialize semaphore. * - * @param s Semaphore. + * @param sem Semaphore. * @param val Maximal number of threads allowed to enter critical section. + * */ -void semaphore_initialize(semaphore_t *s, int val) +void semaphore_initialize(semaphore_t *sem, int val) { - ipl_t ipl; - - waitq_initialize(&s->wq); - - ipl = interrupts_disable(); - - spinlock_lock(&s->wq.lock); - s->wq.missed_wakeups = val; - spinlock_unlock(&s->wq.lock); - - interrupts_restore(ipl); + waitq_initialize(&sem->wq); + waitq_count_set(&sem->wq, val); } @@ -70,6 +62,6 @@ * Conditional mode and mode with timeout can be requested. * - * @param s Semaphore. - * @param usec Timeout in microseconds. + * @param sem Semaphore. + * @param usec Timeout in microseconds. * @param flags Select mode of operation. * @@ -78,8 +70,9 @@ * * @return See comment for waitq_sleep_timeout(). + * */ -int _semaphore_down_timeout(semaphore_t *s, uint32_t usec, int flags) +int _semaphore_down_timeout(semaphore_t *sem, uint32_t usec, unsigned int flags) { - return waitq_sleep_timeout(&s->wq, usec, flags); + return waitq_sleep_timeout(&sem->wq, usec, flags); } @@ -89,8 +82,20 @@ * * @param s Semaphore. + * */ -void semaphore_up(semaphore_t *s) +void semaphore_up(semaphore_t *sem) { - waitq_wakeup(&s->wq, WAKEUP_FIRST); + waitq_wakeup(&sem->wq, WAKEUP_FIRST); +} + +/** Get the semaphore counter value. + * + * @param sem Semaphore. + * @return The number of threads that can down the semaphore + * without blocking. + */ +int semaphore_count_get(semaphore_t *sem) +{ + return waitq_count_get(&sem->wq); } Index: kernel/generic/src/synch/spinlock.c =================================================================== --- kernel/generic/src/synch/spinlock.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/synch/spinlock.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -128,5 +128,5 @@ void spinlock_unlock_debug(spinlock_t *lock) { - ASSERT(atomic_get(&lock->val) != 0); + ASSERT_SPINLOCK(spinlock_locked(lock), lock); /* @@ -143,7 +143,6 @@ /** Lock spinlock conditionally * - * Lock spinlock conditionally. - * If the spinlock is not available at the moment, - * signal failure. + * Lock spinlock conditionally. If the spinlock is not available + * at the moment, signal failure. * * @param lock Pointer to spinlock_t structure. @@ -168,6 +167,171 @@ } +/** Find out whether the spinlock is currently locked. + * + * @param lock Spinlock. + * @return True if the spinlock is locked, false otherwise. + */ +bool spinlock_locked(spinlock_t *lock) +{ + return atomic_get(&lock->val) != 0; +} + #endif +/** Initialize interrupts-disabled spinlock + * + * @param lock IRQ spinlock to be initialized. + * @param name IRQ spinlock name. + * + */ +void irq_spinlock_initialize(irq_spinlock_t *lock, const char *name) +{ + spinlock_initialize(&(lock->lock), name); + lock->guard = false; + lock->ipl = 0; +} + +/** Lock interrupts-disabled spinlock + * + * Lock a spinlock which requires disabled interrupts. + * + * @param lock IRQ spinlock to be locked. + * @param irq_dis If true, interrupts are actually disabled + * prior locking the spinlock. If false, interrupts + * are expected to be already disabled. + * + */ +void irq_spinlock_lock(irq_spinlock_t *lock, bool irq_dis) +{ + if (irq_dis) { + ipl_t ipl = interrupts_disable(); + spinlock_lock(&(lock->lock)); + + lock->guard = true; + lock->ipl = ipl; + } else { + ASSERT_IRQ_SPINLOCK(interrupts_disabled(), lock); + + spinlock_lock(&(lock->lock)); + ASSERT_IRQ_SPINLOCK(!lock->guard, lock); + } +} + +/** Unlock interrupts-disabled spinlock + * + * Unlock a spinlock which requires disabled interrupts. + * + * @param lock IRQ spinlock to be unlocked. + * @param irq_res If true, interrupts are restored to previously + * saved interrupt level. + * + */ +void irq_spinlock_unlock(irq_spinlock_t *lock, bool irq_res) +{ + ASSERT_IRQ_SPINLOCK(interrupts_disabled(), lock); + + if (irq_res) { + ASSERT_IRQ_SPINLOCK(lock->guard, lock); + + lock->guard = false; + ipl_t ipl = lock->ipl; + + spinlock_unlock(&(lock->lock)); + interrupts_restore(ipl); + } else { + ASSERT_IRQ_SPINLOCK(!lock->guard, lock); + spinlock_unlock(&(lock->lock)); + } +} + +/** Lock interrupts-disabled spinlock + * + * Lock an interrupts-disabled spinlock conditionally. If the + * spinlock is not available at the moment, signal failure. + * Interrupts are expected to be already disabled. + * + * @param lock IRQ spinlock to be locked conditionally. + * + * @return Zero on failure, non-zero otherwise. + * + */ +int irq_spinlock_trylock(irq_spinlock_t *lock) +{ + ASSERT_IRQ_SPINLOCK(interrupts_disabled(), lock); + int rc = spinlock_trylock(&(lock->lock)); + + ASSERT_IRQ_SPINLOCK(!lock->guard, lock); + return rc; +} + +/** Pass lock from one interrupts-disabled spinlock to another + * + * Pass lock from one IRQ spinlock to another IRQ spinlock + * without enabling interrupts during the process. + * + * The first IRQ spinlock is supposed to be locked. + * + * @param unlock IRQ spinlock to be unlocked. + * @param lock IRQ spinlock to be locked. + * + */ +void irq_spinlock_pass(irq_spinlock_t *unlock, irq_spinlock_t *lock) +{ + ASSERT_IRQ_SPINLOCK(interrupts_disabled(), unlock); + + /* Pass guard from unlock to lock */ + bool guard = unlock->guard; + ipl_t ipl = unlock->ipl; + unlock->guard = false; + + spinlock_unlock(&(unlock->lock)); + spinlock_lock(&(lock->lock)); + + ASSERT_IRQ_SPINLOCK(!lock->guard, lock); + + if (guard) { + lock->guard = true; + lock->ipl = ipl; + } +} + +/** Hand-over-hand locking of interrupts-disabled spinlocks + * + * Implement hand-over-hand locking between two interrupts-disabled + * spinlocks without enabling interrupts during the process. + * + * The first IRQ spinlock is supposed to be locked. + * + * @param unlock IRQ spinlock to be unlocked. + * @param lock IRQ spinlock to be locked. + * + */ +void irq_spinlock_exchange(irq_spinlock_t *unlock, irq_spinlock_t *lock) +{ + ASSERT_IRQ_SPINLOCK(interrupts_disabled(), unlock); + + spinlock_lock(&(lock->lock)); + ASSERT_IRQ_SPINLOCK(!lock->guard, lock); + + /* Pass guard from unlock to lock */ + if (unlock->guard) { + lock->guard = true; + lock->ipl = unlock->ipl; + unlock->guard = false; + } + + spinlock_unlock(&(unlock->lock)); +} + +/** Find out whether the IRQ spinlock is currently locked. + * + * @param lock IRQ spinlock. + * @return True if the IRQ spinlock is locked, false otherwise. + */ +bool irq_spinlock_locked(irq_spinlock_t *ilock) +{ + return spinlock_locked(&ilock->lock); +} + /** @} */ Index: kernel/generic/src/synch/waitq.c =================================================================== --- kernel/generic/src/synch/waitq.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/synch/waitq.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,5 +33,5 @@ /** * @file - * @brief Wait queue. + * @brief Wait queue. * * Wait queue is the basic synchronization primitive upon which all @@ -41,4 +41,5 @@ * fashion. Conditional operation as well as timeouts and interruptions * are supported. + * */ @@ -56,5 +57,5 @@ #include -static void waitq_sleep_timed_out(void *data); +static void waitq_sleep_timed_out(void *); /** Initialize wait queue @@ -62,9 +63,10 @@ * Initialize wait queue. * - * @param wq Pointer to wait queue to be initialized. + * @param wq Pointer to wait queue to be initialized. + * */ void waitq_initialize(waitq_t *wq) { - spinlock_initialize(&wq->lock, "waitq_lock"); + irq_spinlock_initialize(&wq->lock, "wq.lock"); list_initialize(&wq->head); wq->missed_wakeups = 0; @@ -81,41 +83,44 @@ * timeout at all. * - * @param data Pointer to the thread that called waitq_sleep_timeout(). + * @param data Pointer to the thread that called waitq_sleep_timeout(). + * */ void waitq_sleep_timed_out(void *data) { - thread_t *t = (thread_t *) data; - waitq_t *wq; + thread_t *thread = (thread_t *) data; bool do_wakeup = false; DEADLOCK_PROBE_INIT(p_wqlock); - - spinlock_lock(&threads_lock); - if (!thread_exists(t)) + + irq_spinlock_lock(&threads_lock, false); + if (!thread_exists(thread)) goto out; - + grab_locks: - spinlock_lock(&t->lock); - if ((wq = t->sleep_queue)) { /* assignment */ - if (!spinlock_trylock(&wq->lock)) { - spinlock_unlock(&t->lock); + irq_spinlock_lock(&thread->lock, false); + + waitq_t *wq; + if ((wq = thread->sleep_queue)) { /* Assignment */ + if (!irq_spinlock_trylock(&wq->lock)) { + irq_spinlock_unlock(&thread->lock, false); DEADLOCK_PROBE(p_wqlock, DEADLOCK_THRESHOLD); - goto grab_locks; /* avoid deadlock */ - } - - list_remove(&t->wq_link); - t->saved_context = t->sleep_timeout_context; + /* Avoid deadlock */ + goto grab_locks; + } + + list_remove(&thread->wq_link); + thread->saved_context = thread->sleep_timeout_context; do_wakeup = true; - t->sleep_queue = NULL; - spinlock_unlock(&wq->lock); - } - - t->timeout_pending = false; - spinlock_unlock(&t->lock); + thread->sleep_queue = NULL; + irq_spinlock_unlock(&wq->lock, false); + } + + thread->timeout_pending = false; + irq_spinlock_unlock(&thread->lock, false); if (do_wakeup) - thread_ready(t); - + thread_ready(thread); + out: - spinlock_unlock(&threads_lock); + irq_spinlock_unlock(&threads_lock, false); } @@ -125,52 +130,54 @@ * If the thread is not found sleeping, no action is taken. * - * @param t Thread to be interrupted. - */ -void waitq_interrupt_sleep(thread_t *t) -{ + * @param thread Thread to be interrupted. + * + */ +void waitq_interrupt_sleep(thread_t *thread) +{ + bool do_wakeup = false; + DEADLOCK_PROBE_INIT(p_wqlock); + + irq_spinlock_lock(&threads_lock, true); + if (!thread_exists(thread)) + goto out; + +grab_locks: + irq_spinlock_lock(&thread->lock, false); + waitq_t *wq; - bool do_wakeup = false; - ipl_t ipl; - DEADLOCK_PROBE_INIT(p_wqlock); - - ipl = interrupts_disable(); - spinlock_lock(&threads_lock); - if (!thread_exists(t)) - goto out; - -grab_locks: - spinlock_lock(&t->lock); - if ((wq = t->sleep_queue)) { /* assignment */ - if (!(t->sleep_interruptible)) { + if ((wq = thread->sleep_queue)) { /* Assignment */ + if (!(thread->sleep_interruptible)) { /* * The sleep cannot be interrupted. + * */ - spinlock_unlock(&t->lock); + irq_spinlock_unlock(&thread->lock, false); goto out; } - - if (!spinlock_trylock(&wq->lock)) { - spinlock_unlock(&t->lock); + + if (!irq_spinlock_trylock(&wq->lock)) { + irq_spinlock_unlock(&thread->lock, false); DEADLOCK_PROBE(p_wqlock, DEADLOCK_THRESHOLD); - goto grab_locks; /* avoid deadlock */ - } - - if (t->timeout_pending && timeout_unregister(&t->sleep_timeout)) - t->timeout_pending = false; - - list_remove(&t->wq_link); - t->saved_context = t->sleep_interruption_context; + /* Avoid deadlock */ + goto grab_locks; + } + + if ((thread->timeout_pending) && + (timeout_unregister(&thread->sleep_timeout))) + thread->timeout_pending = false; + + list_remove(&thread->wq_link); + thread->saved_context = thread->sleep_interruption_context; do_wakeup = true; - t->sleep_queue = NULL; - spinlock_unlock(&wq->lock); - } - spinlock_unlock(&t->lock); - + thread->sleep_queue = NULL; + irq_spinlock_unlock(&wq->lock, false); + } + irq_spinlock_unlock(&thread->lock, false); + if (do_wakeup) - thread_ready(t); - + thread_ready(thread); + out: - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&threads_lock, true); } @@ -180,31 +187,35 @@ * is sleeping interruptibly. * - * @param wq Pointer to wait queue. + * @param wq Pointer to wait queue. + * */ void waitq_unsleep(waitq_t *wq) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&wq->lock); - + irq_spinlock_lock(&wq->lock, true); + if (!list_empty(&wq->head)) { - thread_t *t; - - t = list_get_instance(wq->head.next, thread_t, wq_link); - spinlock_lock(&t->lock); - ASSERT(t->sleep_interruptible); - if (t->timeout_pending && timeout_unregister(&t->sleep_timeout)) - t->timeout_pending = false; - list_remove(&t->wq_link); - t->saved_context = t->sleep_interruption_context; - t->sleep_queue = NULL; - spinlock_unlock(&t->lock); - thread_ready(t); - } - - spinlock_unlock(&wq->lock); - interrupts_restore(ipl); -} + thread_t *thread = list_get_instance(wq->head.next, thread_t, wq_link); + + irq_spinlock_lock(&thread->lock, false); + + ASSERT(thread->sleep_interruptible); + + if ((thread->timeout_pending) && + (timeout_unregister(&thread->sleep_timeout))) + thread->timeout_pending = false; + + list_remove(&thread->wq_link); + thread->saved_context = thread->sleep_interruption_context; + thread->sleep_queue = NULL; + + irq_spinlock_unlock(&thread->lock, false); + thread_ready(thread); + } + + irq_spinlock_unlock(&wq->lock, true); +} + +#define PARAM_NON_BLOCKING(flags, usec) \ + (((flags) & SYNCH_FLAGS_NON_BLOCKING) && ((usec) == 0)) /** Sleep until either wakeup, timeout or interruption occurs @@ -218,14 +229,14 @@ * and all the *_timeout() functions use it. * - * @param wq Pointer to wait queue. - * @param usec Timeout in microseconds. - * @param flags Specify mode of the sleep. + * @param wq Pointer to wait queue. + * @param usec Timeout in microseconds. + * @param flags Specify mode of the sleep. * * The sleep can be interrupted only if the * SYNCH_FLAGS_INTERRUPTIBLE bit is specified in flags. - * + * * If usec is greater than zero, regardless of the value of the * SYNCH_FLAGS_NON_BLOCKING bit in flags, the call will not return until either - * timeout, interruption or wakeup comes. + * timeout, interruption or wakeup comes. * * If usec is zero and the SYNCH_FLAGS_NON_BLOCKING bit is not set in flags, @@ -235,29 +246,22 @@ * call will immediately return, reporting either success or failure. * - * @return Returns one of ESYNCH_WOULD_BLOCK, ESYNCH_TIMEOUT, - * ESYNCH_INTERRUPTED, ESYNCH_OK_ATOMIC and - * ESYNCH_OK_BLOCKED. - * - * @li ESYNCH_WOULD_BLOCK means that the sleep failed because at the time of - * the call there was no pending wakeup. - * - * @li ESYNCH_TIMEOUT means that the sleep timed out. - * - * @li ESYNCH_INTERRUPTED means that somebody interrupted the sleeping thread. - * - * @li ESYNCH_OK_ATOMIC means that the sleep succeeded and that there was - * a pending wakeup at the time of the call. The caller was not put - * asleep at all. - * - * @li ESYNCH_OK_BLOCKED means that the sleep succeeded; the full sleep was - * attempted. - */ -int waitq_sleep_timeout(waitq_t *wq, uint32_t usec, int flags) -{ - ipl_t ipl; - int rc; - - ipl = waitq_sleep_prepare(wq); - rc = waitq_sleep_timeout_unsafe(wq, usec, flags); + * @return ESYNCH_WOULD_BLOCK, meaning that the sleep failed because at the + * time of the call there was no pending wakeup + * @return ESYNCH_TIMEOUT, meaning that the sleep timed out. + * @return ESYNCH_INTERRUPTED, meaning that somebody interrupted the sleeping + * thread. + * @return ESYNCH_OK_ATOMIC, meaning that the sleep succeeded and that there + * was a pending wakeup at the time of the call. The caller was not put + * asleep at all. + * @return ESYNCH_OK_BLOCKED, meaning that the sleep succeeded; the full sleep + * was attempted. + * + */ +int waitq_sleep_timeout(waitq_t *wq, uint32_t usec, unsigned int flags) +{ + ASSERT((!PREEMPTION_DISABLED) || (PARAM_NON_BLOCKING(flags, usec))); + + ipl_t ipl = waitq_sleep_prepare(wq); + int rc = waitq_sleep_timeout_unsafe(wq, usec, flags); waitq_sleep_finish(wq, rc, ipl); return rc; @@ -269,7 +273,8 @@ * and interrupts disabled. * - * @param wq Wait queue. - * - * @return Interrupt level as it existed on entry to this function. + * @param wq Wait queue. + * + * @return Interrupt level as it existed on entry to this function. + * */ ipl_t waitq_sleep_prepare(waitq_t *wq) @@ -279,6 +284,6 @@ restart: ipl = interrupts_disable(); - - if (THREAD) { /* needed during system initiailzation */ + + if (THREAD) { /* Needed during system initiailzation */ /* * Busy waiting for a delayed timeout. @@ -287,15 +292,18 @@ * Simply, the thread is not allowed to go to sleep if * there are timeouts in progress. + * */ - spinlock_lock(&THREAD->lock); + irq_spinlock_lock(&THREAD->lock, false); + if (THREAD->timeout_pending) { - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); interrupts_restore(ipl); goto restart; } - spinlock_unlock(&THREAD->lock); - } - - spinlock_lock(&wq->lock); + + irq_spinlock_unlock(&THREAD->lock, false); + } + + irq_spinlock_lock(&wq->lock, false); return ipl; } @@ -307,7 +315,8 @@ * lock is released. * - * @param wq Wait queue. - * @param rc Return code of waitq_sleep_timeout_unsafe(). - * @param ipl Interrupt level returned by waitq_sleep_prepare(). + * @param wq Wait queue. + * @param rc Return code of waitq_sleep_timeout_unsafe(). + * @param ipl Interrupt level returned by waitq_sleep_prepare(). + * */ void waitq_sleep_finish(waitq_t *wq, int rc, ipl_t ipl) @@ -316,9 +325,10 @@ case ESYNCH_WOULD_BLOCK: case ESYNCH_OK_ATOMIC: - spinlock_unlock(&wq->lock); + irq_spinlock_unlock(&wq->lock, false); break; default: break; } + interrupts_restore(ipl); } @@ -330,20 +340,20 @@ * and followed by a call to waitq_sleep_finish(). * - * @param wq See waitq_sleep_timeout(). - * @param usec See waitq_sleep_timeout(). - * @param flags See waitq_sleep_timeout(). - * - * @return See waitq_sleep_timeout(). - */ -int waitq_sleep_timeout_unsafe(waitq_t *wq, uint32_t usec, int flags) -{ - /* checks whether to go to sleep at all */ + * @param wq See waitq_sleep_timeout(). + * @param usec See waitq_sleep_timeout(). + * @param flags See waitq_sleep_timeout(). + * + * @return See waitq_sleep_timeout(). + * + */ +int waitq_sleep_timeout_unsafe(waitq_t *wq, uint32_t usec, unsigned int flags) +{ + /* Checks whether to go to sleep at all */ if (wq->missed_wakeups) { wq->missed_wakeups--; return ESYNCH_OK_ATOMIC; - } - else { - if ((flags & SYNCH_FLAGS_NON_BLOCKING) && (usec == 0)) { - /* return immediatelly instead of going to sleep */ + } else { + if (PARAM_NON_BLOCKING(flags, usec)) { + /* Return immediatelly instead of going to sleep */ return ESYNCH_WOULD_BLOCK; } @@ -352,22 +362,24 @@ /* * Now we are firmly decided to go to sleep. + * */ - spinlock_lock(&THREAD->lock); - + irq_spinlock_lock(&THREAD->lock, false); + if (flags & SYNCH_FLAGS_INTERRUPTIBLE) { - /* * If the thread was already interrupted, * don't go to sleep at all. + * */ if (THREAD->interrupted) { - spinlock_unlock(&THREAD->lock); - spinlock_unlock(&wq->lock); + irq_spinlock_unlock(&THREAD->lock, false); + irq_spinlock_unlock(&wq->lock, false); return ESYNCH_INTERRUPTED; } - + /* * Set context that will be restored if the sleep * of this thread is ever interrupted. + * */ THREAD->sleep_interruptible = true; @@ -375,12 +387,10 @@ /* Short emulation of scheduler() return code. */ THREAD->last_cycle = get_cycle(); - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); return ESYNCH_INTERRUPTED; } - - } else { + } else THREAD->sleep_interruptible = false; - } - + if (usec) { /* We use the timeout variant. */ @@ -388,28 +398,29 @@ /* Short emulation of scheduler() return code. */ THREAD->last_cycle = get_cycle(); - spinlock_unlock(&THREAD->lock); + irq_spinlock_unlock(&THREAD->lock, false); return ESYNCH_TIMEOUT; } + THREAD->timeout_pending = true; timeout_register(&THREAD->sleep_timeout, (uint64_t) usec, waitq_sleep_timed_out, THREAD); } - + list_append(&THREAD->wq_link, &wq->head); - + /* * Suspend execution. + * */ THREAD->state = Sleeping; THREAD->sleep_queue = wq; - - spinlock_unlock(&THREAD->lock); - + + irq_spinlock_unlock(&THREAD->lock, false); + /* wq->lock is released in scheduler_separated_stack() */ - scheduler(); + scheduler(); return ESYNCH_OK_BLOCKED; } - /** Wake up first thread sleeping in a wait queue @@ -421,18 +432,13 @@ * timeout. * - * @param wq Pointer to wait queue. - * @param mode Wakeup mode. + * @param wq Pointer to wait queue. + * @param mode Wakeup mode. + * */ void waitq_wakeup(waitq_t *wq, wakeup_mode_t mode) { - ipl_t ipl; - - ipl = interrupts_disable(); - spinlock_lock(&wq->lock); - + irq_spinlock_lock(&wq->lock, true); _waitq_wakeup_unsafe(wq, mode); - - spinlock_unlock(&wq->lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&wq->lock, true); } @@ -442,26 +448,30 @@ * assumes wq->lock is already locked and interrupts are already disabled. * - * @param wq Pointer to wait queue. - * @param mode If mode is WAKEUP_FIRST, then the longest waiting - * thread, if any, is woken up. If mode is WAKEUP_ALL, then - * all waiting threads, if any, are woken up. If there are - * no waiting threads to be woken up, the missed wakeup is - * recorded in the wait queue. + * @param wq Pointer to wait queue. + * @param mode If mode is WAKEUP_FIRST, then the longest waiting + * thread, if any, is woken up. If mode is WAKEUP_ALL, then + * all waiting threads, if any, are woken up. If there are + * no waiting threads to be woken up, the missed wakeup is + * recorded in the wait queue. + * */ void _waitq_wakeup_unsafe(waitq_t *wq, wakeup_mode_t mode) { - thread_t *t; size_t count = 0; -loop: + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&wq->lock)); + +loop: if (list_empty(&wq->head)) { wq->missed_wakeups++; - if (count && mode == WAKEUP_ALL) + if ((count) && (mode == WAKEUP_ALL)) wq->missed_wakeups--; + return; } - + count++; - t = list_get_instance(wq->head.next, thread_t, wq_link); + thread_t *thread = list_get_instance(wq->head.next, thread_t, wq_link); /* @@ -475,24 +485,55 @@ * invariant must hold: * - * t->sleep_queue != NULL <=> t sleeps in a wait queue + * thread->sleep_queue != NULL <=> thread sleeps in a wait queue * * For an observer who locks the thread, the invariant * holds only when the lock is held prior to removing * it from the wait queue. + * */ - spinlock_lock(&t->lock); - list_remove(&t->wq_link); - - if (t->timeout_pending && timeout_unregister(&t->sleep_timeout)) - t->timeout_pending = false; - t->sleep_queue = NULL; - spinlock_unlock(&t->lock); - - thread_ready(t); - + irq_spinlock_lock(&thread->lock, false); + list_remove(&thread->wq_link); + + if ((thread->timeout_pending) && + (timeout_unregister(&thread->sleep_timeout))) + thread->timeout_pending = false; + + thread->sleep_queue = NULL; + irq_spinlock_unlock(&thread->lock, false); + + thread_ready(thread); + if (mode == WAKEUP_ALL) goto loop; } +/** Get the missed wakeups count. + * + * @param wq Pointer to wait queue. + * @return The wait queue's missed_wakeups count. + */ +int waitq_count_get(waitq_t *wq) +{ + int cnt; + + irq_spinlock_lock(&wq->lock, true); + cnt = wq->missed_wakeups; + irq_spinlock_unlock(&wq->lock, true); + + return cnt; +} + +/** Set the missed wakeups count. + * + * @param wq Pointer to wait queue. + * @param val New value of the missed_wakeups count. + */ +void waitq_count_set(waitq_t *wq, int val) +{ + irq_spinlock_lock(&wq->lock, true); + wq->missed_wakeups = val; + irq_spinlock_unlock(&wq->lock, true); +} + /** @} */ Index: kernel/generic/src/syscall/syscall.c =================================================================== --- kernel/generic/src/syscall/syscall.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/syscall/syscall.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -59,19 +59,20 @@ unative_t a4, unative_t a5, unative_t a6, unative_t id) { - unative_t rc; - /* Do userpace accounting */ + irq_spinlock_lock(&THREAD->lock, true); thread_update_accounting(true); - + irq_spinlock_unlock(&THREAD->lock, true); + #ifdef CONFIG_UDEBUG /* * Early check for undebugged tasks. We do not lock anything as this * test need not be precise in either direction. + * */ - if (THREAD->udebug.active) { + if (THREAD->udebug.active) udebug_syscall_event(a1, a2, a3, a4, a5, a6, id, 0, false); - } #endif + unative_t rc; if (id < SYSCALL_END) { rc = syscall_table[id](a1, a2, a3, a4, a5, a6); @@ -88,5 +89,5 @@ if (THREAD->udebug.active) { udebug_syscall_event(a1, a2, a3, a4, a5, a6, id, rc, true); - + /* * Stopping point needed for tasks that only invoke @@ -98,7 +99,9 @@ } #endif - + /* Do kernel accounting */ + irq_spinlock_lock(&THREAD->lock, true); thread_update_accounting(false); + irq_spinlock_unlock(&THREAD->lock, true); return rc; Index: kernel/generic/src/sysinfo/stats.c =================================================================== --- kernel/generic/src/sysinfo/stats.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/sysinfo/stats.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -110,10 +110,7 @@ } - /* Each CPU structure is locked separatelly */ - ipl_t ipl = interrupts_disable(); - size_t i; for (i = 0; i < config.cpu_count; i++) { - spinlock_lock(&cpus[i].lock); + irq_spinlock_lock(&cpus[i].lock, true); stats_cpus[i].id = cpus[i].id; @@ -123,8 +120,6 @@ stats_cpus[i].idle_ticks = cpus[i].idle_ticks; - spinlock_unlock(&cpus[i].lock); - } - - interrupts_restore(ipl); + irq_spinlock_unlock(&cpus[i].lock, true); + } return ((void *) stats_cpus); @@ -200,6 +195,4 @@ * * Summarize task information into task statistics. - * Task lock should be held and interrupts disabled - * before executing this function. * * @param task Task. @@ -209,4 +202,7 @@ static void produce_stats_task(task_t *task, stats_task_t *stats_task) { + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&task->lock)); + stats_task->task_id = task->taskid; str_cpy(stats_task->name, TASK_NAME_BUFLEN, task->name); @@ -235,5 +231,5 @@ /* Interrupts are already disabled */ - spinlock_lock(&(task->lock)); + irq_spinlock_lock(&(task->lock), false); /* Record the statistics and increment the iterator */ @@ -241,5 +237,5 @@ (*iterator)++; - spinlock_unlock(&(task->lock)); + irq_spinlock_unlock(&(task->lock), false); return true; @@ -260,6 +256,5 @@ { /* Messing with task structures, avoid deadlock */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); /* First walk the task tree to count the tasks */ @@ -269,7 +264,5 @@ if (count == 0) { /* No tasks found (strange) */ - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&tasks_lock, true); *size = 0; return NULL; @@ -278,6 +271,5 @@ *size = sizeof(stats_task_t) * count; if (dry_run) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return NULL; } @@ -286,7 +278,5 @@ if (stats_tasks == NULL) { /* No free space for allocation */ - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&tasks_lock, true); *size = 0; return NULL; @@ -297,6 +287,5 @@ avltree_walk(&tasks_tree, task_serialize_walker, (void *) &iterator); - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return ((void *) stats_tasks); @@ -306,6 +295,4 @@ * * Summarize thread information into thread statistics. - * Thread lock should be held and interrupts disabled - * before executing this function. * * @param thread Thread. @@ -315,4 +302,7 @@ static void produce_stats_thread(thread_t *thread, stats_thread_t *stats_thread) { + ASSERT(interrupts_disabled()); + ASSERT(irq_spinlock_locked(&thread->lock)); + stats_thread->thread_id = thread->tid; stats_thread->task_id = thread->task->taskid; @@ -346,5 +336,5 @@ /* Interrupts are already disabled */ - spinlock_lock(&thread->lock); + irq_spinlock_lock(&thread->lock, false); /* Record the statistics and increment the iterator */ @@ -352,5 +342,5 @@ (*iterator)++; - spinlock_unlock(&thread->lock); + irq_spinlock_unlock(&thread->lock, false); return true; @@ -371,6 +361,5 @@ { /* Messing with threads structures, avoid deadlock */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&threads_lock); + irq_spinlock_lock(&threads_lock, true); /* First walk the thread tree to count the threads */ @@ -380,7 +369,5 @@ if (count == 0) { /* No threads found (strange) */ - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&threads_lock, true); *size = 0; return NULL; @@ -389,6 +376,5 @@ *size = sizeof(stats_thread_t) * count; if (dry_run) { - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&threads_lock, true); return NULL; } @@ -397,7 +383,5 @@ if (stats_threads == NULL) { /* No free space for allocation */ - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); - + irq_spinlock_unlock(&threads_lock, true); *size = 0; return NULL; @@ -408,6 +392,5 @@ avltree_walk(&threads_tree, thread_serialize_walker, (void *) &iterator); - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&threads_lock, true); return ((void *) stats_threads); @@ -443,12 +426,10 @@ /* Messing with task structures, avoid deadlock */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&tasks_lock); + irq_spinlock_lock(&tasks_lock, true); task_t *task = task_find_by_id(task_id); if (task == NULL) { /* No task with this ID */ - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return ret; } @@ -459,5 +440,5 @@ ret.data.size = sizeof(stats_task_t); - spinlock_unlock(&tasks_lock); + irq_spinlock_unlock(&tasks_lock, true); } else { /* Allocate stats_task_t structure */ @@ -465,6 +446,5 @@ (stats_task_t *) malloc(sizeof(stats_task_t), FRAME_ATOMIC); if (stats_task == NULL) { - spinlock_unlock(&tasks_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&tasks_lock, true); return ret; } @@ -474,15 +454,12 @@ ret.data.data = (void *) stats_task; ret.data.size = sizeof(stats_task_t); - + /* Hand-over-hand locking */ - spinlock_lock(&task->lock); - spinlock_unlock(&tasks_lock); + irq_spinlock_exchange(&tasks_lock, &task->lock); produce_stats_task(task, stats_task); - spinlock_unlock(&task->lock); - } - - interrupts_restore(ipl); + irq_spinlock_unlock(&task->lock, true); + } return ret; @@ -518,12 +495,10 @@ /* Messing with threads structures, avoid deadlock */ - ipl_t ipl = interrupts_disable(); - spinlock_lock(&threads_lock); + irq_spinlock_lock(&threads_lock, true); thread_t *thread = thread_find_by_id(thread_id); if (thread == NULL) { /* No thread with this ID */ - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&threads_lock, true); return ret; } @@ -534,5 +509,5 @@ ret.data.size = sizeof(stats_thread_t); - spinlock_unlock(&threads_lock); + irq_spinlock_unlock(&threads_lock, true); } else { /* Allocate stats_thread_t structure */ @@ -540,6 +515,5 @@ (stats_thread_t *) malloc(sizeof(stats_thread_t), FRAME_ATOMIC); if (stats_thread == NULL) { - spinlock_unlock(&threads_lock); - interrupts_restore(ipl); + irq_spinlock_unlock(&threads_lock, true); return ret; } @@ -551,13 +525,10 @@ /* Hand-over-hand locking */ - spinlock_lock(&thread->lock); - spinlock_unlock(&threads_lock); + irq_spinlock_exchange(&threads_lock, &thread->lock); produce_stats_thread(thread, stats_thread); - spinlock_unlock(&thread->lock); - } - - interrupts_restore(ipl); + irq_spinlock_unlock(&thread->lock, true); + } return ret; @@ -673,5 +644,5 @@ { mutex_initialize(&load_lock, MUTEX_PASSIVE); - + sysinfo_set_item_fn_val("system.uptime", NULL, get_stats_uptime); sysinfo_set_item_fn_data("system.cpus", NULL, get_stats_cpus); Index: kernel/generic/src/sysinfo/sysinfo.c =================================================================== --- kernel/generic/src/sysinfo/sysinfo.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/sysinfo/sysinfo.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -58,5 +58,5 @@ * */ -static int sysinfo_item_constructor(void *obj, int kmflag) +static int sysinfo_item_constructor(void *obj, unsigned int kmflag) { sysinfo_item_t *item = (sysinfo_item_t *) obj; @@ -78,5 +78,5 @@ * */ -static int sysinfo_item_destructor(void *obj) +static size_t sysinfo_item_destructor(void *obj) { sysinfo_item_t *item = (sysinfo_item_t *) obj; Index: kernel/generic/src/time/clock.c =================================================================== --- kernel/generic/src/time/clock.c (revision c5cb943d5d19ad0d75578f54188f2491dfd64937) +++ kernel/generic/src/time/clock.c (revision 1113c9e130ee0cf71775a520394a8c7b73e1cf1e) @@ -33,11 +33,12 @@ /** * @file - * @brief High-level clock interrupt handler. + * @brief High-level clock interrupt handler. * * This file contains the clock() function which is the source * of preemption. It is also responsible for executing expired * timeouts. - */ - + * + */ + #include