| /* | |
| * Copyright (C) 2012 Regents of the University of California | |
| * Copyright (C) 2017 SiFive | |
| * | |
| * This program is free software; you can redistribute it and/or | |
| * modify it under the terms of the GNU General Public License | |
| * as published by the Free Software Foundation, version 2. | |
| * | |
| * This program is distributed in the hope that it will be useful, | |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| * GNU General Public License for more details. | |
| */ | |
| #include <linux/init.h> | |
| #include <linux/linkage.h> | |
| #include <asm/asm.h> | |
| #include <asm/csr.h> | |
| #include <asm/unistd.h> | |
| #include <asm/thread_info.h> | |
| #include <asm/asm-offsets.h> | |
| .text | |
| .altmacro | |
| /* | |
| * Prepares to enter a system call or exception by saving all registers to the | |
| * stack. | |
| */ | |
| .macro SAVE_ALL | |
| LOCAL _restore_kernel_tpsp | |
| LOCAL _save_context | |
| /* | |
| * If coming from userspace, preserve the user thread pointer and load | |
| * the kernel thread pointer. If we came from the kernel, sscratch | |
| * will contain 0, and we should continue on the current TP. | |
| */ | |
| csrrw tp, sscratch, tp | |
| bnez tp, _save_context | |
| _restore_kernel_tpsp: | |
| csrr tp, sscratch | |
| REG_S sp, TASK_TI_KERNEL_SP(tp) | |
| _save_context: | |
| REG_S sp, TASK_TI_USER_SP(tp) | |
| REG_L sp, TASK_TI_KERNEL_SP(tp) | |
| addi sp, sp, -(PT_SIZE_ON_STACK) | |
| REG_S x1, PT_RA(sp) | |
| REG_S x3, PT_GP(sp) | |
| REG_S x5, PT_T0(sp) | |
| REG_S x6, PT_T1(sp) | |
| REG_S x7, PT_T2(sp) | |
| REG_S x8, PT_S0(sp) | |
| REG_S x9, PT_S1(sp) | |
| REG_S x10, PT_A0(sp) | |
| REG_S x11, PT_A1(sp) | |
| REG_S x12, PT_A2(sp) | |
| REG_S x13, PT_A3(sp) | |
| REG_S x14, PT_A4(sp) | |
| REG_S x15, PT_A5(sp) | |
| REG_S x16, PT_A6(sp) | |
| REG_S x17, PT_A7(sp) | |
| REG_S x18, PT_S2(sp) | |
| REG_S x19, PT_S3(sp) | |
| REG_S x20, PT_S4(sp) | |
| REG_S x21, PT_S5(sp) | |
| REG_S x22, PT_S6(sp) | |
| REG_S x23, PT_S7(sp) | |
| REG_S x24, PT_S8(sp) | |
| REG_S x25, PT_S9(sp) | |
| REG_S x26, PT_S10(sp) | |
| REG_S x27, PT_S11(sp) | |
| REG_S x28, PT_T3(sp) | |
| REG_S x29, PT_T4(sp) | |
| REG_S x30, PT_T5(sp) | |
| REG_S x31, PT_T6(sp) | |
| /* | |
| * Disable user-mode memory access as it should only be set in the | |
| * actual user copy routines. | |
| * | |
| * Disable the FPU to detect illegal usage of floating point in kernel | |
| * space. | |
| */ | |
| li t0, SR_SUM | SR_FS | |
| REG_L s0, TASK_TI_USER_SP(tp) | |
| csrrc s1, sstatus, t0 | |
| csrr s2, sepc | |
| csrr s3, sbadaddr | |
| csrr s4, scause | |
| csrr s5, sscratch | |
| REG_S s0, PT_SP(sp) | |
| REG_S s1, PT_SSTATUS(sp) | |
| REG_S s2, PT_SEPC(sp) | |
| REG_S s3, PT_SBADADDR(sp) | |
| REG_S s4, PT_SCAUSE(sp) | |
| REG_S s5, PT_TP(sp) | |
| .endm | |
| /* | |
| * Prepares to return from a system call or exception by restoring all | |
| * registers from the stack. | |
| */ | |
| .macro RESTORE_ALL | |
| REG_L a0, PT_SSTATUS(sp) | |
| REG_L a2, PT_SEPC(sp) | |
| csrw sstatus, a0 | |
| csrw sepc, a2 | |
| REG_L x1, PT_RA(sp) | |
| REG_L x3, PT_GP(sp) | |
| REG_L x4, PT_TP(sp) | |
| REG_L x5, PT_T0(sp) | |
| REG_L x6, PT_T1(sp) | |
| REG_L x7, PT_T2(sp) | |
| REG_L x8, PT_S0(sp) | |
| REG_L x9, PT_S1(sp) | |
| REG_L x10, PT_A0(sp) | |
| REG_L x11, PT_A1(sp) | |
| REG_L x12, PT_A2(sp) | |
| REG_L x13, PT_A3(sp) | |
| REG_L x14, PT_A4(sp) | |
| REG_L x15, PT_A5(sp) | |
| REG_L x16, PT_A6(sp) | |
| REG_L x17, PT_A7(sp) | |
| REG_L x18, PT_S2(sp) | |
| REG_L x19, PT_S3(sp) | |
| REG_L x20, PT_S4(sp) | |
| REG_L x21, PT_S5(sp) | |
| REG_L x22, PT_S6(sp) | |
| REG_L x23, PT_S7(sp) | |
| REG_L x24, PT_S8(sp) | |
| REG_L x25, PT_S9(sp) | |
| REG_L x26, PT_S10(sp) | |
| REG_L x27, PT_S11(sp) | |
| REG_L x28, PT_T3(sp) | |
| REG_L x29, PT_T4(sp) | |
| REG_L x30, PT_T5(sp) | |
| REG_L x31, PT_T6(sp) | |
| REG_L x2, PT_SP(sp) | |
| .endm | |
| ENTRY(handle_exception) | |
| SAVE_ALL | |
| /* | |
| * Set sscratch register to 0, so that if a recursive exception | |
| * occurs, the exception vector knows it came from the kernel | |
| */ | |
| csrw sscratch, x0 | |
| /* Load the global pointer */ | |
| .option push | |
| .option norelax | |
| la gp, __global_pointer$ | |
| .option pop | |
| la ra, ret_from_exception | |
| /* | |
| * MSB of cause differentiates between | |
| * interrupts and exceptions | |
| */ | |
| bge s4, zero, 1f | |
| /* Handle interrupts */ | |
| move a0, sp /* pt_regs */ | |
| move a1, s4 /* scause */ | |
| tail do_IRQ | |
| 1: | |
| /* Exceptions run with interrupts enabled or disabled | |
| depending on the state of sstatus.SR_SPIE */ | |
| andi t0, s1, SR_SPIE | |
| beqz t0, 1f | |
| csrs sstatus, SR_SIE | |
| 1: | |
| /* Handle syscalls */ | |
| li t0, EXC_SYSCALL | |
| beq s4, t0, handle_syscall | |
| /* Handle other exceptions */ | |
| slli t0, s4, RISCV_LGPTR | |
| la t1, excp_vect_table | |
| la t2, excp_vect_table_end | |
| move a0, sp /* pt_regs */ | |
| add t0, t1, t0 | |
| /* Check if exception code lies within bounds */ | |
| bgeu t0, t2, 1f | |
| REG_L t0, 0(t0) | |
| jr t0 | |
| 1: | |
| tail do_trap_unknown | |
| handle_syscall: | |
| /* save the initial A0 value (needed in signal handlers) */ | |
| REG_S a0, PT_ORIG_A0(sp) | |
| /* | |
| * Advance SEPC to avoid executing the original | |
| * scall instruction on sret | |
| */ | |
| addi s2, s2, 0x4 | |
| REG_S s2, PT_SEPC(sp) | |
| /* Trace syscalls, but only if requested by the user. */ | |
| REG_L t0, TASK_TI_FLAGS(tp) | |
| andi t0, t0, _TIF_SYSCALL_TRACE | |
| bnez t0, handle_syscall_trace_enter | |
| check_syscall_nr: | |
| /* Check to make sure we don't jump to a bogus syscall number. */ | |
| li t0, __NR_syscalls | |
| la s0, sys_ni_syscall | |
| /* Syscall number held in a7 */ | |
| bgeu a7, t0, 1f | |
| la s0, sys_call_table | |
| slli t0, a7, RISCV_LGPTR | |
| add s0, s0, t0 | |
| REG_L s0, 0(s0) | |
| 1: | |
| jalr s0 | |
| ret_from_syscall: | |
| /* Set user a0 to kernel a0 */ | |
| REG_S a0, PT_A0(sp) | |
| /* Trace syscalls, but only if requested by the user. */ | |
| REG_L t0, TASK_TI_FLAGS(tp) | |
| andi t0, t0, _TIF_SYSCALL_TRACE | |
| bnez t0, handle_syscall_trace_exit | |
| ret_from_exception: | |
| REG_L s0, PT_SSTATUS(sp) | |
| csrc sstatus, SR_SIE | |
| andi s0, s0, SR_SPP | |
| bnez s0, restore_all | |
| resume_userspace: | |
| /* Interrupts must be disabled here so flags are checked atomically */ | |
| REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */ | |
| andi s1, s0, _TIF_WORK_MASK | |
| bnez s1, work_pending | |
| /* Save unwound kernel stack pointer in thread_info */ | |
| addi s0, sp, PT_SIZE_ON_STACK | |
| REG_S s0, TASK_TI_KERNEL_SP(tp) | |
| /* | |
| * Save TP into sscratch, so we can find the kernel data structures | |
| * again. | |
| */ | |
| csrw sscratch, tp | |
| restore_all: | |
| RESTORE_ALL | |
| sret | |
| work_pending: | |
| /* Enter slow path for supplementary processing */ | |
| la ra, ret_from_exception | |
| andi s1, s0, _TIF_NEED_RESCHED | |
| bnez s1, work_resched | |
| work_notifysig: | |
| /* Handle pending signals and notify-resume requests */ | |
| csrs sstatus, SR_SIE /* Enable interrupts for do_notify_resume() */ | |
| move a0, sp /* pt_regs */ | |
| move a1, s0 /* current_thread_info->flags */ | |
| tail do_notify_resume | |
| work_resched: | |
| tail schedule | |
| /* Slow paths for ptrace. */ | |
| handle_syscall_trace_enter: | |
| move a0, sp | |
| call do_syscall_trace_enter | |
| REG_L a0, PT_A0(sp) | |
| REG_L a1, PT_A1(sp) | |
| REG_L a2, PT_A2(sp) | |
| REG_L a3, PT_A3(sp) | |
| REG_L a4, PT_A4(sp) | |
| REG_L a5, PT_A5(sp) | |
| REG_L a6, PT_A6(sp) | |
| REG_L a7, PT_A7(sp) | |
| j check_syscall_nr | |
| handle_syscall_trace_exit: | |
| move a0, sp | |
| call do_syscall_trace_exit | |
| j ret_from_exception | |
| END(handle_exception) | |
| ENTRY(ret_from_fork) | |
| la ra, ret_from_exception | |
| tail schedule_tail | |
| ENDPROC(ret_from_fork) | |
| ENTRY(ret_from_kernel_thread) | |
| call schedule_tail | |
| /* Call fn(arg) */ | |
| la ra, ret_from_exception | |
| move a0, s1 | |
| jr s0 | |
| ENDPROC(ret_from_kernel_thread) | |
| /* | |
| * Integer register context switch | |
| * The callee-saved registers must be saved and restored. | |
| * | |
| * a0: previous task_struct (must be preserved across the switch) | |
| * a1: next task_struct | |
| * | |
| * The value of a0 and a1 must be preserved by this function, as that's how | |
| * arguments are passed to schedule_tail. | |
| */ | |
| ENTRY(__switch_to) | |
| /* Save context into prev->thread */ | |
| li a4, TASK_THREAD_RA | |
| add a3, a0, a4 | |
| add a4, a1, a4 | |
| REG_S ra, TASK_THREAD_RA_RA(a3) | |
| REG_S sp, TASK_THREAD_SP_RA(a3) | |
| REG_S s0, TASK_THREAD_S0_RA(a3) | |
| REG_S s1, TASK_THREAD_S1_RA(a3) | |
| REG_S s2, TASK_THREAD_S2_RA(a3) | |
| REG_S s3, TASK_THREAD_S3_RA(a3) | |
| REG_S s4, TASK_THREAD_S4_RA(a3) | |
| REG_S s5, TASK_THREAD_S5_RA(a3) | |
| REG_S s6, TASK_THREAD_S6_RA(a3) | |
| REG_S s7, TASK_THREAD_S7_RA(a3) | |
| REG_S s8, TASK_THREAD_S8_RA(a3) | |
| REG_S s9, TASK_THREAD_S9_RA(a3) | |
| REG_S s10, TASK_THREAD_S10_RA(a3) | |
| REG_S s11, TASK_THREAD_S11_RA(a3) | |
| /* Restore context from next->thread */ | |
| REG_L ra, TASK_THREAD_RA_RA(a4) | |
| REG_L sp, TASK_THREAD_SP_RA(a4) | |
| REG_L s0, TASK_THREAD_S0_RA(a4) | |
| REG_L s1, TASK_THREAD_S1_RA(a4) | |
| REG_L s2, TASK_THREAD_S2_RA(a4) | |
| REG_L s3, TASK_THREAD_S3_RA(a4) | |
| REG_L s4, TASK_THREAD_S4_RA(a4) | |
| REG_L s5, TASK_THREAD_S5_RA(a4) | |
| REG_L s6, TASK_THREAD_S6_RA(a4) | |
| REG_L s7, TASK_THREAD_S7_RA(a4) | |
| REG_L s8, TASK_THREAD_S8_RA(a4) | |
| REG_L s9, TASK_THREAD_S9_RA(a4) | |
| REG_L s10, TASK_THREAD_S10_RA(a4) | |
| REG_L s11, TASK_THREAD_S11_RA(a4) | |
| /* Swap the CPU entry around. */ | |
| lw a3, TASK_TI_CPU(a0) | |
| lw a4, TASK_TI_CPU(a1) | |
| sw a3, TASK_TI_CPU(a1) | |
| sw a4, TASK_TI_CPU(a0) | |
| #if TASK_TI != 0 | |
| #error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work." | |
| addi tp, a1, TASK_TI | |
| #else | |
| move tp, a1 | |
| #endif | |
| ret | |
| ENDPROC(__switch_to) | |
| ENTRY(__fstate_save) | |
| li a2, TASK_THREAD_F0 | |
| add a0, a0, a2 | |
| li t1, SR_FS | |
| csrs sstatus, t1 | |
| frcsr t0 | |
| fsd f0, TASK_THREAD_F0_F0(a0) | |
| fsd f1, TASK_THREAD_F1_F0(a0) | |
| fsd f2, TASK_THREAD_F2_F0(a0) | |
| fsd f3, TASK_THREAD_F3_F0(a0) | |
| fsd f4, TASK_THREAD_F4_F0(a0) | |
| fsd f5, TASK_THREAD_F5_F0(a0) | |
| fsd f6, TASK_THREAD_F6_F0(a0) | |
| fsd f7, TASK_THREAD_F7_F0(a0) | |
| fsd f8, TASK_THREAD_F8_F0(a0) | |
| fsd f9, TASK_THREAD_F9_F0(a0) | |
| fsd f10, TASK_THREAD_F10_F0(a0) | |
| fsd f11, TASK_THREAD_F11_F0(a0) | |
| fsd f12, TASK_THREAD_F12_F0(a0) | |
| fsd f13, TASK_THREAD_F13_F0(a0) | |
| fsd f14, TASK_THREAD_F14_F0(a0) | |
| fsd f15, TASK_THREAD_F15_F0(a0) | |
| fsd f16, TASK_THREAD_F16_F0(a0) | |
| fsd f17, TASK_THREAD_F17_F0(a0) | |
| fsd f18, TASK_THREAD_F18_F0(a0) | |
| fsd f19, TASK_THREAD_F19_F0(a0) | |
| fsd f20, TASK_THREAD_F20_F0(a0) | |
| fsd f21, TASK_THREAD_F21_F0(a0) | |
| fsd f22, TASK_THREAD_F22_F0(a0) | |
| fsd f23, TASK_THREAD_F23_F0(a0) | |
| fsd f24, TASK_THREAD_F24_F0(a0) | |
| fsd f25, TASK_THREAD_F25_F0(a0) | |
| fsd f26, TASK_THREAD_F26_F0(a0) | |
| fsd f27, TASK_THREAD_F27_F0(a0) | |
| fsd f28, TASK_THREAD_F28_F0(a0) | |
| fsd f29, TASK_THREAD_F29_F0(a0) | |
| fsd f30, TASK_THREAD_F30_F0(a0) | |
| fsd f31, TASK_THREAD_F31_F0(a0) | |
| sw t0, TASK_THREAD_FCSR_F0(a0) | |
| csrc sstatus, t1 | |
| ret | |
| ENDPROC(__fstate_save) | |
| ENTRY(__fstate_restore) | |
| li a2, TASK_THREAD_F0 | |
| add a0, a0, a2 | |
| li t1, SR_FS | |
| lw t0, TASK_THREAD_FCSR_F0(a0) | |
| csrs sstatus, t1 | |
| fld f0, TASK_THREAD_F0_F0(a0) | |
| fld f1, TASK_THREAD_F1_F0(a0) | |
| fld f2, TASK_THREAD_F2_F0(a0) | |
| fld f3, TASK_THREAD_F3_F0(a0) | |
| fld f4, TASK_THREAD_F4_F0(a0) | |
| fld f5, TASK_THREAD_F5_F0(a0) | |
| fld f6, TASK_THREAD_F6_F0(a0) | |
| fld f7, TASK_THREAD_F7_F0(a0) | |
| fld f8, TASK_THREAD_F8_F0(a0) | |
| fld f9, TASK_THREAD_F9_F0(a0) | |
| fld f10, TASK_THREAD_F10_F0(a0) | |
| fld f11, TASK_THREAD_F11_F0(a0) | |
| fld f12, TASK_THREAD_F12_F0(a0) | |
| fld f13, TASK_THREAD_F13_F0(a0) | |
| fld f14, TASK_THREAD_F14_F0(a0) | |
| fld f15, TASK_THREAD_F15_F0(a0) | |
| fld f16, TASK_THREAD_F16_F0(a0) | |
| fld f17, TASK_THREAD_F17_F0(a0) | |
| fld f18, TASK_THREAD_F18_F0(a0) | |
| fld f19, TASK_THREAD_F19_F0(a0) | |
| fld f20, TASK_THREAD_F20_F0(a0) | |
| fld f21, TASK_THREAD_F21_F0(a0) | |
| fld f22, TASK_THREAD_F22_F0(a0) | |
| fld f23, TASK_THREAD_F23_F0(a0) | |
| fld f24, TASK_THREAD_F24_F0(a0) | |
| fld f25, TASK_THREAD_F25_F0(a0) | |
| fld f26, TASK_THREAD_F26_F0(a0) | |
| fld f27, TASK_THREAD_F27_F0(a0) | |
| fld f28, TASK_THREAD_F28_F0(a0) | |
| fld f29, TASK_THREAD_F29_F0(a0) | |
| fld f30, TASK_THREAD_F30_F0(a0) | |
| fld f31, TASK_THREAD_F31_F0(a0) | |
| fscsr t0 | |
| csrc sstatus, t1 | |
| ret | |
| ENDPROC(__fstate_restore) | |
| .section ".rodata" | |
| /* Exception vector table */ | |
| ENTRY(excp_vect_table) | |
| RISCV_PTR do_trap_insn_misaligned | |
| RISCV_PTR do_trap_insn_fault | |
| RISCV_PTR do_trap_insn_illegal | |
| RISCV_PTR do_trap_break | |
| RISCV_PTR do_trap_load_misaligned | |
| RISCV_PTR do_trap_load_fault | |
| RISCV_PTR do_trap_store_misaligned | |
| RISCV_PTR do_trap_store_fault | |
| RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */ | |
| RISCV_PTR do_trap_ecall_s | |
| RISCV_PTR do_trap_unknown | |
| RISCV_PTR do_trap_ecall_m | |
| RISCV_PTR do_page_fault /* instruction page fault */ | |
| RISCV_PTR do_page_fault /* load page fault */ | |
| RISCV_PTR do_trap_unknown | |
| RISCV_PTR do_page_fault /* store page fault */ | |
| excp_vect_table_end: | |
| END(excp_vect_table) |