/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Split from ftrace_64.S */ #include #include #include #include #include #include #include #include #include /* * * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount() * when ftrace is active. * * We arrive here after a function A calls function B, and we are the trace * function for B. When we enter r1 points to A's stack frame, B has not yet * had a chance to allocate one yet. * * Additionally r2 may point either to the TOC for A, or B, depending on * whether B did a TOC setup sequence before calling us. * * On entry the LR points back to the _mcount() call site, and r0 holds the * saved LR as it was on entry to B, ie. the original return address at the * call site in A. * * Our job is to save the register state into a struct pt_regs (on the stack) * and then arrange for the ftrace function to be called. */ .macro ftrace_regs_entry allregs /* Create a minimal stack frame for representing B */ PPC_STLU r1, -STACK_FRAME_MIN_SIZE(r1) /* Create our stack frame + pt_regs */ PPC_STLU r1,-SWITCH_FRAME_SIZE(r1) .if \allregs == 1 SAVE_GPRS(11, 12, r1) .endif /* Get the _mcount() call site out of LR */ mflr r11 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* Load the ftrace_op */ PPC_LL r12, -(MCOUNT_INSN_SIZE*2 + SZL)(r11) /* Load direct_call from the ftrace_op */ PPC_LL r12, FTRACE_OPS_DIRECT_CALL(r12) PPC_LCMPI r12, 0 .if \allregs == 1 bne .Lftrace_direct_call_regs .else bne .Lftrace_direct_call .endif #endif /* Save the previous LR in pt_regs->link */ PPC_STL r0, _LINK(r1) /* Also save it in A's stack frame */ PPC_STL r0, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE+LRSAVE(r1) /* Save all gprs to pt_regs */ SAVE_GPR(0, r1) SAVE_GPRS(3, 10, r1) #ifdef CONFIG_PPC64 /* Ok to continue? */ lbz r3, PACA_FTRACE_ENABLED(r13) cmpdi r3, 0 beq ftrace_no_trace #endif .if \allregs == 1 SAVE_GPR(2, r1) SAVE_GPRS(13, 31, r1) .else #if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE) SAVE_GPR(14, r1) #endif .endif /* Save previous stack pointer (r1) */ addi r8, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE PPC_STL r8, GPR1(r1) .if \allregs == 1 /* Load special regs for save below */ mfcr r7 mfmsr r8 mfctr r9 mfxer r10 .else /* Clear MSR to flag as ftrace_caller versus frace_regs_caller */ li r8, 0 .endif #ifdef CONFIG_PPC64 /* Save callee's TOC in the ABI compliant location */ std r2, STK_GOT(r1) LOAD_PACA_TOC() /* get kernel TOC in r2 */ #endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS /* r11 points to the instruction following the call to ftrace */ PPC_LL r5, -(MCOUNT_INSN_SIZE*2 + SZL)(r11) PPC_LL r12, FTRACE_OPS_FUNC(r5) mtctr r12 #else /* !CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS */ #ifdef CONFIG_PPC64 LOAD_REG_ADDR(r3, function_trace_op) ld r5,0(r3) #else lis r3,function_trace_op@ha lwz r5,function_trace_op@l(r3) #endif #endif /* Save special regs */ PPC_STL r8, _MSR(r1) .if \allregs == 1 PPC_STL r7, _CCR(r1) PPC_STL r9, _CTR(r1) PPC_STL r10, _XER(r1) .endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* Clear orig_gpr3 to later detect ftrace_direct call */ li r7, 0 PPC_STL r7, ORIG_GPR3(r1) #endif #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE /* Save our real return address in nvr for return */ .if \allregs == 0 SAVE_GPR(15, r1) .endif mr r15, r11 /* * We want the ftrace location in the function, but our lr (in r11) * points at the 'mtlr r0' instruction in the out of line stub. To * recover the ftrace location, we read the branch instruction in the * stub, and adjust our lr by the branch offset. * * See ftrace_init_ool_stub() for the profile sequence. */ lwz r8, MCOUNT_INSN_SIZE(r11) slwi r8, r8, 6 srawi r8, r8, 6 add r3, r11, r8 /* * Override our nip to point past the branch in the original function. * This allows reliable stack trace and the ftrace stack tracer to work as-is. */ addi r11, r3, MCOUNT_INSN_SIZE #else /* Calculate ip from nip-4 into r3 for call below */ subi r3, r11, MCOUNT_INSN_SIZE #endif /* Save NIP as pt_regs->nip */ PPC_STL r11, _NIP(r1) /* Also save it in B's stackframe header for proper unwind */ PPC_STL r11, LRSAVE+SWITCH_FRAME_SIZE(r1) #if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE) mr r14, r11 /* remember old NIP */ #endif /* Put the original return address in r4 as parent_ip */ mr r4, r0 /* Load &pt_regs in r6 for call below */ addi r6, r1, STACK_INT_FRAME_REGS .endm .macro ftrace_regs_exit allregs #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS /* Check orig_gpr3 to detect ftrace_direct call */ PPC_LL r3, ORIG_GPR3(r1) PPC_LCMPI cr1, r3, 0 mtctr r3 #endif /* Restore possibly modified LR */ PPC_LL r0, _LINK(r1) #ifndef CONFIG_PPC_FTRACE_OUT_OF_LINE /* Load ctr with the possibly modified NIP */ PPC_LL r3, _NIP(r1) #ifdef CONFIG_LIVEPATCH_64 cmpd r14, r3 /* has NIP been altered? */ #endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS beq cr1,2f mtlr r3 b 3f #endif 2: mtctr r3 mtlr r0 3: #else /* !CONFIG_PPC_FTRACE_OUT_OF_LINE */ /* Load LR with the possibly modified NIP */ PPC_LL r3, _NIP(r1) cmpd r14, r3 /* has NIP been altered? */ bne- 1f mr r3, r15 1: mtlr r3 .if \allregs == 0 REST_GPR(15, r1) .endif #endif /* Restore gprs */ .if \allregs == 1 REST_GPRS(2, 31, r1) .else REST_GPRS(3, 10, r1) #if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE) REST_GPR(14, r1) #endif .endif #ifdef CONFIG_PPC64 /* Restore callee's TOC */ ld r2, STK_GOT(r1) #endif /* Pop our stack frame */ addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE #ifdef CONFIG_LIVEPATCH_64 /* Based on the cmpd above, if the NIP was altered handle livepatch */ bne- livepatch_handler #endif /* jump after _mcount site */ #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS bnectr cr1 #endif /* * Return with blr to keep the link stack balanced. The function profiling sequence * uses 'mtlr r0' to restore LR. */ blr #else bctr #endif .endm .macro ftrace_regs_func allregs #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS bctrl #else .if \allregs == 1 .globl ftrace_regs_call ftrace_regs_call: .else .globl ftrace_call ftrace_call: .endif /* ftrace_call(r3, r4, r5, r6) */ bl ftrace_stub #endif .endm _GLOBAL(ftrace_regs_caller) ftrace_regs_entry 1 ftrace_regs_func 1 ftrace_regs_exit 1 _GLOBAL(ftrace_caller) ftrace_regs_entry 0 ftrace_regs_func 0 ftrace_regs_exit 0 _GLOBAL(ftrace_stub) blr #ifdef CONFIG_PPC64 ftrace_no_trace: #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE REST_GPR(3, r1) addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE blr #else mflr r3 mtctr r3 REST_GPR(3, r1) addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE mtlr r0 bctr #endif #endif #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS .Lftrace_direct_call_regs: mtctr r12 REST_GPRS(11, 12, r1) addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE bctr .Lftrace_direct_call: mtctr r12 addi r1, r1, SWITCH_FRAME_SIZE+STACK_FRAME_MIN_SIZE bctr SYM_FUNC_START(ftrace_stub_direct_tramp) blr SYM_FUNC_END(ftrace_stub_direct_tramp) #endif #ifdef CONFIG_LIVEPATCH_64 /* * This function runs in the mcount context, between two functions. As * such it can only clobber registers which are volatile and used in * function linkage. * * We get here when a function A, calls another function B, but B has * been live patched with a new function C. * * On entry, we have no stack frame and can not allocate one. * * With PPC_FTRACE_OUT_OF_LINE=n, on entry: * - LR points back to the original caller (in A) * - CTR holds the new NIP in C * - r0, r11 & r12 are free * * With PPC_FTRACE_OUT_OF_LINE=y, on entry: * - r0 points back to the original caller (in A) * - LR holds the new NIP in C * - r11 & r12 are free */ livepatch_handler: ld r12, PACA_THREAD_INFO(r13) /* Allocate 3 x 8 bytes */ ld r11, TI_livepatch_sp(r12) addi r11, r11, 24 std r11, TI_livepatch_sp(r12) /* Store stack end marker */ lis r12, STACK_END_MAGIC@h ori r12, r12, STACK_END_MAGIC@l std r12, -8(r11) /* Save toc & real LR on livepatch stack */ std r2, -24(r11) #ifndef CONFIG_PPC_FTRACE_OUT_OF_LINE mflr r12 std r12, -16(r11) mfctr r12 #else std r0, -16(r11) mflr r12 /* Put ctr in r12 for global entry and branch there */ mtctr r12 #endif bctrl /* * Now we are returning from the patched function to the original * caller A. We are free to use r11, r12 and we can use r2 until we * restore it. */ ld r12, PACA_THREAD_INFO(r13) ld r11, TI_livepatch_sp(r12) /* Check stack marker hasn't been trashed */ lis r2, STACK_END_MAGIC@h ori r2, r2, STACK_END_MAGIC@l ld r12, -8(r11) 1: tdne r12, r2 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 /* Restore LR & toc from livepatch stack */ ld r12, -16(r11) mtlr r12 ld r2, -24(r11) /* Pop livepatch stack frame */ ld r12, PACA_THREAD_INFO(r13) subi r11, r11, 24 std r11, TI_livepatch_sp(r12) /* Return to original caller of live patched function */ blr #endif /* CONFIG_LIVEPATCH */ #ifndef CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY _GLOBAL(mcount) _GLOBAL(_mcount) EXPORT_SYMBOL(_mcount) mflr r12 mtctr r12 mtlr r0 bctr #endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER _GLOBAL(return_to_handler) /* need to save return values */ #ifdef CONFIG_PPC64 stdu r1, -SWITCH_FRAME_SIZE(r1) std r4, GPR4(r1) std r3, GPR3(r1) /* Save previous stack pointer (r1) */ addi r3, r1, SWITCH_FRAME_SIZE std r3, GPR1(r1) /* save TOC */ std r2, 24(r1) std r31, 32(r1) mr r31, r1 /* pass ftrace_regs/pt_regs to ftrace_return_to_handler */ addi r3, r1, STACK_INT_FRAME_REGS /* * We might be called from a module. * Switch to our TOC to run inside the core kernel. */ LOAD_PACA_TOC() #else stwu r1, -SWITCH_FRAME_SIZE(r1) stw r4, GPR4(r1) stw r3, GPR3(r1) addi r3, r1, SWITCH_FRAME_SIZE stw r3, GPR1(r1) /* pass ftrace_regs/pt_regs to ftrace_return_to_handler */ addi r3, r1, STACK_INT_FRAME_REGS #endif bl ftrace_return_to_handler nop /* return value has real return address */ mtlr r3 #ifdef CONFIG_PPC64 ld r4, GPR4(r1) ld r3, GPR3(r1) ld r2, 24(r1) ld r31, 32(r1) ld r1, 0(r1) #else lwz r3, GPR3(r1) lwz r4, GPR4(r1) addi r1, r1, SWITCH_FRAME_SIZE #endif /* Jump back to real return address */ blr #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE SYM_DATA(ftrace_ool_stub_text_count, .long CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE) SYM_START(ftrace_ool_stub_text, SYM_L_GLOBAL, .balign SZL) .space CONFIG_PPC_FTRACE_OUT_OF_LINE_NUM_RESERVE * FTRACE_OOL_STUB_SIZE SYM_CODE_END(ftrace_ool_stub_text) #endif .pushsection ".tramp.ftrace.text","aw",@progbits; .globl ftrace_tramp_text ftrace_tramp_text: .space 32 .popsection .pushsection ".tramp.ftrace.init","aw",@progbits; .globl ftrace_tramp_init ftrace_tramp_init: .space 32 .popsection