Skip to content

Commit

Permalink
powerpc64/ftrace: Implement support for ftrace_regs_caller()
Browse files Browse the repository at this point in the history
With -mprofile-kernel, we always save the full register state in
ftrace_caller(). While this works, this is inefficient if we're not
interested in the register state, such as when we're using the function
tracer.

Rename the existing ftrace_caller() as ftrace_regs_caller() and provide
a simpler implementation for ftrace_caller() that is used when registers
are not required to be saved.

Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
  • Loading branch information
Naveen N. Rao authored and Michael Ellerman committed May 3, 2018
1 parent 9ef4042 commit ae30cc0
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 26 deletions.
2 changes: 0 additions & 2 deletions arch/powerpc/include/asm/ftrace.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,6 @@
extern void _mcount(void);

#ifdef CONFIG_DYNAMIC_FTRACE
# define FTRACE_ADDR ((unsigned long)ftrace_caller)
# define FTRACE_REGS_ADDR FTRACE_ADDR
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/* reloction of mcount call site is the same as the address */
Expand Down
3 changes: 3 additions & 0 deletions arch/powerpc/include/asm/module.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ struct mod_arch_specific {
#ifdef CONFIG_DYNAMIC_FTRACE
unsigned long toc;
unsigned long tramp;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
unsigned long tramp_regs;
#endif
#endif

/* For module function descriptor dereference */
Expand Down
28 changes: 21 additions & 7 deletions arch/powerpc/kernel/module_64.c
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ static unsigned long get_stubs_size(const Elf64_Ehdr *hdr,
#ifdef CONFIG_DYNAMIC_FTRACE
/* make the trampoline to the ftrace_caller */
relocs++;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
/* an additional one for ftrace_regs_caller */
relocs++;
#endif
#endif

pr_debug("Looks like a total of %lu stubs, max\n", relocs);
Expand Down Expand Up @@ -765,7 +769,8 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
* via the paca (in r13). The target (ftrace_caller()) is responsible for
* saving and restoring the toc before returning.
*/
static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
struct module *me, unsigned long addr)
{
struct ppc64_stub_entry *entry;
unsigned int i, num_stubs;
Expand All @@ -792,32 +797,41 @@ static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module
memcpy(entry->jump, stub_insns, sizeof(stub_insns));

/* Stub uses address relative to kernel toc (from the paca) */
reladdr = (unsigned long)ftrace_caller - kernel_toc_addr();
reladdr = addr - kernel_toc_addr();
if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
pr_err("%s: Address of ftrace_caller out of range of kernel_toc.\n", me->name);
pr_err("%s: Address of %ps out of range of kernel_toc.\n",
me->name, (void *)addr);
return 0;
}

entry->jump[1] |= PPC_HA(reladdr);
entry->jump[2] |= PPC_LO(reladdr);

/* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */
entry->funcdata = func_desc((unsigned long)ftrace_caller);
entry->funcdata = func_desc(addr);
entry->magic = STUB_MAGIC;

return (unsigned long)entry;
}
#else
static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, struct module *me)
static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs,
struct module *me, unsigned long addr)
{
return stub_for_addr(sechdrs, (unsigned long)ftrace_caller, me);
return stub_for_addr(sechdrs, addr, me);
}
#endif

int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs)
{
mod->arch.toc = my_r2(sechdrs, mod);
mod->arch.tramp = create_ftrace_stub(sechdrs, mod);
mod->arch.tramp = create_ftrace_stub(sechdrs, mod,
(unsigned long)ftrace_caller);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
mod->arch.tramp_regs = create_ftrace_stub(sechdrs, mod,
(unsigned long)ftrace_regs_caller);
if (!mod->arch.tramp_regs)
return -ENOENT;
#endif

if (!mod->arch.tramp)
return -ENOENT;
Expand Down
184 changes: 172 additions & 12 deletions arch/powerpc/kernel/trace/ftrace.c
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,8 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned int op[2];
void *ip = (void *)rec->ip;
unsigned long entry, ptr, tramp;
struct module *mod = rec->arch.mod;

/* read where this goes */
if (probe_kernel_read(op, ip, sizeof(op)))
Expand All @@ -368,34 +370,51 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
return -EINVAL;
}

/* If we never set up a trampoline to ftrace_caller, then bail */
if (!rec->arch.mod->arch.tramp) {
/* If we never set up ftrace trampoline(s), then bail */
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
if (!mod->arch.tramp || !mod->arch.tramp_regs) {
#else
if (!mod->arch.tramp) {
#endif
pr_err("No ftrace trampoline\n");
return -EINVAL;
}

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
if (rec->flags & FTRACE_FL_REGS)
tramp = mod->arch.tramp_regs;
else
#endif
tramp = mod->arch.tramp;

if (module_trampoline_target(mod, tramp, &ptr)) {
pr_err("Failed to get trampoline target\n");
return -EFAULT;
}

pr_devel("trampoline target %lx", ptr);

entry = ppc_global_function_entry((void *)addr);
/* This should match what was called */
if (ptr != entry) {
pr_err("addr %lx does not match expected %lx\n", ptr, entry);
return -EINVAL;
}

/* Ensure branch is within 24 bits */
if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
if (!create_branch(ip, tramp, BRANCH_SET_LINK)) {
pr_err("Branch out of range\n");
return -EINVAL;
}

if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
if (patch_branch(ip, tramp, BRANCH_SET_LINK)) {
pr_err("REL24 out of range!\n");
return -EINVAL;
}

return 0;
}

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
unsigned long addr)
{
return ftrace_make_call(rec, addr);
}
#endif

#else /* !CONFIG_PPC64: */
static int
__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
Expand Down Expand Up @@ -472,6 +491,137 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
#endif /* CONFIG_MODULES */
}

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
#ifdef CONFIG_MODULES
static int
__ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
unsigned long addr)
{
unsigned int op;
unsigned long ip = rec->ip;
unsigned long entry, ptr, tramp;
struct module *mod = rec->arch.mod;

/* If we never set up ftrace trampolines, then bail */
if (!mod->arch.tramp || !mod->arch.tramp_regs) {
pr_err("No ftrace trampoline\n");
return -EINVAL;
}

/* read where this goes */
if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
pr_err("Fetching opcode failed.\n");
return -EFAULT;
}

/* Make sure that that this is still a 24bit jump */
if (!is_bl_op(op)) {
pr_err("Not expected bl: opcode is %x\n", op);
return -EINVAL;
}

/* lets find where the pointer goes */
tramp = find_bl_target(ip, op);
entry = ppc_global_function_entry((void *)old_addr);

pr_devel("ip:%lx jumps to %lx", ip, tramp);

if (tramp != entry) {
/* old_addr is not within range, so we must have used a trampoline */
if (module_trampoline_target(mod, tramp, &ptr)) {
pr_err("Failed to get trampoline target\n");
return -EFAULT;
}

pr_devel("trampoline target %lx", ptr);

/* This should match what was called */
if (ptr != entry) {
pr_err("addr %lx does not match expected %lx\n", ptr, entry);
return -EINVAL;
}
}

/* The new target may be within range */
if (test_24bit_addr(ip, addr)) {
/* within range */
if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
pr_err("REL24 out of range!\n");
return -EINVAL;
}

return 0;
}

if (rec->flags & FTRACE_FL_REGS)
tramp = mod->arch.tramp_regs;
else
tramp = mod->arch.tramp;

if (module_trampoline_target(mod, tramp, &ptr)) {
pr_err("Failed to get trampoline target\n");
return -EFAULT;
}

pr_devel("trampoline target %lx", ptr);

entry = ppc_global_function_entry((void *)addr);
/* This should match what was called */
if (ptr != entry) {
pr_err("addr %lx does not match expected %lx\n", ptr, entry);
return -EINVAL;
}

/* Ensure branch is within 24 bits */
if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
pr_err("Branch out of range\n");
return -EINVAL;
}

if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
pr_err("REL24 out of range!\n");
return -EINVAL;
}

return 0;
}
#endif

int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
unsigned long addr)
{
unsigned long ip = rec->ip;
unsigned int old, new;

/*
* If the calling address is more that 24 bits away,
* then we had to use a trampoline to make the call.
* Otherwise just update the call site.
*/
if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) {
/* within range */
old = ftrace_call_replace(ip, old_addr, 1);
new = ftrace_call_replace(ip, addr, 1);
return ftrace_modify_code(ip, old, new);
}

#ifdef CONFIG_MODULES
/*
* Out of range jumps are called from modules.
*/
if (!rec->arch.mod) {
pr_err("No module loaded\n");
return -EINVAL;
}

return __ftrace_modify_call(rec, old_addr, addr);
#else
/* We should not get here without modules */
return -EINVAL;
#endif /* CONFIG_MODULES */
}
#endif

int ftrace_update_ftrace_func(ftrace_func_t func)
{
unsigned long ip = (unsigned long)(&ftrace_call);
Expand All @@ -482,6 +632,16 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
new = ftrace_call_replace(ip, (unsigned long)func, 1);
ret = ftrace_modify_code(ip, old, new);

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
/* Also update the regs callback function */
if (!ret) {
ip = (unsigned long)(&ftrace_regs_call);
old = *(unsigned int *)&ftrace_regs_call;
new = ftrace_call_replace(ip, (unsigned long)func, 1);
ret = ftrace_modify_code(ip, old, new);
}
#endif

return ret;
}

Expand Down
Loading

0 comments on commit ae30cc0

Please sign in to comment.