Skip to content

Commit

Permalink
x86: Exception hooks for userspace RCU extended QS
Browse files Browse the repository at this point in the history 
Add necessary hooks to x86 exception for userspace
RCU extended quiescent state support.

This includes traps, page fault, debug exceptions, etc...

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Alessio Igor Bogani <abogani@kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Avi Kivity <avi@redhat.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Geoff Levand <geoff@infradead.org>
Cc: Gilad Ben Yossef <gilad@benyossef.com>
Cc: Hakan Akkan <hakanakkan@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Kevin Hilman <khilman@ti.com>
Cc: Max Krasnyansky <maxk@qualcomm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Hemminger <shemminger@vyatta.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sven-Thorsten Dietrich <thebigcorporation@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  • Loading branch information
Frederic Weisbecker committed Sep 26, 2012
1 parent ef3f628 commit 6ba3c97
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 28 deletions.
20 changes: 20 additions & 0 deletions arch/x86/include/asm/rcu.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#ifndef _ASM_X86_RCU_H
#define _ASM_X86_RCU_H

#include <linux/rcupdate.h>
#include <asm/ptrace.h>

static inline void exception_enter(struct pt_regs *regs)
{
rcu_user_exit();
}

static inline void exception_exit(struct pt_regs *regs)
{
#ifdef CONFIG_RCU_USER_QS
if (user_mode(regs))
rcu_user_enter();
#endif
}

#endif
81 changes: 55 additions & 26 deletions arch/x86/kernel/traps.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/mce.h>
#include <asm/rcu.h>

#include <asm/mach_traps.h>

Expand Down Expand Up @@ -180,11 +181,15 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
#define DO_ERROR(trapnr, signr, str, name) \
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
{ \
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
== NOTIFY_STOP) \
exception_enter(regs); \
if (notify_die(DIE_TRAP, str, regs, error_code, \
trapnr, signr) == NOTIFY_STOP) { \
exception_exit(regs); \
return; \
} \
conditional_sti(regs); \
do_trap(trapnr, signr, str, regs, error_code, NULL); \
exception_exit(regs); \
}

#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
Expand All @@ -195,11 +200,15 @@ dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
info.si_errno = 0; \
info.si_code = sicode; \
info.si_addr = (void __user *)siaddr; \
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
== NOTIFY_STOP) \
exception_enter(regs); \
if (notify_die(DIE_TRAP, str, regs, error_code, \
trapnr, signr) == NOTIFY_STOP) { \
exception_exit(regs); \
return; \
} \
conditional_sti(regs); \
do_trap(trapnr, signr, str, regs, error_code, &info); \
exception_exit(regs); \
}

DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
Expand All @@ -222,19 +231,22 @@ DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check,
/* Runs on IST stack */
dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
{
exception_enter(regs);
if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
X86_TRAP_SS, SIGBUS) == NOTIFY_STOP)
return;
preempt_conditional_sti(regs);
do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
preempt_conditional_cli(regs);
X86_TRAP_SS, SIGBUS) != NOTIFY_STOP) {
preempt_conditional_sti(regs);
do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
preempt_conditional_cli(regs);
}
exception_exit(regs);
}

dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
{
static const char str[] = "double fault";
struct task_struct *tsk = current;

exception_enter(regs);
/* Return not checked because double check cannot be ignored */
notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);

Expand All @@ -255,27 +267,28 @@ do_general_protection(struct pt_regs *regs, long error_code)
{
struct task_struct *tsk;

exception_enter(regs);
conditional_sti(regs);

#ifdef CONFIG_X86_32
if (regs->flags & X86_VM_MASK) {
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
return;
goto exit;
}
#endif

tsk = current;
if (!user_mode(regs)) {
if (fixup_exception(regs))
return;
goto exit;

tsk->thread.error_code = error_code;
tsk->thread.trap_nr = X86_TRAP_GP;
if (!notify_die(DIE_GPF, "general protection fault", regs, error_code,
X86_TRAP_GP, SIGSEGV) == NOTIFY_STOP)
if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
die("general protection fault", regs, error_code);
return;
goto exit;
}

tsk->thread.error_code = error_code;
Expand All @@ -291,7 +304,8 @@ do_general_protection(struct pt_regs *regs, long error_code)
}

force_sig(SIGSEGV, tsk);
return;
exit:
exception_exit(regs);
}

/* May run on IST stack. */
Expand All @@ -306,15 +320,16 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
ftrace_int3_handler(regs))
return;
#endif
exception_enter(regs);
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP)
return;
goto exit;
#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */

if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP)
return;
goto exit;

/*
* Let others (NMI) know that the debug stack is in use
Expand All @@ -325,6 +340,8 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
preempt_conditional_cli(regs);
debug_stack_usage_dec();
exit:
exception_exit(regs);
}

#ifdef CONFIG_X86_64
Expand Down Expand Up @@ -385,6 +402,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
unsigned long dr6;
int si_code;

exception_enter(regs);

get_debugreg(dr6, 6);

/* Filter out all the reserved bits which are preset to 1 */
Expand All @@ -400,7 +419,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)

/* Catch kmemcheck conditions first of all! */
if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
return;
goto exit;

/* DR6 may or may not be cleared by the CPU */
set_debugreg(0, 6);
Expand All @@ -415,7 +434,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)

if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code,
SIGTRAP) == NOTIFY_STOP)
return;
goto exit;

/*
* Let others (NMI) know that the debug stack is in use
Expand All @@ -431,7 +450,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
X86_TRAP_DB);
preempt_conditional_cli(regs);
debug_stack_usage_dec();
return;
goto exit;
}

/*
Expand All @@ -452,7 +471,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
preempt_conditional_cli(regs);
debug_stack_usage_dec();

return;
exit:
exception_exit(regs);
}

/*
Expand Down Expand Up @@ -549,14 +569,17 @@ dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
#ifdef CONFIG_X86_32
ignore_fpu_irq = 1;
#endif

exception_enter(regs);
math_error(regs, error_code, X86_TRAP_MF);
exception_exit(regs);
}

dotraplinkage void
do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
{
exception_enter(regs);
math_error(regs, error_code, X86_TRAP_XF);
exception_exit(regs);
}

dotraplinkage void
Expand Down Expand Up @@ -623,6 +646,7 @@ EXPORT_SYMBOL_GPL(math_state_restore);
dotraplinkage void __kprobes
do_device_not_available(struct pt_regs *regs, long error_code)
{
exception_enter(regs);
#ifdef CONFIG_MATH_EMULATION
if (read_cr0() & X86_CR0_EM) {
struct math_emu_info info = { };
Expand All @@ -631,30 +655,35 @@ do_device_not_available(struct pt_regs *regs, long error_code)

info.regs = regs;
math_emulate(&info);
exception_exit(regs);
return;
}
#endif
math_state_restore(); /* interrupts still off */
#ifdef CONFIG_X86_32
conditional_sti(regs);
#endif
exception_exit(regs);
}

#ifdef CONFIG_X86_32
dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
{
siginfo_t info;

exception_enter(regs);
local_irq_enable();

info.si_signo = SIGILL;
info.si_errno = 0;
info.si_code = ILL_BADSTK;
info.si_addr = NULL;
if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
X86_TRAP_IRET, SIGILL) == NOTIFY_STOP)
return;
do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
&info);
X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
&info);
}
exception_exit(regs);
}
#endif

Expand Down
13 changes: 11 additions & 2 deletions arch/x86/mm/fault.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <asm/pgalloc.h> /* pgd_*(), ... */
#include <asm/kmemcheck.h> /* kmemcheck_*(), ... */
#include <asm/fixmap.h> /* VSYSCALL_START */
#include <asm/rcu.h> /* exception_enter(), ... */

/*
* Page fault error code bits:
Expand Down Expand Up @@ -1000,8 +1001,8 @@ static int fault_in_kernel_space(unsigned long address)
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
dotraplinkage void __kprobes
do_page_fault(struct pt_regs *regs, unsigned long error_code)
static void __kprobes
__do_page_fault(struct pt_regs *regs, unsigned long error_code)
{
struct vm_area_struct *vma;
struct task_struct *tsk;
Expand Down Expand Up @@ -1209,3 +1210,11 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)

up_read(&mm->mmap_sem);
}

dotraplinkage void __kprobes
do_page_fault(struct pt_regs *regs, unsigned long error_code)
{
exception_enter(regs);
__do_page_fault(regs, error_code);
exception_exit(regs);
}

0 comments on commit 6ba3c97

Please sign in to comment.