Skip to content

Commit

Permalink
x86/entry: Use generic syscall exit functionality
Browse files Browse the repository at this point in the history 
Replace the x86 variant with the generic version. Provide the relevant
architecture specific helper functions and defines.

Use a temporary define for idtentry_exit_user which will be cleaned up
seperately.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Kees Cook <keescook@chromium.org>
Link: https://lkml.kernel.org/r/20200722220520.494648601@linutronix.de
  • Loading branch information
Thomas Gleixner committed Jul 24, 2020
1 parent 27d6b4d commit 167fd21
Show file tree
Hide file tree
Showing 7 changed files with 54 additions and 222 deletions.
221 changes: 5 additions & 216 deletions arch/x86/entry/common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,8 @@
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/tracehook.h>
#include <linux/audit.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <linux/context_tracking.h>
#include <linux/user-return-notifier.h>
#include <linux/nospec.h>
#include <linux/uprobes.h>
#include <linux/livepatch.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>

Expand All @@ -42,191 +35,6 @@
#include <asm/syscall.h>
#include <asm/irq_stack.h>

#include <trace/events/syscalls.h>

/**
* exit_to_user_mode - Fixup state when exiting to user mode
*
* Syscall exit enables interrupts, but the kernel state is interrupts
* disabled when this is invoked. Also tell RCU about it.
*
* 1) Trace interrupts on state
* 2) Invoke context tracking if enabled to adjust RCU state
* 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on.
* 4) Tell lockdep that interrupts are enabled
*/
static __always_inline void exit_to_user_mode(void)
{
instrumentation_begin();
trace_hardirqs_on_prepare();
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
instrumentation_end();

user_enter_irqoff();
mds_user_clear_cpu_buffers();
lockdep_hardirqs_on(CALLER_ADDR0);
}

#define EXIT_TO_USERMODE_LOOP_FLAGS \
(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
_TIF_NEED_RESCHED | _TIF_PATCH_PENDING)

static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
{
/*
* In order to return to user mode, we need to have IRQs off with
* none of EXIT_TO_USERMODE_LOOP_FLAGS set. Several of these flags
* can be set at any time on preemptible kernels if we have IRQs on,
* so we need to loop. Disabling preemption wouldn't help: doing the
* work to clear some of the flags can sleep.
*/
while (true) {
/* We have work to do. */
local_irq_enable();

if (cached_flags & _TIF_NEED_RESCHED)
schedule();

if (cached_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);

if (cached_flags & _TIF_PATCH_PENDING)
klp_update_patch_state(current);

/* deal with pending signal delivery */
if (cached_flags & _TIF_SIGPENDING)
do_signal(regs);

if (cached_flags & _TIF_NOTIFY_RESUME) {
clear_thread_flag(TIF_NOTIFY_RESUME);
tracehook_notify_resume(regs);
rseq_handle_notify_resume(NULL, regs);
}

/* Disable IRQs and retry */
local_irq_disable();

cached_flags = READ_ONCE(current_thread_info()->flags);

if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
break;
}
}

static void __prepare_exit_to_usermode(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
u32 cached_flags;

addr_limit_user_check();

lockdep_assert_irqs_disabled();
lockdep_sys_exit();

cached_flags = READ_ONCE(ti->flags);

if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
exit_to_usermode_loop(regs, cached_flags);

/* Reload ti->flags; we may have rescheduled above. */
cached_flags = READ_ONCE(ti->flags);

if (cached_flags & _TIF_USER_RETURN_NOTIFY)
fire_user_return_notifiers();

if (unlikely(cached_flags & _TIF_IO_BITMAP))
tss_update_io_bitmap();

fpregs_assert_state_consistent();
if (unlikely(cached_flags & _TIF_NEED_FPU_LOAD))
switch_fpu_return();

#ifdef CONFIG_COMPAT
/*
* Compat syscalls set TS_COMPAT. Make sure we clear it before
* returning to user mode. We need to clear it *after* signal
* handling, because syscall restart has a fixup for compat
* syscalls. The fixup is exercised by the ptrace_syscall_32
* selftest.
*
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
* special case only applies after poking regs and before the
* very next return to user mode.
*/
ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
#endif
}

static noinstr void prepare_exit_to_usermode(struct pt_regs *regs)
{
instrumentation_begin();
__prepare_exit_to_usermode(regs);
instrumentation_end();
exit_to_user_mode();
}

#define SYSCALL_EXIT_WORK_FLAGS \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)

static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
{
bool step;

audit_syscall_exit(regs);

if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
trace_sys_exit(regs, regs->ax);

/*
* If TIF_SYSCALL_EMU is set, we only get here because of
* TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
* We already reported this syscall instruction in
* syscall_trace_enter().
*/
step = unlikely(
(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
== _TIF_SINGLESTEP);
if (step || cached_flags & _TIF_SYSCALL_TRACE)
tracehook_report_syscall_exit(regs, step);
}

static void __syscall_return_slowpath(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
u32 cached_flags = READ_ONCE(ti->flags);

CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
local_irq_enable();

rseq_syscall(regs);

/*
* First do one-time work. If these work items are enabled, we
* want to run them exactly once per syscall exit with IRQs on.
*/
if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
syscall_slow_exit_work(regs, cached_flags);

local_irq_disable();
__prepare_exit_to_usermode(regs);
}

/*
* Called with IRQs on and fully valid regs. Returns with IRQs off in a
* state such that we can immediately switch to user mode.
*/
__visible noinstr void syscall_return_slowpath(struct pt_regs *regs)
{
instrumentation_begin();
__syscall_return_slowpath(regs);
instrumentation_end();
exit_to_user_mode();
}

#ifdef CONFIG_X86_64
__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
{
Expand All @@ -245,7 +53,7 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
#endif
}
instrumentation_end();
syscall_return_slowpath(regs);
syscall_exit_to_user_mode(regs);
}
#endif

Expand Down Expand Up @@ -284,7 +92,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
unsigned int nr = syscall_32_enter(regs);

do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs);
syscall_exit_to_user_mode(regs);
}

static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
Expand All @@ -310,13 +118,13 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
if (res) {
/* User code screwed up. */
regs->ax = -EFAULT;
syscall_return_slowpath(regs);
syscall_exit_to_user_mode(regs);
return false;
}

/* Now this is just like a normal syscall. */
do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs);
syscall_exit_to_user_mode(regs);
return true;
}

Expand Down Expand Up @@ -524,7 +332,7 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)

/* Check whether this returns to user mode */
if (user_mode(regs)) {
prepare_exit_to_usermode(regs);
irqentry_exit_to_user_mode(regs);
} else if (regs->flags & X86_EFLAGS_IF) {
/*
* If RCU was not watching on entry this needs to be done
Expand Down Expand Up @@ -555,25 +363,6 @@ void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
}
}

/**
* idtentry_exit_user - Handle return from exception to user mode
* @regs: Pointer to pt_regs (exception entry regs)
*
* Runs the necessary preemption and work checks and returns to the caller
* with interrupts disabled and no further work pending.
*
* This is the last action before returning to the low level ASM code which
* just needs to return to the appropriate context.
*
* Counterpart to idtentry_enter_user().
*/
void noinstr idtentry_exit_user(struct pt_regs *regs)
{
lockdep_assert_irqs_disabled();

prepare_exit_to_usermode(regs);
}

#ifdef CONFIG_XEN_PV
#ifndef CONFIG_PREEMPTION
/*
Expand Down
2 changes: 1 addition & 1 deletion arch/x86/entry/entry_32.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -846,7 +846,7 @@ SYM_CODE_START(ret_from_fork)
2:
/* When we fork, we trace the syscall return in the child, too. */
movl %esp, %eax
call syscall_return_slowpath
call syscall_exit_to_user_mode
jmp .Lsyscall_32_done

/* kernel thread */
Expand Down
2 changes: 1 addition & 1 deletion arch/x86/entry/entry_64.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ SYM_CODE_START(ret_from_fork)
2:
UNWIND_HINT_REGS
movq %rsp, %rdi
call syscall_return_slowpath /* returns with IRQs disabled */
call syscall_exit_to_user_mode /* returns with IRQs disabled */
jmp swapgs_restore_regs_and_return_to_usermode

1:
Expand Down
44 changes: 44 additions & 0 deletions arch/x86/include/asm/entry-common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@
#ifndef _ASM_X86_ENTRY_COMMON_H
#define _ASM_X86_ENTRY_COMMON_H

#include <linux/user-return-notifier.h>

#include <asm/nospec-branch.h>
#include <asm/io_bitmap.h>
#include <asm/fpu/api.h>

/* Check that the stack and regs on entry from user mode are sane. */
static __always_inline void arch_check_user_regs(struct pt_regs *regs)
{
Expand Down Expand Up @@ -29,4 +35,42 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs)
}
#define arch_check_user_regs arch_check_user_regs

#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP)

static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
unsigned long ti_work)
{
if (ti_work & _TIF_USER_RETURN_NOTIFY)
fire_user_return_notifiers();

if (unlikely(ti_work & _TIF_IO_BITMAP))
tss_update_io_bitmap();

fpregs_assert_state_consistent();
if (unlikely(ti_work & _TIF_NEED_FPU_LOAD))
switch_fpu_return();

#ifdef CONFIG_COMPAT
/*
* Compat syscalls set TS_COMPAT. Make sure we clear it before
* returning to user mode. We need to clear it *after* signal
* handling, because syscall restart has a fixup for compat
* syscalls. The fixup is exercised by the ptrace_syscall_32
* selftest.
*
* We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
* special case only applies after poking regs and before the
* very next return to user mode.
*/
current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
#endif
}
#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare

static __always_inline void arch_exit_to_user_mode(void)
{
mds_user_clear_cpu_buffers();
}
#define arch_exit_to_user_mode arch_exit_to_user_mode

#endif
3 changes: 1 addition & 2 deletions arch/x86/include/asm/idtentry.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@

/* Temporary define */
#define idtentry_enter_user irqentry_enter_from_user_mode

void idtentry_exit_user(struct pt_regs *regs);
#define idtentry_exit_user irqentry_exit_to_user_mode

typedef struct idtentry_state {
bool exit_rcu;
Expand Down
1 change: 0 additions & 1 deletion arch/x86/include/asm/signal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ typedef sigset_t compat_sigset_t;
#endif /* __ASSEMBLY__ */
#include <uapi/asm/signal.h>
#ifndef __ASSEMBLY__
extern void do_signal(struct pt_regs *regs);

#define __ARCH_HAS_SA_RESTORER

Expand Down
3 changes: 2 additions & 1 deletion arch/x86/kernel/signal.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <linux/user-return-notifier.h>
#include <linux/uprobes.h>
#include <linux/context_tracking.h>
#include <linux/entry-common.h>
#include <linux/syscalls.h>

#include <asm/processor.h>
Expand Down Expand Up @@ -803,7 +804,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*/
void do_signal(struct pt_regs *regs)
void arch_do_signal(struct pt_regs *regs)
{
struct ksignal ksig;

Expand Down

0 comments on commit 167fd21

Please sign in to comment.