Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
4facb95
Documentation
LICENSES
arch
alpha
arc
arm
arm64
c6x
csky
h8300
hexagon
ia64
m68k
microblaze
mips
nds32
nios2
openrisc
parisc
powerpc
riscv
s390
sh
sparc
um
x86
boot
configs
crypto
entry
syscalls
vdso
vsyscall
Makefile
calling.h
common.c
entry_32.S
entry_64.S
entry_64_compat.S
syscall_32.c
syscall_64.c
syscall_x32.c
thunk_32.S
thunk_64.S
events
hyperv
ia32
include
kernel
kvm
lib
math-emu
mm
net
oprofile
pci
platform
power
purgatory
ras
realmode
tools
um
video
xen
.gitignore
Kbuild
Kconfig
Kconfig.assembler
Kconfig.cpu
Kconfig.debug
Makefile
Makefile.um
Makefile_32.cpu
xtensa
.gitignore
Kconfig
block
certs
crypto
drivers
fs
include
init
ipc
kernel
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.clang-format
.cocciconfig
.get_maintainer.ignore
.gitattributes
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
Breadcrumbs
linux
/
arch
/
x86
/
entry
/
common.c
Blame
Blame
Latest commit
History
History
317 lines (274 loc) · 8.26 KB
Breadcrumbs
linux
/
arch
/
x86
/
entry
/
common.c
Top
File metadata and controls
Code
Blame
317 lines (274 loc) · 8.26 KB
Raw
// SPDX-License-Identifier: GPL-2.0-only /* * common.c - C code for kernel entry and exit * Copyright (c) 2015 Andrew Lutomirski * * Based on asm and ptrace code by many authors. The code here originated * in ptrace.c and signal.c. */ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> #include <linux/entry-common.h> #include <linux/mm.h> #include <linux/smp.h> #include <linux/errno.h> #include <linux/ptrace.h> #include <linux/export.h> #include <linux/nospec.h> #include <linux/syscalls.h> #include <linux/uaccess.h> #ifdef CONFIG_XEN_PV #include <xen/xen-ops.h> #include <xen/events.h> #endif #include <asm/desc.h> #include <asm/traps.h> #include <asm/vdso.h> #include <asm/cpufeature.h> #include <asm/fpu/api.h> #include <asm/nospec-branch.h> #include <asm/io_bitmap.h> #include <asm/syscall.h> #include <asm/irq_stack.h> #ifdef CONFIG_X86_64 __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) { nr = syscall_enter_from_user_mode(regs, nr); instrumentation_begin(); if (likely(nr < NR_syscalls)) { nr = array_index_nospec(nr, NR_syscalls); regs->ax = sys_call_table[nr](regs); #ifdef CONFIG_X86_X32_ABI } else if (likely((nr & __X32_SYSCALL_BIT) && (nr & ~__X32_SYSCALL_BIT) < X32_NR_syscalls)) { nr = array_index_nospec(nr & ~__X32_SYSCALL_BIT, X32_NR_syscalls); regs->ax = x32_sys_call_table[nr](regs); #endif } instrumentation_end(); syscall_exit_to_user_mode(regs); } #endif #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION) static __always_inline unsigned int syscall_32_enter(struct pt_regs *regs) { if (IS_ENABLED(CONFIG_IA32_EMULATION)) current_thread_info()->status |= TS_COMPAT; return (unsigned int)regs->orig_ax; } /* * Invoke a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL. */ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, unsigned int nr) { if (likely(nr < IA32_NR_syscalls)) { instrumentation_begin(); nr = array_index_nospec(nr, IA32_NR_syscalls); regs->ax = ia32_sys_call_table[nr](regs); instrumentation_end(); } } /* Handles int $0x80 */ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs) { unsigned int nr = syscall_32_enter(regs); /* * Subtlety here: if ptrace pokes something larger than 2^32-1 into * orig_ax, the unsigned int return value truncates it. This may * or may not be necessary, but it matches the old asm behavior. */ nr = (unsigned int)syscall_enter_from_user_mode(regs, nr); do_syscall_32_irqs_on(regs, nr); syscall_exit_to_user_mode(regs); } static noinstr bool __do_fast_syscall_32(struct pt_regs *regs) { unsigned int nr = syscall_32_enter(regs); int res; /* * This cannot use syscall_enter_from_user_mode() as it has to * fetch EBP before invoking any of the syscall entry work * functions. */ syscall_enter_from_user_mode_prepare(regs); instrumentation_begin(); /* Fetch EBP from where the vDSO stashed it. */ if (IS_ENABLED(CONFIG_X86_64)) { /* * Micro-optimization: the pointer we're following is * explicitly 32 bits, so it can't be out of range. */ res = __get_user(*(u32 *)®s->bp, (u32 __user __force *)(unsigned long)(u32)regs->sp); } else { res = get_user(*(u32 *)®s->bp, (u32 __user __force *)(unsigned long)(u32)regs->sp); } instrumentation_end(); if (res) { /* User code screwed up. */ regs->ax = -EFAULT; syscall_exit_to_user_mode(regs); return false; } /* The case truncates any ptrace induced syscall nr > 2^32 -1 */ nr = (unsigned int)syscall_enter_from_user_mode_work(regs, nr); /* Now this is just like a normal syscall. */ do_syscall_32_irqs_on(regs, nr); syscall_exit_to_user_mode(regs); return true; } /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */ __visible noinstr long do_fast_syscall_32(struct pt_regs *regs) { /* * Called using the internal vDSO SYSENTER/SYSCALL32 calling * convention. Adjust regs so it looks like we entered using int80. */ unsigned long landing_pad = (unsigned long)current->mm->context.vdso + vdso_image_32.sym_int80_landing_pad; /* * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward * so that 'regs->ip -= 2' lands back on an int $0x80 instruction. * Fix it up. */ regs->ip = landing_pad; /* Invoke the syscall. If it failed, keep it simple: use IRET. */ if (!__do_fast_syscall_32(regs)) return 0; #ifdef CONFIG_X86_64 /* * Opportunistic SYSRETL: if possible, try to return using SYSRETL. * SYSRETL is available on all 64-bit CPUs, so we don't need to * bother with SYSEXIT. * * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP, * because the ECX fixup above will ensure that this is essentially * never the case. */ return regs->cs == __USER32_CS && regs->ss == __USER_DS && regs->ip == landing_pad && (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0; #else /* * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT. * * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP, * because the ECX fixup above will ensure that this is essentially * never the case. * * We don't allow syscalls at all from VM86 mode, but we still * need to check VM, because we might be returning from sys_vm86. */ return static_cpu_has(X86_FEATURE_SEP) && regs->cs == __USER_CS && regs->ss == __USER_DS && regs->ip == landing_pad && (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0; #endif } /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */ __visible noinstr long do_SYSENTER_32(struct pt_regs *regs) { /* SYSENTER loses RSP, but the vDSO saved it in RBP. */ regs->sp = regs->bp; /* SYSENTER clobbers EFLAGS.IF. Assume it was set in usermode. */ regs->flags |= X86_EFLAGS_IF; return do_fast_syscall_32(regs); } #endif SYSCALL_DEFINE0(ni_syscall) { return -ENOSYS; } noinstr bool idtentry_enter_nmi(struct pt_regs *regs) { bool irq_state = lockdep_hardirqs_enabled(); __nmi_enter(); lockdep_hardirqs_off(CALLER_ADDR0); lockdep_hardirq_enter(); rcu_nmi_enter(); instrumentation_begin(); trace_hardirqs_off_finish(); ftrace_nmi_enter(); instrumentation_end(); return irq_state; } noinstr void idtentry_exit_nmi(struct pt_regs *regs, bool restore) { instrumentation_begin(); ftrace_nmi_exit(); if (restore) { trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(CALLER_ADDR0); } instrumentation_end(); rcu_nmi_exit(); lockdep_hardirq_exit(); if (restore) lockdep_hardirqs_on(CALLER_ADDR0); __nmi_exit(); } #ifdef CONFIG_XEN_PV #ifndef CONFIG_PREEMPTION /* * Some hypercalls issued by the toolstack can take many 10s of * seconds. Allow tasks running hypercalls via the privcmd driver to * be voluntarily preempted even if full kernel preemption is * disabled. * * Such preemptible hypercalls are bracketed by * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end() * calls. */ DEFINE_PER_CPU(bool, xen_in_preemptible_hcall); EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall); /* * In case of scheduling the flag must be cleared and restored after * returning from schedule as the task might move to a different CPU. */ static __always_inline bool get_and_clear_inhcall(void) { bool inhcall = __this_cpu_read(xen_in_preemptible_hcall); __this_cpu_write(xen_in_preemptible_hcall, false); return inhcall; } static __always_inline void restore_inhcall(bool inhcall) { __this_cpu_write(xen_in_preemptible_hcall, inhcall); } #else static __always_inline bool get_and_clear_inhcall(void) { return false; } static __always_inline void restore_inhcall(bool inhcall) { } #endif static void __xen_pv_evtchn_do_upcall(void) { irq_enter_rcu(); inc_irq_stat(irq_hv_callback_count); xen_hvm_evtchn_do_upcall(); irq_exit_rcu(); } __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) { struct pt_regs *old_regs; bool inhcall; irqentry_state_t state; state = irqentry_enter(regs); old_regs = set_irq_regs(regs); instrumentation_begin(); run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, NULL, regs); instrumentation_begin(); set_irq_regs(old_regs); inhcall = get_and_clear_inhcall(); if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { instrumentation_begin(); irqentry_exit_cond_resched(); instrumentation_end(); restore_inhcall(inhcall); } else { irqentry_exit(regs, state); } } #endif /* CONFIG_XEN_PV */
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
You can’t perform that action at this time.