Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
72f40a2
Documentation
LICENSES
arch
alpha
arc
arm
arm64
c6x
csky
h8300
hexagon
ia64
m68k
microblaze
mips
nds32
nios2
openrisc
parisc
powerpc
riscv
s390
sh
sparc
um
x86
boot
configs
crypto
entry
events
hyperv
ia32
include
asm
crypto
e820
fpu
numachip
trace
uv
vdso
xen
GEN-for-each-reg.h
Kbuild
acenv.h
acpi.h
agp.h
alternative-asm.h
alternative.h
amd_nb.h
apb_timer.h
apic.h
apicdef.h
apm.h
arch_hweight.h
archrandom.h
asm-offsets.h
asm-prototypes.h
asm.h
atomic.h
atomic64_32.h
atomic64_64.h
audit.h
barrier.h
bios_ebda.h
bitops.h
boot.h
bootparam_utils.h
bug.h
bugs.h
cache.h
cacheflush.h
cacheinfo.h
ce4100.h
checksum.h
checksum_32.h
checksum_64.h
clocksource.h
cmdline.h
cmpxchg.h
cmpxchg_32.h
cmpxchg_64.h
compat.h
cpu.h
cpu_device_id.h
cpu_entry_area.h
cpufeature.h
cpufeatures.h
cpuidle_haltpoll.h
cpumask.h
crash.h
current.h
debugreg.h
delay.h
desc.h
desc_defs.h
device.h
disabled-features.h
div64.h
dma-mapping.h
dma.h
dmi.h
doublefault.h
dwarf2.h
edac.h
efi.h
elf.h
emergency-restart.h
emulate_prefix.h
enclu.h
entry-common.h
espfix.h
exec.h
extable.h
fb.h
fixmap.h
floppy.h
frame.h
fsgsbase.h
ftrace.h
futex.h
gart.h
genapic.h
geode.h
hardirq.h
highmem.h
hpet.h
hugetlb.h
hw_breakpoint.h
hw_irq.h
hyperv-tlfs.h
hypervisor.h
i8259.h
ia32.h
ia32_unistd.h
idtentry.h
imr.h
inat.h
inat_types.h
init.h
insn-eval.h
insn.h
inst.h
intel-family.h
intel-mid.h
intel_ds.h
intel_mid_vrtc.h
intel_pconfig.h
intel_pt.h
intel_punit_ipc.h
intel_scu_ipc.h
intel_scu_ipc_legacy.h
intel_telemetry.h
invpcid.h
io.h
io_apic.h
io_bitmap.h
iomap.h
iommu.h
iommu_table.h
iosf_mbi.h
irq.h
irq_remapping.h
irq_stack.h
irq_vectors.h
irq_work.h
irqdomain.h
irqflags.h
ist.h
jailhouse_para.h
jump_label.h
kasan.h
kaslr.h
kbdleds.h
kdebug.h
kexec-bzimage64.h
kexec.h
kgdb.h
kprobes.h
kvm_host.h
kvm_page_track.h
kvm_para.h
kvm_types.h
kvm_vcpu_regs.h
kvmclock.h
linkage.h
livepatch.h
local.h
mach_timer.h
mach_traps.h
math_emu.h
mc146818rtc.h
mce.h
mem_encrypt.h
memtype.h
microcode.h
microcode_amd.h
microcode_intel.h
misc.h
mmconfig.h
mmu.h
mmu_context.h
mmx.h
mmzone.h
mmzone_32.h
mmzone_64.h
module.h
mpspec.h
mpspec_def.h
mshyperv.h
msi.h
msr-index.h
msr-trace.h
msr.h
mtrr.h
mwait.h
nmi.h
nops.h
nospec-branch.h
numa.h
numa_32.h
olpc.h
olpc_ofw.h
orc_lookup.h
orc_types.h
page.h
page_32.h
page_32_types.h
page_64.h
page_64_types.h
page_types.h
paravirt.h
paravirt_types.h
parport.h
pci-direct.h
pci-functions.h
pci.h
pci_x86.h
percpu.h
perf_event.h
perf_event_p4.h
pgalloc.h
pgtable-2level.h
pgtable-2level_types.h
pgtable-3level.h
pgtable-3level_types.h
pgtable-invert.h
pgtable.h
pgtable_32.h
pgtable_32_areas.h
pgtable_32_types.h
pgtable_64.h
pgtable_64_types.h
pgtable_areas.h
pgtable_types.h
pkeys.h
platform_sst_audio.h
pm-trace.h
posix_types.h
preempt.h
probe_roms.h
processor-cyrix.h
processor-flags.h
processor.h
prom.h
proto.h
pti.h
ptrace.h
purgatory.h
pvclock-abi.h
pvclock.h
qrwlock.h
qspinlock.h
qspinlock_paravirt.h
realmode.h
reboot.h
reboot_fixups.h
required-features.h
resctrl.h
rmwcc.h
seccomp.h
sections.h
segment.h
serial.h
set_memory.h
setup.h
setup_arch.h
sev-es.h
shmparam.h
sigcontext.h
sigframe.h
sighandling.h
signal.h
simd.h
smap.h
smp.h
softirq_stack.h
sparsemem.h
spec-ctrl.h
special_insns.h
spinlock.h
spinlock_types.h
sta2x11.h
stackprotector.h
stacktrace.h
static_call.h
string.h
string_32.h
string_64.h
suspend.h
suspend_32.h
suspend_64.h
svm.h
swiotlb.h
switch_to.h
sync_bitops.h
sync_core.h
syscall.h
syscall_wrapper.h
syscalls.h
sysfb.h
text-patching.h
thread_info.h
time.h
timer.h
timex.h
tlb.h
tlbbatch.h
tlbflush.h
topology.h
trace_clock.h
trap_pf.h
trapnr.h
traps.h
tsc.h
uaccess.h
uaccess_32.h
uaccess_64.h
umip.h
unaligned.h
unistd.h
unwind.h
unwind_hints.h
uprobes.h
user.h
user32.h
user_32.h
user_64.h
vdso.h
vermagic.h
vga.h
vgtod.h
virtext.h
vm86.h
vmalloc.h
vmware.h
vmx.h
vmxfeatures.h
vsyscall.h
vvar.h
word-at-a-time.h
x86_init.h
xor.h
xor_32.h
xor_64.h
xor_avx.h
uapi
kernel
kvm
lib
math-emu
mm
net
oprofile
pci
platform
power
purgatory
ras
realmode
tools
um
video
xen
.gitignore
Kbuild
Kconfig
Kconfig.assembler
Kconfig.cpu
Kconfig.debug
Makefile
Makefile.um
Makefile_32.cpu
xtensa
.gitignore
Kconfig
block
certs
crypto
drivers
fs
include
init
ipc
kernel
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.clang-format
.cocciconfig
.get_maintainer.ignore
.gitattributes
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
Breadcrumbs
linux
/
arch
/
x86
/
include
/
asm
/
irq_stack.h
Blame
Blame
Latest commit
History
History
223 lines (204 loc) · 7.08 KB
Breadcrumbs
linux
/
arch
/
x86
/
include
/
asm
/
irq_stack.h
Top
File metadata and controls
Code
Blame
223 lines (204 loc) · 7.08 KB
Raw
/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_IRQ_STACK_H #define _ASM_X86_IRQ_STACK_H #include <linux/ptrace.h> #include <asm/processor.h> #ifdef CONFIG_X86_64 /* * Macro to inline switching to an interrupt stack and invoking function * calls from there. The following rules apply: * * - Ordering: * * 1. Write the stack pointer into the top most place of the irq * stack. This ensures that the various unwinders can link back to the * original stack. * * 2. Switch the stack pointer to the top of the irq stack. * * 3. Invoke whatever needs to be done (@asm_call argument) * * 4. Pop the original stack pointer from the top of the irq stack * which brings it back to the original stack where it left off. * * - Function invocation: * * To allow flexible usage of the macro, the actual function code including * the store of the arguments in the call ABI registers is handed in via * the @asm_call argument. * * - Local variables: * * @tos: * The @tos variable holds a pointer to the top of the irq stack and * _must_ be allocated in a non-callee saved register as this is a * restriction coming from objtool. * * Note, that (tos) is both in input and output constraints to ensure * that the compiler does not assume that R11 is left untouched in * case this macro is used in some place where the per cpu interrupt * stack pointer is used again afterwards * * - Function arguments: * The function argument(s), if any, have to be defined in register * variables at the place where this is invoked. Storing the * argument(s) in the proper register(s) is part of the @asm_call * * - Constraints: * * The constraints have to be done very carefully because the compiler * does not know about the assembly call. * * output: * As documented already above the @tos variable is required to be in * the output constraints to make the compiler aware that R11 cannot be * reused after the asm() statement. * * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is * required as well as this prevents certain creative GCC variants from * misplacing the ASM code. * * input: * - func: * Immediate, which tells the compiler that the function is referenced. * * - tos: * Register. The actual register is defined by the variable declaration. * * - function arguments: * The constraints are handed in via the 'argconstr' argument list. They * describe the register arguments which are used in @asm_call. * * clobbers: * Function calls can clobber anything except the callee-saved * registers. Tell the compiler. */ #define call_on_irqstack(func, asm_call, argconstr...) \ { \ register void *tos asm("r11"); \ \ tos = ((void *)__this_cpu_read(hardirq_stack_ptr)); \ \ asm_inline volatile( \ "movq %%rsp, (%[tos]) \n" \ "movq %[tos], %%rsp \n" \ \ asm_call \ \ "popq %%rsp \n" \ \ : "+r" (tos), ASM_CALL_CONSTRAINT \ : [__func] "i" (func), [tos] "r" (tos) argconstr \ : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \ "memory" \ ); \ } /* Macros to assert type correctness for run_*_on_irqstack macros */ #define assert_function_type(func, proto) \ static_assert(__builtin_types_compatible_p(typeof(&func), proto)) #define assert_arg_type(arg, proto) \ static_assert(__builtin_types_compatible_p(typeof(arg), proto)) /* * Macro to invoke system vector and device interrupt C handlers. */ #define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \ { \ /* \ * User mode entry and interrupt on the irq stack do not \ * switch stacks. If from user mode the task stack is empty. \ */ \ if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \ irq_enter_rcu(); \ func(c_args); \ irq_exit_rcu(); \ } else { \ /* \ * Mark the irq stack inuse _before_ and unmark _after_ \ * switching stacks. Interrupts are disabled in both \ * places. Invoke the stack switch macro with the call \ * sequence which matches the above direct invocation. \ */ \ __this_cpu_write(hardirq_stack_inuse, true); \ call_on_irqstack(func, asm_call, constr); \ __this_cpu_write(hardirq_stack_inuse, false); \ } \ } /* * Function call sequence for __call_on_irqstack() for system vectors. * * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input * mechanism because these functions are global and cannot be optimized out * when compiling a particular source file which uses one of these macros. * * The argument (regs) does not need to be pushed or stashed in a callee * saved register to be safe vs. the irq_enter_rcu() call because the * clobbers already prevent the compiler from storing it in a callee * clobbered register. As the compiler has to preserve @regs for the final * call to idtentry_exit() anyway, it's likely that it does not cause extra * effort for this asm magic. */ #define ASM_CALL_SYSVEC \ "call irq_enter_rcu \n" \ "movq %[arg1], %%rdi \n" \ "call %P[__func] \n" \ "call irq_exit_rcu \n" #define SYSVEC_CONSTRAINTS , [arg1] "r" (regs) #define run_sysvec_on_irqstack_cond(func, regs) \ { \ assert_function_type(func, void (*)(struct pt_regs *)); \ assert_arg_type(regs, struct pt_regs *); \ \ call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \ SYSVEC_CONSTRAINTS, regs); \ } /* * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store * @regs and @vector in callee saved registers. */ #define ASM_CALL_IRQ \ "call irq_enter_rcu \n" \ "movq %[arg1], %%rdi \n" \ "movl %[arg2], %%esi \n" \ "call %P[__func] \n" \ "call irq_exit_rcu \n" #define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector) #define run_irq_on_irqstack_cond(func, regs, vector) \ { \ assert_function_type(func, void (*)(struct pt_regs *, u32)); \ assert_arg_type(regs, struct pt_regs *); \ assert_arg_type(vector, u32); \ \ call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \ IRQ_CONSTRAINTS, regs, vector); \ } #define ASM_CALL_SOFTIRQ \ "call %P[__func] \n" /* * Macro to invoke __do_softirq on the irq stack. This is only called from * task context when bottom halfs are about to be reenabled and soft * interrupts are pending to be processed. The interrupt stack cannot be in * use here. */ #define do_softirq_own_stack() \ { \ __this_cpu_write(hardirq_stack_inuse, true); \ call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ __this_cpu_write(hardirq_stack_inuse, false); \ } #else /* CONFIG_X86_64 */ /* System vector handlers always run on the stack they interrupted. */ #define run_sysvec_on_irqstack_cond(func, regs) \ { \ irq_enter_rcu(); \ func(regs); \ irq_exit_rcu(); \ } /* Switches to the irq stack within func() */ #define run_irq_on_irqstack_cond(func, regs, vector) \ { \ irq_enter_rcu(); \ func(regs, vector); \ irq_exit_rcu(); \ } #endif /* !CONFIG_X86_64 */ #endif
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
You can’t perform that action at this time.