Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
7f26482
Documentation
LICENSES
arch
block
certs
crypto
drivers
fs
include
init
ipc
kernel
bpf
cgroup
configs
debug
dma
events
gcov
irq
livepatch
locking
Makefile
lock_events.c
lock_events.h
lock_events_list.h
lockdep.c
lockdep_internals.h
lockdep_proc.c
lockdep_states.h
locktorture.c
mcs_spinlock.h
mutex-debug.c
mutex-debug.h
mutex.c
mutex.h
osq_lock.c
percpu-rwsem.c
qrwlock.c
qspinlock.c
qspinlock_paravirt.h
qspinlock_stat.h
rtmutex-debug.c
rtmutex-debug.h
rtmutex.c
rtmutex.h
rtmutex_common.h
rwsem.c
rwsem.h
semaphore.c
spinlock.c
spinlock_debug.c
test-ww_mutex.c
power
printk
rcu
sched
time
trace
.gitignore
Kconfig.freezer
Kconfig.hz
Kconfig.locks
Kconfig.preempt
Makefile
acct.c
async.c
audit.c
audit.h
audit_fsnotify.c
audit_tree.c
audit_watch.c
auditfilter.c
auditsc.c
backtracetest.c
bounds.c
capability.c
compat.c
configs.c
context_tracking.c
cpu.c
cpu_pm.c
crash_core.c
crash_dump.c
cred.c
delayacct.c
dma.c
elfcore.c
exec_domain.c
exit.c
extable.c
fail_function.c
fork.c
freezer.c
futex.c
gen_kheaders.sh
groups.c
hung_task.c
iomem.c
irq_work.c
jump_label.c
kallsyms.c
kcmp.c
kcov.c
kexec.c
kexec_core.c
kexec_elf.c
kexec_file.c
kexec_internal.h
kheaders.c
kmod.c
kprobes.c
ksysfs.c
kthread.c
latencytop.c
module-internal.h
module.c
module_signature.c
module_signing.c
notifier.c
nsproxy.c
padata.c
panic.c
params.c
pid.c
pid_namespace.c
profile.c
ptrace.c
range.c
reboot.c
relay.c
resource.c
rseq.c
seccomp.c
signal.c
smp.c
smpboot.c
smpboot.h
softirq.c
stackleak.c
stacktrace.c
stop_machine.c
sys.c
sys_ni.c
sysctl-test.c
sysctl.c
sysctl_binary.c
task_work.c
taskstats.c
test_kprobes.c
torture.c
tracepoint.c
tsacct.c
ucount.c
uid16.c
uid16.h
umh.c
up.c
user-return-notifier.c
user.c
user_namespace.c
utsname.c
utsname_sysctl.c
watchdog.c
watchdog_hld.c
workqueue.c
workqueue_internal.h
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.clang-format
.cocciconfig
.get_maintainer.ignore
.gitattributes
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
Breadcrumbs
linux
/
kernel
/
locking
/
percpu-rwsem.c
Blame
Blame
Latest commit
History
History
283 lines (238 loc) · 7.45 KB
Breadcrumbs
linux
/
kernel
/
locking
/
percpu-rwsem.c
Top
File metadata and controls
Code
Blame
283 lines (238 loc) · 7.45 KB
Raw
// SPDX-License-Identifier: GPL-2.0-only #include <linux/atomic.h> #include <linux/percpu.h> #include <linux/wait.h> #include <linux/lockdep.h> #include <linux/percpu-rwsem.h> #include <linux/rcupdate.h> #include <linux/sched.h> #include <linux/sched/task.h> #include <linux/errno.h> int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, const char *name, struct lock_class_key *key) { sem->read_count = alloc_percpu(int); if (unlikely(!sem->read_count)) return -ENOMEM; rcu_sync_init(&sem->rss); rcuwait_init(&sem->writer); init_waitqueue_head(&sem->waiters); atomic_set(&sem->block, 0); #ifdef CONFIG_DEBUG_LOCK_ALLOC debug_check_no_locks_freed((void *)sem, sizeof(*sem)); lockdep_init_map(&sem->dep_map, name, key, 0); #endif return 0; } EXPORT_SYMBOL_GPL(__percpu_init_rwsem); void percpu_free_rwsem(struct percpu_rw_semaphore *sem) { /* * XXX: temporary kludge. The error path in alloc_super() * assumes that percpu_free_rwsem() is safe after kzalloc(). */ if (!sem->read_count) return; rcu_sync_dtor(&sem->rss); free_percpu(sem->read_count); sem->read_count = NULL; /* catch use after free bugs */ } EXPORT_SYMBOL_GPL(percpu_free_rwsem); static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem) { __this_cpu_inc(*sem->read_count); /* * Due to having preemption disabled the decrement happens on * the same CPU as the increment, avoiding the * increment-on-one-CPU-and-decrement-on-another problem. * * If the reader misses the writer's assignment of sem->block, then the * writer is guaranteed to see the reader's increment. * * Conversely, any readers that increment their sem->read_count after * the writer looks are guaranteed to see the sem->block value, which * in turn means that they are guaranteed to immediately decrement * their sem->read_count, so that it doesn't matter that the writer * missed them. */ smp_mb(); /* A matches D */ /* * If !sem->block the critical section starts here, matched by the * release in percpu_up_write(). */ if (likely(!atomic_read_acquire(&sem->block))) return true; __this_cpu_dec(*sem->read_count); /* Prod writer to re-evaluate readers_active_check() */ rcuwait_wake_up(&sem->writer); return false; } static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem) { if (atomic_read(&sem->block)) return false; return atomic_xchg(&sem->block, 1) == 0; } static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader) { if (reader) { bool ret; preempt_disable(); ret = __percpu_down_read_trylock(sem); preempt_enable(); return ret; } return __percpu_down_write_trylock(sem); } /* * The return value of wait_queue_entry::func means: * * <0 - error, wakeup is terminated and the error is returned * 0 - no wakeup, a next waiter is tried * >0 - woken, if EXCLUSIVE, counted towards @nr_exclusive. * * We use EXCLUSIVE for both readers and writers to preserve FIFO order, * and play games with the return value to allow waking multiple readers. * * Specifically, we wake readers until we've woken a single writer, or until a * trylock fails. */ static int percpu_rwsem_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, int wake_flags, void *key) { struct task_struct *p = get_task_struct(wq_entry->private); bool reader = wq_entry->flags & WQ_FLAG_CUSTOM; struct percpu_rw_semaphore *sem = key; /* concurrent against percpu_down_write(), can get stolen */ if (!__percpu_rwsem_trylock(sem, reader)) return 1; list_del_init(&wq_entry->entry); smp_store_release(&wq_entry->private, NULL); wake_up_process(p); put_task_struct(p); return !reader; /* wake (readers until) 1 writer */ } static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader) { DEFINE_WAIT_FUNC(wq_entry, percpu_rwsem_wake_function); bool wait; spin_lock_irq(&sem->waiters.lock); /* * Serialize against the wakeup in percpu_up_write(), if we fail * the trylock, the wakeup must see us on the list. */ wait = !__percpu_rwsem_trylock(sem, reader); if (wait) { wq_entry.flags |= WQ_FLAG_EXCLUSIVE | reader * WQ_FLAG_CUSTOM; __add_wait_queue_entry_tail(&sem->waiters, &wq_entry); } spin_unlock_irq(&sem->waiters.lock); while (wait) { set_current_state(TASK_UNINTERRUPTIBLE); if (!smp_load_acquire(&wq_entry.private)) break; schedule(); } __set_current_state(TASK_RUNNING); } bool __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) { if (__percpu_down_read_trylock(sem)) return true; if (try) return false; preempt_enable(); percpu_rwsem_wait(sem, /* .reader = */ true); preempt_disable(); return true; } EXPORT_SYMBOL_GPL(__percpu_down_read); void __percpu_up_read(struct percpu_rw_semaphore *sem) { smp_mb(); /* B matches C */ /* * In other words, if they see our decrement (presumably to aggregate * zero, as that is the only time it matters) they will also see our * critical section. */ __this_cpu_dec(*sem->read_count); /* Prod writer to re-evaluate readers_active_check() */ rcuwait_wake_up(&sem->writer); } EXPORT_SYMBOL_GPL(__percpu_up_read); #define per_cpu_sum(var) \ ({ \ typeof(var) __sum = 0; \ int cpu; \ compiletime_assert_atomic_type(__sum); \ for_each_possible_cpu(cpu) \ __sum += per_cpu(var, cpu); \ __sum; \ }) /* * Return true if the modular sum of the sem->read_count per-CPU variable is * zero. If this sum is zero, then it is stable due to the fact that if any * newly arriving readers increment a given counter, they will immediately * decrement that same counter. * * Assumes sem->block is set. */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { if (per_cpu_sum(*sem->read_count) != 0) return false; /* * If we observed the decrement; ensure we see the entire critical * section. */ smp_mb(); /* C matches B */ return true; } void percpu_down_write(struct percpu_rw_semaphore *sem) { rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); /* Notify readers to take the slow path. */ rcu_sync_enter(&sem->rss); /* * Try set sem->block; this provides writer-writer exclusion. * Having sem->block set makes new readers block. */ if (!__percpu_down_write_trylock(sem)) percpu_rwsem_wait(sem, /* .reader = */ false); /* smp_mb() implied by __percpu_down_write_trylock() on success -- D matches A */ /* * If they don't see our store of sem->block, then we are guaranteed to * see their sem->read_count increment, and therefore will wait for * them. */ /* Wait for all active readers to complete. */ rcuwait_wait_event(&sem->writer, readers_active_check(sem)); } EXPORT_SYMBOL_GPL(percpu_down_write); void percpu_up_write(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); /* * Signal the writer is done, no fast path yet. * * One reason that we cannot just immediately flip to readers_fast is * that new readers might fail to see the results of this writer's * critical section. * * Therefore we force it through the slow path which guarantees an * acquire and thereby guarantees the critical section's consistency. */ atomic_set_release(&sem->block, 0); /* * Prod any pending reader/writer to make progress. */ __wake_up(&sem->waiters, TASK_NORMAL, 1, sem); /* * Once this completes (at least one RCU-sched grace period hence) the * reader fast path will be available again. Safe to use outside the * exclusive write lock because its counting. */ rcu_sync_exit(&sem->rss); } EXPORT_SYMBOL_GPL(percpu_up_write);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
You can’t perform that action at this time.