Skip to content

Commit

Permalink
Merge branches 'doc.2020.06.29a', 'fixes.2020.06.29a', 'kfree_rcu.202…
Browse files Browse the repository at this point in the history
…0.06.29a', 'rcu-tasks.2020.06.29a', 'scale.2020.06.29a', 'srcu.2020.06.29a' and 'torture.2020.06.29a' into HEAD

doc.2020.06.29a:  Documentation updates.
fixes.2020.06.29a:  Miscellaneous fixes.
kfree_rcu.2020.06.29a:  kfree_rcu() updates.
rcu-tasks.2020.06.29a:  RCU Tasks updates.
scale.2020.06.29a:  Read-side scalability tests.
srcu.2020.06.29a:  SRCU updates.
torture.2020.06.29a:  Torture-test updates.
  • Loading branch information
Paul E. McKenney committed Jun 29, 2020
7 parents d93d97c + 24692fa + da4fc00 + 30d8aa5 + f71d831 + bde50d8 + 7a6bbea commit 13625c0
Show file tree
Hide file tree
Showing 47 changed files with 1,886 additions and 267 deletions.
68 changes: 68 additions & 0 deletions Documentation/admin-guide/kernel-parameters.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4038,6 +4038,14 @@
latencies, which will choose a value aligned
with the appropriate hardware boundaries.

rcutree.rcu_min_cached_objs= [KNL]
Minimum number of objects which are cached and
maintained per one CPU. Object size is equal
to PAGE_SIZE. The cache allows to reduce the
pressure to page allocator, also it makes the
whole algorithm to behave better in low memory
condition.

rcutree.jiffies_till_first_fqs= [KNL]
Set delay from grace-period initialization to
first attempt to force quiescent states.
Expand Down Expand Up @@ -4258,6 +4266,20 @@
Set time (jiffies) between CPU-hotplug operations,
or zero to disable CPU-hotplug testing.

rcutorture.read_exit= [KNL]
Set the number of read-then-exit kthreads used
to test the interaction of RCU updaters and
task-exit processing.

rcutorture.read_exit_burst= [KNL]
The number of times in a given read-then-exit
episode that a set of read-then-exit kthreads
is spawned.

rcutorture.read_exit_delay= [KNL]
The delay, in seconds, between successive
read-then-exit testing episodes.

rcutorture.shuffle_interval= [KNL]
Set task-shuffle interval (s). Shuffling tasks
allows some CPUs to go into dyntick-idle mode
Expand Down Expand Up @@ -4407,6 +4429,45 @@
reboot_cpu is s[mp]#### with #### being the processor
to be used for rebooting.

refscale.holdoff= [KNL]
Set test-start holdoff period. The purpose of
this parameter is to delay the start of the
test until boot completes in order to avoid
interference.

refscale.loops= [KNL]
Set the number of loops over the synchronization
primitive under test. Increasing this number
reduces noise due to loop start/end overhead,
but the default has already reduced the per-pass
noise to a handful of picoseconds on ca. 2020
x86 laptops.

refscale.nreaders= [KNL]
Set number of readers. The default value of -1
selects N, where N is roughly 75% of the number
of CPUs. A value of zero is an interesting choice.

refscale.nruns= [KNL]
Set number of runs, each of which is dumped onto
the console log.

refscale.readdelay= [KNL]
Set the read-side critical-section duration,
measured in microseconds.

refscale.scale_type= [KNL]
Specify the read-protection implementation to test.

refscale.shutdown= [KNL]
Shut down the system at the end of the performance
test. This defaults to 1 (shut it down) when
rcuperf is built into the kernel and to 0 (leave
it running) when rcuperf is built as a module.

refscale.verbose= [KNL]
Enable additional printk() statements.

relax_domain_level=
[KNL, SMP] Set scheduler's default relax_domain_level.
See Documentation/admin-guide/cgroup-v1/cpusets.rst.
Expand Down Expand Up @@ -5082,6 +5143,13 @@
Prevent the CPU-hotplug component of torturing
until after init has spawned.

torture.ftrace_dump_at_shutdown= [KNL]
Dump the ftrace buffer at torture-test shutdown,
even if there were no errors. This can be a
very costly operation when many torture tests
are running concurrently, especially on systems
with rotating-rust storage.

tp720= [HW,PS2]

tpm_suspend_pcr=[HW,TPM]
Expand Down
2 changes: 2 additions & 0 deletions fs/btrfs/extent_io.c
Original file line number Diff line number Diff line change
Expand Up @@ -4515,6 +4515,8 @@ int try_release_extent_mapping(struct page *page, gfp_t mask)

/* once for us */
free_extent_map(em);

cond_resched(); /* Allow large-extent preemption. */
}
}
return try_release_extent_state(tree, page, mask);
Expand Down
2 changes: 1 addition & 1 deletion include/linux/rculist.h
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ static inline void hlist_replace_rcu(struct hlist_node *old,
* @right: The hlist head on the right
*
* The lists start out as [@left ][node1 ... ] and
[@right ][node2 ... ]
* [@right ][node2 ... ]
* The lists end up as [@left ][node2 ... ]
* [@right ][node1 ... ]
*/
Expand Down
53 changes: 46 additions & 7 deletions include/linux/rcupdate.h
Original file line number Diff line number Diff line change
Expand Up @@ -828,17 +828,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)

/*
* Does the specified offset indicate that the corresponding rcu_head
* structure can be handled by kfree_rcu()?
* structure can be handled by kvfree_rcu()?
*/
#define __is_kfree_rcu_offset(offset) ((offset) < 4096)
#define __is_kvfree_rcu_offset(offset) ((offset) < 4096)

/*
* Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
*/
#define __kfree_rcu(head, offset) \
#define __kvfree_rcu(head, offset) \
do { \
BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
kfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \
BUILD_BUG_ON(!__is_kvfree_rcu_offset(offset)); \
kvfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \
} while (0)

/**
Expand All @@ -857,7 +857,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
* Because the functions are not allowed in the low-order 4096 bytes of
* kernel virtual memory, offsets up to 4095 bytes can be accommodated.
* If the offset is larger than 4095 bytes, a compile-time error will
* be generated in __kfree_rcu(). If this error is triggered, you can
* be generated in __kvfree_rcu(). If this error is triggered, you can
* either fall back to use of call_rcu() or rearrange the structure to
* position the rcu_head structure into the first 4096 bytes.
*
Expand All @@ -872,7 +872,46 @@ do { \
typeof (ptr) ___p = (ptr); \
\
if (___p) \
__kfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \
__kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \
} while (0)

/**
* kvfree_rcu() - kvfree an object after a grace period.
*
* This macro consists of one or two arguments and it is
* based on whether an object is head-less or not. If it
* has a head then a semantic stays the same as it used
* to be before:
*
* kvfree_rcu(ptr, rhf);
*
* where @ptr is a pointer to kvfree(), @rhf is the name
* of the rcu_head structure within the type of @ptr.
*
* When it comes to head-less variant, only one argument
* is passed and that is just a pointer which has to be
* freed after a grace period. Therefore the semantic is
*
* kvfree_rcu(ptr);
*
* where @ptr is a pointer to kvfree().
*
* Please note, head-less way of freeing is permitted to
* use from a context that has to follow might_sleep()
* annotation. Otherwise, please switch and embed the
* rcu_head structure within the type of @ptr.
*/
#define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__, \
kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__)

#define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME
#define kvfree_rcu_arg_2(ptr, rhf) kfree_rcu(ptr, rhf)
#define kvfree_rcu_arg_1(ptr) \
do { \
typeof(ptr) ___p = (ptr); \
\
if (___p) \
kvfree_call_rcu(NULL, (rcu_callback_t) (___p)); \
} while (0)

/*
Expand Down
4 changes: 2 additions & 2 deletions include/linux/rcupdate_trace.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ void rcu_read_unlock_trace_special(struct task_struct *t, int nesting);
/**
* rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
*
* When synchronize_rcu_trace() is invoked by one task, then that task
* is guaranteed to block until all other tasks exit their read-side
* When synchronize_rcu_tasks_trace() is invoked by one task, then that
* task is guaranteed to block until all other tasks exit their read-side
* critical sections. Similarly, if call_rcu_trace() is invoked on one
* task while other tasks are within RCU read-side critical sections,
* invocation of the corresponding RCU callback is deferred until after
Expand Down
20 changes: 18 additions & 2 deletions include/linux/rcutiny.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,25 @@ static inline void synchronize_rcu_expedited(void)
synchronize_rcu();
}

static inline void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
/*
* Add one more declaration of kvfree() here. It is
* not so straight forward to just include <linux/mm.h>
* where it is defined due to getting many compile
* errors caused by that include.
*/
extern void kvfree(const void *addr);

static inline void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
{
call_rcu(head, func);
if (head) {
call_rcu(head, func);
return;
}

// kvfree_rcu(one_arg) call.
might_sleep();
synchronize_rcu();
kvfree((void *) func);
}

void rcu_qs(void);
Expand Down
2 changes: 1 addition & 1 deletion include/linux/rcutree.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ static inline void rcu_virt_note_context_switch(int cpu)
}

void synchronize_rcu_expedited(void);
void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func);
void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func);

void rcu_barrier(void);
bool rcu_eqs_special_set(int cpu);
Expand Down
5 changes: 5 additions & 0 deletions include/linux/torture.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ struct torture_random_state {
#define DEFINE_TORTURE_RANDOM_PERCPU(name) \
DEFINE_PER_CPU(struct torture_random_state, name)
unsigned long torture_random(struct torture_random_state *trsp);
static inline void torture_random_init(struct torture_random_state *trsp)
{
trsp->trs_state = 0;
trsp->trs_count = 0;
}

/* Task shuffler, which causes CPUs to occasionally go idle. */
void torture_shuffle_task_register(struct task_struct *tp);
Expand Down
19 changes: 10 additions & 9 deletions include/trace/events/rcu.h
Original file line number Diff line number Diff line change
Expand Up @@ -435,11 +435,12 @@ TRACE_EVENT_RCU(rcu_fqs,
#endif /* #if defined(CONFIG_TREE_RCU) */

/*
* Tracepoint for dyntick-idle entry/exit events. These take a string
* as argument: "Start" for entering dyntick-idle mode, "Startirq" for
* entering it from irq/NMI, "End" for leaving it, "Endirq" for leaving it
* to irq/NMI, "--=" for events moving towards idle, and "++=" for events
* moving away from idle.
* Tracepoint for dyntick-idle entry/exit events. These take 2 strings
* as argument:
* polarity: "Start", "End", "StillNonIdle" for entering, exiting or still not
* being in dyntick-idle mode.
* context: "USER" or "IDLE" or "IRQ".
* NMIs nested in IRQs are inferred with dynticks_nesting > 1 in IRQ context.
*
* These events also take a pair of numbers, which indicate the nesting
* depth before and after the event of interest, and a third number that is
Expand Down Expand Up @@ -506,13 +507,13 @@ TRACE_EVENT_RCU(rcu_callback,

/*
* Tracepoint for the registration of a single RCU callback of the special
* kfree() form. The first argument is the RCU type, the second argument
* kvfree() form. The first argument is the RCU type, the second argument
* is a pointer to the RCU callback, the third argument is the offset
* of the callback within the enclosing RCU-protected data structure,
* the fourth argument is the number of lazy callbacks queued, and the
* fifth argument is the total number of callbacks queued.
*/
TRACE_EVENT_RCU(rcu_kfree_callback,
TRACE_EVENT_RCU(rcu_kvfree_callback,

TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset,
long qlen),
Expand Down Expand Up @@ -596,12 +597,12 @@ TRACE_EVENT_RCU(rcu_invoke_callback,

/*
* Tracepoint for the invocation of a single RCU callback of the special
* kfree() form. The first argument is the RCU flavor, the second
* kvfree() form. The first argument is the RCU flavor, the second
* argument is a pointer to the RCU callback, and the third argument
* is the offset of the callback within the enclosing RCU-protected
* data structure.
*/
TRACE_EVENT_RCU(rcu_invoke_kfree_callback,
TRACE_EVENT_RCU(rcu_invoke_kvfree_callback,

TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),

Expand Down
4 changes: 1 addition & 3 deletions kernel/locking/lockdep.c
Original file line number Diff line number Diff line change
Expand Up @@ -5851,9 +5851,7 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
pr_warn("\n%srcu_scheduler_active = %d, debug_locks = %d\n",
!rcu_lockdep_current_cpu_online()
? "RCU used illegally from offline CPU!\n"
: !rcu_is_watching()
? "RCU used illegally from idle CPU!\n"
: "",
: "",
rcu_scheduler_active, debug_locks);

/*
Expand Down
14 changes: 7 additions & 7 deletions kernel/locking/locktorture.c
Original file line number Diff line number Diff line change
Expand Up @@ -631,13 +631,13 @@ static int lock_torture_writer(void *arg)
cxt.cur_ops->writelock();
if (WARN_ON_ONCE(lock_is_write_held))
lwsp->n_lock_fail++;
lock_is_write_held = 1;
lock_is_write_held = true;
if (WARN_ON_ONCE(lock_is_read_held))
lwsp->n_lock_fail++; /* rare, but... */

lwsp->n_lock_acquired++;
cxt.cur_ops->write_delay(&rand);
lock_is_write_held = 0;
lock_is_write_held = false;
cxt.cur_ops->writeunlock();

stutter_wait("lock_torture_writer");
Expand Down Expand Up @@ -665,13 +665,13 @@ static int lock_torture_reader(void *arg)
schedule_timeout_uninterruptible(1);

cxt.cur_ops->readlock();
lock_is_read_held = 1;
lock_is_read_held = true;
if (WARN_ON_ONCE(lock_is_write_held))
lrsp->n_lock_fail++; /* rare, but... */

lrsp->n_lock_acquired++;
cxt.cur_ops->read_delay(&rand);
lock_is_read_held = 0;
lock_is_read_held = false;
cxt.cur_ops->readunlock();

stutter_wait("lock_torture_reader");
Expand All @@ -686,7 +686,7 @@ static int lock_torture_reader(void *arg)
static void __torture_print_stats(char *page,
struct lock_stress_stats *statp, bool write)
{
bool fail = 0;
bool fail = false;
int i, n_stress;
long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
long long sum = 0;
Expand Down Expand Up @@ -904,7 +904,7 @@ static int __init lock_torture_init(void)

/* Initialize the statistics so that each run gets its own numbers. */
if (nwriters_stress) {
lock_is_write_held = 0;
lock_is_write_held = false;
cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
sizeof(*cxt.lwsa),
GFP_KERNEL);
Expand Down Expand Up @@ -935,7 +935,7 @@ static int __init lock_torture_init(void)
}

if (nreaders_stress) {
lock_is_read_held = 0;
lock_is_read_held = false;
cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
sizeof(*cxt.lrsa),
GFP_KERNEL);
Expand Down
19 changes: 19 additions & 0 deletions kernel/rcu/Kconfig.debug
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,25 @@ config RCU_TORTURE_TEST
Say M if you want the RCU torture tests to build as a module.
Say N if you are unsure.

config RCU_REF_SCALE_TEST
tristate "Scalability tests for read-side synchronization (RCU and others)"
depends on DEBUG_KERNEL
select TORTURE_TEST
select SRCU
select TASKS_RCU
select TASKS_RUDE_RCU
select TASKS_TRACE_RCU
default n
help
This option provides a kernel module that runs performance tests
useful comparing RCU with various read-side synchronization mechanisms.
The kernel module may be built after the fact on the running kernel to be
tested, if desired.

Say Y here if you want these performance tests built into the kernel.
Say M if you want to build it as a module instead.
Say N if you are unsure.

config RCU_CPU_STALL_TIMEOUT
int "RCU CPU stall timeout in seconds"
depends on RCU_STALL_COMMON
Expand Down
Loading

0 comments on commit 13625c0

Please sign in to comment.