From 66d50da2ae04bf5182468814502c0822223ccc8a Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 29 Feb 2008 18:46:50 +0100 Subject: [PATCH] --- yaml --- r: 86601 b: refs/heads/master c: 2232c2d8e0a6a31061dec311f3d1cf7624bc14f1 h: refs/heads/master i: 86599: 2e37dab2f1499d38f7b12d8f43d0ed0610cc8b69 v: v3 --- [refs] | 2 +- .../feature-removal-schedule.txt | 12 - trunk/arch/x86/kernel/init_task.c | 1 - trunk/arch/x86/kernel/process_32.c | 2 - trunk/arch/x86/kernel/process_64.c | 2 - trunk/arch/x86/kernel/ptrace.c | 12 - trunk/arch/x86/kernel/tls.c | 8 +- trunk/arch/x86/kernel/vsyscall_64.c | 11 +- trunk/arch/x86/mm/ioremap.c | 2 +- trunk/arch/x86/mm/pageattr.c | 21 +- trunk/arch/x86/vdso/Makefile | 6 +- trunk/arch/x86/xen/enlighten.c | 1 - trunk/include/asm-x86/pgtable_32.h | 4 +- trunk/include/asm-x86/pgtable_64.h | 6 +- trunk/include/asm-x86/ptrace-abi.h | 8 +- trunk/include/linux/hardirq.h | 10 + trunk/include/linux/rcuclassic.h | 3 + trunk/include/linux/rcupreempt.h | 22 ++ trunk/kernel/rcupreempt.c | 224 +++++++++++++++++- trunk/kernel/softirq.c | 1 + trunk/kernel/time/tick-sched.c | 3 + 21 files changed, 283 insertions(+), 78 deletions(-) diff --git a/[refs] b/[refs] index 818b8e3e6f8f..6592fb16c44c 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: d40e705903397445c6861a0a56c23e5b2e8f9b9a +refs/heads/master: 2232c2d8e0a6a31061dec311f3d1cf7624bc14f1 diff --git a/trunk/Documentation/feature-removal-schedule.txt b/trunk/Documentation/feature-removal-schedule.txt index c1d1fd0c299b..ba899ff2a8f9 100644 --- a/trunk/Documentation/feature-removal-schedule.txt +++ b/trunk/Documentation/feature-removal-schedule.txt @@ -316,15 +316,3 @@ Why: Largely unmaintained and almost entirely unused. File system is largely pointless as without a lot of work only the most trivial of Solaris binaries can work with the emulation code. Who: David S. Miller - ---------------------------- - -What: init_mm export -When: 2.6.26 -Why: Not used in-tree. The current out-of-tree users used it to - work around problems in the CPA code which should be resolved - by now. One usecase was described to provide verification code - of the CPA operation. That's a good idea in general, but such - code / infrastructure should be in the kernel and not in some - out-of-tree driver. -Who: Thomas Gleixner diff --git a/trunk/arch/x86/kernel/init_task.c b/trunk/arch/x86/kernel/init_task.c index 3d01e47777db..5b3ce7934363 100644 --- a/trunk/arch/x86/kernel/init_task.c +++ b/trunk/arch/x86/kernel/init_task.c @@ -15,7 +15,6 @@ static struct files_struct init_files = INIT_FILES; static struct signal_struct init_signals = INIT_SIGNALS(init_signals); static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand); struct mm_struct init_mm = INIT_MM(init_mm); -EXPORT_UNUSED_SYMBOL(init_mm); /* will be removed in 2.6.26 */ /* * Initial thread structure. diff --git a/trunk/arch/x86/kernel/process_32.c b/trunk/arch/x86/kernel/process_32.c index be3c7a299f02..a7d50a547dc2 100644 --- a/trunk/arch/x86/kernel/process_32.c +++ b/trunk/arch/x86/kernel/process_32.c @@ -603,13 +603,11 @@ __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, } #endif -#ifdef X86_BTS if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS); if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS)) ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES); -#endif if (!test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { diff --git a/trunk/arch/x86/kernel/process_64.c b/trunk/arch/x86/kernel/process_64.c index 3baf9b9f4c87..43f287744f9f 100644 --- a/trunk/arch/x86/kernel/process_64.c +++ b/trunk/arch/x86/kernel/process_64.c @@ -604,13 +604,11 @@ static inline void __switch_to_xtra(struct task_struct *prev_p, memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); } -#ifdef X86_BTS if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS); if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS)) ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES); -#endif } /* diff --git a/trunk/arch/x86/kernel/ptrace.c b/trunk/arch/x86/kernel/ptrace.c index f41fdc98efb1..d862e396b099 100644 --- a/trunk/arch/x86/kernel/ptrace.c +++ b/trunk/arch/x86/kernel/ptrace.c @@ -544,8 +544,6 @@ static int ptrace_set_debugreg(struct task_struct *child, return 0; } -#ifdef X86_BTS - static int ptrace_bts_get_size(struct task_struct *child) { if (!child->thread.ds_area_msr) @@ -828,7 +826,6 @@ void ptrace_bts_take_timestamp(struct task_struct *tsk, ptrace_bts_write_record(tsk, &rec); } -#endif /* X86_BTS */ /* * Called by kernel/ptrace.c when detaching.. @@ -842,9 +839,7 @@ void ptrace_disable(struct task_struct *child) clear_tsk_thread_flag(child, TIF_SYSCALL_EMU); #endif if (child->thread.ds_area_msr) { -#ifdef X86_BTS ptrace_bts_realloc(child, 0, 0); -#endif child->thread.debugctlmsr &= ~ds_debugctl_mask(); if (!child->thread.debugctlmsr) clear_tsk_thread_flag(child, TIF_DEBUGCTLMSR); @@ -966,10 +961,6 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data) break; #endif - /* - * These bits need more cooking - not enabled yet: - */ -#ifdef X86_BTS case PTRACE_BTS_CONFIG: ret = ptrace_bts_config (child, data, (struct ptrace_bts_config __user *)addr); @@ -997,7 +988,6 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data) ret = ptrace_bts_drain (child, data, (struct bts_struct __user *) addr); break; -#endif default: ret = ptrace_request(child, request, addr, data); @@ -1236,14 +1226,12 @@ asmlinkage long sys32_ptrace(long request, u32 pid, u32 addr, u32 data) case PTRACE_SETOPTIONS: case PTRACE_SET_THREAD_AREA: case PTRACE_GET_THREAD_AREA: -#ifdef X86_BTS case PTRACE_BTS_CONFIG: case PTRACE_BTS_STATUS: case PTRACE_BTS_SIZE: case PTRACE_BTS_GET: case PTRACE_BTS_CLEAR: case PTRACE_BTS_DRAIN: -#endif return sys_ptrace(request, pid, addr, data); default: diff --git a/trunk/arch/x86/kernel/tls.c b/trunk/arch/x86/kernel/tls.c index 022bcaa3b42e..6dfd4e76661a 100644 --- a/trunk/arch/x86/kernel/tls.c +++ b/trunk/arch/x86/kernel/tls.c @@ -91,9 +91,7 @@ int do_set_thread_area(struct task_struct *p, int idx, asmlinkage int sys_set_thread_area(struct user_desc __user *u_info) { - int ret = do_set_thread_area(current, -1, u_info, 1); - prevent_tail_call(ret); - return ret; + return do_set_thread_area(current, -1, u_info, 1); } @@ -141,9 +139,7 @@ int do_get_thread_area(struct task_struct *p, int idx, asmlinkage int sys_get_thread_area(struct user_desc __user *u_info) { - int ret = do_get_thread_area(current, -1, u_info); - prevent_tail_call(ret); - return ret; + return do_get_thread_area(current, -1, u_info); } int regset_tls_active(struct task_struct *target, diff --git a/trunk/arch/x86/kernel/vsyscall_64.c b/trunk/arch/x86/kernel/vsyscall_64.c index edff4c985485..b6be812fac05 100644 --- a/trunk/arch/x86/kernel/vsyscall_64.c +++ b/trunk/arch/x86/kernel/vsyscall_64.c @@ -222,19 +222,10 @@ long __vsyscall(3) venosys_1(void) } #ifdef CONFIG_SYSCTL - -static int -vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp, - void __user *buffer, size_t *lenp, loff_t *ppos) -{ - return proc_dointvec(ctl, write, filp, buffer, lenp, ppos); -} - static ctl_table kernel_table2[] = { { .procname = "vsyscall64", .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = vsyscall_sysctl_change }, + .mode = 0644 }, {} }; diff --git a/trunk/arch/x86/mm/ioremap.c b/trunk/arch/x86/mm/ioremap.c index ac3c959e271d..882328efc3db 100644 --- a/trunk/arch/x86/mm/ioremap.c +++ b/trunk/arch/x86/mm/ioremap.c @@ -162,7 +162,7 @@ static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size, area->phys_addr = phys_addr; vaddr = (unsigned long) area->addr; if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) { - free_vm_area(area); + remove_vm_area((void *)(vaddr & PAGE_MASK)); return NULL; } diff --git a/trunk/arch/x86/mm/pageattr.c b/trunk/arch/x86/mm/pageattr.c index 7049294fb469..14e48b5a94ba 100644 --- a/trunk/arch/x86/mm/pageattr.c +++ b/trunk/arch/x86/mm/pageattr.c @@ -26,7 +26,6 @@ struct cpa_data { pgprot_t mask_set; pgprot_t mask_clr; int numpages; - int processed; int flushtlb; unsigned long pfn; }; @@ -291,8 +290,8 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, */ nextpage_addr = (address + psize) & pmask; numpages = (nextpage_addr - address) >> PAGE_SHIFT; - if (numpages < cpa->processed) - cpa->processed = numpages; + if (numpages < cpa->numpages) + cpa->numpages = numpages; /* * We are safe now. Check whether the new pgprot is the same: @@ -319,7 +318,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, */ addr = address + PAGE_SIZE; pfn++; - for (i = 1; i < cpa->processed; i++, addr += PAGE_SIZE, pfn++) { + for (i = 1; i < cpa->numpages; i++, addr += PAGE_SIZE, pfn++) { pgprot_t chk_prot = static_protections(new_prot, addr, pfn); if (pgprot_val(chk_prot) != pgprot_val(new_prot)) @@ -343,7 +342,7 @@ try_preserve_large_page(pte_t *kpte, unsigned long address, * that we limited the number of possible pages already to * the number of pages in the large page. */ - if (address == (nextpage_addr - psize) && cpa->processed == numpages) { + if (address == (nextpage_addr - psize) && cpa->numpages == numpages) { /* * The address is aligned and the number of pages * covers the full page. @@ -573,7 +572,7 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) set_pte_atomic(kpte, new_pte); cpa->flushtlb = 1; } - cpa->processed = 1; + cpa->numpages = 1; return 0; } @@ -584,7 +583,7 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) do_split = try_preserve_large_page(kpte, address, cpa); /* * When the range fits into the existing large page, - * return. cp->processed and cpa->tlbflush have been updated in + * return. cp->numpages and cpa->tlbflush have been updated in * try_large_page: */ if (do_split <= 0) @@ -663,7 +662,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) * Store the remaining nr of pages for the large page * preservation check. */ - cpa->numpages = cpa->processed = numpages; + cpa->numpages = numpages; ret = __change_page_attr(cpa, checkalias); if (ret) @@ -680,9 +679,9 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) * CPA operation. Either a large page has been * preserved or a single page update happened. */ - BUG_ON(cpa->processed > numpages); - numpages -= cpa->processed; - cpa->vaddr += cpa->processed * PAGE_SIZE; + BUG_ON(cpa->numpages > numpages); + numpages -= cpa->numpages; + cpa->vaddr += cpa->numpages * PAGE_SIZE; } return 0; } diff --git a/trunk/arch/x86/vdso/Makefile b/trunk/arch/x86/vdso/Makefile index 0a8f4742ef51..b8bd0c4aa02e 100644 --- a/trunk/arch/x86/vdso/Makefile +++ b/trunk/arch/x86/vdso/Makefile @@ -48,11 +48,9 @@ obj-$(VDSO64-y) += vdso-syms.lds # Match symbols in the DSO that look like VDSO*; produce a file of constants. # sed-vdsosym := -e 's/^00*/0/' \ - -e 's/^\([0-9a-fA-F]*\) . \(VDSO[a-zA-Z0-9_]*\)$$/\2 = 0x\1;/p' + -e 's/^\([[:xdigit:]]*\) . \(VDSO[[:alnum:]_]*\)$$/\2 = 0x\1;/p' quiet_cmd_vdsosym = VDSOSYM $@ -define cmd_vdsosym - $(NM) $< | LC_ALL=C sed -n $(sed-vdsosym) | LC_ALL=C sort > $@ -endef + cmd_vdsosym = $(NM) $< | sed -n $(sed-vdsosym) | LC_ALL=C sort > $@ $(obj)/%-syms.lds: $(obj)/%.so.dbg FORCE $(call if_changed,vdsosym) diff --git a/trunk/arch/x86/xen/enlighten.c b/trunk/arch/x86/xen/enlighten.c index 8b9ee27805fd..49e5358f481a 100644 --- a/trunk/arch/x86/xen/enlighten.c +++ b/trunk/arch/x86/xen/enlighten.c @@ -153,7 +153,6 @@ static void xen_cpuid(unsigned int *ax, unsigned int *bx, if (*ax == 1) maskedx = ~((1 << X86_FEATURE_APIC) | /* disable APIC */ (1 << X86_FEATURE_ACPI) | /* disable ACPI */ - (1 << X86_FEATURE_SEP) | /* disable SEP */ (1 << X86_FEATURE_ACC)); /* thermal monitoring */ asm(XEN_EMULATE_PREFIX "cpuid" diff --git a/trunk/include/asm-x86/pgtable_32.h b/trunk/include/asm-x86/pgtable_32.h index b478efa971e0..a842c7222b1e 100644 --- a/trunk/include/asm-x86/pgtable_32.h +++ b/trunk/include/asm-x86/pgtable_32.h @@ -91,9 +91,7 @@ extern unsigned long pg0[]; /* To avoid harmful races, pmd_none(x) should check only the lower when PAE */ #define pmd_none(x) (!(unsigned long)pmd_val(x)) #define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) -#define pmd_bad(x) ((pmd_val(x) \ - & ~(PAGE_MASK | _PAGE_USER | _PAGE_PSE | _PAGE_NX)) \ - != _KERNPG_TABLE) +#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE) #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) diff --git a/trunk/include/asm-x86/pgtable_64.h b/trunk/include/asm-x86/pgtable_64.h index 0a9258333cbd..0a0b77bc736a 100644 --- a/trunk/include/asm-x86/pgtable_64.h +++ b/trunk/include/asm-x86/pgtable_64.h @@ -153,14 +153,12 @@ static inline unsigned long pgd_bad(pgd_t pgd) static inline unsigned long pud_bad(pud_t pud) { - return pud_val(pud) & - ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER | _PAGE_PSE | _PAGE_NX); + return pud_val(pud) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); } static inline unsigned long pmd_bad(pmd_t pmd) { - return pmd_val(pmd) & - ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER | _PAGE_PSE | _PAGE_NX); + return pmd_val(pmd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); } #define pte_none(x) (!pte_val(x)) diff --git a/trunk/include/asm-x86/ptrace-abi.h b/trunk/include/asm-x86/ptrace-abi.h index f224eb3c3157..81a8ee4c55fc 100644 --- a/trunk/include/asm-x86/ptrace-abi.h +++ b/trunk/include/asm-x86/ptrace-abi.h @@ -89,13 +89,13 @@ */ struct ptrace_bts_config { /* requested or actual size of BTS buffer in bytes */ - __u32 size; + u32 size; /* bitmask of below flags */ - __u32 flags; + u32 flags; /* buffer overflow signal */ - __u32 signal; + u32 signal; /* actual size of bts_struct in bytes */ - __u32 bts_size; + u32 bts_size; }; #endif diff --git a/trunk/include/linux/hardirq.h b/trunk/include/linux/hardirq.h index 2961ec788046..49829988bfa0 100644 --- a/trunk/include/linux/hardirq.h +++ b/trunk/include/linux/hardirq.h @@ -109,6 +109,14 @@ static inline void account_system_vtime(struct task_struct *tsk) } #endif +#if defined(CONFIG_PREEMPT_RCU) && defined(CONFIG_NO_HZ) +extern void rcu_irq_enter(void); +extern void rcu_irq_exit(void); +#else +# define rcu_irq_enter() do { } while (0) +# define rcu_irq_exit() do { } while (0) +#endif /* CONFIG_PREEMPT_RCU */ + /* * It is safe to do non-atomic ops on ->hardirq_context, * because NMI handlers may not preempt and the ops are @@ -117,6 +125,7 @@ static inline void account_system_vtime(struct task_struct *tsk) */ #define __irq_enter() \ do { \ + rcu_irq_enter(); \ account_system_vtime(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ @@ -135,6 +144,7 @@ extern void irq_enter(void); trace_hardirq_exit(); \ account_system_vtime(current); \ sub_preempt_count(HARDIRQ_OFFSET); \ + rcu_irq_exit(); \ } while (0) /* diff --git a/trunk/include/linux/rcuclassic.h b/trunk/include/linux/rcuclassic.h index 4d6624260b4c..b3dccd68629e 100644 --- a/trunk/include/linux/rcuclassic.h +++ b/trunk/include/linux/rcuclassic.h @@ -160,5 +160,8 @@ extern void rcu_restart_cpu(int cpu); extern long rcu_batches_completed(void); extern long rcu_batches_completed_bh(void); +#define rcu_enter_nohz() do { } while (0) +#define rcu_exit_nohz() do { } while (0) + #endif /* __KERNEL__ */ #endif /* __LINUX_RCUCLASSIC_H */ diff --git a/trunk/include/linux/rcupreempt.h b/trunk/include/linux/rcupreempt.h index 60c2a033b19e..01152ed532c8 100644 --- a/trunk/include/linux/rcupreempt.h +++ b/trunk/include/linux/rcupreempt.h @@ -82,5 +82,27 @@ extern struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu); struct softirq_action; +#ifdef CONFIG_NO_HZ +DECLARE_PER_CPU(long, dynticks_progress_counter); + +static inline void rcu_enter_nohz(void) +{ + __get_cpu_var(dynticks_progress_counter)++; + WARN_ON(__get_cpu_var(dynticks_progress_counter) & 0x1); + mb(); +} + +static inline void rcu_exit_nohz(void) +{ + mb(); + __get_cpu_var(dynticks_progress_counter)++; + WARN_ON(!(__get_cpu_var(dynticks_progress_counter) & 0x1)); +} + +#else /* CONFIG_NO_HZ */ +#define rcu_enter_nohz() do { } while (0) +#define rcu_exit_nohz() do { } while (0) +#endif /* CONFIG_NO_HZ */ + #endif /* __KERNEL__ */ #endif /* __LINUX_RCUPREEMPT_H */ diff --git a/trunk/kernel/rcupreempt.c b/trunk/kernel/rcupreempt.c index 987cfb7ade89..c7c52096df48 100644 --- a/trunk/kernel/rcupreempt.c +++ b/trunk/kernel/rcupreempt.c @@ -23,6 +23,10 @@ * to Suparna Bhattacharya for pushing me completely away * from atomic instructions on the read side. * + * - Added handling of Dynamic Ticks + * Copyright 2007 - Paul E. Mckenney + * - Steven Rostedt + * * Papers: http://www.rdrop.com/users/paulmck/RCU * * Design Document: http://lwn.net/Articles/253651/ @@ -409,6 +413,212 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp) } } +#ifdef CONFIG_NO_HZ + +DEFINE_PER_CPU(long, dynticks_progress_counter) = 1; +static DEFINE_PER_CPU(long, rcu_dyntick_snapshot); +static DEFINE_PER_CPU(int, rcu_update_flag); + +/** + * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI. + * + * If the CPU was idle with dynamic ticks active, this updates the + * dynticks_progress_counter to let the RCU handling know that the + * CPU is active. + */ +void rcu_irq_enter(void) +{ + int cpu = smp_processor_id(); + + if (per_cpu(rcu_update_flag, cpu)) + per_cpu(rcu_update_flag, cpu)++; + + /* + * Only update if we are coming from a stopped ticks mode + * (dynticks_progress_counter is even). + */ + if (!in_interrupt() && + (per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) { + /* + * The following might seem like we could have a race + * with NMI/SMIs. But this really isn't a problem. + * Here we do a read/modify/write, and the race happens + * when an NMI/SMI comes in after the read and before + * the write. But NMI/SMIs will increment this counter + * twice before returning, so the zero bit will not + * be corrupted by the NMI/SMI which is the most important + * part. + * + * The only thing is that we would bring back the counter + * to a postion that it was in during the NMI/SMI. + * But the zero bit would be set, so the rest of the + * counter would again be ignored. + * + * On return from the IRQ, the counter may have the zero + * bit be 0 and the counter the same as the return from + * the NMI/SMI. If the state machine was so unlucky to + * see that, it still doesn't matter, since all + * RCU read-side critical sections on this CPU would + * have already completed. + */ + per_cpu(dynticks_progress_counter, cpu)++; + /* + * The following memory barrier ensures that any + * rcu_read_lock() primitives in the irq handler + * are seen by other CPUs to follow the above + * increment to dynticks_progress_counter. This is + * required in order for other CPUs to correctly + * determine when it is safe to advance the RCU + * grace-period state machine. + */ + smp_mb(); /* see above block comment. */ + /* + * Since we can't determine the dynamic tick mode from + * the dynticks_progress_counter after this routine, + * we use a second flag to acknowledge that we came + * from an idle state with ticks stopped. + */ + per_cpu(rcu_update_flag, cpu)++; + /* + * If we take an NMI/SMI now, they will also increment + * the rcu_update_flag, and will not update the + * dynticks_progress_counter on exit. That is for + * this IRQ to do. + */ + } +} + +/** + * rcu_irq_exit - Called from exiting Hard irq context. + * + * If the CPU was idle with dynamic ticks active, update the + * dynticks_progress_counter to put let the RCU handling be + * aware that the CPU is going back to idle with no ticks. + */ +void rcu_irq_exit(void) +{ + int cpu = smp_processor_id(); + + /* + * rcu_update_flag is set if we interrupted the CPU + * when it was idle with ticks stopped. + * Once this occurs, we keep track of interrupt nesting + * because a NMI/SMI could also come in, and we still + * only want the IRQ that started the increment of the + * dynticks_progress_counter to be the one that modifies + * it on exit. + */ + if (per_cpu(rcu_update_flag, cpu)) { + if (--per_cpu(rcu_update_flag, cpu)) + return; + + /* This must match the interrupt nesting */ + WARN_ON(in_interrupt()); + + /* + * If an NMI/SMI happens now we are still + * protected by the dynticks_progress_counter being odd. + */ + + /* + * The following memory barrier ensures that any + * rcu_read_unlock() primitives in the irq handler + * are seen by other CPUs to preceed the following + * increment to dynticks_progress_counter. This + * is required in order for other CPUs to determine + * when it is safe to advance the RCU grace-period + * state machine. + */ + smp_mb(); /* see above block comment. */ + per_cpu(dynticks_progress_counter, cpu)++; + WARN_ON(per_cpu(dynticks_progress_counter, cpu) & 0x1); + } +} + +static void dyntick_save_progress_counter(int cpu) +{ + per_cpu(rcu_dyntick_snapshot, cpu) = + per_cpu(dynticks_progress_counter, cpu); +} + +static inline int +rcu_try_flip_waitack_needed(int cpu) +{ + long curr; + long snap; + + curr = per_cpu(dynticks_progress_counter, cpu); + snap = per_cpu(rcu_dyntick_snapshot, cpu); + smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ + + /* + * If the CPU remained in dynticks mode for the entire time + * and didn't take any interrupts, NMIs, SMIs, or whatever, + * then it cannot be in the middle of an rcu_read_lock(), so + * the next rcu_read_lock() it executes must use the new value + * of the counter. So we can safely pretend that this CPU + * already acknowledged the counter. + */ + + if ((curr == snap) && ((curr & 0x1) == 0)) + return 0; + + /* + * If the CPU passed through or entered a dynticks idle phase with + * no active irq handlers, then, as above, we can safely pretend + * that this CPU already acknowledged the counter. + */ + + if ((curr - snap) > 2 || (snap & 0x1) == 0) + return 0; + + /* We need this CPU to explicitly acknowledge the counter flip. */ + + return 1; +} + +static inline int +rcu_try_flip_waitmb_needed(int cpu) +{ + long curr; + long snap; + + curr = per_cpu(dynticks_progress_counter, cpu); + snap = per_cpu(rcu_dyntick_snapshot, cpu); + smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ + + /* + * If the CPU remained in dynticks mode for the entire time + * and didn't take any interrupts, NMIs, SMIs, or whatever, + * then it cannot have executed an RCU read-side critical section + * during that time, so there is no need for it to execute a + * memory barrier. + */ + + if ((curr == snap) && ((curr & 0x1) == 0)) + return 0; + + /* + * If the CPU either entered or exited an outermost interrupt, + * SMI, NMI, or whatever handler, then we know that it executed + * a memory barrier when doing so. So we don't need another one. + */ + if (curr != snap) + return 0; + + /* We need the CPU to execute a memory barrier. */ + + return 1; +} + +#else /* !CONFIG_NO_HZ */ + +# define dyntick_save_progress_counter(cpu) do { } while (0) +# define rcu_try_flip_waitack_needed(cpu) (1) +# define rcu_try_flip_waitmb_needed(cpu) (1) + +#endif /* CONFIG_NO_HZ */ + /* * Get here when RCU is idle. Decide whether we need to * move out of idle state, and return non-zero if so. @@ -447,8 +657,10 @@ rcu_try_flip_idle(void) /* Now ask each CPU for acknowledgement of the flip. */ - for_each_cpu_mask(cpu, rcu_cpu_online_map) + for_each_cpu_mask(cpu, rcu_cpu_online_map) { per_cpu(rcu_flip_flag, cpu) = rcu_flipped; + dyntick_save_progress_counter(cpu); + } return 1; } @@ -464,7 +676,8 @@ rcu_try_flip_waitack(void) RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); for_each_cpu_mask(cpu, rcu_cpu_online_map) - if (per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { + if (rcu_try_flip_waitack_needed(cpu) && + per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); return 0; } @@ -509,8 +722,10 @@ rcu_try_flip_waitzero(void) smp_mb(); /* ^^^^^^^^^^^^ */ /* Call for a memory barrier from each CPU. */ - for_each_cpu_mask(cpu, rcu_cpu_online_map) + for_each_cpu_mask(cpu, rcu_cpu_online_map) { per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; + dyntick_save_progress_counter(cpu); + } RCU_TRACE_ME(rcupreempt_trace_try_flip_z2); return 1; @@ -528,7 +743,8 @@ rcu_try_flip_waitmb(void) RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); for_each_cpu_mask(cpu, rcu_cpu_online_map) - if (per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { + if (rcu_try_flip_waitmb_needed(cpu) && + per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); return 0; } diff --git a/trunk/kernel/softirq.c b/trunk/kernel/softirq.c index 5b3aea5f471e..31e9f2a47928 100644 --- a/trunk/kernel/softirq.c +++ b/trunk/kernel/softirq.c @@ -313,6 +313,7 @@ void irq_exit(void) /* Make sure that timer wheel updates are propagated */ if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) tick_nohz_stop_sched_tick(); + rcu_irq_exit(); #endif preempt_enable_no_resched(); } diff --git a/trunk/kernel/time/tick-sched.c b/trunk/kernel/time/tick-sched.c index fa9bb73dbdb4..2968298f8f36 100644 --- a/trunk/kernel/time/tick-sched.c +++ b/trunk/kernel/time/tick-sched.c @@ -282,6 +282,7 @@ void tick_nohz_stop_sched_tick(void) ts->idle_tick = ts->sched_timer.expires; ts->tick_stopped = 1; ts->idle_jiffies = last_jiffies; + rcu_enter_nohz(); } /* @@ -375,6 +376,8 @@ void tick_nohz_restart_sched_tick(void) return; } + rcu_exit_nohz(); + /* Update jiffies first */ select_nohz_load_balancer(0); now = ktime_get();