From 8bbce9bdbf82b19f3cedb9d4d665a6342c871baa Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 9 Jul 2008 11:39:02 +0200 Subject: [PATCH] --- yaml --- r: 100098 b: refs/heads/master c: d028203c0446c7f65ed2b22342a56f03c6c4a6c1 h: refs/heads/master v: v3 --- [refs] | 2 +- trunk/arch/mips/mm/c-r3k.c | 6 +- trunk/arch/mips/mm/page.c | 61 +-- trunk/arch/mips/mm/sc-rm7k.c | 4 +- trunk/arch/powerpc/kernel/legacy_serial.c | 5 +- trunk/arch/um/Makefile | 1 - trunk/arch/um/Makefile-i386 | 7 + trunk/arch/um/Makefile-x86_64 | 3 + trunk/arch/x86/Kconfig | 6 +- trunk/arch/x86/Kconfig.cpu | 2 +- trunk/arch/x86/ia32/ia32entry.S | 25 +- trunk/arch/x86/kernel/acpi/boot.c | 23 +- trunk/arch/x86/kernel/asm-offsets_64.c | 2 +- trunk/arch/x86/kernel/e820.c | 6 - trunk/arch/x86/kernel/entry_64.S | 23 +- trunk/arch/x86/kernel/io_apic_64.c | 5 + trunk/arch/x86/kernel/tsc.c | 1 - trunk/arch/x86/lib/Makefile | 4 +- trunk/arch/x86/lib/copy_user_64.S | 429 ++++++++++------- trunk/arch/x86/lib/copy_user_nocache_64.S | 283 +++++++---- trunk/arch/x86/lib/{delay.c => delay_32.c} | 17 +- trunk/arch/x86/lib/delay_64.c | 85 ++++ trunk/arch/x86/lib/getuser_32.S | 78 +++ .../arch/x86/lib/{getuser.S => getuser_64.S} | 87 ++-- .../arch/x86/lib/{putuser.S => putuser_32.S} | 73 +-- trunk/arch/x86/lib/putuser_64.S | 106 +++++ trunk/arch/x86/lib/usercopy_64.c | 23 - trunk/arch/x86/mm/init_64.c | 106 ++--- trunk/arch/x86/pci/common.c | 8 +- trunk/drivers/ide/arm/palm_bk3710.c | 38 +- trunk/drivers/ide/ide-probe.c | 8 +- trunk/drivers/ide/pci/it8213.c | 3 +- trunk/drivers/ide/pci/ns87415.c | 6 +- trunk/drivers/net/ehea/ehea.h | 8 +- trunk/drivers/net/ehea/ehea_main.c | 42 +- trunk/drivers/net/forcedeth.c | 15 +- trunk/drivers/net/fs_enet/mac-fcc.c | 3 + trunk/drivers/net/ibm_newemac/core.c | 8 + trunk/drivers/net/pasemi_mac.c | 2 +- trunk/drivers/net/wan/hdlc_fr.c | 1 + trunk/drivers/net/wireless/iwlwifi/iwl-3945.c | 6 +- trunk/drivers/net/wireless/iwlwifi/iwl-4965.c | 6 +- .../net/wireless/iwlwifi/iwl3945-base.c | 3 +- .../net/wireless/iwlwifi/iwl4965-base.c | 3 +- trunk/drivers/net/wireless/libertas/if_usb.c | 1 + trunk/fs/nfs/dir.c | 2 +- trunk/fs/reiserfs/inode.c | 2 + trunk/include/asm-powerpc/hugetlb.h | 6 - trunk/include/asm-powerpc/pgtable-ppc64.h | 10 + trunk/include/asm-s390/pgtable.h | 5 +- trunk/include/asm-x86/asm.h | 9 +- trunk/include/asm-x86/delay.h | 4 + trunk/include/asm-x86/fixmap_64.h | 5 - trunk/include/asm-x86/gart.h | 34 +- trunk/include/asm-x86/genapic_32.h | 4 - trunk/include/asm-x86/genapic_64.h | 4 - trunk/include/asm-x86/iommu.h | 31 +- trunk/include/asm-x86/uaccess.h | 448 ------------------ trunk/include/asm-x86/uaccess_32.h | 422 +++++++++++++++++ trunk/include/asm-x86/uaccess_64.h | 263 +++++++++- trunk/include/linux/ide.h | 15 + trunk/lib/vsprintf.c | 128 +++-- trunk/net/bridge/br_if.c | 10 +- trunk/net/can/af_can.c | 10 + trunk/net/can/bcm.c | 23 +- trunk/net/can/raw.c | 3 + trunk/net/ipv4/tcp.c | 4 +- trunk/net/sunrpc/rpcb_clnt.c | 23 +- 68 files changed, 1938 insertions(+), 1161 deletions(-) rename trunk/arch/x86/lib/{delay.c => delay_32.c} (96%) create mode 100644 trunk/arch/x86/lib/delay_64.c create mode 100644 trunk/arch/x86/lib/getuser_32.S rename trunk/arch/x86/lib/{getuser.S => getuser_64.S} (53%) rename trunk/arch/x86/lib/{putuser.S => putuser_32.S} (54%) create mode 100644 trunk/arch/x86/lib/putuser_64.S diff --git a/[refs] b/[refs] index 06fc8bef0830..75fa1d9a01b3 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: a6784ad7eafa74a085c8458909eda0699a8fcf57 +refs/heads/master: d028203c0446c7f65ed2b22342a56f03c6c4a6c1 diff --git a/trunk/arch/mips/mm/c-r3k.c b/trunk/arch/mips/mm/c-r3k.c index 76935e320214..27a5b466c85c 100644 --- a/trunk/arch/mips/mm/c-r3k.c +++ b/trunk/arch/mips/mm/c-r3k.c @@ -26,7 +26,7 @@ static unsigned long icache_size, dcache_size; /* Size in bytes */ static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ -unsigned long __init r3k_cache_size(unsigned long ca_flags) +unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags) { unsigned long flags, status, dummy, size; volatile unsigned long *p; @@ -61,7 +61,7 @@ unsigned long __init r3k_cache_size(unsigned long ca_flags) return size * sizeof(*p); } -unsigned long __init r3k_cache_lsize(unsigned long ca_flags) +unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags) { unsigned long flags, status, lsize, i; volatile unsigned long *p; @@ -90,7 +90,7 @@ unsigned long __init r3k_cache_lsize(unsigned long ca_flags) return lsize * sizeof(*p); } -static void __init r3k_probe_cache(void) +static void __cpuinit r3k_probe_cache(void) { dcache_size = r3k_cache_size(ST0_ISC); if (dcache_size) diff --git a/trunk/arch/mips/mm/page.c b/trunk/arch/mips/mm/page.c index 1edf0cbbeede..1417c6494858 100644 --- a/trunk/arch/mips/mm/page.c +++ b/trunk/arch/mips/mm/page.c @@ -235,13 +235,12 @@ static void __cpuinit set_prefetch_parameters(void) } /* * Too much unrolling will overflow the available space in - * clear_space_array / copy_page_array. 8 words sounds generous, - * but a R4000 with 128 byte L2 line length can exceed even that. + * clear_space_array / copy_page_array. */ - half_clear_loop_size = min(8 * clear_word_size, + half_clear_loop_size = min(16 * clear_word_size, max(cache_line_size >> 1, 4 * clear_word_size)); - half_copy_loop_size = min(8 * copy_word_size, + half_copy_loop_size = min(16 * copy_word_size, max(cache_line_size >> 1, 4 * copy_word_size)); } @@ -263,21 +262,23 @@ static inline void __cpuinit build_clear_pref(u32 **buf, int off) if (pref_bias_clear_store) { uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off, A0); - } else if (cpu_has_cache_cdex_s) { - uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); - } else if (cpu_has_cache_cdex_p) { - if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { - uasm_i_nop(buf); - uasm_i_nop(buf); - uasm_i_nop(buf); - uasm_i_nop(buf); - } + } else if (cache_line_size == (half_clear_loop_size << 1)) { + if (cpu_has_cache_cdex_s) { + uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); + } else if (cpu_has_cache_cdex_p) { + if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { + uasm_i_nop(buf); + uasm_i_nop(buf); + uasm_i_nop(buf); + uasm_i_nop(buf); + } - if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) - uasm_i_lw(buf, ZERO, ZERO, AT); + if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) + uasm_i_lw(buf, ZERO, ZERO, AT); - uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); - } + uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); + } + } } void __cpuinit build_clear_page(void) @@ -403,20 +404,22 @@ static inline void build_copy_store_pref(u32 **buf, int off) if (pref_bias_copy_store) { uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, A0); - } else if (cpu_has_cache_cdex_s) { - uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); - } else if (cpu_has_cache_cdex_p) { - if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { - uasm_i_nop(buf); - uasm_i_nop(buf); - uasm_i_nop(buf); - uasm_i_nop(buf); - } + } else if (cache_line_size == (half_copy_loop_size << 1)) { + if (cpu_has_cache_cdex_s) { + uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); + } else if (cpu_has_cache_cdex_p) { + if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { + uasm_i_nop(buf); + uasm_i_nop(buf); + uasm_i_nop(buf); + uasm_i_nop(buf); + } - if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) - uasm_i_lw(buf, ZERO, ZERO, AT); + if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) + uasm_i_lw(buf, ZERO, ZERO, AT); - uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); + uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); + } } } diff --git a/trunk/arch/mips/mm/sc-rm7k.c b/trunk/arch/mips/mm/sc-rm7k.c index fc227f3b1199..e3abfb2d7e86 100644 --- a/trunk/arch/mips/mm/sc-rm7k.c +++ b/trunk/arch/mips/mm/sc-rm7k.c @@ -86,7 +86,7 @@ static void rm7k_sc_inv(unsigned long addr, unsigned long size) /* * This function is executed in uncached address space. */ -static __init void __rm7k_sc_enable(void) +static __cpuinit void __rm7k_sc_enable(void) { int i; @@ -107,7 +107,7 @@ static __init void __rm7k_sc_enable(void) } } -static __init void rm7k_sc_enable(void) +static __cpuinit void rm7k_sc_enable(void) { if (read_c0_config() & RM7K_CONF_SE) return; diff --git a/trunk/arch/powerpc/kernel/legacy_serial.c b/trunk/arch/powerpc/kernel/legacy_serial.c index cf37f5ca4b71..4d96e1db55ee 100644 --- a/trunk/arch/powerpc/kernel/legacy_serial.c +++ b/trunk/arch/powerpc/kernel/legacy_serial.c @@ -33,13 +33,14 @@ static struct legacy_serial_info { phys_addr_t taddr; } legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS]; -static struct __initdata of_device_id parents[] = { +static struct __initdata of_device_id legacy_serial_parents[] = { {.type = "soc",}, {.type = "tsi-bridge",}, {.type = "opb", }, {.compatible = "ibm,opb",}, {.compatible = "simple-bus",}, {.compatible = "wrs,epld-localbus",}, + {}, }; static unsigned int legacy_serial_count; @@ -327,7 +328,7 @@ void __init find_legacy_serial_ports(void) struct device_node *parent = of_get_parent(np); if (!parent) continue; - if (of_match_node(parents, parent) != NULL) { + if (of_match_node(legacy_serial_parents, parent) != NULL) { index = add_legacy_soc_port(np, np); if (index >= 0 && np == stdout) legacy_serial_console = index; diff --git a/trunk/arch/um/Makefile b/trunk/arch/um/Makefile index dbeab15e7bb7..ca40397017b9 100644 --- a/trunk/arch/um/Makefile +++ b/trunk/arch/um/Makefile @@ -77,7 +77,6 @@ include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS) KERNEL_DEFINES = $(strip -Derrno=kernel_errno -Dsigprocmask=kernel_sigprocmask \ -Dmktime=kernel_mktime $(ARCH_KERNEL_DEFINES)) KBUILD_CFLAGS += $(KERNEL_DEFINES) -KBUILD_CFLAGS += $(call cc-option,-fno-unit-at-a-time,) PHONY += linux diff --git a/trunk/arch/um/Makefile-i386 b/trunk/arch/um/Makefile-i386 index 561e373bd850..302cbe504543 100644 --- a/trunk/arch/um/Makefile-i386 +++ b/trunk/arch/um/Makefile-i386 @@ -32,4 +32,11 @@ cflags-y += $(call cc-option,-mpreferred-stack-boundary=2) # an unresolved reference. cflags-y += -ffreestanding +# Disable unit-at-a-time mode on pre-gcc-4.0 compilers, it makes gcc use +# a lot more stack due to the lack of sharing of stacklots. Also, gcc +# 4.3.0 needs -funit-at-a-time for extern inline functions. +KBUILD_CFLAGS += $(shell if [ $(call cc-version) -lt 0400 ] ; then \ + echo $(call cc-option,-fno-unit-at-a-time); \ + else echo $(call cc-option,-funit-at-a-time); fi ;) + KBUILD_CFLAGS += $(cflags-y) diff --git a/trunk/arch/um/Makefile-x86_64 b/trunk/arch/um/Makefile-x86_64 index 8ed362f93582..a9cd7e77a7ab 100644 --- a/trunk/arch/um/Makefile-x86_64 +++ b/trunk/arch/um/Makefile-x86_64 @@ -21,3 +21,6 @@ HEADER_ARCH := x86 LINK-$(CONFIG_LD_SCRIPT_DYN) += -Wl,-rpath,/lib64 LINK-y += -m64 + +# Do unit-at-a-time unconditionally on x86_64, following the host +KBUILD_CFLAGS += $(call cc-option,-funit-at-a-time) diff --git a/trunk/arch/x86/Kconfig b/trunk/arch/x86/Kconfig index 3b1b1da78dae..bb0c0d0f6db7 100644 --- a/trunk/arch/x86/Kconfig +++ b/trunk/arch/x86/Kconfig @@ -283,7 +283,7 @@ config X86_VOYAGER config X86_VISWS bool "SGI 320/540 (Visual Workstation)" - depends on X86_32 && PCI + depends on X86_32 && !PCI help The SGI Visual Workstation series is an IA32-based workstation based on SGI systems chips with some legacy PC hardware attached. @@ -355,7 +355,7 @@ config X86_RDC321X config X86_VSMP bool "Support for ScaleMP vSMP" select PARAVIRT - depends on X86_64 && PCI + depends on X86_64 && !PCI help Support for ScaleMP vSMP systems. Say 'Y' here if this kernel is supposed to run on these EM64T-based machines. Only choose this option @@ -575,7 +575,7 @@ config SWIOTLB 3 GB of memory. If unsure, say Y. config IOMMU_HELPER - def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU) + def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB) config MAXSMP bool "Configure Maximum number of SMP Processors and NUMA Nodes" depends on X86_64 && SMP diff --git a/trunk/arch/x86/Kconfig.cpu b/trunk/arch/x86/Kconfig.cpu index abff1b84ed5b..3d22bb8175b4 100644 --- a/trunk/arch/x86/Kconfig.cpu +++ b/trunk/arch/x86/Kconfig.cpu @@ -344,7 +344,7 @@ config X86_F00F_BUG config X86_WP_WORKS_OK def_bool y - depends on !M386 + depends on X86_32 && !M386 config X86_INVLPG def_bool y diff --git a/trunk/arch/x86/ia32/ia32entry.S b/trunk/arch/x86/ia32/ia32entry.S index 20371d0635e4..24e4d4928d65 100644 --- a/trunk/arch/x86/ia32/ia32entry.S +++ b/trunk/arch/x86/ia32/ia32entry.S @@ -116,7 +116,7 @@ ENTRY(ia32_sysenter_target) pushfq CFI_ADJUST_CFA_OFFSET 8 /*CFI_REL_OFFSET rflags,0*/ - movl 8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d + movl 8*3-THREAD_SIZE+threadinfo_sysenter_return(%rsp), %r10d CFI_REGISTER rip,r10 pushq $__USER32_CS CFI_ADJUST_CFA_OFFSET 8 @@ -136,9 +136,8 @@ ENTRY(ia32_sysenter_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,TI_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ - TI_flags(%r10) + orl $TS_COMPAT,threadinfo_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) CFI_REMEMBER_STATE jnz sysenter_tracesys sysenter_do_call: @@ -150,9 +149,9 @@ sysenter_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,TI_flags(%r10) + testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,TI_status(%r10) + andl $~TS_COMPAT,threadinfo_status(%r10) /* clear IF, that popfq doesn't enable interrupts early */ andl $~0x200,EFLAGS-R11(%rsp) movl RIP-R11(%rsp),%edx /* User %eip */ @@ -241,9 +240,8 @@ ENTRY(ia32_cstar_target) .quad 1b,ia32_badarg .previous GET_THREAD_INFO(%r10) - orl $TS_COMPAT,TI_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ - TI_flags(%r10) + orl $TS_COMPAT,threadinfo_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) CFI_REMEMBER_STATE jnz cstar_tracesys cstar_do_call: @@ -255,9 +253,9 @@ cstar_do_call: GET_THREAD_INFO(%r10) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - testl $_TIF_ALLWORK_MASK,TI_flags(%r10) + testl $_TIF_ALLWORK_MASK,threadinfo_flags(%r10) jnz int_ret_from_sys_call - andl $~TS_COMPAT,TI_status(%r10) + andl $~TS_COMPAT,threadinfo_status(%r10) RESTORE_ARGS 1,-ARG_SKIP,1,1,1 movl RIP-ARGOFFSET(%rsp),%ecx CFI_REGISTER rip,rcx @@ -335,9 +333,8 @@ ENTRY(ia32_syscall) this could be a problem. */ SAVE_ARGS 0,0,1 GET_THREAD_INFO(%r10) - orl $TS_COMPAT,TI_status(%r10) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ - TI_flags(%r10) + orl $TS_COMPAT,threadinfo_status(%r10) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) jnz ia32_tracesys ia32_do_syscall: cmpl $(IA32_NR_syscalls-1),%eax diff --git a/trunk/arch/x86/kernel/acpi/boot.c b/trunk/arch/x86/kernel/acpi/boot.c index a31a579a47ca..bf7b4f7f60e1 100644 --- a/trunk/arch/x86/kernel/acpi/boot.c +++ b/trunk/arch/x86/kernel/acpi/boot.c @@ -109,6 +109,21 @@ static u64 acpi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; */ enum acpi_irq_model_id acpi_irq_model = ACPI_IRQ_MODEL_PIC; +#ifdef CONFIG_X86_64 + +/* rely on all ACPI tables being in the direct mapping */ +char *__init __acpi_map_table(unsigned long phys_addr, unsigned long size) +{ + if (!phys_addr || !size) + return NULL; + + if (phys_addr+size <= (max_pfn_mapped << PAGE_SHIFT) + PAGE_SIZE) + return __va(phys_addr); + + return NULL; +} + +#else /* * Temporarily use the virtual area starting from FIX_IO_APIC_BASE_END, @@ -127,15 +142,11 @@ char *__init __acpi_map_table(unsigned long phys, unsigned long size) unsigned long base, offset, mapped_size; int idx; - if (!phys || !size) - return NULL; - - if (phys+size <= (max_pfn_mapped << PAGE_SHIFT)) + if (phys + size < 8 * 1024 * 1024) return __va(phys); offset = phys & (PAGE_SIZE - 1); mapped_size = PAGE_SIZE - offset; - clear_fixmap(FIX_ACPI_END); set_fixmap(FIX_ACPI_END, phys); base = fix_to_virt(FIX_ACPI_END); @@ -147,13 +158,13 @@ char *__init __acpi_map_table(unsigned long phys, unsigned long size) if (--idx < FIX_ACPI_BEGIN) return NULL; /* cannot handle this */ phys += PAGE_SIZE; - clear_fixmap(idx); set_fixmap(idx, phys); mapped_size += PAGE_SIZE; } return ((unsigned char *)base + offset); } +#endif #ifdef CONFIG_PCI_MMCONFIG /* The physical address of the MMCONFIG aperture. Set from ACPI tables. */ diff --git a/trunk/arch/x86/kernel/asm-offsets_64.c b/trunk/arch/x86/kernel/asm-offsets_64.c index bacf5deeec2d..3295e7c08fe7 100644 --- a/trunk/arch/x86/kernel/asm-offsets_64.c +++ b/trunk/arch/x86/kernel/asm-offsets_64.c @@ -34,7 +34,7 @@ int main(void) ENTRY(pid); BLANK(); #undef ENTRY -#define ENTRY(entry) DEFINE(TI_ ## entry, offsetof(struct thread_info, entry)) +#define ENTRY(entry) DEFINE(threadinfo_ ## entry, offsetof(struct thread_info, entry)) ENTRY(flags); ENTRY(addr_limit); ENTRY(preempt_count); diff --git a/trunk/arch/x86/kernel/e820.c b/trunk/arch/x86/kernel/e820.c index 9836a079cfd9..2e08619a9c5c 100644 --- a/trunk/arch/x86/kernel/e820.c +++ b/trunk/arch/x86/kernel/e820.c @@ -1066,9 +1066,6 @@ unsigned long __init e820_end(void) struct e820entry *ei = &e820.map[i]; unsigned long end_pfn; - if (ei->type != E820_RAM) - continue; - end_pfn = (ei->addr + ei->size) >> PAGE_SHIFT; if (end_pfn > last_pfn) last_pfn = end_pfn; @@ -1196,9 +1193,6 @@ static int __init parse_memmap_opt(char *p) char *oldp; u64 start_at, mem_size; - if (!p) - return -EINVAL; - if (!strcmp(p, "exactmap")) { #ifdef CONFIG_CRASH_DUMP /* diff --git a/trunk/arch/x86/kernel/entry_64.S b/trunk/arch/x86/kernel/entry_64.S index 466b9284ed2f..07d69f262337 100644 --- a/trunk/arch/x86/kernel/entry_64.S +++ b/trunk/arch/x86/kernel/entry_64.S @@ -168,13 +168,13 @@ ENTRY(ret_from_fork) CFI_ADJUST_CFA_OFFSET -4 call schedule_tail GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx) jnz rff_trace rff_action: RESTORE_REST testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread? je int_ret_from_sys_call - testl $_TIF_IA32,TI_flags(%rcx) + testl $_TIF_IA32,threadinfo_flags(%rcx) jnz int_ret_from_sys_call RESTORE_TOP_OF_STACK %rdi,ARGOFFSET jmp ret_from_sys_call @@ -243,8 +243,7 @@ ENTRY(system_call_after_swapgs) movq %rcx,RIP-ARGOFFSET(%rsp) CFI_REL_OFFSET rip,RIP-ARGOFFSET GET_THREAD_INFO(%rcx) - testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \ - TI_flags(%rcx) + testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) jnz tracesys cmpq $__NR_syscall_max,%rax ja badsys @@ -263,7 +262,7 @@ sysret_check: GET_THREAD_INFO(%rcx) DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF - movl TI_flags(%rcx),%edx + movl threadinfo_flags(%rcx),%edx andl %edi,%edx jnz sysret_careful CFI_REMEMBER_STATE @@ -348,10 +347,10 @@ int_ret_from_sys_call: int_with_check: LOCKDEP_SYS_EXIT_IRQ GET_THREAD_INFO(%rcx) - movl TI_flags(%rcx),%edx + movl threadinfo_flags(%rcx),%edx andl %edi,%edx jnz int_careful - andl $~TS_COMPAT,TI_status(%rcx) + andl $~TS_COMPAT,threadinfo_status(%rcx) jmp retint_swapgs /* Either reschedule or signal or syscall exit tracking needed. */ @@ -559,7 +558,7 @@ retint_with_reschedule: movl $_TIF_WORK_MASK,%edi retint_check: LOCKDEP_SYS_EXIT_IRQ - movl TI_flags(%rcx),%edx + movl threadinfo_flags(%rcx),%edx andl %edi,%edx CFI_REMEMBER_STATE jnz retint_careful @@ -655,9 +654,9 @@ retint_signal: /* Returning to kernel space. Check if we need preemption */ /* rcx: threadinfo. interrupts off. */ ENTRY(retint_kernel) - cmpl $0,TI_preempt_count(%rcx) + cmpl $0,threadinfo_preempt_count(%rcx) jnz retint_restore_args - bt $TIF_NEED_RESCHED,TI_flags(%rcx) + bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx) jnc retint_restore_args bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ jnc retint_restore_args @@ -820,7 +819,7 @@ paranoid_restore\trace: jmp irq_return paranoid_userspace\trace: GET_THREAD_INFO(%rcx) - movl TI_flags(%rcx),%ebx + movl threadinfo_flags(%rcx),%ebx andl $_TIF_WORK_MASK,%ebx jz paranoid_swapgs\trace movq %rsp,%rdi /* &pt_regs */ @@ -918,7 +917,7 @@ error_exit: testl %eax,%eax jne retint_kernel LOCKDEP_SYS_EXIT_IRQ - movl TI_flags(%rcx),%edx + movl threadinfo_flags(%rcx),%edx movl $_TIF_WORK_MASK,%edi andl %edi,%edx jnz retint_careful diff --git a/trunk/arch/x86/kernel/io_apic_64.c b/trunk/arch/x86/kernel/io_apic_64.c index 848411753c7c..0494cdb270c5 100644 --- a/trunk/arch/x86/kernel/io_apic_64.c +++ b/trunk/arch/x86/kernel/io_apic_64.c @@ -1724,6 +1724,11 @@ static inline void __init check_timer(void) apic2 = apic1; } + replace_pin_at_irq(0, 0, 0, apic1, pin1); + apic1 = 0; + pin1 = 0; + setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); + if (pin1 != -1) { /* * Ok, does IRQ0 through the IOAPIC work? diff --git a/trunk/arch/x86/kernel/tsc.c b/trunk/arch/x86/kernel/tsc.c index 4a775d001957..3c36f92160c9 100644 --- a/trunk/arch/x86/kernel/tsc.c +++ b/trunk/arch/x86/kernel/tsc.c @@ -513,7 +513,6 @@ void __init tsc_init(void) */ for_each_possible_cpu(cpu) set_cyc2ns_scale(cpu_khz, cpu); - use_tsc_delay(); if (tsc_disabled > 0) return; diff --git a/trunk/arch/x86/lib/Makefile b/trunk/arch/x86/lib/Makefile index 83226e0a7ce4..76f60f52a885 100644 --- a/trunk/arch/x86/lib/Makefile +++ b/trunk/arch/x86/lib/Makefile @@ -4,8 +4,8 @@ obj-$(CONFIG_SMP) := msr-on-cpu.o -lib-y := delay.o -lib-y += usercopy_$(BITS).o getuser.o putuser.o +lib-y := delay_$(BITS).o +lib-y += usercopy_$(BITS).o getuser_$(BITS).o putuser_$(BITS).o lib-y += memcpy_$(BITS).o ifeq ($(CONFIG_X86_32),y) diff --git a/trunk/arch/x86/lib/copy_user_64.S b/trunk/arch/x86/lib/copy_user_64.S index e5afb4ad3f1c..ee1c3f635157 100644 --- a/trunk/arch/x86/lib/copy_user_64.S +++ b/trunk/arch/x86/lib/copy_user_64.S @@ -1,10 +1,8 @@ -/* - * Copyright 2008 Vitaly Mayatskikh - * Copyright 2002 Andi Kleen, SuSE Labs. +/* Copyright 2002 Andi Kleen, SuSE Labs. * Subject to the GNU Public License v2. - * - * Functions to copy from and to user space. - */ + * + * Functions to copy from and to user space. + */ #include #include @@ -22,88 +20,60 @@ .long \orig-1f /* by default jump to orig */ 1: .section .altinstr_replacement,"ax" -2: .byte 0xe9 /* near jump with 32bit immediate */ +2: .byte 0xe9 /* near jump with 32bit immediate */ .long \alt-1b /* offset */ /* or alternatively to alt */ .previous .section .altinstructions,"a" .align 8 .quad 0b .quad 2b - .byte \feature /* when feature is set */ + .byte \feature /* when feature is set */ .byte 5 .byte 5 .previous .endm - .macro ALIGN_DESTINATION -#ifdef FIX_ALIGNMENT - /* check for bad alignment of destination */ - movl %edi,%ecx - andl $7,%ecx - jz 102f /* already aligned */ - subl $8,%ecx - negl %ecx - subl %ecx,%edx -100: movb (%rsi),%al -101: movb %al,(%rdi) - incq %rsi - incq %rdi - decl %ecx - jnz 100b -102: - .section .fixup,"ax" -103: addl %r8d,%edx /* ecx is zerorest also */ - jmp copy_user_handle_tail - .previous - - .section __ex_table,"a" - .align 8 - .quad 100b,103b - .quad 101b,103b - .previous -#endif - .endm - -/* Standard copy_to_user with segment limit checking */ +/* Standard copy_to_user with segment limit checking */ ENTRY(copy_to_user) CFI_STARTPROC GET_THREAD_INFO(%rax) movq %rdi,%rcx addq %rdx,%rcx - jc bad_to_user - cmpq TI_addr_limit(%rax),%rcx + jc bad_to_user + cmpq threadinfo_addr_limit(%rax),%rcx jae bad_to_user + xorl %eax,%eax /* clear zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string CFI_ENDPROC -/* Standard copy_from_user with segment limit checking */ -ENTRY(copy_from_user) +ENTRY(copy_user_generic) CFI_STARTPROC - GET_THREAD_INFO(%rax) - movq %rsi,%rcx - addq %rdx,%rcx - jc bad_from_user - cmpq TI_addr_limit(%rax),%rcx - jae bad_from_user + movl $1,%ecx /* set zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string CFI_ENDPROC -ENDPROC(copy_from_user) -ENTRY(copy_user_generic) +ENTRY(__copy_from_user_inatomic) CFI_STARTPROC + xorl %ecx,%ecx /* clear zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string CFI_ENDPROC -ENDPROC(copy_user_generic) -ENTRY(__copy_from_user_inatomic) +/* Standard copy_from_user with segment limit checking */ +ENTRY(copy_from_user) CFI_STARTPROC + GET_THREAD_INFO(%rax) + movq %rsi,%rcx + addq %rdx,%rcx + jc bad_from_user + cmpq threadinfo_addr_limit(%rax),%rcx + jae bad_from_user + movl $1,%ecx /* set zero flag */ ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string CFI_ENDPROC -ENDPROC(__copy_from_user_inatomic) - +ENDPROC(copy_from_user) + .section .fixup,"ax" /* must zero dest */ -ENTRY(bad_from_user) bad_from_user: CFI_STARTPROC movl %edx,%ecx @@ -111,158 +81,271 @@ bad_from_user: rep stosb bad_to_user: - movl %edx,%eax + movl %edx,%eax ret CFI_ENDPROC -ENDPROC(bad_from_user) +END(bad_from_user) .previous - + + /* * copy_user_generic_unrolled - memory copy with exception handling. - * This version is for CPUs like P4 that don't have efficient micro - * code for rep movsq - * - * Input: + * This version is for CPUs like P4 that don't have efficient micro code for rep movsq + * + * Input: * rdi destination * rsi source * rdx count + * ecx zero flag -- if true zero destination on error * - * Output: - * eax uncopied bytes or 0 if successfull. + * Output: + * eax uncopied bytes or 0 if successful. */ ENTRY(copy_user_generic_unrolled) CFI_STARTPROC - cmpl $8,%edx - jb 20f /* less then 8 bytes, go to byte copy loop */ - ALIGN_DESTINATION - movl %edx,%ecx - andl $63,%edx - shrl $6,%ecx - jz 17f -1: movq (%rsi),%r8 -2: movq 1*8(%rsi),%r9 -3: movq 2*8(%rsi),%r10 -4: movq 3*8(%rsi),%r11 -5: movq %r8,(%rdi) -6: movq %r9,1*8(%rdi) -7: movq %r10,2*8(%rdi) -8: movq %r11,3*8(%rdi) -9: movq 4*8(%rsi),%r8 -10: movq 5*8(%rsi),%r9 -11: movq 6*8(%rsi),%r10 -12: movq 7*8(%rsi),%r11 -13: movq %r8,4*8(%rdi) -14: movq %r9,5*8(%rdi) -15: movq %r10,6*8(%rdi) -16: movq %r11,7*8(%rdi) + pushq %rbx + CFI_ADJUST_CFA_OFFSET 8 + CFI_REL_OFFSET rbx, 0 + pushq %rcx + CFI_ADJUST_CFA_OFFSET 8 + CFI_REL_OFFSET rcx, 0 + xorl %eax,%eax /*zero for the exception handler */ + +#ifdef FIX_ALIGNMENT + /* check for bad alignment of destination */ + movl %edi,%ecx + andl $7,%ecx + jnz .Lbad_alignment +.Lafter_bad_alignment: +#endif + + movq %rdx,%rcx + + movl $64,%ebx + shrq $6,%rdx + decq %rdx + js .Lhandle_tail + + .p2align 4 +.Lloop: +.Ls1: movq (%rsi),%r11 +.Ls2: movq 1*8(%rsi),%r8 +.Ls3: movq 2*8(%rsi),%r9 +.Ls4: movq 3*8(%rsi),%r10 +.Ld1: movq %r11,(%rdi) +.Ld2: movq %r8,1*8(%rdi) +.Ld3: movq %r9,2*8(%rdi) +.Ld4: movq %r10,3*8(%rdi) + +.Ls5: movq 4*8(%rsi),%r11 +.Ls6: movq 5*8(%rsi),%r8 +.Ls7: movq 6*8(%rsi),%r9 +.Ls8: movq 7*8(%rsi),%r10 +.Ld5: movq %r11,4*8(%rdi) +.Ld6: movq %r8,5*8(%rdi) +.Ld7: movq %r9,6*8(%rdi) +.Ld8: movq %r10,7*8(%rdi) + + decq %rdx + leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi - decl %ecx - jnz 1b -17: movl %edx,%ecx - andl $7,%edx + + jns .Lloop + + .p2align 4 +.Lhandle_tail: + movl %ecx,%edx + andl $63,%ecx shrl $3,%ecx - jz 20f -18: movq (%rsi),%r8 -19: movq %r8,(%rdi) - leaq 8(%rsi),%rsi - leaq 8(%rdi),%rdi + jz .Lhandle_7 + movl $8,%ebx + .p2align 4 +.Lloop_8: +.Ls9: movq (%rsi),%r8 +.Ld9: movq %r8,(%rdi) decl %ecx - jnz 18b -20: andl %edx,%edx - jz 23f + leaq 8(%rdi),%rdi + leaq 8(%rsi),%rsi + jnz .Lloop_8 + +.Lhandle_7: movl %edx,%ecx -21: movb (%rsi),%al -22: movb %al,(%rdi) - incq %rsi + andl $7,%ecx + jz .Lende + .p2align 4 +.Lloop_1: +.Ls10: movb (%rsi),%bl +.Ld10: movb %bl,(%rdi) incq %rdi + incq %rsi decl %ecx - jnz 21b -23: xor %eax,%eax + jnz .Lloop_1 + + CFI_REMEMBER_STATE +.Lende: + popq %rcx + CFI_ADJUST_CFA_OFFSET -8 + CFI_RESTORE rcx + popq %rbx + CFI_ADJUST_CFA_OFFSET -8 + CFI_RESTORE rbx ret + CFI_RESTORE_STATE - .section .fixup,"ax" -30: shll $6,%ecx - addl %ecx,%edx - jmp 60f -40: leal (%edx,%ecx,8),%edx - jmp 60f -50: movl %ecx,%edx -60: jmp copy_user_handle_tail /* ecx is zerorest also */ - .previous +#ifdef FIX_ALIGNMENT + /* align destination */ + .p2align 4 +.Lbad_alignment: + movl $8,%r9d + subl %ecx,%r9d + movl %r9d,%ecx + cmpq %r9,%rdx + jz .Lhandle_7 + js .Lhandle_7 +.Lalign_1: +.Ls11: movb (%rsi),%bl +.Ld11: movb %bl,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz .Lalign_1 + subq %r9,%rdx + jmp .Lafter_bad_alignment +#endif + /* table sorted by exception address */ .section __ex_table,"a" .align 8 - .quad 1b,30b - .quad 2b,30b - .quad 3b,30b - .quad 4b,30b - .quad 5b,30b - .quad 6b,30b - .quad 7b,30b - .quad 8b,30b - .quad 9b,30b - .quad 10b,30b - .quad 11b,30b - .quad 12b,30b - .quad 13b,30b - .quad 14b,30b - .quad 15b,30b - .quad 16b,30b - .quad 18b,40b - .quad 19b,40b - .quad 21b,50b - .quad 22b,50b + .quad .Ls1,.Ls1e /* Ls1-Ls4 have copied zero bytes */ + .quad .Ls2,.Ls1e + .quad .Ls3,.Ls1e + .quad .Ls4,.Ls1e + .quad .Ld1,.Ls1e /* Ld1-Ld4 have copied 0-24 bytes */ + .quad .Ld2,.Ls2e + .quad .Ld3,.Ls3e + .quad .Ld4,.Ls4e + .quad .Ls5,.Ls5e /* Ls5-Ls8 have copied 32 bytes */ + .quad .Ls6,.Ls5e + .quad .Ls7,.Ls5e + .quad .Ls8,.Ls5e + .quad .Ld5,.Ls5e /* Ld5-Ld8 have copied 32-56 bytes */ + .quad .Ld6,.Ls6e + .quad .Ld7,.Ls7e + .quad .Ld8,.Ls8e + .quad .Ls9,.Le_quad + .quad .Ld9,.Le_quad + .quad .Ls10,.Le_byte + .quad .Ld10,.Le_byte +#ifdef FIX_ALIGNMENT + .quad .Ls11,.Lzero_rest + .quad .Ld11,.Lzero_rest +#endif + .quad .Le5,.Le_zero .previous + + /* eax: zero, ebx: 64 */ +.Ls1e: addl $8,%eax /* eax is bytes left uncopied within the loop (Ls1e: 64 .. Ls8e: 8) */ +.Ls2e: addl $8,%eax +.Ls3e: addl $8,%eax +.Ls4e: addl $8,%eax +.Ls5e: addl $8,%eax +.Ls6e: addl $8,%eax +.Ls7e: addl $8,%eax +.Ls8e: addl $8,%eax + addq %rbx,%rdi /* +64 */ + subq %rax,%rdi /* correct destination with computed offset */ + + shlq $6,%rdx /* loop counter * 64 (stride length) */ + addq %rax,%rdx /* add offset to loopcnt */ + andl $63,%ecx /* remaining bytes */ + addq %rcx,%rdx /* add them */ + jmp .Lzero_rest + + /* exception on quad word loop in tail handling */ + /* ecx: loopcnt/8, %edx: length, rdi: correct */ +.Le_quad: + shll $3,%ecx + andl $7,%edx + addl %ecx,%edx + /* edx: bytes to zero, rdi: dest, eax:zero */ +.Lzero_rest: + cmpl $0,(%rsp) + jz .Le_zero + movq %rdx,%rcx +.Le_byte: + xorl %eax,%eax +.Le5: rep + stosb + /* when there is another exception while zeroing the rest just return */ +.Le_zero: + movq %rdx,%rax + jmp .Lende CFI_ENDPROC -ENDPROC(copy_user_generic_unrolled) +ENDPROC(copy_user_generic) -/* Some CPUs run faster using the string copy instructions. - * This is also a lot simpler. Use them when possible. - * - * Only 4GB of copy is supported. This shouldn't be a problem - * because the kernel normally only writes from/to page sized chunks - * even if user space passed a longer buffer. - * And more would be dangerous because both Intel and AMD have - * errata with rep movsq > 4GB. If someone feels the need to fix - * this please consider this. - * - * Input: - * rdi destination - * rsi source - * rdx count - * - * Output: - * eax uncopied bytes or 0 if successful. - */ + + /* Some CPUs run faster using the string copy instructions. + This is also a lot simpler. Use them when possible. + Patch in jmps to this code instead of copying it fully + to avoid unwanted aliasing in the exception tables. */ + + /* rdi destination + * rsi source + * rdx count + * ecx zero flag + * + * Output: + * eax uncopied bytes or 0 if successfull. + * + * Only 4GB of copy is supported. This shouldn't be a problem + * because the kernel normally only writes from/to page sized chunks + * even if user space passed a longer buffer. + * And more would be dangerous because both Intel and AMD have + * errata with rep movsq > 4GB. If someone feels the need to fix + * this please consider this. + */ ENTRY(copy_user_generic_string) CFI_STARTPROC - andl %edx,%edx - jz 4f - cmpl $8,%edx - jb 2f /* less than 8 bytes, go to byte copy loop */ - ALIGN_DESTINATION + movl %ecx,%r8d /* save zero flag */ movl %edx,%ecx shrl $3,%ecx - andl $7,%edx -1: rep - movsq -2: movl %edx,%ecx -3: rep + andl $7,%edx + jz 10f +1: rep + movsq + movl %edx,%ecx +2: rep movsb -4: xorl %eax,%eax +9: movl %ecx,%eax ret - .section .fixup,"ax" -11: leal (%edx,%ecx,8),%ecx -12: movl %ecx,%edx /* ecx is zerorest also */ - jmp copy_user_handle_tail - .previous + /* multiple of 8 byte */ +10: rep + movsq + xor %eax,%eax + ret + + /* exception handling */ +3: lea (%rdx,%rcx,8),%rax /* exception on quad loop */ + jmp 6f +5: movl %ecx,%eax /* exception on byte loop */ + /* eax: left over bytes */ +6: testl %r8d,%r8d /* zero flag set? */ + jz 7f + movl %eax,%ecx /* initialize x86 loop counter */ + push %rax + xorl %eax,%eax +8: rep + stosb /* zero the rest */ +11: pop %rax +7: ret + CFI_ENDPROC +END(copy_user_generic_c) .section __ex_table,"a" - .align 8 - .quad 1b,11b - .quad 3b,12b + .quad 1b,3b + .quad 2b,5b + .quad 8b,11b + .quad 10b,3b .previous - CFI_ENDPROC -ENDPROC(copy_user_generic_string) diff --git a/trunk/arch/x86/lib/copy_user_nocache_64.S b/trunk/arch/x86/lib/copy_user_nocache_64.S index 93353d6a5263..9d3d1ab83763 100644 --- a/trunk/arch/x86/lib/copy_user_nocache_64.S +++ b/trunk/arch/x86/lib/copy_user_nocache_64.S @@ -1,6 +1,4 @@ -/* - * Copyright 2008 Vitaly Mayatskikh - * Copyright 2002 Andi Kleen, SuSE Labs. +/* Copyright 2002 Andi Kleen, SuSE Labs. * Subject to the GNU Public License v2. * * Functions to copy from and to user space. @@ -14,125 +12,204 @@ #include #include #include - - .macro ALIGN_DESTINATION -#ifdef FIX_ALIGNMENT - /* check for bad alignment of destination */ - movl %edi,%ecx - andl $7,%ecx - jz 102f /* already aligned */ - subl $8,%ecx - negl %ecx - subl %ecx,%edx -100: movb (%rsi),%al -101: movb %al,(%rdi) - incq %rsi - incq %rdi - decl %ecx - jnz 100b -102: - .section .fixup,"ax" -103: addl %r8d,%edx /* ecx is zerorest also */ - jmp copy_user_handle_tail - .previous - - .section __ex_table,"a" - .align 8 - .quad 100b,103b - .quad 101b,103b - .previous -#endif - .endm +#include /* * copy_user_nocache - Uncached memory copy with exception handling * This will force destination/source out of cache for more performance. + * + * Input: + * rdi destination + * rsi source + * rdx count + * rcx zero flag when 1 zero on exception + * + * Output: + * eax uncopied bytes or 0 if successful. */ ENTRY(__copy_user_nocache) CFI_STARTPROC - cmpl $8,%edx - jb 20f /* less then 8 bytes, go to byte copy loop */ - ALIGN_DESTINATION - movl %edx,%ecx - andl $63,%edx - shrl $6,%ecx - jz 17f -1: movq (%rsi),%r8 -2: movq 1*8(%rsi),%r9 -3: movq 2*8(%rsi),%r10 -4: movq 3*8(%rsi),%r11 -5: movnti %r8,(%rdi) -6: movnti %r9,1*8(%rdi) -7: movnti %r10,2*8(%rdi) -8: movnti %r11,3*8(%rdi) -9: movq 4*8(%rsi),%r8 -10: movq 5*8(%rsi),%r9 -11: movq 6*8(%rsi),%r10 -12: movq 7*8(%rsi),%r11 -13: movnti %r8,4*8(%rdi) -14: movnti %r9,5*8(%rdi) -15: movnti %r10,6*8(%rdi) -16: movnti %r11,7*8(%rdi) + pushq %rbx + CFI_ADJUST_CFA_OFFSET 8 + CFI_REL_OFFSET rbx, 0 + pushq %rcx /* save zero flag */ + CFI_ADJUST_CFA_OFFSET 8 + CFI_REL_OFFSET rcx, 0 + + xorl %eax,%eax /* zero for the exception handler */ + +#ifdef FIX_ALIGNMENT + /* check for bad alignment of destination */ + movl %edi,%ecx + andl $7,%ecx + jnz .Lbad_alignment +.Lafter_bad_alignment: +#endif + + movq %rdx,%rcx + + movl $64,%ebx + shrq $6,%rdx + decq %rdx + js .Lhandle_tail + + .p2align 4 +.Lloop: +.Ls1: movq (%rsi),%r11 +.Ls2: movq 1*8(%rsi),%r8 +.Ls3: movq 2*8(%rsi),%r9 +.Ls4: movq 3*8(%rsi),%r10 +.Ld1: movnti %r11,(%rdi) +.Ld2: movnti %r8,1*8(%rdi) +.Ld3: movnti %r9,2*8(%rdi) +.Ld4: movnti %r10,3*8(%rdi) + +.Ls5: movq 4*8(%rsi),%r11 +.Ls6: movq 5*8(%rsi),%r8 +.Ls7: movq 6*8(%rsi),%r9 +.Ls8: movq 7*8(%rsi),%r10 +.Ld5: movnti %r11,4*8(%rdi) +.Ld6: movnti %r8,5*8(%rdi) +.Ld7: movnti %r9,6*8(%rdi) +.Ld8: movnti %r10,7*8(%rdi) + + dec %rdx + leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi - decl %ecx - jnz 1b -17: movl %edx,%ecx - andl $7,%edx + + jns .Lloop + + .p2align 4 +.Lhandle_tail: + movl %ecx,%edx + andl $63,%ecx shrl $3,%ecx - jz 20f -18: movq (%rsi),%r8 -19: movnti %r8,(%rdi) - leaq 8(%rsi),%rsi - leaq 8(%rdi),%rdi + jz .Lhandle_7 + movl $8,%ebx + .p2align 4 +.Lloop_8: +.Ls9: movq (%rsi),%r8 +.Ld9: movnti %r8,(%rdi) decl %ecx - jnz 18b -20: andl %edx,%edx - jz 23f + leaq 8(%rdi),%rdi + leaq 8(%rsi),%rsi + jnz .Lloop_8 + +.Lhandle_7: movl %edx,%ecx -21: movb (%rsi),%al -22: movb %al,(%rdi) - incq %rsi + andl $7,%ecx + jz .Lende + .p2align 4 +.Lloop_1: +.Ls10: movb (%rsi),%bl +.Ld10: movb %bl,(%rdi) incq %rdi + incq %rsi decl %ecx - jnz 21b -23: xorl %eax,%eax + jnz .Lloop_1 + + CFI_REMEMBER_STATE +.Lende: + popq %rcx + CFI_ADJUST_CFA_OFFSET -8 + CFI_RESTORE %rcx + popq %rbx + CFI_ADJUST_CFA_OFFSET -8 + CFI_RESTORE rbx sfence ret + CFI_RESTORE_STATE - .section .fixup,"ax" -30: shll $6,%ecx - addl %ecx,%edx - jmp 60f -40: leal (%edx,%ecx,8),%edx - jmp 60f -50: movl %ecx,%edx -60: sfence - movl %r8d,%ecx - jmp copy_user_handle_tail - .previous +#ifdef FIX_ALIGNMENT + /* align destination */ + .p2align 4 +.Lbad_alignment: + movl $8,%r9d + subl %ecx,%r9d + movl %r9d,%ecx + cmpq %r9,%rdx + jz .Lhandle_7 + js .Lhandle_7 +.Lalign_1: +.Ls11: movb (%rsi),%bl +.Ld11: movb %bl,(%rdi) + incq %rsi + incq %rdi + decl %ecx + jnz .Lalign_1 + subq %r9,%rdx + jmp .Lafter_bad_alignment +#endif + /* table sorted by exception address */ .section __ex_table,"a" - .quad 1b,30b - .quad 2b,30b - .quad 3b,30b - .quad 4b,30b - .quad 5b,30b - .quad 6b,30b - .quad 7b,30b - .quad 8b,30b - .quad 9b,30b - .quad 10b,30b - .quad 11b,30b - .quad 12b,30b - .quad 13b,30b - .quad 14b,30b - .quad 15b,30b - .quad 16b,30b - .quad 18b,40b - .quad 19b,40b - .quad 21b,50b - .quad 22b,50b + .align 8 + .quad .Ls1,.Ls1e /* .Ls[1-4] - 0 bytes copied */ + .quad .Ls2,.Ls1e + .quad .Ls3,.Ls1e + .quad .Ls4,.Ls1e + .quad .Ld1,.Ls1e /* .Ld[1-4] - 0..24 bytes coped */ + .quad .Ld2,.Ls2e + .quad .Ld3,.Ls3e + .quad .Ld4,.Ls4e + .quad .Ls5,.Ls5e /* .Ls[5-8] - 32 bytes copied */ + .quad .Ls6,.Ls5e + .quad .Ls7,.Ls5e + .quad .Ls8,.Ls5e + .quad .Ld5,.Ls5e /* .Ld[5-8] - 32..56 bytes copied */ + .quad .Ld6,.Ls6e + .quad .Ld7,.Ls7e + .quad .Ld8,.Ls8e + .quad .Ls9,.Le_quad + .quad .Ld9,.Le_quad + .quad .Ls10,.Le_byte + .quad .Ld10,.Le_byte +#ifdef FIX_ALIGNMENT + .quad .Ls11,.Lzero_rest + .quad .Ld11,.Lzero_rest +#endif + .quad .Le5,.Le_zero .previous + + /* eax: zero, ebx: 64 */ +.Ls1e: addl $8,%eax /* eax: bytes left uncopied: Ls1e: 64 .. Ls8e: 8 */ +.Ls2e: addl $8,%eax +.Ls3e: addl $8,%eax +.Ls4e: addl $8,%eax +.Ls5e: addl $8,%eax +.Ls6e: addl $8,%eax +.Ls7e: addl $8,%eax +.Ls8e: addl $8,%eax + addq %rbx,%rdi /* +64 */ + subq %rax,%rdi /* correct destination with computed offset */ + + shlq $6,%rdx /* loop counter * 64 (stride length) */ + addq %rax,%rdx /* add offset to loopcnt */ + andl $63,%ecx /* remaining bytes */ + addq %rcx,%rdx /* add them */ + jmp .Lzero_rest + + /* exception on quad word loop in tail handling */ + /* ecx: loopcnt/8, %edx: length, rdi: correct */ +.Le_quad: + shll $3,%ecx + andl $7,%edx + addl %ecx,%edx + /* edx: bytes to zero, rdi: dest, eax:zero */ +.Lzero_rest: + cmpl $0,(%rsp) /* zero flag set? */ + jz .Le_zero + movq %rdx,%rcx +.Le_byte: + xorl %eax,%eax +.Le5: rep + stosb + /* when there is another exception while zeroing the rest just return */ +.Le_zero: + movq %rdx,%rax + jmp .Lende CFI_ENDPROC ENDPROC(__copy_user_nocache) + + diff --git a/trunk/arch/x86/lib/delay.c b/trunk/arch/x86/lib/delay_32.c similarity index 96% rename from trunk/arch/x86/lib/delay.c rename to trunk/arch/x86/lib/delay_32.c index f4568605d7d5..ef691316f8b6 100644 --- a/trunk/arch/x86/lib/delay.c +++ b/trunk/arch/x86/lib/delay_32.c @@ -29,7 +29,7 @@ /* simple loop based delay: */ static void delay_loop(unsigned long loops) { - asm volatile( + __asm__ __volatile__( " test %0,%0 \n" " jz 3f \n" " jmp 1f \n" @@ -38,9 +38,9 @@ static void delay_loop(unsigned long loops) "1: jmp 2f \n" ".align 16 \n" - "2: dec %0 \n" + "2: decl %0 \n" " jnz 2b \n" - "3: dec %0 \n" + "3: decl %0 \n" : /* we don't need output */ :"a" (loops) @@ -98,7 +98,7 @@ void use_tsc_delay(void) int __devinit read_current_timer(unsigned long *timer_val) { if (delay_fn == delay_tsc) { - rdtscll(*timer_val); + rdtscl(*timer_val); return 0; } return -1; @@ -108,30 +108,31 @@ void __delay(unsigned long loops) { delay_fn(loops); } -EXPORT_SYMBOL(__delay); inline void __const_udelay(unsigned long xloops) { int d0; xloops *= 4; - asm("mull %%edx" + __asm__("mull %0" :"=d" (xloops), "=&a" (d0) :"1" (xloops), "0" (cpu_data(raw_smp_processor_id()).loops_per_jiffy * (HZ/4))); __delay(++xloops); } -EXPORT_SYMBOL(__const_udelay); void __udelay(unsigned long usecs) { __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ } -EXPORT_SYMBOL(__udelay); void __ndelay(unsigned long nsecs) { __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ } + +EXPORT_SYMBOL(__delay); +EXPORT_SYMBOL(__const_udelay); +EXPORT_SYMBOL(__udelay); EXPORT_SYMBOL(__ndelay); diff --git a/trunk/arch/x86/lib/delay_64.c b/trunk/arch/x86/lib/delay_64.c new file mode 100644 index 000000000000..4c441be92641 --- /dev/null +++ b/trunk/arch/x86/lib/delay_64.c @@ -0,0 +1,85 @@ +/* + * Precise Delay Loops for x86-64 + * + * Copyright (C) 1993 Linus Torvalds + * Copyright (C) 1997 Martin Mares + * + * The __delay function must _NOT_ be inlined as its execution time + * depends wildly on alignment on many x86 processors. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef CONFIG_SMP +#include +#endif + +int __devinit read_current_timer(unsigned long *timer_value) +{ + rdtscll(*timer_value); + return 0; +} + +void __delay(unsigned long loops) +{ + unsigned bclock, now; + int cpu; + + preempt_disable(); + cpu = smp_processor_id(); + rdtscl(bclock); + for (;;) { + rdtscl(now); + if ((now - bclock) >= loops) + break; + + /* Allow RT tasks to run */ + preempt_enable(); + rep_nop(); + preempt_disable(); + + /* + * It is possible that we moved to another CPU, and + * since TSC's are per-cpu we need to calculate + * that. The delay must guarantee that we wait "at + * least" the amount of time. Being moved to another + * CPU could make the wait longer but we just need to + * make sure we waited long enough. Rebalance the + * counter for this CPU. + */ + if (unlikely(cpu != smp_processor_id())) { + loops -= (now - bclock); + cpu = smp_processor_id(); + rdtscl(bclock); + } + } + preempt_enable(); +} +EXPORT_SYMBOL(__delay); + +inline void __const_udelay(unsigned long xloops) +{ + __delay(((xloops * HZ * + cpu_data(raw_smp_processor_id()).loops_per_jiffy) >> 32) + 1); +} +EXPORT_SYMBOL(__const_udelay); + +void __udelay(unsigned long usecs) +{ + __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ +} +EXPORT_SYMBOL(__udelay); + +void __ndelay(unsigned long nsecs) +{ + __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ +} +EXPORT_SYMBOL(__ndelay); diff --git a/trunk/arch/x86/lib/getuser_32.S b/trunk/arch/x86/lib/getuser_32.S new file mode 100644 index 000000000000..6d84b53f12a2 --- /dev/null +++ b/trunk/arch/x86/lib/getuser_32.S @@ -0,0 +1,78 @@ +/* + * __get_user functions. + * + * (C) Copyright 1998 Linus Torvalds + * + * These functions have a non-standard call interface + * to make them more efficient, especially as they + * return an error value in addition to the "real" + * return value. + */ +#include +#include +#include + + +/* + * __get_user_X + * + * Inputs: %eax contains the address + * + * Outputs: %eax is error code (0 or -EFAULT) + * %edx contains zero-extended value + * + * These functions should not modify any other registers, + * as they get called from within inline assembly. + */ + +.text +ENTRY(__get_user_1) + CFI_STARTPROC + GET_THREAD_INFO(%edx) + cmpl TI_addr_limit(%edx),%eax + jae bad_get_user +1: movzbl (%eax),%edx + xorl %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_1) + +ENTRY(__get_user_2) + CFI_STARTPROC + addl $1,%eax + jc bad_get_user + GET_THREAD_INFO(%edx) + cmpl TI_addr_limit(%edx),%eax + jae bad_get_user +2: movzwl -1(%eax),%edx + xorl %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_2) + +ENTRY(__get_user_4) + CFI_STARTPROC + addl $3,%eax + jc bad_get_user + GET_THREAD_INFO(%edx) + cmpl TI_addr_limit(%edx),%eax + jae bad_get_user +3: movl -3(%eax),%edx + xorl %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__get_user_4) + +bad_get_user: + CFI_STARTPROC + xorl %edx,%edx + movl $-14,%eax + ret + CFI_ENDPROC +END(bad_get_user) + +.section __ex_table,"a" + .long 1b,bad_get_user + .long 2b,bad_get_user + .long 3b,bad_get_user +.previous diff --git a/trunk/arch/x86/lib/getuser.S b/trunk/arch/x86/lib/getuser_64.S similarity index 53% rename from trunk/arch/x86/lib/getuser.S rename to trunk/arch/x86/lib/getuser_64.S index ad374003742f..5448876261f8 100644 --- a/trunk/arch/x86/lib/getuser.S +++ b/trunk/arch/x86/lib/getuser_64.S @@ -3,7 +3,6 @@ * * (C) Copyright 1998 Linus Torvalds * (C) Copyright 2005 Andi Kleen - * (C) Copyright 2008 Glauber Costa * * These functions have a non-standard call interface * to make them more efficient, especially as they @@ -14,13 +13,14 @@ /* * __get_user_X * - * Inputs: %[r|e]ax contains the address. + * Inputs: %rcx contains the address. * The register is modified, but all changes are undone * before returning because the C code doesn't know about it. * - * Outputs: %[r|e]ax is error code (0 or -EFAULT) - * %[r|e]dx contains zero-extended value - * + * Outputs: %rax is error code (0 or -EFAULT) + * %rdx contains zero-extended value + * + * %r8 is destroyed. * * These functions should not modify any other registers, * as they get called from within inline assembly. @@ -32,73 +32,78 @@ #include #include #include -#include .text ENTRY(__get_user_1) CFI_STARTPROC - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX + GET_THREAD_INFO(%r8) + cmpq threadinfo_addr_limit(%r8),%rcx jae bad_get_user -1: movzb (%_ASM_AX),%edx - xor %eax,%eax +1: movzb (%rcx),%edx + xorl %eax,%eax ret CFI_ENDPROC ENDPROC(__get_user_1) ENTRY(__get_user_2) CFI_STARTPROC - add $1,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -2: movzwl -1(%_ASM_AX),%edx - xor %eax,%eax + GET_THREAD_INFO(%r8) + addq $1,%rcx + jc 20f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 20f + decq %rcx +2: movzwl (%rcx),%edx + xorl %eax,%eax ret +20: decq %rcx + jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_2) ENTRY(__get_user_4) CFI_STARTPROC - add $3,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -3: mov -3(%_ASM_AX),%edx - xor %eax,%eax + GET_THREAD_INFO(%r8) + addq $3,%rcx + jc 30f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 30f + subq $3,%rcx +3: movl (%rcx),%edx + xorl %eax,%eax ret +30: subq $3,%rcx + jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_4) -#ifdef CONFIG_X86_64 ENTRY(__get_user_8) CFI_STARTPROC - add $7,%_ASM_AX - jc bad_get_user - GET_THREAD_INFO(%_ASM_DX) - cmp TI_addr_limit(%_ASM_DX),%_ASM_AX - jae bad_get_user -4: movq -7(%_ASM_AX),%_ASM_DX - xor %eax,%eax + GET_THREAD_INFO(%r8) + addq $7,%rcx + jc 40f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 40f + subq $7,%rcx +4: movq (%rcx),%rdx + xorl %eax,%eax ret +40: subq $7,%rcx + jmp bad_get_user CFI_ENDPROC ENDPROC(__get_user_8) -#endif bad_get_user: CFI_STARTPROC - xor %edx,%edx - mov $(-EFAULT),%_ASM_AX + xorl %edx,%edx + movq $(-EFAULT),%rax ret CFI_ENDPROC END(bad_get_user) .section __ex_table,"a" - _ASM_PTR 1b,bad_get_user - _ASM_PTR 2b,bad_get_user - _ASM_PTR 3b,bad_get_user -#ifdef CONFIG_X86_64 - _ASM_PTR 4b,bad_get_user -#endif + .quad 1b,bad_get_user + .quad 2b,bad_get_user + .quad 3b,bad_get_user + .quad 4b,bad_get_user +.previous diff --git a/trunk/arch/x86/lib/putuser.S b/trunk/arch/x86/lib/putuser_32.S similarity index 54% rename from trunk/arch/x86/lib/putuser.S rename to trunk/arch/x86/lib/putuser_32.S index 36b0d15ae6e9..f58fba109d18 100644 --- a/trunk/arch/x86/lib/putuser.S +++ b/trunk/arch/x86/lib/putuser_32.S @@ -2,8 +2,6 @@ * __put_user functions. * * (C) Copyright 2005 Linus Torvalds - * (C) Copyright 2005 Andi Kleen - * (C) Copyright 2008 Glauber Costa * * These functions have a non-standard call interface * to make them more efficient, especially as they @@ -13,8 +11,6 @@ #include #include #include -#include -#include /* @@ -30,68 +26,73 @@ */ #define ENTER CFI_STARTPROC ; \ - GET_THREAD_INFO(%_ASM_BX) -#define EXIT ret ; \ + pushl %ebx ; \ + CFI_ADJUST_CFA_OFFSET 4 ; \ + CFI_REL_OFFSET ebx, 0 ; \ + GET_THREAD_INFO(%ebx) +#define EXIT popl %ebx ; \ + CFI_ADJUST_CFA_OFFSET -4 ; \ + CFI_RESTORE ebx ; \ + ret ; \ CFI_ENDPROC .text ENTRY(__put_user_1) ENTER - cmp TI_addr_limit(%_ASM_BX),%_ASM_CX + cmpl TI_addr_limit(%ebx),%ecx jae bad_put_user -1: movb %al,(%_ASM_CX) - xor %eax,%eax +1: movb %al,(%ecx) + xorl %eax,%eax EXIT ENDPROC(__put_user_1) ENTRY(__put_user_2) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $1,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX + movl TI_addr_limit(%ebx),%ebx + subl $1,%ebx + cmpl %ebx,%ecx jae bad_put_user -2: movw %ax,(%_ASM_CX) - xor %eax,%eax +2: movw %ax,(%ecx) + xorl %eax,%eax EXIT ENDPROC(__put_user_2) ENTRY(__put_user_4) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $3,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX + movl TI_addr_limit(%ebx),%ebx + subl $3,%ebx + cmpl %ebx,%ecx jae bad_put_user -3: movl %eax,(%_ASM_CX) - xor %eax,%eax +3: movl %eax,(%ecx) + xorl %eax,%eax EXIT ENDPROC(__put_user_4) ENTRY(__put_user_8) ENTER - mov TI_addr_limit(%_ASM_BX),%_ASM_BX - sub $7,%_ASM_BX - cmp %_ASM_BX,%_ASM_CX + movl TI_addr_limit(%ebx),%ebx + subl $7,%ebx + cmpl %ebx,%ecx jae bad_put_user -4: mov %_ASM_AX,(%_ASM_CX) -#ifdef CONFIG_X86_32 -5: movl %edx,4(%_ASM_CX) -#endif - xor %eax,%eax +4: movl %eax,(%ecx) +5: movl %edx,4(%ecx) + xorl %eax,%eax EXIT ENDPROC(__put_user_8) bad_put_user: - CFI_STARTPROC - movl $-EFAULT,%eax + CFI_STARTPROC simple + CFI_DEF_CFA esp, 2*4 + CFI_OFFSET eip, -1*4 + CFI_OFFSET ebx, -2*4 + movl $-14,%eax EXIT END(bad_put_user) .section __ex_table,"a" - _ASM_PTR 1b,bad_put_user - _ASM_PTR 2b,bad_put_user - _ASM_PTR 3b,bad_put_user - _ASM_PTR 4b,bad_put_user -#ifdef CONFIG_X86_32 - _ASM_PTR 5b,bad_put_user -#endif + .long 1b,bad_put_user + .long 2b,bad_put_user + .long 3b,bad_put_user + .long 4b,bad_put_user + .long 5b,bad_put_user .previous diff --git a/trunk/arch/x86/lib/putuser_64.S b/trunk/arch/x86/lib/putuser_64.S new file mode 100644 index 000000000000..4989f5a8fa9b --- /dev/null +++ b/trunk/arch/x86/lib/putuser_64.S @@ -0,0 +1,106 @@ +/* + * __put_user functions. + * + * (C) Copyright 1998 Linus Torvalds + * (C) Copyright 2005 Andi Kleen + * + * These functions have a non-standard call interface + * to make them more efficient, especially as they + * return an error value in addition to the "real" + * return value. + */ + +/* + * __put_user_X + * + * Inputs: %rcx contains the address + * %rdx contains new value + * + * Outputs: %rax is error code (0 or -EFAULT) + * + * %r8 is destroyed. + * + * These functions should not modify any other registers, + * as they get called from within inline assembly. + */ + +#include +#include +#include +#include +#include +#include + + .text +ENTRY(__put_user_1) + CFI_STARTPROC + GET_THREAD_INFO(%r8) + cmpq threadinfo_addr_limit(%r8),%rcx + jae bad_put_user +1: movb %dl,(%rcx) + xorl %eax,%eax + ret + CFI_ENDPROC +ENDPROC(__put_user_1) + +ENTRY(__put_user_2) + CFI_STARTPROC + GET_THREAD_INFO(%r8) + addq $1,%rcx + jc 20f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 20f + decq %rcx +2: movw %dx,(%rcx) + xorl %eax,%eax + ret +20: decq %rcx + jmp bad_put_user + CFI_ENDPROC +ENDPROC(__put_user_2) + +ENTRY(__put_user_4) + CFI_STARTPROC + GET_THREAD_INFO(%r8) + addq $3,%rcx + jc 30f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 30f + subq $3,%rcx +3: movl %edx,(%rcx) + xorl %eax,%eax + ret +30: subq $3,%rcx + jmp bad_put_user + CFI_ENDPROC +ENDPROC(__put_user_4) + +ENTRY(__put_user_8) + CFI_STARTPROC + GET_THREAD_INFO(%r8) + addq $7,%rcx + jc 40f + cmpq threadinfo_addr_limit(%r8),%rcx + jae 40f + subq $7,%rcx +4: movq %rdx,(%rcx) + xorl %eax,%eax + ret +40: subq $7,%rcx + jmp bad_put_user + CFI_ENDPROC +ENDPROC(__put_user_8) + +bad_put_user: + CFI_STARTPROC + movq $(-EFAULT),%rax + ret + CFI_ENDPROC +END(bad_put_user) + +.section __ex_table,"a" + .quad 1b,bad_put_user + .quad 2b,bad_put_user + .quad 3b,bad_put_user + .quad 4b,bad_put_user +.previous diff --git a/trunk/arch/x86/lib/usercopy_64.c b/trunk/arch/x86/lib/usercopy_64.c index f4df6e7c718b..0c89d1bb0287 100644 --- a/trunk/arch/x86/lib/usercopy_64.c +++ b/trunk/arch/x86/lib/usercopy_64.c @@ -158,26 +158,3 @@ unsigned long copy_in_user(void __user *to, const void __user *from, unsigned le } EXPORT_SYMBOL(copy_in_user); -/* - * Try to copy last bytes and clear the rest if needed. - * Since protection fault in copy_from/to_user is not a normal situation, - * it is not necessary to optimize tail handling. - */ -unsigned long -copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest) -{ - char c; - unsigned zero_len; - - for (; len; --len) { - if (__get_user_nocheck(c, from++, sizeof(char))) - break; - if (__put_user_nocheck(c, to++, sizeof(char))) - break; - } - - for (c = 0, zero_len = len; zerorest && zero_len; --zero_len) - if (__put_user_nocheck(c, to++, sizeof(char))) - break; - return len; -} diff --git a/trunk/arch/x86/mm/init_64.c b/trunk/arch/x86/mm/init_64.c index 48548ef7ddf8..51f69b39b752 100644 --- a/trunk/arch/x86/mm/init_64.c +++ b/trunk/arch/x86/mm/init_64.c @@ -302,13 +302,11 @@ static __meminit void unmap_low_page(void *adr) early_iounmap(adr, PAGE_SIZE); } -static unsigned long __meminit +static void __meminit phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end) { unsigned pages = 0; - unsigned long last_map_addr = end; int i; - pte_t *pte = pte_page + pte_index(addr); for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) { @@ -328,20 +326,17 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end) printk(" pte=%p addr=%lx pte=%016lx\n", pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte); set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL)); - last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE; pages++; } update_page_count(PG_LEVEL_4K, pages); - - return last_map_addr; } -static unsigned long __meminit +static void __meminit phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end) { pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); - return phys_pte_init(pte, address, end); + phys_pte_init(pte, address, end); } static unsigned long __meminit @@ -349,7 +344,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, unsigned long page_size_mask) { unsigned long pages = 0; - unsigned long last_map_addr = end; int i = pmd_index(address); @@ -368,8 +362,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, if (pmd_val(*pmd)) { if (!pmd_large(*pmd)) - last_map_addr = phys_pte_update(pmd, address, - end); + phys_pte_update(pmd, address, end); continue; } @@ -377,18 +370,17 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, pages++; set_pte((pte_t *)pmd, pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); - last_map_addr = (address & PMD_MASK) + PMD_SIZE; continue; } pte = alloc_low_page(&pte_phys); - last_map_addr = phys_pte_init(pte, address, end); + phys_pte_init(pte, address, end); unmap_low_page(pte); pmd_populate_kernel(&init_mm, pmd, __va(pte_phys)); } update_page_count(PG_LEVEL_2M, pages); - return last_map_addr; + return address; } static unsigned long __meminit @@ -667,32 +659,6 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start, return last_map_addr; } - -struct map_range { - unsigned long start; - unsigned long end; - unsigned page_size_mask; -}; - -#define NR_RANGE_MR 5 - -static int save_mr(struct map_range *mr, int nr_range, - unsigned long start_pfn, unsigned long end_pfn, - unsigned long page_size_mask) -{ - - if (start_pfn < end_pfn) { - if (nr_range >= NR_RANGE_MR) - panic("run out of range for init_memory_mapping\n"); - mr[nr_range].start = start_pfn<> PAGE_SHIFT; end_pfn = ((start + (PMD_SIZE - 1)) >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); - nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); + if (start_pfn < end_pfn) + last_map_addr = kernel_physical_mapping_init( + start_pfn<>PMD_SHIFT) @@ -741,40 +706,37 @@ unsigned long __init_refok init_memory_mapping(unsigned long start, << (PUD_SHIFT - PAGE_SHIFT); if (end_pfn > ((end>>PUD_SHIFT)<<(PUD_SHIFT - PAGE_SHIFT))) end_pfn = ((end>>PUD_SHIFT)<<(PUD_SHIFT - PAGE_SHIFT)); - nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, - page_size_mask & (1<>PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); - nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, - page_size_mask & - ((1<>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); - nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, - page_size_mask & (1<>PAGE_SHIFT; - nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); - - for (i = 0; i < nr_range; i++) - printk(KERN_DEBUG " %010lx - %010lx page %s\n", - mr[i].start, mr[i].end, - (mr[i].page_size_mask & (1<cycle, min_cycle); /* DMA Data Setup */ - t0 = DIV_ROUND_UP(cycletime, ide_palm_clk); - td = DIV_ROUND_UP(t->active, ide_palm_clk); + t0 = DIV_ROUND_UP(cycletime, ideclk_period); + td = DIV_ROUND_UP(t->active, ideclk_period); tkw = t0 - td - 1; td -= 1; @@ -168,9 +166,9 @@ static void palm_bk3710_setpiomode(void __iomem *base, ide_drive_t *mate, struct ide_timing *t; /* PIO Data Setup */ - t0 = DIV_ROUND_UP(cycletime, ide_palm_clk); + t0 = DIV_ROUND_UP(cycletime, ideclk_period); t2 = DIV_ROUND_UP(ide_timing_find_mode(XFER_PIO_0 + mode)->active, - ide_palm_clk); + ideclk_period); t2i = t0 - t2 - 1; t2 -= 1; @@ -192,8 +190,8 @@ static void palm_bk3710_setpiomode(void __iomem *base, ide_drive_t *mate, /* TASKFILE Setup */ t = ide_timing_find_mode(XFER_PIO_0 + mode); - t0 = DIV_ROUND_UP(t->cyc8b, ide_palm_clk); - t2 = DIV_ROUND_UP(t->act8b, ide_palm_clk); + t0 = DIV_ROUND_UP(t->cyc8b, ideclk_period); + t2 = DIV_ROUND_UP(t->act8b, ideclk_period); t2i = t0 - t2 - 1; t2 -= 1; @@ -350,22 +348,22 @@ static const struct ide_port_info __devinitdata palm_bk3710_port_info = { static int __devinit palm_bk3710_probe(struct platform_device *pdev) { - struct clk *clkp; + struct clk *clk; struct resource *mem, *irq; ide_hwif_t *hwif; - unsigned long base; + unsigned long base, rate; int i; hw_regs_t hw; u8 idx[4] = { 0xff, 0xff, 0xff, 0xff }; - clkp = clk_get(NULL, "IDECLK"); - if (IS_ERR(clkp)) + clk = clk_get(NULL, "IDECLK"); + if (IS_ERR(clk)) return -ENODEV; - ideclkp = clkp; - clk_enable(ideclkp); - ide_palm_clk = clk_get_rate(ideclkp)/100000; - ide_palm_clk = (10000/ide_palm_clk) + 1; + clk_enable(clk); + rate = clk_get_rate(clk); + ideclk_period = 1000000000UL / rate; + /* Register the IDE interface with Linux ATA Interface */ memset(&hw, 0, sizeof(hw)); diff --git a/trunk/drivers/ide/ide-probe.c b/trunk/drivers/ide/ide-probe.c index d27061b39324..26e68b65b7cf 100644 --- a/trunk/drivers/ide/ide-probe.c +++ b/trunk/drivers/ide/ide-probe.c @@ -1218,16 +1218,12 @@ static void drive_release_dev (struct device *dev) complete(&drive->gendev_rel_comp); } -#ifndef ide_default_irq -#define ide_default_irq(irq) 0 -#endif - static int hwif_init(ide_hwif_t *hwif) { int old_irq; if (!hwif->irq) { - hwif->irq = ide_default_irq(hwif->io_ports.data_addr); + hwif->irq = __ide_default_irq(hwif->io_ports.data_addr); if (!hwif->irq) { printk("%s: DISABLED, NO IRQ\n", hwif->name); return 0; @@ -1257,7 +1253,7 @@ static int hwif_init(ide_hwif_t *hwif) * It failed to initialise. Find the default IRQ for * this port and try that. */ - hwif->irq = ide_default_irq(hwif->io_ports.data_addr); + hwif->irq = __ide_default_irq(hwif->io_ports.data_addr); if (!hwif->irq) { printk("%s: Disabled unable to get IRQ %d.\n", hwif->name, old_irq); diff --git a/trunk/drivers/ide/pci/it8213.c b/trunk/drivers/ide/pci/it8213.c index 9053c8771e6e..2b71bdf74e73 100644 --- a/trunk/drivers/ide/pci/it8213.c +++ b/trunk/drivers/ide/pci/it8213.c @@ -184,8 +184,7 @@ static const struct ide_port_info it8213_chipsets[] __devinitdata = { static int __devinit it8213_init_one(struct pci_dev *dev, const struct pci_device_id *id) { - ide_setup_pci_device(dev, &it8213_chipsets[id->driver_data]); - return 0; + return ide_setup_pci_device(dev, &it8213_chipsets[id->driver_data]); } static const struct pci_device_id it8213_pci_tbl[] = { diff --git a/trunk/drivers/ide/pci/ns87415.c b/trunk/drivers/ide/pci/ns87415.c index fec4955f449b..a7a41bb82778 100644 --- a/trunk/drivers/ide/pci/ns87415.c +++ b/trunk/drivers/ide/pci/ns87415.c @@ -225,10 +225,6 @@ static int ns87415_dma_setup(ide_drive_t *drive) return 1; } -#ifndef ide_default_irq -#define ide_default_irq(irq) 0 -#endif - static void __devinit init_hwif_ns87415 (ide_hwif_t *hwif) { struct pci_dev *dev = to_pci_dev(hwif->dev); @@ -288,7 +284,7 @@ static void __devinit init_hwif_ns87415 (ide_hwif_t *hwif) } if (!using_inta) - hwif->irq = ide_default_irq(hwif->io_ports.data_addr); + hwif->irq = __ide_default_irq(hwif->io_ports.data_addr); else if (!hwif->irq && hwif->mate && hwif->mate->irq) hwif->irq = hwif->mate->irq; /* share IRQ with mate */ diff --git a/trunk/drivers/net/ehea/ehea.h b/trunk/drivers/net/ehea/ehea.h index fe872fbd671e..e01926b7b5b7 100644 --- a/trunk/drivers/net/ehea/ehea.h +++ b/trunk/drivers/net/ehea/ehea.h @@ -40,7 +40,7 @@ #include #define DRV_NAME "ehea" -#define DRV_VERSION "EHEA_0091" +#define DRV_VERSION "EHEA_0092" /* eHEA capability flags */ #define DLPAR_PORT_ADD_REM 1 @@ -452,7 +452,7 @@ struct ehea_bcmc_reg_entry { struct ehea_bcmc_reg_array { struct ehea_bcmc_reg_entry *arr; int num_entries; - struct mutex lock; + spinlock_t lock; }; #define EHEA_PORT_UP 1 @@ -478,6 +478,7 @@ struct ehea_port { int num_add_tx_qps; int num_mcs; int resets; + u64 flags; u64 mac_addr; u32 logical_port_id; u32 port_speed; @@ -501,7 +502,8 @@ struct port_res_cfg { }; enum ehea_flag_bits { - __EHEA_STOP_XFER + __EHEA_STOP_XFER, + __EHEA_DISABLE_PORT_RESET }; void ehea_set_ethtool_ops(struct net_device *netdev); diff --git a/trunk/drivers/net/ehea/ehea_main.c b/trunk/drivers/net/ehea/ehea_main.c index 075fd547421e..0920b796bd78 100644 --- a/trunk/drivers/net/ehea/ehea_main.c +++ b/trunk/drivers/net/ehea/ehea_main.c @@ -118,6 +118,7 @@ static struct of_device_id ehea_device_table[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, ehea_device_table); static struct of_platform_driver ehea_driver = { .name = "ehea", @@ -137,6 +138,12 @@ void ehea_dump(void *adr, int len, char *msg) } } +void ehea_schedule_port_reset(struct ehea_port *port) +{ + if (!test_bit(__EHEA_DISABLE_PORT_RESET, &port->flags)) + schedule_work(&port->reset_task); +} + static void ehea_update_firmware_handles(void) { struct ehea_fw_handle_entry *arr = NULL; @@ -241,7 +248,7 @@ static void ehea_update_bcmc_registrations(void) } if (num_registrations) { - arr = kzalloc(num_registrations * sizeof(*arr), GFP_KERNEL); + arr = kzalloc(num_registrations * sizeof(*arr), GFP_ATOMIC); if (!arr) return; /* Keep the existing array */ } else @@ -301,7 +308,7 @@ static struct net_device_stats *ehea_get_stats(struct net_device *dev) memset(stats, 0, sizeof(*stats)); - cb2 = kzalloc(PAGE_SIZE, GFP_KERNEL); + cb2 = kzalloc(PAGE_SIZE, GFP_ATOMIC); if (!cb2) { ehea_error("no mem for cb2"); goto out; @@ -587,7 +594,7 @@ static int ehea_treat_poll_error(struct ehea_port_res *pr, int rq, "Resetting port.", pr->qp->init_attr.qp_nr); ehea_dump(cqe, sizeof(*cqe), "CQE"); } - schedule_work(&pr->port->reset_task); + ehea_schedule_port_reset(pr->port); return 1; } @@ -616,7 +623,7 @@ static int get_skb_hdr(struct sk_buff *skb, void **iphdr, *tcph = tcp_hdr(skb); /* check if ip header and tcp header are complete */ - if (iph->tot_len < ip_len + tcp_hdrlen(skb)) + if (ntohs(iph->tot_len) < ip_len + tcp_hdrlen(skb)) return -1; *hdr_flags = LRO_IPV4 | LRO_TCP; @@ -765,7 +772,7 @@ static struct ehea_cqe *ehea_proc_cqes(struct ehea_port_res *pr, int my_quota) ehea_error("Send Completion Error: Resetting port"); if (netif_msg_tx_err(pr->port)) ehea_dump(cqe, sizeof(*cqe), "Send CQE"); - schedule_work(&pr->port->reset_task); + ehea_schedule_port_reset(pr->port); break; } @@ -885,7 +892,7 @@ static irqreturn_t ehea_qp_aff_irq_handler(int irq, void *param) eqe = ehea_poll_eq(port->qp_eq); } - schedule_work(&port->reset_task); + ehea_schedule_port_reset(port); return IRQ_HANDLED; } @@ -1763,7 +1770,7 @@ static int ehea_set_mac_addr(struct net_device *dev, void *sa) memcpy(dev->dev_addr, mac_addr->sa_data, dev->addr_len); - mutex_lock(&ehea_bcmc_regs.lock); + spin_lock(&ehea_bcmc_regs.lock); /* Deregister old MAC in pHYP */ if (port->state == EHEA_PORT_UP) { @@ -1785,7 +1792,7 @@ static int ehea_set_mac_addr(struct net_device *dev, void *sa) out_upregs: ehea_update_bcmc_registrations(); - mutex_unlock(&ehea_bcmc_regs.lock); + spin_unlock(&ehea_bcmc_regs.lock); out_free: kfree(cb0); out: @@ -1947,7 +1954,7 @@ static void ehea_set_multicast_list(struct net_device *dev) } ehea_promiscuous(dev, 0); - mutex_lock(&ehea_bcmc_regs.lock); + spin_lock(&ehea_bcmc_regs.lock); if (dev->flags & IFF_ALLMULTI) { ehea_allmulti(dev, 1); @@ -1978,7 +1985,7 @@ static void ehea_set_multicast_list(struct net_device *dev) } out: ehea_update_bcmc_registrations(); - mutex_unlock(&ehea_bcmc_regs.lock); + spin_unlock(&ehea_bcmc_regs.lock); return; } @@ -2497,7 +2504,7 @@ static int ehea_up(struct net_device *dev) } } - mutex_lock(&ehea_bcmc_regs.lock); + spin_lock(&ehea_bcmc_regs.lock); ret = ehea_broadcast_reg_helper(port, H_REG_BCMC); if (ret) { @@ -2520,7 +2527,7 @@ static int ehea_up(struct net_device *dev) ehea_info("Failed starting %s. ret=%i", dev->name, ret); ehea_update_bcmc_registrations(); - mutex_unlock(&ehea_bcmc_regs.lock); + spin_unlock(&ehea_bcmc_regs.lock); ehea_update_firmware_handles(); mutex_unlock(&ehea_fw_handles.lock); @@ -2575,7 +2582,7 @@ static int ehea_down(struct net_device *dev) mutex_lock(&ehea_fw_handles.lock); - mutex_lock(&ehea_bcmc_regs.lock); + spin_lock(&ehea_bcmc_regs.lock); ehea_drop_multicast_list(dev); ehea_broadcast_reg_helper(port, H_DEREG_BCMC); @@ -2584,7 +2591,7 @@ static int ehea_down(struct net_device *dev) port->state = EHEA_PORT_DOWN; ehea_update_bcmc_registrations(); - mutex_unlock(&ehea_bcmc_regs.lock); + spin_unlock(&ehea_bcmc_regs.lock); ret = ehea_clean_all_portres(port); if (ret) @@ -2605,13 +2612,14 @@ static int ehea_stop(struct net_device *dev) if (netif_msg_ifdown(port)) ehea_info("disabling port %s", dev->name); + set_bit(__EHEA_DISABLE_PORT_RESET, &port->flags); cancel_work_sync(&port->reset_task); - mutex_lock(&port->port_lock); netif_stop_queue(dev); port_napi_disable(port); ret = ehea_down(dev); mutex_unlock(&port->port_lock); + clear_bit(__EHEA_DISABLE_PORT_RESET, &port->flags); return ret; } @@ -2941,7 +2949,7 @@ static void ehea_tx_watchdog(struct net_device *dev) if (netif_carrier_ok(dev) && !test_bit(__EHEA_STOP_XFER, &ehea_driver_flags)) - schedule_work(&port->reset_task); + ehea_schedule_port_reset(port); } int ehea_sense_adapter_attr(struct ehea_adapter *adapter) @@ -3590,7 +3598,7 @@ int __init ehea_module_init(void) memset(&ehea_bcmc_regs, 0, sizeof(ehea_bcmc_regs)); mutex_init(&ehea_fw_handles.lock); - mutex_init(&ehea_bcmc_regs.lock); + spin_lock_init(&ehea_bcmc_regs.lock); ret = check_module_parm(); if (ret) diff --git a/trunk/drivers/net/forcedeth.c b/trunk/drivers/net/forcedeth.c index 2cb244763292..20d4fe96a81c 100644 --- a/trunk/drivers/net/forcedeth.c +++ b/trunk/drivers/net/forcedeth.c @@ -4194,12 +4194,23 @@ static int nv_set_settings(struct net_device *dev, struct ethtool_cmd *ecmd) netif_carrier_off(dev); if (netif_running(dev)) { + unsigned long flags; + nv_disable_irq(dev); netif_tx_lock_bh(dev); - spin_lock(&np->lock); + /* with plain spinlock lockdep complains */ + spin_lock_irqsave(&np->lock, flags); /* stop engines */ + /* FIXME: + * this can take some time, and interrupts are disabled + * due to spin_lock_irqsave, but let's hope no daemon + * is going to change the settings very often... + * Worst case: + * NV_RXSTOP_DELAY1MAX + NV_TXSTOP_DELAY1MAX + * + some minor delays, which is up to a second approximately + */ nv_stop_rxtx(dev); - spin_unlock(&np->lock); + spin_unlock_irqrestore(&np->lock, flags); netif_tx_unlock_bh(dev); } diff --git a/trunk/drivers/net/fs_enet/mac-fcc.c b/trunk/drivers/net/fs_enet/mac-fcc.c index e36321152d50..8268b3535b30 100644 --- a/trunk/drivers/net/fs_enet/mac-fcc.c +++ b/trunk/drivers/net/fs_enet/mac-fcc.c @@ -463,6 +463,9 @@ static void restart(struct net_device *dev) else C32(fccp, fcc_fpsmr, FCC_PSMR_FDE | FCC_PSMR_LPB); + /* Restore multicast and promiscuous settings */ + set_multicast_list(dev); + S32(fccp, fcc_gfmr, FCC_GFMR_ENR | FCC_GFMR_ENT); } diff --git a/trunk/drivers/net/ibm_newemac/core.c b/trunk/drivers/net/ibm_newemac/core.c index 5d2108c5ac7c..babc79ad490b 100644 --- a/trunk/drivers/net/ibm_newemac/core.c +++ b/trunk/drivers/net/ibm_newemac/core.c @@ -1636,6 +1636,12 @@ static int emac_poll_rx(void *param, int budget) goto next; } + if (len < ETH_HLEN) { + ++dev->estats.rx_dropped_stack; + emac_recycle_rx_skb(dev, slot, len); + goto next; + } + if (len && len < EMAC_RX_COPY_THRESH) { struct sk_buff *copy_skb = alloc_skb(len + EMAC_RX_SKB_HEADROOM + 2, GFP_ATOMIC); @@ -2719,6 +2725,8 @@ static int __devinit emac_probe(struct of_device *ofdev, /* Clean rings */ memset(dev->tx_desc, 0, NUM_TX_BUFF * sizeof(struct mal_descriptor)); memset(dev->rx_desc, 0, NUM_RX_BUFF * sizeof(struct mal_descriptor)); + memset(dev->tx_skb, 0, NUM_TX_BUFF * sizeof(struct sk_buff *)); + memset(dev->rx_skb, 0, NUM_RX_BUFF * sizeof(struct sk_buff *)); /* Attach to ZMII, if needed */ if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII) && diff --git a/trunk/drivers/net/pasemi_mac.c b/trunk/drivers/net/pasemi_mac.c index 3b2a6c598088..993d87c9296f 100644 --- a/trunk/drivers/net/pasemi_mac.c +++ b/trunk/drivers/net/pasemi_mac.c @@ -277,7 +277,7 @@ static int get_skb_hdr(struct sk_buff *skb, void **iphdr, *tcph = tcp_hdr(skb); /* check if ip header and tcp header are complete */ - if (iph->tot_len < ip_len + tcp_hdrlen(skb)) + if (ntohs(iph->tot_len) < ip_len + tcp_hdrlen(skb)) return -1; *hdr_flags = LRO_IPV4 | LRO_TCP; diff --git a/trunk/drivers/net/wan/hdlc_fr.c b/trunk/drivers/net/wan/hdlc_fr.c index 520bb0b1a9a2..6d35155c7145 100644 --- a/trunk/drivers/net/wan/hdlc_fr.c +++ b/trunk/drivers/net/wan/hdlc_fr.c @@ -1008,6 +1008,7 @@ static int fr_rx(struct sk_buff *skb) stats->rx_bytes += skb->len; if (pvc->state.becn) stats->rx_compressed++; + skb->dev = dev; netif_rx(skb); return NET_RX_SUCCESS; } else { diff --git a/trunk/drivers/net/wireless/iwlwifi/iwl-3945.c b/trunk/drivers/net/wireless/iwlwifi/iwl-3945.c index 62a3d8f8563e..f5387a7a76c0 100644 --- a/trunk/drivers/net/wireless/iwlwifi/iwl-3945.c +++ b/trunk/drivers/net/wireless/iwlwifi/iwl-3945.c @@ -588,8 +588,12 @@ static void iwl3945_add_radiotap(struct iwl3945_priv *priv, if (rate == -1) iwl3945_rt->rt_rate = 0; - else + else { + if (stats->band == IEEE80211_BAND_5GHZ) + rate += IWL_FIRST_OFDM_RATE; + iwl3945_rt->rt_rate = iwl3945_rates[rate].ieee; + } /* antenna number */ antenna = phy_flags_hw & RX_RES_PHY_FLAGS_ANTENNA_MSK; diff --git a/trunk/drivers/net/wireless/iwlwifi/iwl-4965.c b/trunk/drivers/net/wireless/iwlwifi/iwl-4965.c index bf19eb8aafd0..de330ae0ca95 100644 --- a/trunk/drivers/net/wireless/iwlwifi/iwl-4965.c +++ b/trunk/drivers/net/wireless/iwlwifi/iwl-4965.c @@ -3528,8 +3528,12 @@ static void iwl4965_add_radiotap(struct iwl_priv *priv, if (rate == -1) iwl4965_rt->rt_rate = 0; - else + else { + if (stats->band == IEEE80211_BAND_5GHZ) + rate += IWL_FIRST_OFDM_RATE; + iwl4965_rt->rt_rate = iwl4965_rates[rate].ieee; + } /* * "antenna number" diff --git a/trunk/drivers/net/wireless/iwlwifi/iwl3945-base.c b/trunk/drivers/net/wireless/iwlwifi/iwl3945-base.c index b1b3c523185d..6027e1119c3f 100644 --- a/trunk/drivers/net/wireless/iwlwifi/iwl3945-base.c +++ b/trunk/drivers/net/wireless/iwlwifi/iwl3945-base.c @@ -6687,7 +6687,8 @@ static int iwl3945_mac_tx(struct ieee80211_hw *hw, struct sk_buff *skb, if (priv->iw_mode == IEEE80211_IF_TYPE_MNTR) { IWL_DEBUG_MAC80211("leave - monitor\n"); - return -1; + dev_kfree_skb_any(skb); + return 0; } IWL_DEBUG_TX("dev->xmit(%d bytes) at rate 0x%02x\n", skb->len, diff --git a/trunk/drivers/net/wireless/iwlwifi/iwl4965-base.c b/trunk/drivers/net/wireless/iwlwifi/iwl4965-base.c index 5ed16ce78468..0bd55bb19739 100644 --- a/trunk/drivers/net/wireless/iwlwifi/iwl4965-base.c +++ b/trunk/drivers/net/wireless/iwlwifi/iwl4965-base.c @@ -6237,7 +6237,8 @@ static int iwl4965_mac_tx(struct ieee80211_hw *hw, struct sk_buff *skb, if (priv->iw_mode == IEEE80211_IF_TYPE_MNTR) { IWL_DEBUG_MAC80211("leave - monitor\n"); - return -1; + dev_kfree_skb_any(skb); + return 0; } IWL_DEBUG_TX("dev->xmit(%d bytes) at rate 0x%02x\n", skb->len, diff --git a/trunk/drivers/net/wireless/libertas/if_usb.c b/trunk/drivers/net/wireless/libertas/if_usb.c index 8032df72aaab..36288b29abf7 100644 --- a/trunk/drivers/net/wireless/libertas/if_usb.c +++ b/trunk/drivers/net/wireless/libertas/if_usb.c @@ -925,6 +925,7 @@ static struct usb_driver if_usb_driver = { .id_table = if_usb_table, .suspend = if_usb_suspend, .resume = if_usb_resume, + .reset_resume = if_usb_resume, }; static int __init if_usb_init_module(void) diff --git a/trunk/fs/nfs/dir.c b/trunk/fs/nfs/dir.c index 58d43daec084..982a2064fe4c 100644 --- a/trunk/fs/nfs/dir.c +++ b/trunk/fs/nfs/dir.c @@ -204,7 +204,7 @@ int nfs_readdir_filler(nfs_readdir_descriptor_t *desc, struct page *page) * Note: assumes we have exclusive access to this mapping either * through inode->i_mutex or some other mechanism. */ - if (page->index == 0 && invalidate_inode_pages2_range(inode->i_mapping, PAGE_CACHE_SIZE, -1) < 0) { + if (invalidate_inode_pages2_range(inode->i_mapping, page->index + 1, -1) < 0) { /* Should never happen */ nfs_zap_mapping(inode, inode->i_mapping); } diff --git a/trunk/fs/reiserfs/inode.c b/trunk/fs/reiserfs/inode.c index 57917932212e..192269698a8a 100644 --- a/trunk/fs/reiserfs/inode.c +++ b/trunk/fs/reiserfs/inode.c @@ -45,6 +45,8 @@ void reiserfs_delete_inode(struct inode *inode) goto out; reiserfs_update_inode_transaction(inode); + reiserfs_discard_prealloc(&th, inode); + err = reiserfs_delete_object(&th, inode); /* Do quota update inside a transaction for journaled quotas. We must do that diff --git a/trunk/include/asm-powerpc/hugetlb.h b/trunk/include/asm-powerpc/hugetlb.h index 649c6c3b87b3..be32ff02f4a0 100644 --- a/trunk/include/asm-powerpc/hugetlb.h +++ b/trunk/include/asm-powerpc/hugetlb.h @@ -49,12 +49,6 @@ static inline pte_t huge_pte_wrprotect(pte_t pte) return pte_wrprotect(pte); } -static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - ptep_set_wrprotect(mm, addr, ptep); -} - static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte, int dirty) diff --git a/trunk/include/asm-powerpc/pgtable-ppc64.h b/trunk/include/asm-powerpc/pgtable-ppc64.h index cc6a43ba41d0..7686569a0bef 100644 --- a/trunk/include/asm-powerpc/pgtable-ppc64.h +++ b/trunk/include/asm-powerpc/pgtable-ppc64.h @@ -314,6 +314,16 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, old = pte_update(mm, addr, ptep, _PAGE_RW, 0); } +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long old; + + if ((pte_val(*ptep) & _PAGE_RW) == 0) + return; + old = pte_update(mm, addr, ptep, _PAGE_RW, 1); +} + /* * We currently remove entries from the hashtable regardless of whether * the entry was young or dirty. The generic routines only flush if the diff --git a/trunk/include/asm-s390/pgtable.h b/trunk/include/asm-s390/pgtable.h index c7f4f8e3e297..bd0ea191dfa9 100644 --- a/trunk/include/asm-s390/pgtable.h +++ b/trunk/include/asm-s390/pgtable.h @@ -223,6 +223,9 @@ extern char empty_zero_page[PAGE_SIZE]; #define _PAGE_SPECIAL 0x004 /* SW associated with special page */ #define __HAVE_ARCH_PTE_SPECIAL +/* Set of bits not changed in pte_modify */ +#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL) + /* Six different types of pages. */ #define _PAGE_TYPE_EMPTY 0x400 #define _PAGE_TYPE_NONE 0x401 @@ -681,7 +684,7 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - pte_val(pte) &= PAGE_MASK; + pte_val(pte) &= _PAGE_CHG_MASK; pte_val(pte) |= pgprot_val(newprot); return pte; } diff --git a/trunk/include/asm-x86/asm.h b/trunk/include/asm-x86/asm.h index 97220321f39d..70939820c55f 100644 --- a/trunk/include/asm-x86/asm.h +++ b/trunk/include/asm-x86/asm.h @@ -3,10 +3,8 @@ #ifdef __ASSEMBLY__ # define __ASM_FORM(x) x -# define __ASM_EX_SEC .section __ex_table #else # define __ASM_FORM(x) " " #x " " -# define __ASM_EX_SEC " .section __ex_table,\"a\"\n" #endif #ifdef CONFIG_X86_32 @@ -16,7 +14,6 @@ #endif #define __ASM_SIZE(inst) __ASM_SEL(inst##l, inst##q) -#define __ASM_REG(reg) __ASM_SEL(e##reg, r##reg) #define _ASM_PTR __ASM_SEL(.long, .quad) #define _ASM_ALIGN __ASM_SEL(.balign 4, .balign 8) @@ -27,14 +24,10 @@ #define _ASM_ADD __ASM_SIZE(add) #define _ASM_SUB __ASM_SIZE(sub) #define _ASM_XADD __ASM_SIZE(xadd) -#define _ASM_AX __ASM_REG(ax) -#define _ASM_BX __ASM_REG(bx) -#define _ASM_CX __ASM_REG(cx) -#define _ASM_DX __ASM_REG(dx) /* Exception table entry */ # define _ASM_EXTABLE(from,to) \ - __ASM_EX_SEC \ + " .section __ex_table,\"a\"\n" \ _ASM_ALIGN "\n" \ _ASM_PTR #from "," #to "\n" \ " .previous\n" diff --git a/trunk/include/asm-x86/delay.h b/trunk/include/asm-x86/delay.h index 409a649204aa..bb80880c834b 100644 --- a/trunk/include/asm-x86/delay.h +++ b/trunk/include/asm-x86/delay.h @@ -26,6 +26,10 @@ extern void __delay(unsigned long loops); ((n) > 20000 ? __bad_ndelay() : __const_udelay((n) * 5ul)) : \ __ndelay(n)) +#ifdef CONFIG_X86_32 void use_tsc_delay(void); +#else +#define use_tsc_delay() {} +#endif #endif /* _ASM_X86_DELAY_H */ diff --git a/trunk/include/asm-x86/fixmap_64.h b/trunk/include/asm-x86/fixmap_64.h index 6a4789d57e6c..1a0b61eb02ff 100644 --- a/trunk/include/asm-x86/fixmap_64.h +++ b/trunk/include/asm-x86/fixmap_64.h @@ -12,7 +12,6 @@ #define _ASM_FIXMAP_64_H #include -#include #include #include #include @@ -50,10 +49,6 @@ enum fixed_addresses { #ifdef CONFIG_PARAVIRT FIX_PARAVIRT_BOOTMAP, #endif -#ifdef CONFIG_ACPI - FIX_ACPI_BEGIN, - FIX_ACPI_END = FIX_ACPI_BEGIN + FIX_ACPI_PAGES - 1, -#endif #ifdef CONFIG_PROVIDE_OHCI1394_DMA_INIT FIX_OHCI1394_BASE, #endif diff --git a/trunk/include/asm-x86/gart.h b/trunk/include/asm-x86/gart.h index 33b9aeeb35a2..417f76ea677b 100644 --- a/trunk/include/asm-x86/gart.h +++ b/trunk/include/asm-x86/gart.h @@ -1,14 +1,40 @@ -#ifndef _ASM_X8664_GART_H -#define _ASM_X8664_GART_H 1 +#ifndef _ASM_X8664_IOMMU_H +#define _ASM_X8664_IOMMU_H 1 #include -#include +extern void pci_iommu_shutdown(void); +extern void no_iommu_init(void); +extern int force_iommu, no_iommu; +extern int iommu_detected; +extern int agp_amd64_init(void); +#ifdef CONFIG_GART_IOMMU +extern void gart_iommu_init(void); +extern void gart_iommu_shutdown(void); +extern void __init gart_parse_options(char *); +extern void early_gart_iommu_check(void); +extern void gart_iommu_hole_init(void); extern void set_up_gart_resume(u32, u32); - extern int fallback_aper_order; extern int fallback_aper_force; +extern int gart_iommu_aperture; +extern int gart_iommu_aperture_allowed; +extern int gart_iommu_aperture_disabled; extern int fix_aperture; +#else +#define gart_iommu_aperture 0 +#define gart_iommu_aperture_allowed 0 +#define gart_iommu_aperture_disabled 1 + +static inline void early_gart_iommu_check(void) +{ +} + +static inline void gart_iommu_shutdown(void) +{ +} + +#endif /* PTE bits. */ #define GPTE_VALID 1 diff --git a/trunk/include/asm-x86/genapic_32.h b/trunk/include/asm-x86/genapic_32.h index 33a73f5ed222..8d4c8bdb9065 100644 --- a/trunk/include/asm-x86/genapic_32.h +++ b/trunk/include/asm-x86/genapic_32.h @@ -119,10 +119,6 @@ enum uv_system_type {UV_NONE, UV_LEGACY_APIC, UV_X2APIC, UV_NON_UNIQUE_APIC}; #define is_uv_system() 0 #define uv_wakeup_secondary(a, b) 1 -#ifdef CONFIG_X86_IO_APIC extern void force_mask_ioapic_irq_2(void); -#else -static inline void force_mask_ioapic_irq_2(void) { } -#endif #endif diff --git a/trunk/include/asm-x86/genapic_64.h b/trunk/include/asm-x86/genapic_64.h index 647e4e5c2580..082ad020e412 100644 --- a/trunk/include/asm-x86/genapic_64.h +++ b/trunk/include/asm-x86/genapic_64.h @@ -46,10 +46,6 @@ extern int uv_wakeup_secondary(int phys_apicid, unsigned int start_rip); extern void setup_apic_routing(void); -#ifdef CONFIG_X86_IO_APIC extern void force_mask_ioapic_irq_2(void); -#else -static inline void force_mask_ioapic_irq_2(void) { } -#endif #endif diff --git a/trunk/include/asm-x86/iommu.h b/trunk/include/asm-x86/iommu.h index 068c9a40aa5b..07862fdd23c0 100644 --- a/trunk/include/asm-x86/iommu.h +++ b/trunk/include/asm-x86/iommu.h @@ -1,34 +1,29 @@ -#ifndef _ASM_X8664_IOMMU_H -#define _ASM_X8664_IOMMU_H 1 +#ifndef _ASM_X8664_GART_H +#define _ASM_X8664_GART_H 1 extern void pci_iommu_shutdown(void); extern void no_iommu_init(void); extern int force_iommu, no_iommu; extern int iommu_detected; - -#ifdef CONFIG_GART_IOMMU -extern int gart_iommu_aperture; -extern int gart_iommu_aperture_allowed; -extern int gart_iommu_aperture_disabled; - -extern void early_gart_iommu_check(void); +#ifdef CONFIG_IOMMU extern void gart_iommu_init(void); extern void gart_iommu_shutdown(void); extern void __init gart_parse_options(char *); -extern void gart_iommu_hole_init(void); - +extern void iommu_hole_init(void); +extern int fallback_aper_order; +extern int fallback_aper_force; +extern int iommu_aperture; +extern int iommu_aperture_allowed; +extern int iommu_aperture_disabled; +extern int fix_aperture; #else -#define gart_iommu_aperture 0 -#define gart_iommu_aperture_allowed 0 -#define gart_iommu_aperture_disabled 1 - -static inline void early_gart_iommu_check(void) -{ -} +#define iommu_aperture 0 +#define iommu_aperture_allowed 0 static inline void gart_iommu_shutdown(void) { } + #endif #endif diff --git a/trunk/include/asm-x86/uaccess.h b/trunk/include/asm-x86/uaccess.h index f6fa4d841bbc..9fefd2947e78 100644 --- a/trunk/include/asm-x86/uaccess.h +++ b/trunk/include/asm-x86/uaccess.h @@ -1,453 +1,5 @@ -#ifndef _ASM_UACCES_H_ -#define _ASM_UACCES_H_ -/* - * User space memory access functions - */ -#include -#include -#include -#include -#include -#include -#include - -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) - -#define KERNEL_DS MAKE_MM_SEG(-1UL) -#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) - -#define get_ds() (KERNEL_DS) -#define get_fs() (current_thread_info()->addr_limit) -#define set_fs(x) (current_thread_info()->addr_limit = (x)) - -#define segment_eq(a, b) ((a).seg == (b).seg) - -#define __addr_ok(addr) \ - ((unsigned long __force)(addr) < \ - (current_thread_info()->addr_limit.seg)) - -/* - * Test whether a block of memory is a valid user space address. - * Returns 0 if the range is valid, nonzero otherwise. - * - * This is equivalent to the following test: - * (u33)addr + (u33)size >= (u33)current->addr_limit.seg (u65 for x86_64) - * - * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry... - */ - -#define __range_not_ok(addr, size) \ -({ \ - unsigned long flag, roksum; \ - __chk_user_ptr(addr); \ - asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \ - : "=&r" (flag), "=r" (roksum) \ - : "1" (addr), "g" ((long)(size)), \ - "rm" (current_thread_info()->addr_limit.seg)); \ - flag; \ -}) - -/** - * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block may be valid, false (zero) - * if it is definitely invalid. - * - * Note that, depending on architecture, this function probably just - * checks that the pointer is in the user space range - after calling - * this function, memory access functions may still return -EFAULT. - */ -#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0)) - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned long insn, fixup; -}; - -extern int fixup_exception(struct pt_regs *regs); - -/* - * These are the main single-value transfer routines. They automatically - * use the right size if we just have the right pointer type. - * - * This gets kind of ugly. We want to return _two_ values in "get_user()" - * and yet we don't want to do any pointers, because that is too much - * of a performance impact. Thus we have a few rather ugly macros here, - * and hide all the ugliness from the user. - * - * The "__xxx" versions of the user access functions are versions that - * do not verify the address space, that must have been done previously - * with a separate "access_ok()" call (this is used when we do multiple - * accesses to the same area of user memory). - */ - -extern int __get_user_1(void); -extern int __get_user_2(void); -extern int __get_user_4(void); -extern int __get_user_8(void); -extern int __get_user_bad(void); - -#define __get_user_x(size, ret, x, ptr) \ - asm volatile("call __get_user_" #size \ - : "=a" (ret),"=d" (x) \ - : "0" (ptr)) \ - -/* Careful: we have to cast the result to the type of the pointer - * for sign reasons */ - -/** - * get_user: - Get a simple variable from user space. - * @x: Variable to store result. - * @ptr: Source address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple variable from user space to kernel - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and the result of - * dereferencing @ptr must be assignable to @x without a cast. - * - * Returns zero on success, or -EFAULT on error. - * On error, the variable @x is set to zero. - */ -#ifdef CONFIG_X86_32 -#define __get_user_8(__ret_gu, __val_gu, ptr) \ - __get_user_x(X, __ret_gu, __val_gu, ptr) -#else -#define __get_user_8(__ret_gu, __val_gu, ptr) \ - __get_user_x(8, __ret_gu, __val_gu, ptr) -#endif - -#define get_user(x, ptr) \ -({ \ - int __ret_gu; \ - unsigned long __val_gu; \ - __chk_user_ptr(ptr); \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __get_user_x(1, __ret_gu, __val_gu, ptr); \ - break; \ - case 2: \ - __get_user_x(2, __ret_gu, __val_gu, ptr); \ - break; \ - case 4: \ - __get_user_x(4, __ret_gu, __val_gu, ptr); \ - break; \ - case 8: \ - __get_user_8(__ret_gu, __val_gu, ptr); \ - break; \ - default: \ - __get_user_x(X, __ret_gu, __val_gu, ptr); \ - break; \ - } \ - (x) = (__typeof__(*(ptr)))__val_gu; \ - __ret_gu; \ -}) - -#define __put_user_x(size, x, ptr, __ret_pu) \ - asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ - :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - - - -#ifdef CONFIG_X86_32 -#define __put_user_u64(x, addr, err) \ - asm volatile("1: movl %%eax,0(%2)\n" \ - "2: movl %%edx,4(%2)\n" \ - "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: movl %3,%0\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "=r" (err) \ - : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) - -#define __put_user_x8(x, ptr, __ret_pu) \ - asm volatile("call __put_user_8" : "=a" (__ret_pu) \ - : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") -#else -#define __put_user_u64(x, ptr, retval) \ - __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT) -#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) -#endif - -extern void __put_user_bad(void); - -/* - * Strange magic calling convention: pointer in %ecx, - * value in %eax(:%edx), return value in %eax. clobbers %rbx - */ -extern void __put_user_1(void); -extern void __put_user_2(void); -extern void __put_user_4(void); -extern void __put_user_8(void); - -#ifdef CONFIG_X86_WP_WORKS_OK - -/** - * put_user: - Write a simple value into user space. - * @x: Value to copy to user space. - * @ptr: Destination address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple value from kernel space to user - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and @x must be assignable - * to the result of dereferencing @ptr. - * - * Returns zero on success, or -EFAULT on error. - */ -#define put_user(x, ptr) \ -({ \ - int __ret_pu; \ - __typeof__(*(ptr)) __pu_val; \ - __chk_user_ptr(ptr); \ - __pu_val = x; \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __put_user_x(1, __pu_val, ptr, __ret_pu); \ - break; \ - case 2: \ - __put_user_x(2, __pu_val, ptr, __ret_pu); \ - break; \ - case 4: \ - __put_user_x(4, __pu_val, ptr, __ret_pu); \ - break; \ - case 8: \ - __put_user_x8(__pu_val, ptr, __ret_pu); \ - break; \ - default: \ - __put_user_x(X, __pu_val, ptr, __ret_pu); \ - break; \ - } \ - __ret_pu; \ -}) - -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ - break; \ - case 2: \ - __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ - break; \ - case 4: \ - __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\ - break; \ - case 8: \ - __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - -#else - -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - __typeof__(*(ptr))__pus_tmp = x; \ - retval = 0; \ - \ - if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ - retval = errret; \ -} while (0) - -#define put_user(x, ptr) \ -({ \ - int __ret_pu; \ - __typeof__(*(ptr))__pus_tmp = x; \ - __ret_pu = 0; \ - if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ - sizeof(*(ptr))) != 0)) \ - __ret_pu = -EFAULT; \ - __ret_pu; \ -}) -#endif - -#ifdef CONFIG_X86_32 -#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad() -#else -#define __get_user_asm_u64(x, ptr, retval, errret) \ - __get_user_asm(x, ptr, retval, "q", "", "=r", errret) -#endif - -#define __get_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ - break; \ - case 2: \ - __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ - break; \ - case 4: \ - __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ - break; \ - case 8: \ - __get_user_asm_u64(x, ptr, retval, errret); \ - break; \ - default: \ - (x) = __get_user_bad(); \ - } \ -} while (0) - -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r" (err), ltype(x) \ - : "m" (__m(addr)), "i" (errret), "0" (err)) - -#define __put_user_nocheck(x, ptr, size) \ -({ \ - long __pu_err; \ - __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ - __pu_err; \ -}) - -#define __get_user_nocheck(x, ptr, size) \ -({ \ - long __gu_err; \ - unsigned long __gu_val; \ - __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ - (x) = (__force __typeof__(*(ptr)))__gu_val; \ - __gu_err; \ -}) - -/* FIXME: this hack is definitely wrong -AK */ -struct __large_struct { unsigned long buf[100]; }; -#define __m(x) (*(struct __large_struct __user *)(x)) - -/* - * Tell gcc we read from memory instead of writing: this is because - * we do not write to any memory gcc knows about, so there are no - * aliasing issues. - */ -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r"(err) \ - : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) -/** - * __get_user: - Get a simple variable from user space, with less checking. - * @x: Variable to store result. - * @ptr: Source address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple variable from user space to kernel - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and the result of - * dereferencing @ptr must be assignable to @x without a cast. - * - * Caller must check the pointer with access_ok() before calling this - * function. - * - * Returns zero on success, or -EFAULT on error. - * On error, the variable @x is set to zero. - */ - -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) -/** - * __put_user: - Write a simple value into user space, with less checking. - * @x: Value to copy to user space. - * @ptr: Destination address, in user space. - * - * Context: User context only. This function may sleep. - * - * This macro copies a single simple value from kernel space to user - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. - * - * @ptr must have pointer-to-simple-variable type, and @x must be assignable - * to the result of dereferencing @ptr. - * - * Caller must check the pointer with access_ok() before calling this - * function. - * - * Returns zero on success, or -EFAULT on error. - */ - -#define __put_user(x, ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - -#define __get_user_unaligned __get_user -#define __put_user_unaligned __put_user - -/* - * movsl can be slow when source and dest are not both 8-byte aligned - */ -#ifdef CONFIG_X86_INTEL_USERCOPY -extern struct movsl_mask { - int mask; -} ____cacheline_aligned_in_smp movsl_mask; -#endif - -#define ARCH_HAS_NOCACHE_UACCESS 1 - #ifdef CONFIG_X86_32 # include "uaccess_32.h" #else -# define ARCH_HAS_SEARCH_EXTABLE # include "uaccess_64.h" #endif - -#endif diff --git a/trunk/include/asm-x86/uaccess_32.h b/trunk/include/asm-x86/uaccess_32.h index 6fdef39a0bcb..8e7595c1f34e 100644 --- a/trunk/include/asm-x86/uaccess_32.h +++ b/trunk/include/asm-x86/uaccess_32.h @@ -11,6 +11,426 @@ #include #include +#define VERIFY_READ 0 +#define VERIFY_WRITE 1 + +/* + * The fs value determines whether argument validity checking should be + * performed or not. If get_fs() == USER_DS, checking is performed, with + * get_fs() == KERNEL_DS, checking is bypassed. + * + * For historical reasons, these macros are grossly misnamed. + */ + +#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) + + +#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFUL) +#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) + +#define get_ds() (KERNEL_DS) +#define get_fs() (current_thread_info()->addr_limit) +#define set_fs(x) (current_thread_info()->addr_limit = (x)) + +#define segment_eq(a, b) ((a).seg == (b).seg) + +/* + * movsl can be slow when source and dest are not both 8-byte aligned + */ +#ifdef CONFIG_X86_INTEL_USERCOPY +extern struct movsl_mask { + int mask; +} ____cacheline_aligned_in_smp movsl_mask; +#endif + +#define __addr_ok(addr) \ + ((unsigned long __force)(addr) < \ + (current_thread_info()->addr_limit.seg)) + +/* + * Test whether a block of memory is a valid user space address. + * Returns 0 if the range is valid, nonzero otherwise. + * + * This is equivalent to the following test: + * (u33)addr + (u33)size >= (u33)current->addr_limit.seg + * + * This needs 33-bit arithmetic. We have a carry... + */ +#define __range_ok(addr, size) \ +({ \ + unsigned long flag, roksum; \ + __chk_user_ptr(addr); \ + asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ + :"=&r" (flag), "=r" (roksum) \ + :"1" (addr), "g" ((int)(size)), \ + "rm" (current_thread_info()->addr_limit.seg)); \ + flag; \ +}) + +/** + * access_ok: - Checks if a user space pointer is valid + * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that + * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe + * to write to a block, it is always safe to read from it. + * @addr: User space pointer to start of block to check + * @size: Size of block to check + * + * Context: User context only. This function may sleep. + * + * Checks if a pointer to a block of memory in user space is valid. + * + * Returns true (nonzero) if the memory block may be valid, false (zero) + * if it is definitely invalid. + * + * Note that, depending on architecture, this function probably just + * checks that the pointer is in the user space range - after calling + * this function, memory access functions may still return -EFAULT. + */ +#define access_ok(type, addr, size) (likely(__range_ok(addr, size) == 0)) + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned long insn, fixup; +}; + +extern int fixup_exception(struct pt_regs *regs); + +/* + * These are the main single-value transfer routines. They automatically + * use the right size if we just have the right pointer type. + * + * This gets kind of ugly. We want to return _two_ values in "get_user()" + * and yet we don't want to do any pointers, because that is too much + * of a performance impact. Thus we have a few rather ugly macros here, + * and hide all the ugliness from the user. + * + * The "__xxx" versions of the user access functions are versions that + * do not verify the address space, that must have been done previously + * with a separate "access_ok()" call (this is used when we do multiple + * accesses to the same area of user memory). + */ + +extern void __get_user_1(void); +extern void __get_user_2(void); +extern void __get_user_4(void); + +#define __get_user_x(size, ret, x, ptr) \ + asm volatile("call __get_user_" #size \ + :"=a" (ret),"=d" (x) \ + :"0" (ptr)) + + +/* Careful: we have to cast the result to the type of the pointer + * for sign reasons */ + +/** + * get_user: - Get a simple variable from user space. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define get_user(x, ptr) \ +({ \ + int __ret_gu; \ + unsigned long __val_gu; \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __get_user_x(1, __ret_gu, __val_gu, ptr); \ + break; \ + case 2: \ + __get_user_x(2, __ret_gu, __val_gu, ptr); \ + break; \ + case 4: \ + __get_user_x(4, __ret_gu, __val_gu, ptr); \ + break; \ + default: \ + __get_user_x(X, __ret_gu, __val_gu, ptr); \ + break; \ + } \ + (x) = (__typeof__(*(ptr)))__val_gu; \ + __ret_gu; \ +}) + +extern void __put_user_bad(void); + +/* + * Strange magic calling convention: pointer in %ecx, + * value in %eax(:%edx), return value in %eax, no clobbers. + */ +extern void __put_user_1(void); +extern void __put_user_2(void); +extern void __put_user_4(void); +extern void __put_user_8(void); + +#define __put_user_1(x, ptr) \ + asm volatile("call __put_user_1" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_2(x, ptr) \ + asm volatile("call __put_user_2" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_4(x, ptr) \ + asm volatile("call __put_user_4" : "=a" (__ret_pu) \ + : "0" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_8(x, ptr) \ + asm volatile("call __put_user_8" : "=a" (__ret_pu) \ + : "A" ((typeof(*(ptr)))(x)), "c" (ptr)) + +#define __put_user_X(x, ptr) \ + asm volatile("call __put_user_X" : "=a" (__ret_pu) \ + : "c" (ptr)) + +/** + * put_user: - Write a simple value into user space. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Returns zero on success, or -EFAULT on error. + */ +#ifdef CONFIG_X86_WP_WORKS_OK + +#define put_user(x, ptr) \ +({ \ + int __ret_pu; \ + __typeof__(*(ptr)) __pu_val; \ + __chk_user_ptr(ptr); \ + __pu_val = x; \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __put_user_1(__pu_val, ptr); \ + break; \ + case 2: \ + __put_user_2(__pu_val, ptr); \ + break; \ + case 4: \ + __put_user_4(__pu_val, ptr); \ + break; \ + case 8: \ + __put_user_8(__pu_val, ptr); \ + break; \ + default: \ + __put_user_X(__pu_val, ptr); \ + break; \ + } \ + __ret_pu; \ +}) + +#else +#define put_user(x, ptr) \ +({ \ + int __ret_pu; \ + __typeof__(*(ptr))__pus_tmp = x; \ + __ret_pu = 0; \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \ + sizeof(*(ptr))) != 0)) \ + __ret_pu = -EFAULT; \ + __ret_pu; \ +}) + + +#endif + +/** + * __get_user: - Get a simple variable from user space, with less checking. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) + + +/** + * __put_user: - Write a simple value into user space, with less checking. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + */ +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + +#define __put_user_nocheck(x, ptr, size) \ +({ \ + long __pu_err; \ + __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ + __pu_err; \ +}) + + +#define __put_user_u64(x, addr, err) \ + asm volatile("1: movl %%eax,0(%2)\n" \ + "2: movl %%edx,4(%2)\n" \ + "3:\n" \ + ".section .fixup,\"ax\"\n" \ + "4: movl %3,%0\n" \ + " jmp 3b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ + : "=r" (err) \ + : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err)) + +#ifdef CONFIG_X86_WP_WORKS_OK + +#define __put_user_size(x, ptr, size, retval, errret) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ + break; \ + case 2: \ + __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ + break; \ + case 4: \ + __put_user_asm(x, ptr, retval, "l", "", "ir", errret); \ + break; \ + case 8: \ + __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \ + break; \ + default: \ + __put_user_bad(); \ + } \ +} while (0) + +#else + +#define __put_user_size(x, ptr, size, retval, errret) \ +do { \ + __typeof__(*(ptr))__pus_tmp = x; \ + retval = 0; \ + \ + if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \ + retval = errret; \ +} while (0) + +#endif +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + +/* + * Tell gcc we read from memory instead of writing: this is because + * we do not write to any memory gcc knows about, so there are no + * aliasing issues. + */ +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + asm volatile("1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r"(err) \ + : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err)) + + +#define __get_user_nocheck(x, ptr, size) \ +({ \ + long __gu_err; \ + unsigned long __gu_val; \ + __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ + (x) = (__typeof__(*(ptr)))__gu_val; \ + __gu_err; \ +}) + +extern long __get_user_bad(void); + +#define __get_user_size(x, ptr, size, retval, errret) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ + break; \ + case 2: \ + __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ + break; \ + case 4: \ + __get_user_asm(x, ptr, retval, "l", "", "=r", errret); \ + break; \ + default: \ + (x) = __get_user_bad(); \ + } \ +} while (0) + +#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + asm volatile("1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype (x) \ + : "m" (__m(addr)), "i" (errret), "0" (err)) + + unsigned long __must_check __copy_to_user_ll (void __user *to, const void *from, unsigned long n); unsigned long __must_check __copy_from_user_ll @@ -156,6 +576,8 @@ __copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_from_user_ll(to, from, n); } +#define ARCH_HAS_NOCACHE_UACCESS + static __always_inline unsigned long __copy_from_user_nocache(void *to, const void __user *from, unsigned long n) { diff --git a/trunk/include/asm-x86/uaccess_64.h b/trunk/include/asm-x86/uaccess_64.h index 515d4dce96b5..b8a2f4339903 100644 --- a/trunk/include/asm-x86/uaccess_64.h +++ b/trunk/include/asm-x86/uaccess_64.h @@ -9,6 +9,265 @@ #include #include +#define VERIFY_READ 0 +#define VERIFY_WRITE 1 + +/* + * The fs value determines whether argument validity checking should be + * performed or not. If get_fs() == USER_DS, checking is performed, with + * get_fs() == KERNEL_DS, checking is bypassed. + * + * For historical reasons, these macros are grossly misnamed. + */ + +#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) + +#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFFFFFFFFFUL) +#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) + +#define get_ds() (KERNEL_DS) +#define get_fs() (current_thread_info()->addr_limit) +#define set_fs(x) (current_thread_info()->addr_limit = (x)) + +#define segment_eq(a, b) ((a).seg == (b).seg) + +#define __addr_ok(addr) (!((unsigned long)(addr) & \ + (current_thread_info()->addr_limit.seg))) + +/* + * Uhhuh, this needs 65-bit arithmetic. We have a carry.. + */ +#define __range_not_ok(addr, size) \ +({ \ + unsigned long flag, roksum; \ + __chk_user_ptr(addr); \ + asm("# range_ok\n\r" \ + "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ + : "=&r" (flag), "=r" (roksum) \ + : "1" (addr), "g" ((long)(size)), \ + "g" (current_thread_info()->addr_limit.seg)); \ + flag; \ +}) + +#define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0) + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned long insn, fixup; +}; + +extern int fixup_exception(struct pt_regs *regs); + +#define ARCH_HAS_SEARCH_EXTABLE + +/* + * These are the main single-value transfer routines. They automatically + * use the right size if we just have the right pointer type. + * + * This gets kind of ugly. We want to return _two_ values in "get_user()" + * and yet we don't want to do any pointers, because that is too much + * of a performance impact. Thus we have a few rather ugly macros here, + * and hide all the ugliness from the user. + * + * The "__xxx" versions of the user access functions are versions that + * do not verify the address space, that must have been done previously + * with a separate "access_ok()" call (this is used when we do multiple + * accesses to the same area of user memory). + */ + +#define __get_user_x(size, ret, x, ptr) \ + asm volatile("call __get_user_" #size \ + : "=a" (ret),"=d" (x) \ + : "c" (ptr) \ + : "r8") + +/* Careful: we have to cast the result to the type of the pointer + * for sign reasons */ + +#define get_user(x, ptr) \ +({ \ + unsigned long __val_gu; \ + int __ret_gu; \ + __chk_user_ptr(ptr); \ + switch (sizeof(*(ptr))) { \ + case 1: \ + __get_user_x(1, __ret_gu, __val_gu, ptr); \ + break; \ + case 2: \ + __get_user_x(2, __ret_gu, __val_gu, ptr); \ + break; \ + case 4: \ + __get_user_x(4, __ret_gu, __val_gu, ptr); \ + break; \ + case 8: \ + __get_user_x(8, __ret_gu, __val_gu, ptr); \ + break; \ + default: \ + __get_user_bad(); \ + break; \ + } \ + (x) = (__force typeof(*(ptr)))__val_gu; \ + __ret_gu; \ +}) + +extern void __put_user_1(void); +extern void __put_user_2(void); +extern void __put_user_4(void); +extern void __put_user_8(void); +extern void __put_user_bad(void); + +#define __put_user_x(size, ret, x, ptr) \ + asm volatile("call __put_user_" #size \ + :"=a" (ret) \ + :"c" (ptr),"d" (x) \ + :"r8") + +#define put_user(x, ptr) \ + __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + +#define __get_user_unaligned __get_user +#define __put_user_unaligned __put_user + +#define __put_user_nocheck(x, ptr, size) \ +({ \ + int __pu_err; \ + __put_user_size((x), (ptr), (size), __pu_err); \ + __pu_err; \ +}) + + +#define __put_user_check(x, ptr, size) \ +({ \ + int __pu_err; \ + typeof(*(ptr)) __user *__pu_addr = (ptr); \ + switch (size) { \ + case 1: \ + __put_user_x(1, __pu_err, x, __pu_addr); \ + break; \ + case 2: \ + __put_user_x(2, __pu_err, x, __pu_addr); \ + break; \ + case 4: \ + __put_user_x(4, __pu_err, x, __pu_addr); \ + break; \ + case 8: \ + __put_user_x(8, __pu_err, x, __pu_addr); \ + break; \ + default: \ + __put_user_bad(); \ + } \ + __pu_err; \ +}) + +#define __put_user_size(x, ptr, size, retval) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\ + break; \ + case 2: \ + __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\ + break; \ + case 4: \ + __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\ + break; \ + case 8: \ + __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \ + break; \ + default: \ + __put_user_bad(); \ + } \ +} while (0) + +/* FIXME: this hack is definitely wrong -AK */ +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + +/* + * Tell gcc we read from memory instead of writing: this is because + * we do not write to any memory gcc knows about, so there are no + * aliasing issues. + */ +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ + asm volatile("1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup, \"ax\"\n" \ + "3: mov %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r"(err) \ + : ltype (x), "m" (__m(addr)), "i" (errno), "0" (err)) + + +#define __get_user_nocheck(x, ptr, size) \ +({ \ + int __gu_err; \ + unsigned long __gu_val; \ + __get_user_size(__gu_val, (ptr), (size), __gu_err); \ + (x) = (__force typeof(*(ptr)))__gu_val; \ + __gu_err; \ +}) + +extern int __get_user_1(void); +extern int __get_user_2(void); +extern int __get_user_4(void); +extern int __get_user_8(void); +extern int __get_user_bad(void); + +#define __get_user_size(x, ptr, size, retval) \ +do { \ + retval = 0; \ + __chk_user_ptr(ptr); \ + switch (size) { \ + case 1: \ + __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\ + break; \ + case 2: \ + __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\ + break; \ + case 4: \ + __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\ + break; \ + case 8: \ + __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \ + break; \ + default: \ + (x) = __get_user_bad(); \ + } \ +} while (0) + +#define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ + asm volatile("1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup, \"ax\"\n" \ + "3: mov %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : "=r" (err), ltype (x) \ + : "m" (__m(addr)), "i"(errno), "0"(err)) + /* * Copy To/From Userspace */ @@ -178,6 +437,7 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) return copy_user_generic((__force void *)dst, src, size); } +#define ARCH_HAS_NOCACHE_UACCESS 1 extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); @@ -195,7 +455,4 @@ static inline int __copy_from_user_inatomic_nocache(void *dst, return __copy_user_nocache(dst, src, size, 0); } -unsigned long -copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest); - #endif /* __X86_64_UACCESS_H */ diff --git a/trunk/include/linux/ide.h b/trunk/include/linux/ide.h index 9918772bf274..eddb6daadf4a 100644 --- a/trunk/include/linux/ide.h +++ b/trunk/include/linux/ide.h @@ -189,6 +189,21 @@ static inline void ide_std_init_ports(hw_regs_t *hw, hw->io_ports.ctl_addr = ctl_addr; } +/* for IDE PCI controllers in legacy mode, temporary */ +static inline int __ide_default_irq(unsigned long base) +{ + switch (base) { +#ifdef CONFIG_IA64 + case 0x1f0: return isa_irq_to_vector(14); + case 0x170: return isa_irq_to_vector(15); +#else + case 0x1f0: return 14; + case 0x170: return 15; +#endif + } + return 0; +} + #include #if !defined(MAX_HWIFS) || defined(CONFIG_EMBEDDED) diff --git a/trunk/lib/vsprintf.c b/trunk/lib/vsprintf.c index 6021757a4496..1dc2d1d18fa8 100644 --- a/trunk/lib/vsprintf.c +++ b/trunk/lib/vsprintf.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include /* for PAGE_SIZE */ #include @@ -482,6 +484,89 @@ static char *number(char *buf, char *end, unsigned long long num, int base, int return buf; } +static char *string(char *buf, char *end, char *s, int field_width, int precision, int flags) +{ + int len, i; + + if ((unsigned long)s < PAGE_SIZE) + s = ""; + + len = strnlen(s, precision); + + if (!(flags & LEFT)) { + while (len < field_width--) { + if (buf < end) + *buf = ' '; + ++buf; + } + } + for (i = 0; i < len; ++i) { + if (buf < end) + *buf = *s; + ++buf; ++s; + } + while (len < field_width--) { + if (buf < end) + *buf = ' '; + ++buf; + } + return buf; +} + +static inline void *dereference_function_descriptor(void *ptr) +{ +#if defined(CONFIG_IA64) || defined(CONFIG_PPC64) + void *p; + if (!probe_kernel_address(ptr, p)) + ptr = p; +#endif + return ptr; +} + +static char *symbol_string(char *buf, char *end, void *ptr, int field_width, int precision, int flags) +{ + unsigned long value = (unsigned long) ptr; +#ifdef CONFIG_KALLSYMS + char sym[KSYM_SYMBOL_LEN]; + sprint_symbol(sym, value); + return string(buf, end, sym, field_width, precision, flags); +#else + field_width = 2*sizeof(void *); + flags |= SPECIAL | SMALL | ZEROPAD; + return number(buf, end, value, 16, field_width, precision, flags); +#endif +} + +/* + * Show a '%p' thing. A kernel extension is that the '%p' is followed + * by an extra set of alphanumeric characters that are extended format + * specifiers. + * + * Right now we just handle 'F' (for symbolic Function descriptor pointers) + * and 'S' (for Symbolic direct pointers), but this can easily be + * extended in the future (network address types etc). + * + * The difference between 'S' and 'F' is that on ia64 and ppc64 function + * pointers are really function descriptors, which contain a pointer the + * real address. + */ +static char *pointer(const char *fmt, char *buf, char *end, void *ptr, int field_width, int precision, int flags) +{ + switch (*fmt) { + case 'F': + ptr = dereference_function_descriptor(ptr); + /* Fallthrough */ + case 'S': + return symbol_string(buf, end, ptr, field_width, precision, flags); + } + flags |= SMALL; + if (field_width == -1) { + field_width = 2*sizeof(void *); + flags |= ZEROPAD; + } + return number(buf, end, (unsigned long) ptr, 16, field_width, precision, flags); +} + /** * vsnprintf - Format a string and place it in a buffer * @buf: The buffer to place the result into @@ -502,11 +587,9 @@ static char *number(char *buf, char *end, unsigned long long num, int base, int */ int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) { - int len; unsigned long long num; - int i, base; + int base; char *str, *end, c; - const char *s; int flags; /* flags to number() */ @@ -622,43 +705,18 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) continue; case 's': - s = va_arg(args, char *); - if ((unsigned long)s < PAGE_SIZE) - s = ""; - - len = strnlen(s, precision); - - if (!(flags & LEFT)) { - while (len < field_width--) { - if (str < end) - *str = ' '; - ++str; - } - } - for (i = 0; i < len; ++i) { - if (str < end) - *str = *s; - ++str; ++s; - } - while (len < field_width--) { - if (str < end) - *str = ' '; - ++str; - } + str = string(str, end, va_arg(args, char *), field_width, precision, flags); continue; case 'p': - flags |= SMALL; - if (field_width == -1) { - field_width = 2*sizeof(void *); - flags |= ZEROPAD; - } - str = number(str, end, - (unsigned long) va_arg(args, void *), - 16, field_width, precision, flags); + str = pointer(fmt+1, str, end, + va_arg(args, void *), + field_width, precision, flags); + /* Skip all alphanumeric pointer suffixes */ + while (isalnum(fmt[1])) + fmt++; continue; - case 'n': /* FIXME: * What does C99 say about the overflow case here? */ diff --git a/trunk/net/bridge/br_if.c b/trunk/net/bridge/br_if.c index c2397f503b0f..f38cc5317b88 100644 --- a/trunk/net/bridge/br_if.c +++ b/trunk/net/bridge/br_if.c @@ -442,12 +442,16 @@ int br_del_if(struct net_bridge *br, struct net_device *dev) void __exit br_cleanup_bridges(void) { - struct net_device *dev, *nxt; + struct net_device *dev; rtnl_lock(); - for_each_netdev_safe(&init_net, dev, nxt) - if (dev->priv_flags & IFF_EBRIDGE) +restart: + for_each_netdev(&init_net, dev) { + if (dev->priv_flags & IFF_EBRIDGE) { del_br(dev->priv); + goto restart; + } + } rtnl_unlock(); } diff --git a/trunk/net/can/af_can.c b/trunk/net/can/af_can.c index 7e8ca2836452..484bbf6dd032 100644 --- a/trunk/net/can/af_can.c +++ b/trunk/net/can/af_can.c @@ -205,12 +205,19 @@ static int can_create(struct net *net, struct socket *sock, int protocol) * -ENOBUFS on full driver queue (see net_xmit_errno()) * -ENOMEM when local loopback failed at calling skb_clone() * -EPERM when trying to send on a non-CAN interface + * -EINVAL when the skb->data does not contain a valid CAN frame */ int can_send(struct sk_buff *skb, int loop) { struct sk_buff *newskb = NULL; + struct can_frame *cf = (struct can_frame *)skb->data; int err; + if (skb->len != sizeof(struct can_frame) || cf->can_dlc > 8) { + kfree_skb(skb); + return -EINVAL; + } + if (skb->dev->type != ARPHRD_CAN) { kfree_skb(skb); return -EPERM; @@ -605,6 +612,7 @@ static int can_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { struct dev_rcv_lists *d; + struct can_frame *cf = (struct can_frame *)skb->data; int matches; if (dev->type != ARPHRD_CAN || dev_net(dev) != &init_net) { @@ -612,6 +620,8 @@ static int can_rcv(struct sk_buff *skb, struct net_device *dev, return 0; } + BUG_ON(skb->len != sizeof(struct can_frame) || cf->can_dlc > 8); + /* update statistics */ can_stats.rx_frames++; can_stats.rx_frames_delta++; diff --git a/trunk/net/can/bcm.c b/trunk/net/can/bcm.c index d9a3a9d13bed..72c2ce904f83 100644 --- a/trunk/net/can/bcm.c +++ b/trunk/net/can/bcm.c @@ -298,7 +298,7 @@ static void bcm_send_to_user(struct bcm_op *op, struct bcm_msg_head *head, if (head->nframes) { /* can_frames starting here */ - firstframe = (struct can_frame *) skb_tail_pointer(skb); + firstframe = (struct can_frame *)skb_tail_pointer(skb); memcpy(skb_put(skb, datalen), frames, datalen); @@ -826,6 +826,10 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, for (i = 0; i < msg_head->nframes; i++) { err = memcpy_fromiovec((u8 *)&op->frames[i], msg->msg_iov, CFSIZ); + + if (op->frames[i].can_dlc > 8) + err = -EINVAL; + if (err < 0) return err; @@ -858,6 +862,10 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, for (i = 0; i < msg_head->nframes; i++) { err = memcpy_fromiovec((u8 *)&op->frames[i], msg->msg_iov, CFSIZ); + + if (op->frames[i].can_dlc > 8) + err = -EINVAL; + if (err < 0) { if (op->frames != &op->sframe) kfree(op->frames); @@ -1164,9 +1172,12 @@ static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk) skb->dev = dev; skb->sk = sk; - can_send(skb, 1); /* send with loopback */ + err = can_send(skb, 1); /* send with loopback */ dev_put(dev); + if (err) + return err; + return CFSIZ + MHSIZ; } @@ -1185,6 +1196,10 @@ static int bcm_sendmsg(struct kiocb *iocb, struct socket *sock, if (!bo->bound) return -ENOTCONN; + /* check for valid message length from userspace */ + if (size < MHSIZ || (size - MHSIZ) % CFSIZ) + return -EINVAL; + /* check for alternative ifindex for this bcm_op */ if (!ifindex && msg->msg_name) { @@ -1259,8 +1274,8 @@ static int bcm_sendmsg(struct kiocb *iocb, struct socket *sock, break; case TX_SEND: - /* we need at least one can_frame */ - if (msg_head.nframes < 1) + /* we need exactly one can_frame behind the msg head */ + if ((msg_head.nframes != 1) || (size != CFSIZ + MHSIZ)) ret = -EINVAL; else ret = bcm_tx_send(msg, ifindex, sk); diff --git a/trunk/net/can/raw.c b/trunk/net/can/raw.c index 69877b8e7e9c..3e46ee36a1aa 100644 --- a/trunk/net/can/raw.c +++ b/trunk/net/can/raw.c @@ -632,6 +632,9 @@ static int raw_sendmsg(struct kiocb *iocb, struct socket *sock, } else ifindex = ro->ifindex; + if (size != sizeof(struct can_frame)) + return -EINVAL; + dev = dev_get_by_index(&init_net, ifindex); if (!dev) return -ENXIO; diff --git a/trunk/net/ipv4/tcp.c b/trunk/net/ipv4/tcp.c index 850825dc86e6..1d723de18686 100644 --- a/trunk/net/ipv4/tcp.c +++ b/trunk/net/ipv4/tcp.c @@ -255,6 +255,7 @@ #include #include #include +#include #include #include #include @@ -1208,7 +1209,8 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, return -ENOTCONN; while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { if (offset < skb->len) { - size_t used, len; + int used; + size_t len; len = skb->len - offset; /* Stop reading if we hit a patch of urgent data */ diff --git a/trunk/net/sunrpc/rpcb_clnt.c b/trunk/net/sunrpc/rpcb_clnt.c index 0517967a68bf..e6fb21b19b86 100644 --- a/trunk/net/sunrpc/rpcb_clnt.c +++ b/trunk/net/sunrpc/rpcb_clnt.c @@ -243,10 +243,10 @@ int rpcb_getport_sync(struct sockaddr_in *sin, u32 prog, u32 vers, int prot) } EXPORT_SYMBOL_GPL(rpcb_getport_sync); -static struct rpc_task *rpcb_call_async(struct rpc_clnt *rpcb_clnt, struct rpcbind_args *map, int version) +static struct rpc_task *rpcb_call_async(struct rpc_clnt *rpcb_clnt, struct rpcbind_args *map, struct rpc_procinfo *proc) { struct rpc_message msg = { - .rpc_proc = rpcb_next_version[version].rpc_proc, + .rpc_proc = proc, .rpc_argp = map, .rpc_resp = &map->r_port, }; @@ -271,6 +271,7 @@ static struct rpc_task *rpcb_call_async(struct rpc_clnt *rpcb_clnt, struct rpcbi void rpcb_getport_async(struct rpc_task *task) { struct rpc_clnt *clnt = task->tk_client; + struct rpc_procinfo *proc; u32 bind_version; struct rpc_xprt *xprt = task->tk_xprt; struct rpc_clnt *rpcb_clnt; @@ -280,7 +281,6 @@ void rpcb_getport_async(struct rpc_task *task) struct sockaddr *sap = (struct sockaddr *)&addr; size_t salen; int status; - struct rpcb_info *info; dprintk("RPC: %5u %s(%s, %u, %u, %d)\n", task->tk_pid, __func__, @@ -313,10 +313,12 @@ void rpcb_getport_async(struct rpc_task *task) /* Don't ever use rpcbind v2 for AF_INET6 requests */ switch (sap->sa_family) { case AF_INET: - info = rpcb_next_version; + proc = rpcb_next_version[xprt->bind_index].rpc_proc; + bind_version = rpcb_next_version[xprt->bind_index].rpc_vers; break; case AF_INET6: - info = rpcb_next_version6; + proc = rpcb_next_version6[xprt->bind_index].rpc_proc; + bind_version = rpcb_next_version6[xprt->bind_index].rpc_vers; break; default: status = -EAFNOSUPPORT; @@ -324,14 +326,13 @@ void rpcb_getport_async(struct rpc_task *task) task->tk_pid, __func__); goto bailout_nofree; } - if (info[xprt->bind_index].rpc_proc == NULL) { + if (proc == NULL) { xprt->bind_index = 0; status = -EPFNOSUPPORT; dprintk("RPC: %5u %s: no more getport versions available\n", task->tk_pid, __func__); goto bailout_nofree; } - bind_version = info[xprt->bind_index].rpc_vers; dprintk("RPC: %5u %s: trying rpcbind version %u\n", task->tk_pid, __func__, bind_version); @@ -361,22 +362,20 @@ void rpcb_getport_async(struct rpc_task *task) map->r_addr = rpc_peeraddr2str(rpcb_clnt, RPC_DISPLAY_UNIVERSAL_ADDR); map->r_owner = RPCB_OWNER_STRING; /* ignored for GETADDR */ - child = rpcb_call_async(rpcb_clnt, map, xprt->bind_index); + child = rpcb_call_async(rpcb_clnt, map, proc); rpc_release_client(rpcb_clnt); if (IS_ERR(child)) { status = -EIO; + /* rpcb_map_release() has freed the arguments */ dprintk("RPC: %5u %s: rpc_run_task failed\n", task->tk_pid, __func__); - goto bailout; + goto bailout_nofree; } rpc_put_task(child); task->tk_xprt->stat.bind_count++; return; -bailout: - kfree(map); - xprt_put(xprt); bailout_nofree: rpcb_wake_rpcbind_waiters(xprt, status); bailout_nowake: