Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
db68ce1
Documentation
arch
alpha
arc
arm
arm64
avr32
blackfin
c6x
cris
frv
h8300
hexagon
ia64
m32r
m68k
metag
microblaze
mips
alchemy
ar7
ath25
ath79
bcm47xx
bcm63xx
bmips
boot
cavium-octeon
cobalt
configs
dec
emma
fw
generic
include
asm
dec
emma
fw
ip32
lasat
mach-ar7
mach-ath25
mach-ath79
mach-au1x00
mach-bcm47xx
mach-bcm63xx
mach-bmips
mach-cavium-octeon
mach-cobalt
mach-db1x00
mach-dec
mach-emma2rh
mach-generic
mach-ip22
mach-ip27
mach-ip28
mach-ip32
mach-jazz
mach-jz4740
mach-lantiq
mach-lasat
mach-loongson32
mach-loongson64
mach-malta
mach-netlogic
mach-paravirt
mach-pic32
mach-pistachio
mach-pmcs-msp71xx
mach-pnx833x
mach-ralink
mach-rc32434
mach-rm
mach-sibyte
mach-tx39xx
mach-tx49xx
mach-vr41xx
mach-xilfpga
mips-boards
netlogic
octeon
pci
sgi
sibyte
sn
txx9
vr41xx
xtalk
Kbuild
abi.h
addrspace.h
amon.h
arch_hweight.h
asm-eva.h
asm-offsets.h
asm-prototypes.h
asm.h
asmmacro-32.h
asmmacro-64.h
asmmacro.h
atomic.h
barrier.h
bcache.h
bitops.h
bitrev.h
bmips-spaces.h
bmips.h
bootinfo.h
branch.h
break.h
bug.h
bugs.h
cache.h
cacheflush.h
cacheops.h
cdmm.h
cevt-r4k.h
checksum.h
clock.h
clocksource.h
cmp.h
cmpxchg.h
compat-signal.h
compat.h
compiler.h
cop2.h
cpu-features.h
cpu-info.h
cpu-type.h
cpu.h
debug.h
delay.h
device.h
div64.h
dma-coherence.h
dma-mapping.h
dma.h
ds1287.h
dsemul.h
dsp.h
edac.h
elf.h
errno.h
eva.h
exec.h
extable.h
fb.h
fixmap.h
floppy.h
fpregdef.h
fpu.h
fpu_emulator.h
ftrace.h
futex.h
gio_device.h
gt64120.h
hardirq.h
hazards.h
highmem.h
hpet.h
hugetlb.h
hw_irq.h
i8259.h
ide.h
idle.h
inst.h
io.h
irq.h
irq_cpu.h
irq_gt641xx.h
irq_regs.h
irqflags.h
isadep.h
jazz.h
jazzdma.h
jump_label.h
kdebug.h
kexec.h
kgdb.h
kmap_types.h
kprobes.h
kvm_host.h
kvm_para.h
linkage.h
llsc.h
local.h
m48t37.h
maar.h
machine.h
mc146818-time.h
mc146818rtc.h
mips-cm.h
mips-cpc.h
mips-r2-to-r6-emul.h
mips_machine.h
mips_mt.h
mipsmtregs.h
mipsprom.h
mipsregs.h
mmu.h
mmu_context.h
mmzone.h
module.h
msa.h
msc01_ic.h
nile4.h
paccess.h
page.h
pci.h
perf_event.h
pgalloc.h
pgtable-32.h
pgtable-64.h
pgtable-bits.h
pgtable.h
pm-cps.h
pm.h
pmon.h
prefetch.h
processor.h
prom.h
ptrace.h
r4k-timer.h
r4kcache.h
reboot.h
reg.h
regdef.h
rtlx.h
seccomp.h
setup.h
sgialib.h
sgiarcs.h
shmparam.h
sigcontext.h
signal.h
sim.h
smp-cps.h
smp-ops.h
smp.h
sni.h
socket.h
sparsemem.h
spinlock.h
spinlock_types.h
spram.h
stackframe.h
stackprotector.h
stacktrace.h
string.h
switch_to.h
syscall.h
termios.h
thread_info.h
time.h
timex.h
tlb.h
tlbdebug.h
tlbex.h
tlbflush.h
tlbmisc.h
topology.h
traps.h
txx9irq.h
txx9pio.h
txx9tmr.h
types.h
uaccess.h
uasm.h
unistd.h
uprobes.h
vdso.h
vga.h
vpe.h
war.h
watch.h
wbflush.h
uapi
jazz
jz4740
kernel
kvm
lantiq
lasat
lib
loongson32
loongson64
math-emu
mm
mti-malta
net
netlogic
oprofile
paravirt
pci
pic32
pistachio
pmcs-msp71xx
pnx833x
power
ralink
rb532
sgi-ip22
sgi-ip27
sgi-ip32
sibyte
sni
txx9
vdso
vr41xx
xilfpga
Kbuild
Kbuild.platforms
Kconfig
Kconfig.debug
Makefile
Makefile.postlink
mn10300
nios2
openrisc
parisc
powerpc
s390
score
sh
sparc
tile
um
unicore32
x86
xtensa
.gitignore
Kconfig
block
certs
crypto
drivers
firmware
fs
include
init
ipc
kernel
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.cocciconfig
.get_maintainer.ignore
.gitattributes
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
Breadcrumbs
linux
/
arch
/
mips
/
include
/
asm
/
r4kcache.h
Copy path
Blame
Blame
Latest commit
History
History
750 lines (695 loc) · 25.3 KB
Breadcrumbs
linux
/
arch
/
mips
/
include
/
asm
/
r4kcache.h
Top
File metadata and controls
Code
Blame
750 lines (695 loc) · 25.3 KB
Raw
/* * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive * for more details. * * Inline assembly cache operations. * * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org) * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org) */ #ifndef _ASM_R4KCACHE_H #define _ASM_R4KCACHE_H #include <linux/stringify.h> #include <asm/asm.h> #include <asm/cacheops.h> #include <asm/compiler.h> #include <asm/cpu-features.h> #include <asm/cpu-type.h> #include <asm/mipsmtregs.h> #include <linux/uaccess.h> /* for uaccess_kernel() */ extern void (*r4k_blast_dcache)(void); extern void (*r4k_blast_icache)(void); /* * This macro return a properly sign-extended address suitable as base address * for indexed cache operations. Two issues here: * * - The MIPS32 and MIPS64 specs permit an implementation to directly derive * the index bits from the virtual address. This breaks with tradition * set by the R4000. To keep unpleasant surprises from happening we pick * an address in KSEG0 / CKSEG0. * - We need a properly sign extended address for 64-bit code. To get away * without ifdefs we let the compiler do it by a type cast. */ #define INDEX_BASE CKSEG0 #define cache_op(op,addr) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set "MIPS_ISA_ARCH_LEVEL" \n" \ " cache %0, %1 \n" \ " .set pop \n" \ : \ : "i" (op), "R" (*(unsigned char *)(addr))) #ifdef CONFIG_MIPS_MT #define __iflush_prologue \ unsigned long redundance; \ extern int mt_n_iflushes; \ for (redundance = 0; redundance < mt_n_iflushes; redundance++) { #define __iflush_epilogue \ } #define __dflush_prologue \ unsigned long redundance; \ extern int mt_n_dflushes; \ for (redundance = 0; redundance < mt_n_dflushes; redundance++) { #define __dflush_epilogue \ } #define __inv_dflush_prologue __dflush_prologue #define __inv_dflush_epilogue __dflush_epilogue #define __sflush_prologue { #define __sflush_epilogue } #define __inv_sflush_prologue __sflush_prologue #define __inv_sflush_epilogue __sflush_epilogue #else /* CONFIG_MIPS_MT */ #define __iflush_prologue { #define __iflush_epilogue } #define __dflush_prologue { #define __dflush_epilogue } #define __inv_dflush_prologue { #define __inv_dflush_epilogue } #define __sflush_prologue { #define __sflush_epilogue } #define __inv_sflush_prologue { #define __inv_sflush_epilogue } #endif /* CONFIG_MIPS_MT */ static inline void flush_icache_line_indexed(unsigned long addr) { __iflush_prologue cache_op(Index_Invalidate_I, addr); __iflush_epilogue } static inline void flush_dcache_line_indexed(unsigned long addr) { __dflush_prologue cache_op(Index_Writeback_Inv_D, addr); __dflush_epilogue } static inline void flush_scache_line_indexed(unsigned long addr) { cache_op(Index_Writeback_Inv_SD, addr); } static inline void flush_icache_line(unsigned long addr) { __iflush_prologue switch (boot_cpu_type()) { case CPU_LOONGSON2: cache_op(Hit_Invalidate_I_Loongson2, addr); break; default: cache_op(Hit_Invalidate_I, addr); break; } __iflush_epilogue } static inline void flush_dcache_line(unsigned long addr) { __dflush_prologue cache_op(Hit_Writeback_Inv_D, addr); __dflush_epilogue } static inline void invalidate_dcache_line(unsigned long addr) { __dflush_prologue cache_op(Hit_Invalidate_D, addr); __dflush_epilogue } static inline void invalidate_scache_line(unsigned long addr) { cache_op(Hit_Invalidate_SD, addr); } static inline void flush_scache_line(unsigned long addr) { cache_op(Hit_Writeback_Inv_SD, addr); } #define protected_cache_op(op,addr) \ ({ \ int __err = 0; \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set "MIPS_ISA_ARCH_LEVEL" \n" \ "1: cache %1, (%2) \n" \ "2: .insn \n" \ " .set pop \n" \ " .section .fixup,\"ax\" \n" \ "3: li %0, %3 \n" \ " j 2b \n" \ " .previous \n" \ " .section __ex_table,\"a\" \n" \ " "STR(PTR)" 1b, 3b \n" \ " .previous" \ : "+r" (__err) \ : "i" (op), "r" (addr), "i" (-EFAULT)); \ __err; \ }) #define protected_cachee_op(op,addr) \ ({ \ int __err = 0; \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips0 \n" \ " .set eva \n" \ "1: cachee %1, (%2) \n" \ "2: .insn \n" \ " .set pop \n" \ " .section .fixup,\"ax\" \n" \ "3: li %0, %3 \n" \ " j 2b \n" \ " .previous \n" \ " .section __ex_table,\"a\" \n" \ " "STR(PTR)" 1b, 3b \n" \ " .previous" \ : "+r" (__err) \ : "i" (op), "r" (addr), "i" (-EFAULT)); \ __err; \ }) /* * The next two are for badland addresses like signal trampolines. */ static inline int protected_flush_icache_line(unsigned long addr) { switch (boot_cpu_type()) { case CPU_LOONGSON2: return protected_cache_op(Hit_Invalidate_I_Loongson2, addr); default: #ifdef CONFIG_EVA return protected_cachee_op(Hit_Invalidate_I, addr); #else return protected_cache_op(Hit_Invalidate_I, addr); #endif } } /* * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style * caches. We're talking about one cacheline unnecessarily getting invalidated * here so the penalty isn't overly hard. */ static inline int protected_writeback_dcache_line(unsigned long addr) { #ifdef CONFIG_EVA return protected_cachee_op(Hit_Writeback_Inv_D, addr); #else return protected_cache_op(Hit_Writeback_Inv_D, addr); #endif } static inline int protected_writeback_scache_line(unsigned long addr) { #ifdef CONFIG_EVA return protected_cachee_op(Hit_Writeback_Inv_SD, addr); #else return protected_cache_op(Hit_Writeback_Inv_SD, addr); #endif } /* * This one is RM7000-specific */ static inline void invalidate_tcache_page(unsigned long addr) { cache_op(Page_Invalidate_T, addr); } #ifndef CONFIG_CPU_MIPSR6 #define cache16_unroll32(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips3 \n" \ " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \ " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \ " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \ " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \ " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \ " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \ " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \ " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \ " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \ " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \ " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \ " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \ " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \ " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \ " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #define cache32_unroll32(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips3 \n" \ " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \ " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \ " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \ " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \ " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \ " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \ " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \ " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \ " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \ " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \ " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \ " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \ " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \ " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \ " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #define cache64_unroll32(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips3 \n" \ " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \ " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \ " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \ " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \ " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \ " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \ " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \ " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \ " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \ " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \ " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \ " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \ " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \ " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \ " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #define cache128_unroll32(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips3 \n" \ " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \ " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \ " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \ " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \ " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \ " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \ " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \ " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \ " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \ " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \ " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \ " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \ " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \ " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \ " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \ " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #else /* * MIPS R6 changed the cache opcode and moved to a 8-bit offset field. * This means we now need to increment the base register before we flush * more cache lines */ #define cache16_unroll32(base,op) \ __asm__ __volatile__( \ " .set push\n" \ " .set noreorder\n" \ " .set mips64r6\n" \ " .set noat\n" \ " cache %1, 0x000(%0); cache %1, 0x010(%0)\n" \ " cache %1, 0x020(%0); cache %1, 0x030(%0)\n" \ " cache %1, 0x040(%0); cache %1, 0x050(%0)\n" \ " cache %1, 0x060(%0); cache %1, 0x070(%0)\n" \ " cache %1, 0x080(%0); cache %1, 0x090(%0)\n" \ " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0)\n" \ " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0)\n" \ " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0)\n" \ " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x010($1)\n" \ " cache %1, 0x020($1); cache %1, 0x030($1)\n" \ " cache %1, 0x040($1); cache %1, 0x050($1)\n" \ " cache %1, 0x060($1); cache %1, 0x070($1)\n" \ " cache %1, 0x080($1); cache %1, 0x090($1)\n" \ " cache %1, 0x0a0($1); cache %1, 0x0b0($1)\n" \ " cache %1, 0x0c0($1); cache %1, 0x0d0($1)\n" \ " cache %1, 0x0e0($1); cache %1, 0x0f0($1)\n" \ " .set pop\n" \ : \ : "r" (base), \ "i" (op)); #define cache32_unroll32(base,op) \ __asm__ __volatile__( \ " .set push\n" \ " .set noreorder\n" \ " .set mips64r6\n" \ " .set noat\n" \ " cache %1, 0x000(%0); cache %1, 0x020(%0)\n" \ " cache %1, 0x040(%0); cache %1, 0x060(%0)\n" \ " cache %1, 0x080(%0); cache %1, 0x0a0(%0)\n" \ " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0)\n" \ " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100\n" \ " cache %1, 0x000($1); cache %1, 0x020($1)\n" \ " cache %1, 0x040($1); cache %1, 0x060($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0a0($1)\n" \ " cache %1, 0x0c0($1); cache %1, 0x0e0($1)\n" \ " .set pop\n" \ : \ : "r" (base), \ "i" (op)); #define cache64_unroll32(base,op) \ __asm__ __volatile__( \ " .set push\n" \ " .set noreorder\n" \ " .set mips64r6\n" \ " .set noat\n" \ " cache %1, 0x000(%0); cache %1, 0x040(%0)\n" \ " cache %1, 0x080(%0); cache %1, 0x0c0(%0)\n" \ " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x040($1)\n" \ " cache %1, 0x080($1); cache %1, 0x0c0($1)\n" \ " .set pop\n" \ : \ : "r" (base), \ "i" (op)); #define cache128_unroll32(base,op) \ __asm__ __volatile__( \ " .set push\n" \ " .set noreorder\n" \ " .set mips64r6\n" \ " .set noat\n" \ " cache %1, 0x000(%0); cache %1, 0x080(%0)\n" \ " "__stringify(LONG_ADDIU)" $1, %0, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " "__stringify(LONG_ADDIU)" $1, $1, 0x100 \n" \ " cache %1, 0x000($1); cache %1, 0x080($1)\n" \ " .set pop\n" \ : \ : "r" (base), \ "i" (op)); #endif /* CONFIG_CPU_MIPSR6 */ /* * Perform the cache operation specified by op using a user mode virtual * address while in kernel mode. */ #define cache16_unroll32_user(base,op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips0 \n" \ " .set eva \n" \ " cachee %1, 0x000(%0); cachee %1, 0x010(%0) \n" \ " cachee %1, 0x020(%0); cachee %1, 0x030(%0) \n" \ " cachee %1, 0x040(%0); cachee %1, 0x050(%0) \n" \ " cachee %1, 0x060(%0); cachee %1, 0x070(%0) \n" \ " cachee %1, 0x080(%0); cachee %1, 0x090(%0) \n" \ " cachee %1, 0x0a0(%0); cachee %1, 0x0b0(%0) \n" \ " cachee %1, 0x0c0(%0); cachee %1, 0x0d0(%0) \n" \ " cachee %1, 0x0e0(%0); cachee %1, 0x0f0(%0) \n" \ " cachee %1, 0x100(%0); cachee %1, 0x110(%0) \n" \ " cachee %1, 0x120(%0); cachee %1, 0x130(%0) \n" \ " cachee %1, 0x140(%0); cachee %1, 0x150(%0) \n" \ " cachee %1, 0x160(%0); cachee %1, 0x170(%0) \n" \ " cachee %1, 0x180(%0); cachee %1, 0x190(%0) \n" \ " cachee %1, 0x1a0(%0); cachee %1, 0x1b0(%0) \n" \ " cachee %1, 0x1c0(%0); cachee %1, 0x1d0(%0) \n" \ " cachee %1, 0x1e0(%0); cachee %1, 0x1f0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #define cache32_unroll32_user(base, op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips0 \n" \ " .set eva \n" \ " cachee %1, 0x000(%0); cachee %1, 0x020(%0) \n" \ " cachee %1, 0x040(%0); cachee %1, 0x060(%0) \n" \ " cachee %1, 0x080(%0); cachee %1, 0x0a0(%0) \n" \ " cachee %1, 0x0c0(%0); cachee %1, 0x0e0(%0) \n" \ " cachee %1, 0x100(%0); cachee %1, 0x120(%0) \n" \ " cachee %1, 0x140(%0); cachee %1, 0x160(%0) \n" \ " cachee %1, 0x180(%0); cachee %1, 0x1a0(%0) \n" \ " cachee %1, 0x1c0(%0); cachee %1, 0x1e0(%0) \n" \ " cachee %1, 0x200(%0); cachee %1, 0x220(%0) \n" \ " cachee %1, 0x240(%0); cachee %1, 0x260(%0) \n" \ " cachee %1, 0x280(%0); cachee %1, 0x2a0(%0) \n" \ " cachee %1, 0x2c0(%0); cachee %1, 0x2e0(%0) \n" \ " cachee %1, 0x300(%0); cachee %1, 0x320(%0) \n" \ " cachee %1, 0x340(%0); cachee %1, 0x360(%0) \n" \ " cachee %1, 0x380(%0); cachee %1, 0x3a0(%0) \n" \ " cachee %1, 0x3c0(%0); cachee %1, 0x3e0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); #define cache64_unroll32_user(base, op) \ __asm__ __volatile__( \ " .set push \n" \ " .set noreorder \n" \ " .set mips0 \n" \ " .set eva \n" \ " cachee %1, 0x000(%0); cachee %1, 0x040(%0) \n" \ " cachee %1, 0x080(%0); cachee %1, 0x0c0(%0) \n" \ " cachee %1, 0x100(%0); cachee %1, 0x140(%0) \n" \ " cachee %1, 0x180(%0); cachee %1, 0x1c0(%0) \n" \ " cachee %1, 0x200(%0); cachee %1, 0x240(%0) \n" \ " cachee %1, 0x280(%0); cachee %1, 0x2c0(%0) \n" \ " cachee %1, 0x300(%0); cachee %1, 0x340(%0) \n" \ " cachee %1, 0x380(%0); cachee %1, 0x3c0(%0) \n" \ " cachee %1, 0x400(%0); cachee %1, 0x440(%0) \n" \ " cachee %1, 0x480(%0); cachee %1, 0x4c0(%0) \n" \ " cachee %1, 0x500(%0); cachee %1, 0x540(%0) \n" \ " cachee %1, 0x580(%0); cachee %1, 0x5c0(%0) \n" \ " cachee %1, 0x600(%0); cachee %1, 0x640(%0) \n" \ " cachee %1, 0x680(%0); cachee %1, 0x6c0(%0) \n" \ " cachee %1, 0x700(%0); cachee %1, 0x740(%0) \n" \ " cachee %1, 0x780(%0); cachee %1, 0x7c0(%0) \n" \ " .set pop \n" \ : \ : "r" (base), \ "i" (op)); /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */ #define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize, extra) \ static inline void extra##blast_##pfx##cache##lsize(void) \ { \ unsigned long start = INDEX_BASE; \ unsigned long end = start + current_cpu_data.desc.waysize; \ unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ unsigned long ws_end = current_cpu_data.desc.ways << \ current_cpu_data.desc.waybit; \ unsigned long ws, addr; \ \ __##pfx##flush_prologue \ \ for (ws = 0; ws < ws_end; ws += ws_inc) \ for (addr = start; addr < end; addr += lsize * 32) \ cache##lsize##_unroll32(addr|ws, indexop); \ \ __##pfx##flush_epilogue \ } \ \ static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \ { \ unsigned long start = page; \ unsigned long end = page + PAGE_SIZE; \ \ __##pfx##flush_prologue \ \ do { \ cache##lsize##_unroll32(start, hitop); \ start += lsize * 32; \ } while (start < end); \ \ __##pfx##flush_epilogue \ } \ \ static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \ { \ unsigned long indexmask = current_cpu_data.desc.waysize - 1; \ unsigned long start = INDEX_BASE + (page & indexmask); \ unsigned long end = start + PAGE_SIZE; \ unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \ unsigned long ws_end = current_cpu_data.desc.ways << \ current_cpu_data.desc.waybit; \ unsigned long ws, addr; \ \ __##pfx##flush_prologue \ \ for (ws = 0; ws < ws_end; ws += ws_inc) \ for (addr = start; addr < end; addr += lsize * 32) \ cache##lsize##_unroll32(addr|ws, indexop); \ \ __##pfx##flush_epilogue \ } __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16, ) __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16, ) __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16, ) __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32, ) __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32, ) __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I_Loongson2, 32, loongson2_) __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32, ) __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64, ) __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64, ) __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64, ) __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 128, ) __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 128, ) __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128, ) __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16, ) __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32, ) __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16, ) __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32, ) __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64, ) __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128, ) #define __BUILD_BLAST_USER_CACHE(pfx, desc, indexop, hitop, lsize) \ static inline void blast_##pfx##cache##lsize##_user_page(unsigned long page) \ { \ unsigned long start = page; \ unsigned long end = page + PAGE_SIZE; \ \ __##pfx##flush_prologue \ \ do { \ cache##lsize##_unroll32_user(start, hitop); \ start += lsize * 32; \ } while (start < end); \ \ __##pfx##flush_epilogue \ } __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16) __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16) __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32) __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32) __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64) __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64) /* build blast_xxx_range, protected_blast_xxx_range */ #define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot, extra) \ static inline void prot##extra##blast_##pfx##cache##_range(unsigned long start, \ unsigned long end) \ { \ unsigned long lsize = cpu_##desc##_line_size(); \ unsigned long addr = start & ~(lsize - 1); \ unsigned long aend = (end - 1) & ~(lsize - 1); \ \ __##pfx##flush_prologue \ \ while (1) { \ prot##cache_op(hitop, addr); \ if (addr == aend) \ break; \ addr += lsize; \ } \ \ __##pfx##flush_epilogue \ } #ifndef CONFIG_EVA __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_, ) __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_, ) #else #define __BUILD_PROT_BLAST_CACHE_RANGE(pfx, desc, hitop) \ static inline void protected_blast_##pfx##cache##_range(unsigned long start,\ unsigned long end) \ { \ unsigned long lsize = cpu_##desc##_line_size(); \ unsigned long addr = start & ~(lsize - 1); \ unsigned long aend = (end - 1) & ~(lsize - 1); \ \ __##pfx##flush_prologue \ \ if (!uaccess_kernel()) { \ while (1) { \ protected_cachee_op(hitop, addr); \ if (addr == aend) \ break; \ addr += lsize; \ } \ } else { \ while (1) { \ protected_cache_op(hitop, addr); \ if (addr == aend) \ break; \ addr += lsize; \ } \ \ } \ __##pfx##flush_epilogue \ } __BUILD_PROT_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D) __BUILD_PROT_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I) #endif __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_, ) __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I_Loongson2, \ protected_, loongson2_) __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, , ) __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, , ) __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, , ) /* blast_inv_dcache_range */ __BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, , ) __BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, , ) #endif /* _ASM_R4KCACHE_H */
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
You can’t perform that action at this time.