Skip to content
Navigation Menu
Toggle navigation
Sign in
In this repository
All GitHub Enterprise
↵
Jump to
↵
No suggested jump to results
In this repository
All GitHub Enterprise
↵
Jump to
↵
In this organization
All GitHub Enterprise
↵
Jump to
↵
In this repository
All GitHub Enterprise
↵
Jump to
↵
Sign in
Reseting focus
You signed in with another tab or window.
Reload
to refresh your session.
You signed out in another tab or window.
Reload
to refresh your session.
You switched accounts on another tab or window.
Reload
to refresh your session.
Dismiss alert
{{ message }}
mariux64
/
linux
Public
Notifications
You must be signed in to change notification settings
Fork
0
Star
0
Code
Issues
2
Pull requests
0
Actions
Projects
0
Wiki
Security
Insights
Additional navigation options
Code
Issues
Pull requests
Actions
Projects
Wiki
Security
Insights
Files
5100a9d
Documentation
arch
block
crypto
drivers
accessibility
acpi
amba
ata
atm
auxdisplay
base
bcma
block
bluetooth
bus
cdrom
char
clk
clocksource
connector
cpufreq
cpuidle
crypto
dca
devfreq
dio
dma-buf
dma
edac
eisa
extcon
firewire
firmware
fmc
gpio
gpu
hid
hsi
hv
hwmon
hwspinlock
i2c
ide
idle
iio
infiniband
input
iommu
ipack
irqchip
isdn
leds
lguest
macintosh
mailbox
mcb
md
media
memory
memstick
message
mfd
misc
altera-stapl
c2port
carma
cb710
cxl
Kconfig
Makefile
base.c
context.c
cxl.h
debugfs.c
fault.c
file.c
irq.c
main.c
native.c
pci.c
sysfs.c
echo
eeprom
fuse
genwqe
ibmasm
lis3lv02d
mei
mic
sgi-gru
sgi-xp
ti-st
vmw_vmci
Kconfig
Makefile
ad525x_dpot-i2c.c
ad525x_dpot-spi.c
ad525x_dpot.c
ad525x_dpot.h
apds9802als.c
apds990x.c
arm-charlcd.c
atmel-ssc.c
atmel_tclib.c
bh1770glc.c
bh1780gli.c
bmp085-i2c.c
bmp085-spi.c
bmp085.c
bmp085.h
cs5535-mfgpt.c
ds1682.c
dummy-irq.c
enclosure.c
fsa9480.c
hmc6352.c
hpilo.c
hpilo.h
ics932s401.c
ioc4.c
isl29003.c
isl29020.c
kgdbts.c
lattice-ecp3-config.c
lkdtm.c
pch_phub.c
phantom.c
pti.c
spear13xx_pcie_gadget.c
sram.c
ti_dac7512.c
tifm_7xx1.c
tifm_core.c
tsl2550.c
vexpress-syscfg.c
vmw_balloon.c
mmc
mtd
net
nfc
ntb
nubus
of
oprofile
parisc
parport
pci
pcmcia
phy
pinctrl
platform
pnp
power
powercap
pps
ps3
ptp
pwm
rapidio
ras
regulator
remoteproc
reset
rpmsg
rtc
s390
sbus
scsi
sfi
sh
sn
soc
spi
spmi
ssb
staging
target
tc
thermal
thunderbolt
tty
uio
usb
uwb
vfio
vhost
video
virt
virtio
vlynq
vme
w1
watchdog
xen
zorro
Kconfig
Makefile
firmware
fs
include
init
ipc
kernel
lib
mm
net
samples
scripts
security
sound
tools
usr
virt
.gitignore
.mailmap
COPYING
CREDITS
Kbuild
Kconfig
MAINTAINERS
Makefile
README
REPORTING-BUGS
Breadcrumbs
linux
/
drivers
/
misc
/
cxl
/
fault.c
Blame
Blame
Latest commit
History
History
277 lines (230 loc) · 6.49 KB
Breadcrumbs
linux
/
drivers
/
misc
/
cxl
/
fault.c
Top
File metadata and controls
Code
Blame
277 lines (230 loc) · 6.49 KB
Raw
/* * Copyright 2014 IBM Corp. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ #include <linux/workqueue.h> #include <linux/sched.h> #include <linux/pid.h> #include <linux/mm.h> #include <linux/moduleparam.h> #undef MODULE_PARAM_PREFIX #define MODULE_PARAM_PREFIX "cxl" "." #include <asm/current.h> #include <asm/copro.h> #include <asm/mmu.h> #include "cxl.h" static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, unsigned int *lru) { unsigned int entry; struct cxl_sste *sste, *group = primary_group; for (entry = 0; entry < 8; entry++) { sste = group + entry; if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) return sste; } /* Nothing free, select an entry to cast out */ sste = primary_group + *lru; *lru = (*lru + 1) & 0x7; return sste; } static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) { /* mask is the group index, we search primary and secondary here. */ unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ struct cxl_sste *sste; unsigned int hash; unsigned long flags; if (slb->vsid & SLB_VSID_B_1T) hash = (slb->esid >> SID_SHIFT_1T) & mask; else /* 256M */ hash = (slb->esid >> SID_SHIFT) & mask; spin_lock_irqsave(&ctx->sste_lock, flags); sste = find_free_sste(ctx->sstp + (hash << 3), &ctx->sst_lru); pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", sste - ctx->sstp, slb->vsid, slb->esid); sste->vsid_data = cpu_to_be64(slb->vsid); sste->esid_data = cpu_to_be64(slb->esid); spin_unlock_irqrestore(&ctx->sste_lock, flags); } static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm, u64 ea) { struct copro_slb slb = {0,0}; int rc; if (!(rc = copro_calculate_slb(mm, ea, &slb))) { cxl_load_segment(ctx, &slb); } return rc; } static void cxl_ack_ae(struct cxl_context *ctx) { unsigned long flags; cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0); spin_lock_irqsave(&ctx->lock, flags); ctx->pending_fault = true; ctx->fault_addr = ctx->dar; ctx->fault_dsisr = ctx->dsisr; spin_unlock_irqrestore(&ctx->lock, flags); wake_up_all(&ctx->wq); } static int cxl_handle_segment_miss(struct cxl_context *ctx, struct mm_struct *mm, u64 ea) { int rc; pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea); if ((rc = cxl_fault_segment(ctx, mm, ea))) cxl_ack_ae(ctx); else { mb(); /* Order seg table write to TFC MMIO write */ cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); } return IRQ_HANDLED; } static void cxl_handle_page_fault(struct cxl_context *ctx, struct mm_struct *mm, u64 dsisr, u64 dar) { unsigned flt = 0; int result; unsigned long access, flags; if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) { pr_devel("copro_handle_mm_fault failed: %#x\n", result); return cxl_ack_ae(ctx); } /* * update_mmu_cache() will not have loaded the hash since current->trap * is not a 0x400 or 0x300, so just call hash_page_mm() here. */ access = _PAGE_PRESENT; if (dsisr & CXL_PSL_DSISR_An_S) access |= _PAGE_RW; if ((!ctx->kernel) || ~(dar & (1ULL << 63))) access |= _PAGE_USER; local_irq_save(flags); hash_page_mm(mm, dar, access, 0x300); local_irq_restore(flags); pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe); cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); } void cxl_handle_fault(struct work_struct *fault_work) { struct cxl_context *ctx = container_of(fault_work, struct cxl_context, fault_work); u64 dsisr = ctx->dsisr; u64 dar = ctx->dar; struct task_struct *task; struct mm_struct *mm; if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr || cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar || cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) { /* Most likely explanation is harmless - a dedicated process * has detached and these were cleared by the PSL purge, but * warn about it just in case */ dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n"); return; } pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. " "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar); if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { pr_devel("cxl_handle_fault unable to get task %i\n", pid_nr(ctx->pid)); cxl_ack_ae(ctx); return; } if (!(mm = get_task_mm(task))) { pr_devel("cxl_handle_fault unable to get mm %i\n", pid_nr(ctx->pid)); cxl_ack_ae(ctx); goto out; } if (dsisr & CXL_PSL_DSISR_An_DS) cxl_handle_segment_miss(ctx, mm, dar); else if (dsisr & CXL_PSL_DSISR_An_DM) cxl_handle_page_fault(ctx, mm, dsisr, dar); else WARN(1, "cxl_handle_fault has nothing to handle\n"); mmput(mm); out: put_task_struct(task); } static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) { int rc; struct task_struct *task; struct mm_struct *mm; if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { pr_devel("cxl_prefault_one unable to get task %i\n", pid_nr(ctx->pid)); return; } if (!(mm = get_task_mm(task))) { pr_devel("cxl_prefault_one unable to get mm %i\n", pid_nr(ctx->pid)); put_task_struct(task); return; } rc = cxl_fault_segment(ctx, mm, ea); mmput(mm); put_task_struct(task); } static u64 next_segment(u64 ea, u64 vsid) { if (vsid & SLB_VSID_B_1T) ea |= (1ULL << 40) - 1; else ea |= (1ULL << 28) - 1; return ea + 1; } static void cxl_prefault_vma(struct cxl_context *ctx) { u64 ea, last_esid = 0; struct copro_slb slb; struct vm_area_struct *vma; int rc; struct task_struct *task; struct mm_struct *mm; if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { pr_devel("cxl_prefault_vma unable to get task %i\n", pid_nr(ctx->pid)); return; } if (!(mm = get_task_mm(task))) { pr_devel("cxl_prefault_vm unable to get mm %i\n", pid_nr(ctx->pid)); goto out1; } down_read(&mm->mmap_sem); for (vma = mm->mmap; vma; vma = vma->vm_next) { for (ea = vma->vm_start; ea < vma->vm_end; ea = next_segment(ea, slb.vsid)) { rc = copro_calculate_slb(mm, ea, &slb); if (rc) continue; if (last_esid == slb.esid) continue; cxl_load_segment(ctx, &slb); last_esid = slb.esid; } } up_read(&mm->mmap_sem); mmput(mm); out1: put_task_struct(task); } void cxl_prefault(struct cxl_context *ctx, u64 wed) { switch (ctx->afu->prefault_mode) { case CXL_PREFAULT_WED: cxl_prefault_one(ctx, wed); break; case CXL_PREFAULT_ALL: cxl_prefault_vma(ctx); break; default: break; } }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
You can’t perform that action at this time.