Skip to content

Commit

Permalink
cxl: Abstract the differences between the PSL and XSL
Browse files Browse the repository at this point in the history 
The XSL (Translation Service Layer) is a stripped down version of the
PSL (Power Service Layer) used in some cards such as the Mellanox CX4.

Like the PSL, it implements the CAIA architecture, but has a number of
differences, mostly in it's implementation dependent registers. This
adds an ops structure to abstract these differences to bring initial
support for XSL CAPI devices.

The XSL does not implement the optional architected SERR register,
however while it treats it as a reserved register and should work with
no special treatment, attempting to access it will cause the XSL_FEC
(First Error Capture) register to be filled out, preventing it from
capturing any subsequent errors. Therefore, this patch also prevents the
kernel from trying to set up the SERR register so that the FEC register
may still be useful, and to save one interrupt.

The XSL also uses a special DMA cxl mode, which uses a slightly
different init sequence for the CAPP and PHB. The kernel support for
this will be in a future patch once the corresponding support has been
merged into skiboot.

Co-authored-by: Ian Munsie <imunsie@au1.ibm.com>
Signed-off-by: Ian Munsie <imunsie@au1.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
  • Loading branch information
2 people authored and Michael Ellerman committed Jun 16, 2016
1 parent 292841b commit 6d38261
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 48 deletions.
26 changes: 26 additions & 0 deletions drivers/misc/cxl/cxl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ static const cxl_p1_reg_t CXL_PSL_TLBIA = {0x00A8};
static const cxl_p1_reg_t CXL_PSL_AFUSEL = {0x00B0};

/* 0x00C0:7EFF Implementation dependent area */
/* PSL registers */
static const cxl_p1_reg_t CXL_PSL_FIR1 = {0x0100};
static const cxl_p1_reg_t CXL_PSL_FIR2 = {0x0108};
static const cxl_p1_reg_t CXL_PSL_Timebase = {0x0110};
Expand All @@ -91,6 +92,11 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL = {0x0148};
static const cxl_p1_reg_t CXL_PSL_DSNDCTL = {0x0150};
static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
static const cxl_p1_reg_t CXL_PSL_TRACE = {0x0170};
/* XSL registers (Mellanox CX4) */
static const cxl_p1_reg_t CXL_XSL_Timebase = {0x0100};
static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108};
static const cxl_p1_reg_t CXL_XSL_FEC = {0x0158};
static const cxl_p1_reg_t CXL_XSL_DSNCTL = {0x0168};
/* 0x7F00:7FFF Reserved PCIe MSI-X Pending Bit Array area */
/* 0x8000:FFFF Reserved PCIe MSI-X Table Area */

Expand Down Expand Up @@ -525,6 +531,20 @@ struct cxl_context {
struct rcu_head rcu;
};

struct cxl_service_layer_ops {
int (*adapter_regs_init)(struct cxl *adapter, struct pci_dev *dev);
int (*afu_regs_init)(struct cxl_afu *afu);
int (*register_serr_irq)(struct cxl_afu *afu);
void (*release_serr_irq)(struct cxl_afu *afu);
void (*debugfs_add_adapter_sl_regs)(struct cxl *adapter, struct dentry *dir);
void (*debugfs_add_afu_sl_regs)(struct cxl_afu *afu, struct dentry *dir);
void (*psl_irq_dump_registers)(struct cxl_context *ctx);
void (*err_irq_dump_registers)(struct cxl *adapter);
void (*debugfs_stop_trace)(struct cxl *adapter);
void (*write_timebase_ctrl)(struct cxl *adapter);
u64 (*timebase_read)(struct cxl *adapter);
};

struct cxl_native {
u64 afu_desc_off;
u64 afu_desc_size;
Expand All @@ -533,6 +553,7 @@ struct cxl_native {
irq_hw_number_t err_hwirq;
unsigned int err_virq;
u64 ps_off;
const struct cxl_service_layer_ops *sl_ops;
};

struct cxl_guest {
Expand Down Expand Up @@ -805,6 +826,11 @@ int cxl_tlb_slb_invalidate(struct cxl *adapter);
int cxl_afu_disable(struct cxl_afu *afu);
int cxl_psl_purge(struct cxl_afu *afu);

void cxl_debugfs_add_adapter_psl_regs(struct cxl *adapter, struct dentry *dir);
void cxl_debugfs_add_adapter_xsl_regs(struct cxl *adapter, struct dentry *dir);
void cxl_debugfs_add_afu_psl_regs(struct cxl_afu *afu, struct dentry *dir);
void cxl_native_psl_irq_dump_regs(struct cxl_context *ctx);
void cxl_native_err_irq_dump_regs(struct cxl *adapter);
void cxl_stop_trace(struct cxl *cxl);
int cxl_pci_vphb_add(struct cxl_afu *afu);
void cxl_pci_vphb_remove(struct cxl_afu *afu);
Expand Down
35 changes: 25 additions & 10 deletions drivers/misc/cxl/debugfs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,19 @@ static struct dentry *debugfs_create_io_x64(const char *name, umode_t mode,
return debugfs_create_file(name, mode, parent, (void __force *)value, &fops_io_x64);
}

void cxl_debugfs_add_adapter_psl_regs(struct cxl *adapter, struct dentry *dir)
{
debugfs_create_io_x64("fir1", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR1));
debugfs_create_io_x64("fir2", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR2));
debugfs_create_io_x64("fir_cntl", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR_CNTL));
debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
}

void cxl_debugfs_add_adapter_xsl_regs(struct cxl *adapter, struct dentry *dir)
{
debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC));
}

int cxl_debugfs_adapter_add(struct cxl *adapter)
{
struct dentry *dir;
Expand All @@ -65,13 +78,10 @@ int cxl_debugfs_adapter_add(struct cxl *adapter)
return PTR_ERR(dir);
adapter->debugfs = dir;

debugfs_create_io_x64("fir1", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR1));
debugfs_create_io_x64("fir2", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR2));
debugfs_create_io_x64("fir_cntl", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR_CNTL));
debugfs_create_io_x64("err_ivte", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_ErrIVTE));

debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));

if (adapter->native->sl_ops->debugfs_add_adapter_sl_regs)
adapter->native->sl_ops->debugfs_add_adapter_sl_regs(adapter, dir);
return 0;
}

Expand All @@ -80,6 +90,14 @@ void cxl_debugfs_adapter_remove(struct cxl *adapter)
debugfs_remove_recursive(adapter->debugfs);
}

void cxl_debugfs_add_afu_psl_regs(struct cxl_afu *afu, struct dentry *dir)
{
debugfs_create_io_x64("fir", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_FIR_SLICE_An));
debugfs_create_io_x64("serr", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SERR_An));
debugfs_create_io_x64("afu_debug", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_AFU_DEBUG_An));
debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SLICE_TRACE));
}

int cxl_debugfs_afu_add(struct cxl_afu *afu)
{
struct dentry *dir;
Expand All @@ -94,18 +112,15 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu)
return PTR_ERR(dir);
afu->debugfs = dir;

debugfs_create_io_x64("fir", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_FIR_SLICE_An));
debugfs_create_io_x64("serr", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SERR_An));
debugfs_create_io_x64("afu_debug", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_AFU_DEBUG_An));
debugfs_create_io_x64("sr", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SR_An));

debugfs_create_io_x64("dsisr", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DSISR_An));
debugfs_create_io_x64("dar", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DAR_An));
debugfs_create_io_x64("sstp0", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP0_An));
debugfs_create_io_x64("sstp1", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP1_An));
debugfs_create_io_x64("err_status", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_ErrStat_An));

debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SLICE_TRACE));
if (afu->adapter->native->sl_ops->debugfs_add_afu_sl_regs)
afu->adapter->native->sl_ops->debugfs_add_afu_sl_regs(afu, dir);

return 0;
}
Expand Down
53 changes: 39 additions & 14 deletions drivers/misc/cxl/native.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -795,26 +795,38 @@ static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
return 0;
}

static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
u64 dsisr, u64 errstat)
void cxl_native_psl_irq_dump_regs(struct cxl_context *ctx)
{
u64 fir1, fir2, fir_slice, serr, afu_debug;

fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);

dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
}
dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
}

static irqreturn_t native_handle_psl_slice_error(struct cxl_context *ctx,
u64 dsisr, u64 errstat)
{

dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);

dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
cxl_stop_trace(ctx->afu->adapter);
if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers)
ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx);

if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) {
dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter);
}

return cxl_ops->ack_irq(ctx, 0, errstat);
}
Expand Down Expand Up @@ -892,6 +904,9 @@ static irqreturn_t native_slice_irq_err(int irq, void *data)
struct cxl_afu *afu = data;
u64 fir_slice, errstat, serr, afu_debug;

/*
* slice err interrupt is only used with full PSL (no XSL)
*/
WARN(irq, "CXL SLICE ERROR interrupt %i\n", irq);

serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
Expand All @@ -908,23 +923,33 @@ static irqreturn_t native_slice_irq_err(int irq, void *data)
return IRQ_HANDLED;
}

void cxl_native_err_irq_dump_regs(struct cxl *adapter)
{
u64 fir1, fir2;

fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);

dev_crit(&adapter->dev, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1, fir2);
}

static irqreturn_t native_irq_err(int irq, void *data)
{
struct cxl *adapter = data;
u64 fir1, fir2, err_ivte;
u64 err_ivte;

WARN(1, "CXL ERROR interrupt %i\n", irq);

err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%016llx\n", err_ivte);

dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
cxl_stop_trace(adapter);

fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);
if (adapter->native->sl_ops->debugfs_stop_trace) {
dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
adapter->native->sl_ops->debugfs_stop_trace(adapter);
}

dev_crit(&adapter->dev, "PSL_FIR1: 0x%016llx\nPSL_FIR2: 0x%016llx\n", fir1, fir2);
if (adapter->native->sl_ops->err_irq_dump_registers)
adapter->native->sl_ops->err_irq_dump_registers(adapter);

return IRQ_HANDLED;
}
Expand Down
Loading

0 comments on commit 6d38261

Please sign in to comment.