Skip to content

Commit

Permalink
[PATCH] IB/mthca: Align FW command mailboxes to 4K
Browse files Browse the repository at this point in the history 
Future versions of Mellanox HCA firmware will require command mailboxes to be
aligned to 4K.  Support this by using a pci_pool to allocate all mailboxes.
This has the added benefit of shrinking the source and text of mthca.

Signed-off-by: Roland Dreier <roland@topspin.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
  • Loading branch information
Roland Dreier authored and Linus Torvalds committed Jun 27, 2005
1 parent 80fd823 commit ed87845
Show file tree
Hide file tree
Showing 8 changed files with 329 additions and 422 deletions.
510 changes: 206 additions & 304 deletions drivers/infiniband/hw/mthca/mthca_cmd.c
View file

Large diffs are not rendered by default.

46 changes: 26 additions & 20 deletions drivers/infiniband/hw/mthca/mthca_cmd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@

#include <ib_verbs.h>

#define MTHCA_CMD_MAILBOX_ALIGN 16UL
#define MTHCA_CMD_MAILBOX_EXTRA (MTHCA_CMD_MAILBOX_ALIGN - 1)
#define MTHCA_MAILBOX_SIZE 4096

enum {
/* command completed successfully: */
Expand Down Expand Up @@ -112,6 +111,11 @@ enum {
DEV_LIM_FLAG_UD_MULTI = 1 << 21,
};

struct mthca_mailbox {
dma_addr_t dma;
void *buf;
};

struct mthca_dev_lim {
int max_srq_sz;
int max_qp_sz;
Expand Down Expand Up @@ -242,6 +246,10 @@ void mthca_cmd_use_polling(struct mthca_dev *dev);
void mthca_cmd_event(struct mthca_dev *dev, u16 token,
u8 status, u64 out_param);

struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev,
unsigned int gfp_mask);
void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox);

int mthca_SYS_EN(struct mthca_dev *dev, u8 *status);
int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status);
int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status);
Expand Down Expand Up @@ -272,41 +280,39 @@ int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status);
int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status);
int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages,
u8 *status);
int mthca_SW2HW_MPT(struct mthca_dev *dev, void *mpt_entry,
int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int mpt_index, u8 *status);
int mthca_HW2SW_MPT(struct mthca_dev *dev, void *mpt_entry,
int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int mpt_index, u8 *status);
int mthca_WRITE_MTT(struct mthca_dev *dev, u64 *mtt_entry,
int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int num_mtt, u8 *status);
int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status);
int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap,
int eq_num, u8 *status);
int mthca_SW2HW_EQ(struct mthca_dev *dev, void *eq_context,
int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int eq_num, u8 *status);
int mthca_HW2SW_EQ(struct mthca_dev *dev, void *eq_context,
int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int eq_num, u8 *status);
int mthca_SW2HW_CQ(struct mthca_dev *dev, void *cq_context,
int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int cq_num, u8 *status);
int mthca_HW2SW_CQ(struct mthca_dev *dev, void *cq_context,
int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int cq_num, u8 *status);
int mthca_MODIFY_QP(struct mthca_dev *dev, int trans, u32 num,
int is_ee, void *qp_context, u32 optmask,
int is_ee, struct mthca_mailbox *mailbox, u32 optmask,
u8 *status);
int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee,
void *qp_context, u8 *status);
struct mthca_mailbox *mailbox, u8 *status);
int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
u8 *status);
int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
int port, struct ib_wc* in_wc, struct ib_grh* in_grh,
int port, struct ib_wc *in_wc, struct ib_grh *in_grh,
void *in_mad, void *response_mad, u8 *status);
int mthca_READ_MGM(struct mthca_dev *dev, int index, void *mgm,
u8 *status);
int mthca_WRITE_MGM(struct mthca_dev *dev, int index, void *mgm,
u8 *status);
int mthca_MGID_HASH(struct mthca_dev *dev, void *gid, u16 *hash,
u8 *status);
int mthca_READ_MGM(struct mthca_dev *dev, int index,
struct mthca_mailbox *mailbox, u8 *status);
int mthca_WRITE_MGM(struct mthca_dev *dev, int index,
struct mthca_mailbox *mailbox, u8 *status);
int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
u16 *hash, u8 *status);
int mthca_NOP(struct mthca_dev *dev, u8 *status);

#define MAILBOX_ALIGN(x) ((void *) ALIGN((unsigned long) (x), MTHCA_CMD_MAILBOX_ALIGN))

#endif /* MTHCA_CMD_H */
34 changes: 16 additions & 18 deletions drivers/infiniband/hw/mthca/mthca_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -745,7 +745,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
struct mthca_cq *cq)
{
int size = nent * MTHCA_CQ_ENTRY_SIZE;
void *mailbox = NULL;
struct mthca_mailbox *mailbox;
struct mthca_cq_context *cq_context;
int err = -ENOMEM;
u8 status;
Expand Down Expand Up @@ -779,12 +779,11 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
goto err_out_ci;
}

mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
GFP_KERNEL);
if (!mailbox)
goto err_out_mailbox;
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
if (IS_ERR(mailbox))
goto err_out_arm;

cq_context = MAILBOX_ALIGN(mailbox);
cq_context = mailbox->buf;

err = mthca_alloc_cq_buf(dev, size, cq);
if (err)
Expand Down Expand Up @@ -815,7 +814,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
cq_context->state_db = cpu_to_be32(cq->arm_db_index);
}

err = mthca_SW2HW_CQ(dev, cq_context, cq->cqn, &status);
err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn, &status);
if (err) {
mthca_warn(dev, "SW2HW_CQ failed (%d)\n", err);
goto err_out_free_mr;
Expand All @@ -839,7 +838,7 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,

cq->cons_index = 0;

kfree(mailbox);
mthca_free_mailbox(dev, mailbox);

return 0;

Expand All @@ -848,8 +847,9 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
mthca_free_cq_buf(dev, cq);

err_out_mailbox:
kfree(mailbox);
mthca_free_mailbox(dev, mailbox);

err_out_arm:
if (mthca_is_memfree(dev))
mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);

Expand All @@ -869,28 +869,26 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
void mthca_free_cq(struct mthca_dev *dev,
struct mthca_cq *cq)
{
void *mailbox;
struct mthca_mailbox *mailbox;
int err;
u8 status;

might_sleep();

mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
GFP_KERNEL);
if (!mailbox) {
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
if (IS_ERR(mailbox)) {
mthca_warn(dev, "No memory for mailbox to free CQ.\n");
return;
}

err = mthca_HW2SW_CQ(dev, MAILBOX_ALIGN(mailbox), cq->cqn, &status);
err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn, &status);
if (err)
mthca_warn(dev, "HW2SW_CQ failed (%d)\n", err);
else if (status)
mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n",
status);
mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n", status);

if (0) {
u32 *ctx = MAILBOX_ALIGN(mailbox);
u32 *ctx = mailbox->buf;
int j;

printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
Expand Down Expand Up @@ -922,7 +920,7 @@ void mthca_free_cq(struct mthca_dev *dev,

mthca_table_put(dev, dev->cq_table.table, cq->cqn);
mthca_free(&dev->cq_table.alloc, cq->cqn);
kfree(mailbox);
mthca_free_mailbox(dev, mailbox);
}

int __devinit mthca_init_cq_table(struct mthca_dev *dev)
Expand Down
1 change: 1 addition & 0 deletions drivers/infiniband/hw/mthca/mthca_dev.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ enum {
};

struct mthca_cmd {
struct pci_pool *pool;
int use_events;
struct semaphore hcr_sem;
struct semaphore poll_sem;
Expand Down
37 changes: 18 additions & 19 deletions drivers/infiniband/hw/mthca/mthca_eq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
PAGE_SIZE;
u64 *dma_list = NULL;
dma_addr_t t;
void *mailbox = NULL;
struct mthca_mailbox *mailbox;
struct mthca_eq_context *eq_context;
int err = -ENOMEM;
int i;
Expand All @@ -494,17 +494,16 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
if (!dma_list)
goto err_out_free;

mailbox = kmalloc(sizeof *eq_context + MTHCA_CMD_MAILBOX_EXTRA,
GFP_KERNEL);
if (!mailbox)
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
if (IS_ERR(mailbox))
goto err_out_free;
eq_context = MAILBOX_ALIGN(mailbox);
eq_context = mailbox->buf;

for (i = 0; i < npages; ++i) {
eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,
PAGE_SIZE, &t, GFP_KERNEL);
if (!eq->page_list[i].buf)
goto err_out_free;
goto err_out_free_pages;

dma_list[i] = t;
pci_unmap_addr_set(&eq->page_list[i], mapping, t);
Expand All @@ -517,7 +516,7 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,

eq->eqn = mthca_alloc(&dev->eq_table.alloc);
if (eq->eqn == -1)
goto err_out_free;
goto err_out_free_pages;

err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
dma_list, PAGE_SHIFT, npages,
Expand Down Expand Up @@ -548,7 +547,7 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
eq_context->intr = intr;
eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey);

err = mthca_SW2HW_EQ(dev, eq_context, eq->eqn, &status);
err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn, &status);
if (err) {
mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err);
goto err_out_free_mr;
Expand All @@ -561,7 +560,7 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
}

kfree(dma_list);
kfree(mailbox);
mthca_free_mailbox(dev, mailbox);

eq->eqn_mask = swab32(1 << eq->eqn);
eq->cons_index = 0;
Expand All @@ -579,17 +578,19 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
err_out_free_eq:
mthca_free(&dev->eq_table.alloc, eq->eqn);

err_out_free:
err_out_free_pages:
for (i = 0; i < npages; ++i)
if (eq->page_list[i].buf)
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
eq->page_list[i].buf,
pci_unmap_addr(&eq->page_list[i],
mapping));

mthca_free_mailbox(dev, mailbox);

err_out_free:
kfree(eq->page_list);
kfree(dma_list);
kfree(mailbox);

err_out:
return err;
Expand All @@ -598,20 +599,18 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev,
static void mthca_free_eq(struct mthca_dev *dev,
struct mthca_eq *eq)
{
void *mailbox = NULL;
struct mthca_mailbox *mailbox;
int err;
u8 status;
int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /
PAGE_SIZE;
int i;

mailbox = kmalloc(sizeof (struct mthca_eq_context) + MTHCA_CMD_MAILBOX_EXTRA,
GFP_KERNEL);
if (!mailbox)
mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
if (IS_ERR(mailbox))
return;

err = mthca_HW2SW_EQ(dev, MAILBOX_ALIGN(mailbox),
eq->eqn, &status);
err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn, &status);
if (err)
mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err);
if (status)
Expand All @@ -624,7 +623,7 @@ static void mthca_free_eq(struct mthca_dev *dev,
for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) {
if (i % 4 == 0)
printk("[%02x] ", i * 4);
printk(" %08x", be32_to_cpup(MAILBOX_ALIGN(mailbox) + i * 4));
printk(" %08x", be32_to_cpup(mailbox->buf + i * 4));
if ((i + 1) % 4 == 0)
printk("\n");
}
Expand All @@ -637,7 +636,7 @@ static void mthca_free_eq(struct mthca_dev *dev,
pci_unmap_addr(&eq->page_list[i], mapping));

kfree(eq->page_list);
kfree(mailbox);
mthca_free_mailbox(dev, mailbox);
}

static void mthca_free_irqs(struct mthca_dev *dev)
Expand Down
Loading

0 comments on commit ed87845

Please sign in to comment.