Skip to content

Commit

Permalink
IB/mlx5: Refactor UMR post send format
Browse files Browse the repository at this point in the history 
* Update struct mlx5_wqe_umr_ctrl_seg.
* Currenlty UMR send_flags aim only certain use cases: enabled/disable
  cached MR, modifying XLT for ODP. By making flags independent make UMR
  more flexible allowing arbitrary manipulations.
* Since different UMR formats have different entry sizes UMR request
  should receive exact size of translation table update instead of
  number of entries. Rename field npages to xlt_size in struct mlx5_umr_wr
  and update relevant code accordingly.
* Add support of length64 bit.

Signed-off-by: Artemy Kovalyov <artemyko@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Artemy Kovalyov authored and David S. Miller committed Jan 2, 2017
1 parent bcda1ac commit 3161625
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 118 deletions.
24 changes: 12 additions & 12 deletions drivers/infiniband/hw/mlx5/mlx5_ib.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,12 @@ struct mlx5_ib_flow_db {
* enum ib_send_flags and enum ib_qp_type for low-level driver
*/

#define MLX5_IB_SEND_UMR_UNREG IB_SEND_RESERVED_START
#define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 1)
#define MLX5_IB_SEND_UMR_UPDATE_MTT (IB_SEND_RESERVED_START << 2)

#define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 3)
#define MLX5_IB_SEND_UMR_UPDATE_PD (IB_SEND_RESERVED_START << 4)
#define MLX5_IB_SEND_UMR_UPDATE_ACCESS IB_SEND_RESERVED_END
#define MLX5_IB_SEND_UMR_ENABLE_MR (IB_SEND_RESERVED_START << 0)
#define MLX5_IB_SEND_UMR_DISABLE_MR (IB_SEND_RESERVED_START << 1)
#define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 2)
#define MLX5_IB_SEND_UMR_UPDATE_XLT (IB_SEND_RESERVED_START << 3)
#define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 4)
#define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS IB_SEND_RESERVED_END

#define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
/*
Expand All @@ -190,6 +189,9 @@ struct mlx5_ib_flow_db {
#define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
#define MLX5_IB_WR_UMR IB_WR_RESERVED1

#define MLX5_IB_UMR_OCTOWORD 16
#define MLX5_IB_UMR_XLT_ALIGNMENT 64

/* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
*
* These flags are intended for internal use by the mlx5_ib driver, and they
Expand Down Expand Up @@ -414,13 +416,11 @@ enum mlx5_ib_qp_flags {

struct mlx5_umr_wr {
struct ib_send_wr wr;
union {
u64 virt_addr;
u64 offset;
} target;
u64 virt_addr;
u64 offset;
struct ib_pd *pd;
unsigned int page_shift;
unsigned int npages;
unsigned int xlt_size;
u64 length;
int access_flags;
u32 mkey;
Expand Down
50 changes: 26 additions & 24 deletions drivers/infiniband/hw/mlx5/mr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,15 +774,15 @@ static int dma_map_mr_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
* To avoid copying garbage after the pas array, we allocate
* a little more.
*/
*size = ALIGN(sizeof(u64) * npages, MLX5_UMR_MTT_ALIGNMENT);
*size = ALIGN(sizeof(struct mlx5_mtt) * npages, MLX5_UMR_MTT_ALIGNMENT);
*mr_pas = kmalloc(*size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
if (!(*mr_pas))
return -ENOMEM;

pas = PTR_ALIGN(*mr_pas, MLX5_UMR_ALIGN);
mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);
/* Clear padding after the actual pages. */
memset(pas + npages, 0, *size - npages * sizeof(u64));
memset(pas + npages, 0, *size - npages * sizeof(struct mlx5_mtt));

*dma = dma_map_single(ddev, pas, *size, DMA_TO_DEVICE);
if (dma_mapping_error(ddev, *dma)) {
Expand All @@ -801,7 +801,8 @@ static void prep_umr_wqe_common(struct ib_pd *pd, struct ib_send_wr *wr,
struct mlx5_umr_wr *umrwr = umr_wr(wr);

sg->addr = dma;
sg->length = ALIGN(sizeof(u64) * n, 64);
sg->length = ALIGN(sizeof(struct mlx5_mtt) * n,
MLX5_IB_UMR_XLT_ALIGNMENT);
sg->lkey = dev->umrc.pd->local_dma_lkey;

wr->next = NULL;
Expand All @@ -813,7 +814,7 @@ static void prep_umr_wqe_common(struct ib_pd *pd, struct ib_send_wr *wr,

wr->opcode = MLX5_IB_WR_UMR;

umrwr->npages = n;
umrwr->xlt_size = sg->length;
umrwr->page_shift = page_shift;
umrwr->mkey = key;
}
Expand All @@ -827,9 +828,11 @@ static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,

prep_umr_wqe_common(pd, wr, sg, dma, n, key, page_shift);

wr->send_flags = 0;
wr->send_flags = MLX5_IB_SEND_UMR_ENABLE_MR |
MLX5_IB_SEND_UMR_UPDATE_TRANSLATION |
MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;

umrwr->target.virt_addr = virt_addr;
umrwr->virt_addr = virt_addr;
umrwr->length = len;
umrwr->access_flags = access_flags;
umrwr->pd = pd;
Expand All @@ -840,7 +843,8 @@ static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
{
struct mlx5_umr_wr *umrwr = umr_wr(wr);

wr->send_flags = MLX5_IB_SEND_UMR_UNREG | MLX5_IB_SEND_UMR_FAIL_IF_FREE;
wr->send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
MLX5_IB_SEND_UMR_FAIL_IF_FREE;
wr->opcode = MLX5_IB_WR_UMR;
umrwr->mkey = key;
}
Expand Down Expand Up @@ -993,7 +997,8 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
struct mlx5_umr_wr wr;
struct ib_sge sg;
int err = 0;
const int page_index_alignment = MLX5_UMR_MTT_ALIGNMENT / sizeof(u64);
const int page_index_alignment = MLX5_UMR_MTT_ALIGNMENT /
sizeof(struct mlx5_mtt);
const int page_index_mask = page_index_alignment - 1;
size_t pages_mapped = 0;
size_t pages_to_map = 0;
Expand All @@ -1012,7 +1017,7 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
if (start_page_index + pages_to_map > MLX5_MAX_UMR_PAGES)
return -EINVAL;

size = sizeof(u64) * pages_to_map;
size = sizeof(struct mlx5_mtt) * pages_to_map;
size = min_t(int, PAGE_SIZE, size);
/* We allocate with GFP_ATOMIC to avoid recursion into page-reclaim
* code, when we are called from an invalidation. The pas buffer must
Expand All @@ -1026,7 +1031,7 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
mutex_lock(&mlx5_ib_update_mtt_emergency_buffer_mutex);
memset(pas, 0, size);
}
pages_iter = size / sizeof(u64);
pages_iter = size / sizeof(struct mlx5_mtt);
dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE);
if (dma_mapping_error(ddev, dma)) {
mlx5_ib_err(dev, "unable to map DMA during MTT update.\n");
Expand All @@ -1049,27 +1054,28 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
MLX5_IB_MTT_PRESENT);
/* Clear padding after the pages brought from the
* umem. */
memset(pas + npages, 0, size - npages * sizeof(u64));
memset(pas + npages, 0, size - npages *
sizeof(struct mlx5_mtt));
}

dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);

memset(&wr, 0, sizeof(wr));

sg.addr = dma;
sg.length = ALIGN(npages * sizeof(u64),
sg.length = ALIGN(npages * sizeof(struct mlx5_mtt),
MLX5_UMR_MTT_ALIGNMENT);
sg.lkey = dev->umrc.pd->local_dma_lkey;

wr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE |
MLX5_IB_SEND_UMR_UPDATE_MTT;
MLX5_IB_SEND_UMR_UPDATE_XLT;
wr.wr.sg_list = &sg;
wr.wr.num_sge = 1;
wr.wr.opcode = MLX5_IB_WR_UMR;
wr.npages = sg.length / sizeof(u64);
wr.xlt_size = sg.length;
wr.page_shift = PAGE_SHIFT;
wr.mkey = mr->mmkey.key;
wr.target.offset = start_page_index;
wr.offset = start_page_index * sizeof(struct mlx5_mtt);

err = mlx5_ib_post_send_wait(dev, &wr);
}
Expand Down Expand Up @@ -1272,22 +1278,18 @@ static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, u64 virt_addr,
if (err)
return err;

umrwr.target.virt_addr = virt_addr;
umrwr.virt_addr = virt_addr;
umrwr.length = length;
umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
}

prep_umr_wqe_common(pd, &umrwr.wr, &sg, dma, npages, mr->mmkey.key,
page_shift);

if (flags & IB_MR_REREG_PD) {
if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
umrwr.pd = pd;
umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD;
}

if (flags & IB_MR_REREG_ACCESS) {
umrwr.access_flags = access_flags;
umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_ACCESS;
umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
}

/* post send request to UMR QP */
Expand Down Expand Up @@ -1552,11 +1554,11 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
err = mlx5_alloc_priv_descs(pd->device, mr,
ndescs, sizeof(u64));
ndescs, sizeof(struct mlx5_mtt));
if (err)
goto err_free_in;

mr->desc_size = sizeof(u64);
mr->desc_size = sizeof(struct mlx5_mtt);
mr->max_descs = ndescs;
} else if (mr_type == IB_MR_TYPE_SG_GAPS) {
mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
Expand Down
3 changes: 2 additions & 1 deletion drivers/infiniband/hw/mlx5/odp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
unsigned long end)
{
struct mlx5_ib_mr *mr;
const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / sizeof(u64)) - 1;
const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT /
sizeof(struct mlx5_mtt)) - 1;
u64 idx = 0, blk_start_idx = 0;
int in_block = 0;
u64 addr;
Expand Down
Loading

0 comments on commit 3161625

Please sign in to comment.