Skip to content

Commit

Permalink
net/mlx5: Mkey creation command adjustments
Browse files Browse the repository at this point in the history 
This change updates the mlx5 interface to create mkey
on the device.

The updates in the command mailbox include increasing the
access mode type field to 5 bits in order to support additional
types such as MLX5_MKC_ACCESS_MODE_MEMIC which represents device
memory access type and will be used when registering MR on allocated
device memory.

All the places that use the old access mode format are adjusted as
well.

Signed-off-by: Ariel Levkovich <lariel@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
  • Loading branch information
Ariel Levkovich authored and Jason Gunthorpe committed Apr 5, 2018
1 parent 24da001 commit cdbd0d2
Show file tree
Hide file tree
Showing 6 changed files with 25 additions and 19 deletions.
16 changes: 7 additions & 9 deletions drivers/infiniband/hw/mlx5/cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,23 +99,21 @@ int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,
mlx5_alignment);

do {
while (page_idx < num_memic_hw_pages) {
spin_lock(&memic->memic_lock);
page_idx = bitmap_find_next_zero_area(memic->memic_alloc_pages,
num_memic_hw_pages,
page_idx,
num_pages, 0);

if (page_idx + num_pages <= num_memic_hw_pages)
if (page_idx < num_memic_hw_pages)
bitmap_set(memic->memic_alloc_pages,
page_idx, num_pages);
else
ret = -ENOMEM;

spin_unlock(&memic->memic_lock);

if (ret)
return ret;
if (page_idx >= num_memic_hw_pages)
break;

MLX5_SET64(alloc_memic_in, in, range_start_addr,
hw_start_addr + (page_idx * PAGE_SIZE));
Expand All @@ -138,10 +136,10 @@ int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
*addr = pci_resource_start(dev->pdev, 0) +
MLX5_GET64(alloc_memic_out, out, memic_start_addr);

return ret;
} while (page_idx < num_memic_hw_pages);
return 0;
}

return ret;
return -ENOMEM;
}

int mlx5_cmd_dealloc_memic(struct mlx5_memic *memic, u64 addr, u64 length)
Expand Down
13 changes: 8 additions & 5 deletions drivers/infiniband/hw/mlx5/mr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,9 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)

MLX5_SET(mkc, mkc, free, 1);
MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, access_mode, ent->access_mode);
MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
MLX5_SET(mkc, mkc, access_mode_4_2,
(ent->access_mode >> 2) & 0x7);

MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
Expand Down Expand Up @@ -804,7 +806,7 @@ struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)

mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
Expand Down Expand Up @@ -1171,7 +1173,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,

mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, free, !populate);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
Expand Down Expand Up @@ -1668,7 +1670,8 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
goto err_free_in;
}

MLX5_SET(mkc, mkc, access_mode, mr->access_mode);
MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);
MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);
MLX5_SET(mkc, mkc, umr_en, 1);

mr->ibmr.device = pd->device;
Expand Down Expand Up @@ -1749,7 +1752,7 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, lr, 1);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_KLMS);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
MLX5_SET(mkc, mkc, qpn, 0xffffff);

Expand Down
2 changes: 1 addition & 1 deletion drivers/net/ethernet/mellanox/mlx5/core/en_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
return -ENOMEM;

mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, lr, 1);

Expand Down
2 changes: 1 addition & 1 deletion drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
MLX5_SET(mkc, mkc, umr_en, 1);
MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, lr, 1);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);

MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
Expand Down
2 changes: 1 addition & 1 deletion drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
return -ENOMEM;

mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, lr, 1);

Expand Down
9 changes: 7 additions & 2 deletions include/linux/mlx5/mlx5_ifc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2720,20 +2720,25 @@ enum {
MLX5_MKC_ACCESS_MODE_MTT = 0x1,
MLX5_MKC_ACCESS_MODE_KLMS = 0x2,
MLX5_MKC_ACCESS_MODE_KSM = 0x3,
MLX5_MKC_ACCESS_MODE_MEMIC = 0x5,
};

struct mlx5_ifc_mkc_bits {
u8 reserved_at_0[0x1];
u8 free[0x1];
u8 reserved_at_2[0xd];
u8 reserved_at_2[0x1];
u8 access_mode_4_2[0x3];
u8 reserved_at_6[0x7];
u8 relaxed_ordering_write[0x1];
u8 reserved_at_e[0x1];
u8 small_fence_on_rdma_read_response[0x1];
u8 umr_en[0x1];
u8 a[0x1];
u8 rw[0x1];
u8 rr[0x1];
u8 lw[0x1];
u8 lr[0x1];
u8 access_mode[0x2];
u8 access_mode_1_0[0x2];
u8 reserved_at_18[0x8];

u8 qpn[0x18];
Expand Down

0 comments on commit cdbd0d2

Please sign in to comment.