Skip to content

Commit

Permalink
net: ena: add functions for handling Low Latency Queues in ena_com
Browse files Browse the repository at this point in the history 
This patch introduces APIs for detection, initialization, configuration
and actual usage of low latency queues(LLQ). It extends transmit API with
creation of LLQ descriptors in device memory (which include host buffers
descriptors as well as packet header)

Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Arthur Kiyanovski authored and David S. Miller committed Oct 11, 2018
1 parent a7982b8 commit 689b2bd
Show file tree
Hide file tree
Showing 5 changed files with 474 additions and 80 deletions.
249 changes: 235 additions & 14 deletions drivers/net/ethernet/amazon/ena/ena_com.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@

#define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF

#define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT 4

#define ENA_REGS_ADMIN_INTR_MASK 1

#define ENA_POLL_MS 5
Expand Down Expand Up @@ -352,21 +354,48 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
&io_sq->desc_addr.phys_addr,
GFP_KERNEL);
}
} else {

if (!io_sq->desc_addr.virt_addr) {
pr_err("memory allocation failed");
return -ENOMEM;
}
}

if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
/* Allocate bounce buffers */
io_sq->bounce_buf_ctrl.buffer_size =
ena_dev->llq_info.desc_list_entry_size;
io_sq->bounce_buf_ctrl.buffers_num =
ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
io_sq->bounce_buf_ctrl.next_to_use = 0;

size = io_sq->bounce_buf_ctrl.buffer_size *
io_sq->bounce_buf_ctrl.buffers_num;

dev_node = dev_to_node(ena_dev->dmadev);
set_dev_node(ena_dev->dmadev, ctx->numa_node);
io_sq->desc_addr.virt_addr =
io_sq->bounce_buf_ctrl.base_buffer =
devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
set_dev_node(ena_dev->dmadev, dev_node);
if (!io_sq->desc_addr.virt_addr) {
io_sq->desc_addr.virt_addr =
if (!io_sq->bounce_buf_ctrl.base_buffer)
io_sq->bounce_buf_ctrl.base_buffer =
devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);

if (!io_sq->bounce_buf_ctrl.base_buffer) {
pr_err("bounce buffer memory allocation failed");
return -ENOMEM;
}
}

if (!io_sq->desc_addr.virt_addr) {
pr_err("memory allocation failed");
return -ENOMEM;
memcpy(&io_sq->llq_info, &ena_dev->llq_info,
sizeof(io_sq->llq_info));

/* Initiate the first bounce buffer */
io_sq->llq_buf_ctrl.curr_bounce_buf =
ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl);
memset(io_sq->llq_buf_ctrl.curr_bounce_buf,
0x0, io_sq->llq_info.desc_list_entry_size);
io_sq->llq_buf_ctrl.descs_left_in_line =
io_sq->llq_info.descs_num_before_header;
}

io_sq->tail = 0;
Expand Down Expand Up @@ -554,6 +583,156 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c
return ret;
}

/**
* Set the LLQ configurations of the firmware
*
* The driver provides only the enabled feature values to the device,
* which in turn, checks if they are supported.
*/
static int ena_com_set_llq(struct ena_com_dev *ena_dev)
{
struct ena_com_admin_queue *admin_queue;
struct ena_admin_set_feat_cmd cmd;
struct ena_admin_set_feat_resp resp;
struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
int ret;

memset(&cmd, 0x0, sizeof(cmd));
admin_queue = &ena_dev->admin_queue;

cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
cmd.feat_common.feature_id = ENA_ADMIN_LLQ;

cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl;
cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl;
cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header;
cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl;

ret = ena_com_execute_admin_command(admin_queue,
(struct ena_admin_aq_entry *)&cmd,
sizeof(cmd),
(struct ena_admin_acq_entry *)&resp,
sizeof(resp));

if (unlikely(ret))
pr_err("Failed to set LLQ configurations: %d\n", ret);

return ret;
}

static int ena_com_config_llq_info(struct ena_com_dev *ena_dev,
struct ena_admin_feature_llq_desc *llq_features,
struct ena_llq_configurations *llq_default_cfg)
{
struct ena_com_llq_info *llq_info = &ena_dev->llq_info;
u16 supported_feat;
int rc;

memset(llq_info, 0, sizeof(*llq_info));

supported_feat = llq_features->header_location_ctrl_supported;

if (likely(supported_feat & llq_default_cfg->llq_header_location)) {
llq_info->header_location_ctrl =
llq_default_cfg->llq_header_location;
} else {
pr_err("Invalid header location control, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}

if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) {
supported_feat = llq_features->descriptors_stride_ctrl_supported;
if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) {
llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl;
} else {
if (supported_feat & ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) {
llq_info->desc_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
} else if (supported_feat & ENA_ADMIN_SINGLE_DESC_PER_ENTRY) {
llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY;
} else {
pr_err("Invalid desc_stride_ctrl, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}

pr_err("Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_stride_ctrl, supported_feat,
llq_info->desc_stride_ctrl);
}
} else {
llq_info->desc_stride_ctrl = 0;
}

supported_feat = llq_features->entry_size_ctrl_supported;
if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) {
llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size;
llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value;
} else {
if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_128B) {
llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
llq_info->desc_list_entry_size = 128;
} else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_192B) {
llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B;
llq_info->desc_list_entry_size = 192;
} else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_256B) {
llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
llq_info->desc_list_entry_size = 256;
} else {
pr_err("Invalid entry_size_ctrl, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}

pr_err("Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_ring_entry_size, supported_feat,
llq_info->desc_list_entry_size);
}
if (unlikely(llq_info->desc_list_entry_size & 0x7)) {
/* The desc list entry size should be whole multiply of 8
* This requirement comes from __iowrite64_copy()
*/
pr_err("illegal entry size %d\n",
llq_info->desc_list_entry_size);
return -EINVAL;
}

if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY)
llq_info->descs_per_entry = llq_info->desc_list_entry_size /
sizeof(struct ena_eth_io_tx_desc);
else
llq_info->descs_per_entry = 1;

supported_feat = llq_features->desc_num_before_header_supported;
if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) {
llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header;
} else {
if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2) {
llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1) {
llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1;
} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4) {
llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4;
} else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) {
llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8;
} else {
pr_err("Invalid descs_num_before_header, supported: 0x%x\n",
supported_feat);
return -EINVAL;
}

pr_err("Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_num_decs_before_header,
supported_feat, llq_info->descs_num_before_header);
}

rc = ena_com_set_llq(ena_dev);
if (rc)
pr_err("Cannot set LLQ configuration: %d\n", rc);

return 0;
}

static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
struct ena_com_admin_queue *admin_queue)
{
Expand Down Expand Up @@ -725,15 +904,17 @@ static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
if (io_sq->desc_addr.virt_addr) {
size = io_sq->desc_entry_size * io_sq->q_depth;

if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
dma_free_coherent(ena_dev->dmadev, size,
io_sq->desc_addr.virt_addr,
io_sq->desc_addr.phys_addr);
else
devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
dma_free_coherent(ena_dev->dmadev, size,
io_sq->desc_addr.virt_addr,
io_sq->desc_addr.phys_addr);

io_sq->desc_addr.virt_addr = NULL;
}

if (io_sq->bounce_buf_ctrl.base_buffer) {
devm_kfree(ena_dev->dmadev, io_sq->bounce_buf_ctrl.base_buffer);
io_sq->bounce_buf_ctrl.base_buffer = NULL;
}
}

static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
Expand Down Expand Up @@ -1740,6 +1921,15 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
else
return rc;

rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ);
if (!rc)
memcpy(&get_feat_ctx->llq, &get_resp.u.llq,
sizeof(get_resp.u.llq));
else if (rc == -EOPNOTSUPP)
memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq));
else
return rc;

return 0;
}

Expand Down Expand Up @@ -2708,3 +2898,34 @@ void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
intr_moder_tbl[level].pkts_per_interval;
entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
}

int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
struct ena_admin_feature_llq_desc *llq_features,
struct ena_llq_configurations *llq_default_cfg)
{
int rc;
int size;

if (!llq_features->max_llq_num) {
ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
return 0;
}

rc = ena_com_config_llq_info(ena_dev, llq_features, llq_default_cfg);
if (rc)
return rc;

/* Validate the descriptor is not too big */
size = ena_dev->tx_max_header_size;
size += ena_dev->llq_info.descs_num_before_header *
sizeof(struct ena_eth_io_tx_desc);

if (unlikely(ena_dev->llq_info.desc_list_entry_size < size)) {
pr_err("the size of the LLQ entry is smaller than needed\n");
return -EINVAL;
}

ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;

return 0;
}
28 changes: 28 additions & 0 deletions drivers/net/ethernet/amazon/ena/ena_com.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/gfp.h>
#include <linux/io.h>
#include <linux/sched.h>
#include <linux/sizes.h>
#include <linux/spinlock.h>
Expand Down Expand Up @@ -973,6 +974,16 @@ void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
enum ena_intr_moder_level level,
struct ena_intr_moder_entry *entry);

/* ena_com_config_dev_mode - Configure the placement policy of the device.
* @ena_dev: ENA communication layer struct
* @llq_features: LLQ feature descriptor, retrieve via
* ena_com_get_dev_attr_feat.
* @ena_llq_config: The default driver LLQ parameters configurations
*/
int ena_com_config_dev_mode(struct ena_com_dev *ena_dev,
struct ena_admin_feature_llq_desc *llq_features,
struct ena_llq_configurations *llq_default_config);

static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev)
{
return ena_dev->adaptive_coalescing;
Expand Down Expand Up @@ -1082,4 +1093,21 @@ static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg,
intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
}

static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl)
{
u16 size, buffers_num;
u8 *buf;

size = bounce_buf_ctrl->buffer_size;
buffers_num = bounce_buf_ctrl->buffers_num;

buf = bounce_buf_ctrl->base_buffer +
(bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size;

prefetchw(bounce_buf_ctrl->base_buffer +
(bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size);

return buf;
}

#endif /* !(ENA_COM) */
Loading

0 comments on commit 689b2bd

Please sign in to comment.