Skip to content

Commit

Permalink
net: Add functions netif_reset_xps_queue and netif_set_xps_queue
Browse files Browse the repository at this point in the history 
This patch adds two functions, netif_reset_xps_queue and
netif_set_xps_queue.  The main idea behind these two functions is to
provide a mechanism through which drivers can update their defaults in
regards to XPS.

Currently no such mechanism exists and as a result we cannot use XPS for
things such as ATR which would require a basic configuration to start in
which the Tx queues are mapped to CPUs via a 1:1 mapping.  With this change
I am making it possible for drivers such as ixgbe to be able to use the XPS
feature by controlling the default configuration.

Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Alexander Duyck authored and David S. Miller committed Jan 11, 2013
1 parent 416186f commit 537c00d
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 143 deletions.
13 changes: 13 additions & 0 deletions include/linux/netdevice.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2103,6 +2103,19 @@ static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
__netif_schedule(txq->qdisc);
}

#ifdef CONFIG_XPS
extern void netif_reset_xps_queue(struct net_device *dev, u16 index);
extern int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask,
u16 index);
#else
static inline int netif_set_xps_queue(struct net_device *dev,
struct cpumask *mask,
u16 index)
{
return 0;
}
#endif

/*
* Returns a Tx hash for the given packet when dev->real_num_tx_queues is used
* as a distribution range limit for the returned value.
Expand Down
155 changes: 155 additions & 0 deletions net/core/dev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1857,6 +1857,161 @@ static void netif_setup_tc(struct net_device *dev, unsigned int txq)
}
}

#ifdef CONFIG_XPS
static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P) \
rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))

void netif_reset_xps_queue(struct net_device *dev, u16 index)
{
struct xps_dev_maps *dev_maps;
struct xps_map *map;
int i, pos, nonempty = 0;

mutex_lock(&xps_map_mutex);
dev_maps = xmap_dereference(dev->xps_maps);

if (!dev_maps)
goto out_no_maps;

for_each_possible_cpu(i) {
map = xmap_dereference(dev_maps->cpu_map[i]);
if (!map)
continue;

for (pos = 0; pos < map->len; pos++)
if (map->queues[pos] == index)
break;

if (pos < map->len) {
if (map->len > 1) {
map->queues[pos] = map->queues[--map->len];
} else {
RCU_INIT_POINTER(dev_maps->cpu_map[i], NULL);
kfree_rcu(map, rcu);
map = NULL;
}
}
if (map)
nonempty = 1;
}

if (!nonempty) {
RCU_INIT_POINTER(dev->xps_maps, NULL);
kfree_rcu(dev_maps, rcu);
}

out_no_maps:
mutex_unlock(&xps_map_mutex);
}

int netif_set_xps_queue(struct net_device *dev, struct cpumask *mask, u16 index)
{
int i, cpu, pos, map_len, alloc_len, need_set;
struct xps_map *map, *new_map;
struct xps_dev_maps *dev_maps, *new_dev_maps;
int nonempty = 0;
int numa_node_id = -2;
int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);

new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
if (!new_dev_maps)
return -ENOMEM;

mutex_lock(&xps_map_mutex);

dev_maps = xmap_dereference(dev->xps_maps);

for_each_possible_cpu(cpu) {
map = dev_maps ?
xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
new_map = map;
if (map) {
for (pos = 0; pos < map->len; pos++)
if (map->queues[pos] == index)
break;
map_len = map->len;
alloc_len = map->alloc_len;
} else
pos = map_len = alloc_len = 0;

need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu);
#ifdef CONFIG_NUMA
if (need_set) {
if (numa_node_id == -2)
numa_node_id = cpu_to_node(cpu);
else if (numa_node_id != cpu_to_node(cpu))
numa_node_id = -1;
}
#endif
if (need_set && pos >= map_len) {
/* Need to add queue to this CPU's map */
if (map_len >= alloc_len) {
alloc_len = alloc_len ?
2 * alloc_len : XPS_MIN_MAP_ALLOC;
new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len),
GFP_KERNEL,
cpu_to_node(cpu));
if (!new_map)
goto error;
new_map->alloc_len = alloc_len;
for (i = 0; i < map_len; i++)
new_map->queues[i] = map->queues[i];
new_map->len = map_len;
}
new_map->queues[new_map->len++] = index;
} else if (!need_set && pos < map_len) {
/* Need to remove queue from this CPU's map */
if (map_len > 1)
new_map->queues[pos] =
new_map->queues[--new_map->len];
else
new_map = NULL;
}
RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map);
}

/* Cleanup old maps */
for_each_possible_cpu(cpu) {
map = dev_maps ?
xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map)
kfree_rcu(map, rcu);
if (new_dev_maps->cpu_map[cpu])
nonempty = 1;
}

if (nonempty) {
rcu_assign_pointer(dev->xps_maps, new_dev_maps);
} else {
kfree(new_dev_maps);
RCU_INIT_POINTER(dev->xps_maps, NULL);
}

if (dev_maps)
kfree_rcu(dev_maps, rcu);

netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
(numa_node_id >= 0) ? numa_node_id :
NUMA_NO_NODE);

mutex_unlock(&xps_map_mutex);

return 0;
error:
mutex_unlock(&xps_map_mutex);

if (new_dev_maps)
for_each_possible_cpu(i)
kfree(rcu_dereference_protected(
new_dev_maps->cpu_map[i],
1));
kfree(new_dev_maps);
return -ENOMEM;
}
EXPORT_SYMBOL(netif_set_xps_queue);

#endif
/*
* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
* greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
Expand Down
148 changes: 5 additions & 143 deletions net/core/net-sysfs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1002,68 +1002,24 @@ static ssize_t show_xps_map(struct netdev_queue *queue,
return len;
}

static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P) \
rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))

static void xps_queue_release(struct netdev_queue *queue)
{
struct net_device *dev = queue->dev;
struct xps_dev_maps *dev_maps;
struct xps_map *map;
unsigned long index;
int i, pos, nonempty = 0;

index = get_netdev_queue_index(queue);

mutex_lock(&xps_map_mutex);
dev_maps = xmap_dereference(dev->xps_maps);

if (dev_maps) {
for_each_possible_cpu(i) {
map = xmap_dereference(dev_maps->cpu_map[i]);
if (!map)
continue;

for (pos = 0; pos < map->len; pos++)
if (map->queues[pos] == index)
break;

if (pos < map->len) {
if (map->len > 1)
map->queues[pos] =
map->queues[--map->len];
else {
RCU_INIT_POINTER(dev_maps->cpu_map[i],
NULL);
kfree_rcu(map, rcu);
map = NULL;
}
}
if (map)
nonempty = 1;
}

if (!nonempty) {
RCU_INIT_POINTER(dev->xps_maps, NULL);
kfree_rcu(dev_maps, rcu);
}
}
mutex_unlock(&xps_map_mutex);
netif_reset_xps_queue(dev, index);
}

static ssize_t store_xps_map(struct netdev_queue *queue,
struct netdev_queue_attribute *attribute,
const char *buf, size_t len)
{
struct net_device *dev = queue->dev;
cpumask_var_t mask;
int err, i, cpu, pos, map_len, alloc_len, need_set;
unsigned long index;
struct xps_map *map, *new_map;
struct xps_dev_maps *dev_maps, *new_dev_maps;
int nonempty = 0;
int numa_node_id = -2;
cpumask_var_t mask;
int err;

if (!capable(CAP_NET_ADMIN))
return -EPERM;
Expand All @@ -1079,105 +1035,11 @@ static ssize_t store_xps_map(struct netdev_queue *queue,
return err;
}

new_dev_maps = kzalloc(max_t(unsigned int,
XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES), GFP_KERNEL);
if (!new_dev_maps) {
free_cpumask_var(mask);
return -ENOMEM;
}

mutex_lock(&xps_map_mutex);

dev_maps = xmap_dereference(dev->xps_maps);

for_each_possible_cpu(cpu) {
map = dev_maps ?
xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
new_map = map;
if (map) {
for (pos = 0; pos < map->len; pos++)
if (map->queues[pos] == index)
break;
map_len = map->len;
alloc_len = map->alloc_len;
} else
pos = map_len = alloc_len = 0;

need_set = cpumask_test_cpu(cpu, mask) && cpu_online(cpu);
#ifdef CONFIG_NUMA
if (need_set) {
if (numa_node_id == -2)
numa_node_id = cpu_to_node(cpu);
else if (numa_node_id != cpu_to_node(cpu))
numa_node_id = -1;
}
#endif
if (need_set && pos >= map_len) {
/* Need to add queue to this CPU's map */
if (map_len >= alloc_len) {
alloc_len = alloc_len ?
2 * alloc_len : XPS_MIN_MAP_ALLOC;
new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len),
GFP_KERNEL,
cpu_to_node(cpu));
if (!new_map)
goto error;
new_map->alloc_len = alloc_len;
for (i = 0; i < map_len; i++)
new_map->queues[i] = map->queues[i];
new_map->len = map_len;
}
new_map->queues[new_map->len++] = index;
} else if (!need_set && pos < map_len) {
/* Need to remove queue from this CPU's map */
if (map_len > 1)
new_map->queues[pos] =
new_map->queues[--new_map->len];
else
new_map = NULL;
}
RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], new_map);
}

/* Cleanup old maps */
for_each_possible_cpu(cpu) {
map = dev_maps ?
xmap_dereference(dev_maps->cpu_map[cpu]) : NULL;
if (map && xmap_dereference(new_dev_maps->cpu_map[cpu]) != map)
kfree_rcu(map, rcu);
if (new_dev_maps->cpu_map[cpu])
nonempty = 1;
}

if (nonempty) {
rcu_assign_pointer(dev->xps_maps, new_dev_maps);
} else {
kfree(new_dev_maps);
RCU_INIT_POINTER(dev->xps_maps, NULL);
}

if (dev_maps)
kfree_rcu(dev_maps, rcu);

netdev_queue_numa_node_write(queue, (numa_node_id >= 0) ? numa_node_id :
NUMA_NO_NODE);

mutex_unlock(&xps_map_mutex);
err = netif_set_xps_queue(dev, mask, index);

free_cpumask_var(mask);
return len;

error:
mutex_unlock(&xps_map_mutex);

if (new_dev_maps)
for_each_possible_cpu(i)
kfree(rcu_dereference_protected(
new_dev_maps->cpu_map[i],
1));
kfree(new_dev_maps);
free_cpumask_var(mask);
return -ENOMEM;
return err ? : len;
}

static struct netdev_queue_attribute xps_cpus_attribute =
Expand Down

0 comments on commit 537c00d

Please sign in to comment.