Skip to content

Commit

Permalink
sunvnet: make transmit path zero-copy in the kernel
Browse files Browse the repository at this point in the history 
This patch removes pre-allocated transmit buffers and instead directly maps
pending packets on demand. This saves O(n^2) maximum-sized transmit buffers,
for n hosts on a vswitch, as well as a copy to those buffers.

Single-stream TCP throughput linux-solaris dropped ~5% for 1500-byte MTU,
but linux-linux at 1500-bytes increased ~20%.

Signed-off-by: David L Stevens <david.stevens@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David L Stevens authored and David S. Miller committed Sep 30, 2014
1 parent e4defc7 commit 8e845f4
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 45 deletions.
218 changes: 174 additions & 44 deletions drivers/net/ethernet/sun/sunvnet.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,117 @@ struct vnet_port *tx_port_find(struct vnet *vp, struct sk_buff *skb)
return ret;
}

static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port,
unsigned *pending)
{
struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
struct sk_buff *skb = NULL;
int i, txi;

*pending = 0;

txi = dr->prod-1;
if (txi < 0)
txi = VNET_TX_RING_SIZE-1;

for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
struct vio_net_desc *d;

d = vio_dring_entry(dr, txi);

if (d->hdr.state == VIO_DESC_DONE) {
if (port->tx_bufs[txi].skb) {
BUG_ON(port->tx_bufs[txi].skb->next);

port->tx_bufs[txi].skb->next = skb;
skb = port->tx_bufs[txi].skb;
port->tx_bufs[txi].skb = NULL;

ldc_unmap(port->vio.lp,
port->tx_bufs[txi].cookies,
port->tx_bufs[txi].ncookies);
}
d->hdr.state = VIO_DESC_FREE;
} else if (d->hdr.state == VIO_DESC_READY) {
(*pending)++;
} else if (d->hdr.state == VIO_DESC_FREE) {
break;
}
--txi;
if (txi < 0)
txi = VNET_TX_RING_SIZE-1;
}
return skb;
}

static inline void vnet_free_skbs(struct sk_buff *skb)
{
struct sk_buff *next;

while (skb) {
next = skb->next;
skb->next = NULL;
dev_kfree_skb(skb);
skb = next;
}
}

static void vnet_clean_timer_expire(unsigned long port0)
{
struct vnet_port *port = (struct vnet_port *)port0;
struct sk_buff *freeskbs;
unsigned pending;
unsigned long flags;

spin_lock_irqsave(&port->vio.lock, flags);
freeskbs = vnet_clean_tx_ring(port, &pending);
spin_unlock_irqrestore(&port->vio.lock, flags);

vnet_free_skbs(freeskbs);

if (pending)
(void)mod_timer(&port->clean_timer,
jiffies + VNET_CLEAN_TIMEOUT);
else
del_timer(&port->clean_timer);
}

static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart,
int *plen)
{
struct sk_buff *nskb;
int len, pad;

len = skb->len;
pad = 0;
if (len < ETH_ZLEN) {
pad += ETH_ZLEN - skb->len;
len += pad;
}
len += VNET_PACKET_SKIP;
pad += 8 - (len & 7);
len += 8 - (len & 7);

if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP ||
skb_tailroom(skb) < pad ||
skb_headroom(skb) < VNET_PACKET_SKIP) {
nskb = alloc_and_align_skb(skb->dev, skb->len);
skb_reserve(nskb, VNET_PACKET_SKIP);
if (skb_copy_bits(skb, 0, nskb->data, skb->len)) {
dev_kfree_skb(nskb);
dev_kfree_skb(skb);
return NULL;
}
(void)skb_put(nskb, skb->len);
dev_kfree_skb(skb);
skb = nskb;
}

*pstart = skb->data - VNET_PACKET_SKIP;
*plen = len;
return skb;
}

static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct vnet *vp = netdev_priv(dev);
Expand All @@ -788,12 +899,20 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
struct vio_net_desc *d;
unsigned long flags;
unsigned int len;
void *tx_buf;
int i, err;
struct sk_buff *freeskbs = NULL;
int i, err, txi;
void *start = NULL;
int nlen = 0;
unsigned pending = 0;

if (unlikely(!port))
goto out_dropped;

skb = vnet_skb_shape(skb, &start, &nlen);

if (unlikely(!skb))
goto out_dropped;

spin_lock_irqsave(&port->vio.lock, flags);

dr = &port->vio.drings[VIO_DRIVER_TX_RING];
Expand All @@ -811,14 +930,27 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)

d = vio_dring_cur(dr);

tx_buf = port->tx_bufs[dr->prod].buf;
skb_copy_from_linear_data(skb, tx_buf + VNET_PACKET_SKIP, skb->len);
txi = dr->prod;

freeskbs = vnet_clean_tx_ring(port, &pending);

BUG_ON(port->tx_bufs[txi].skb);

len = skb->len;
if (len < ETH_ZLEN) {
if (len < ETH_ZLEN)
len = ETH_ZLEN;
memset(tx_buf+VNET_PACKET_SKIP+skb->len, 0, len - skb->len);

port->tx_bufs[txi].skb = skb;
skb = NULL;

err = ldc_map_single(port->vio.lp, start, nlen,
port->tx_bufs[txi].cookies, 2,
(LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW));
if (err < 0) {
netdev_info(dev, "tx buffer map error %d\n", err);
goto out_dropped_unlock;
}
port->tx_bufs[txi].ncookies = err;

/* We don't rely on the ACKs to free the skb in vnet_start_xmit(),
* thus it is safe to not set VIO_ACK_ENABLE for each transmission:
Expand All @@ -830,9 +962,9 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
*/
d->hdr.ack = VIO_ACK_DISABLE;
d->size = len;
d->ncookies = port->tx_bufs[dr->prod].ncookies;
d->ncookies = port->tx_bufs[txi].ncookies;
for (i = 0; i < d->ncookies; i++)
d->cookies[i] = port->tx_bufs[dr->prod].cookies[i];
d->cookies[i] = port->tx_bufs[txi].cookies[i];

/* This has to be a non-SMP write barrier because we are writing
* to memory which is shared with the peer LDOM.
Expand Down Expand Up @@ -876,7 +1008,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
port->start_cons = false;

dev->stats.tx_packets++;
dev->stats.tx_bytes += skb->len;
dev->stats.tx_bytes += port->tx_bufs[txi].skb->len;

dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);
if (unlikely(vnet_tx_dring_avail(dr) < 2)) {
Expand All @@ -887,15 +1019,24 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)

spin_unlock_irqrestore(&port->vio.lock, flags);

dev_kfree_skb(skb);
vnet_free_skbs(freeskbs);

(void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT);

return NETDEV_TX_OK;

out_dropped_unlock:
spin_unlock_irqrestore(&port->vio.lock, flags);

out_dropped:
dev_kfree_skb(skb);
if (skb)
dev_kfree_skb(skb);
vnet_free_skbs(freeskbs);
if (pending)
(void)mod_timer(&port->clean_timer,
jiffies + VNET_CLEAN_TIMEOUT);
else
del_timer(&port->clean_timer);
dev->stats.tx_dropped++;
return NETDEV_TX_OK;
}
Expand Down Expand Up @@ -1097,17 +1238,22 @@ static void vnet_port_free_tx_bufs(struct vnet_port *port)
}

for (i = 0; i < VNET_TX_RING_SIZE; i++) {
void *buf = port->tx_bufs[i].buf;
struct vio_net_desc *d;
void *skb = port->tx_bufs[i].skb;

if (!buf)
if (!skb)
continue;

d = vio_dring_entry(dr, i);
if (d->hdr.state == VIO_DESC_READY)
pr_warn("active transmit buffers freed\n");

ldc_unmap(port->vio.lp,
port->tx_bufs[i].cookies,
port->tx_bufs[i].ncookies);

kfree(buf);
port->tx_bufs[i].buf = NULL;
dev_kfree_skb(skb);
port->tx_bufs[i].skb = NULL;
d->hdr.state = VIO_DESC_FREE;
}
}

Expand All @@ -1118,34 +1264,6 @@ static int vnet_port_alloc_tx_bufs(struct vnet_port *port)
int i, err, ncookies;
void *dring;

for (i = 0; i < VNET_TX_RING_SIZE; i++) {
void *buf = kzalloc(VNET_MAXPACKET + 8, GFP_KERNEL);
int map_len = (VNET_MAXPACKET + 7) & ~7;

err = -ENOMEM;
if (!buf)
goto err_out;

err = -EFAULT;
if ((unsigned long)buf & (8UL - 1)) {
pr_err("TX buffer misaligned\n");
kfree(buf);
goto err_out;
}

err = ldc_map_single(port->vio.lp, buf, map_len,
port->tx_bufs[i].cookies, 2,
(LDC_MAP_SHADOW |
LDC_MAP_DIRECT |
LDC_MAP_RW));
if (err < 0) {
kfree(buf);
goto err_out;
}
port->tx_bufs[i].buf = buf;
port->tx_bufs[i].ncookies = err;
}

dr = &port->vio.drings[VIO_DRIVER_TX_RING];

len = (VNET_TX_RING_SIZE *
Expand All @@ -1172,6 +1290,12 @@ static int vnet_port_alloc_tx_bufs(struct vnet_port *port)
dr->pending = VNET_TX_RING_SIZE;
dr->ncookies = ncookies;

for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
struct vio_net_desc *d;

d = vio_dring_entry(dr, i);
d->hdr.state = VIO_DESC_FREE;
}
return 0;

err_out:
Expand Down Expand Up @@ -1203,6 +1327,8 @@ static struct vnet *vnet_new(const u64 *local_mac)
dev = alloc_etherdev(sizeof(*vp));
if (!dev)
return ERR_PTR(-ENOMEM);
dev->needed_headroom = VNET_PACKET_SKIP + 8;
dev->needed_tailroom = 8;

for (i = 0; i < ETH_ALEN; i++)
dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff;
Expand Down Expand Up @@ -1397,6 +1523,9 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id)
pr_info("%s: PORT ( remote-mac %pM%s )\n",
vp->dev->name, port->raddr, switch_port ? " switch-port" : "");

setup_timer(&port->clean_timer, vnet_clean_timer_expire,
(unsigned long)port);

vio_port_up(&port->vio);

mdesc_release(hp);
Expand All @@ -1423,6 +1552,7 @@ static int vnet_port_remove(struct vio_dev *vdev)
unsigned long flags;

del_timer_sync(&port->vio.timer);
del_timer_sync(&port->clean_timer);

spin_lock_irqsave(&vp->lock, flags);
list_del(&port->list);
Expand Down
9 changes: 8 additions & 1 deletion drivers/net/ethernet/sun/sunvnet.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@
*/
#define VNET_TX_TIMEOUT (5 * HZ)

/* length of time (or less) we expect pending descriptors to be marked
* as VIO_DESC_DONE and skbs ready to be freed
*/
#define VNET_CLEAN_TIMEOUT ((HZ/100)+1)

#define VNET_MAXPACKET 1518ULL /* ETH_FRAMELEN + VLAN_HDR */
#define VNET_TX_RING_SIZE 512
#define VNET_TX_WAKEUP_THRESH(dr) ((dr)->pending / 4)
Expand All @@ -22,7 +27,7 @@
#define VNET_PACKET_SKIP 6

struct vnet_tx_entry {
void *buf;
struct sk_buff *skb;
unsigned int ncookies;
struct ldc_trans_cookie cookies[2];
};
Expand All @@ -46,6 +51,8 @@ struct vnet_port {
bool stop_rx;
bool start_cons;

struct timer_list clean_timer;

u64 rmtu;
};

Expand Down

0 comments on commit 8e845f4

Please sign in to comment.