Skip to content

Commit

Permalink
crypto: talitos - align locks on cache lines
Browse files Browse the repository at this point in the history 
align channel access locks onto separate cache lines (for performance
reasons).  This is done by placing per-channel variables into their own
private struct, and using the cacheline_aligned attribute within that
struct.

Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  • Loading branch information
Kim Phillips authored and Herbert Xu committed Aug 13, 2009
1 parent e41256f commit 4b99262
Showing 1 changed file with 58 additions and 83 deletions.
141 changes: 58 additions & 83 deletions drivers/crypto/talitos.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,25 @@ struct talitos_request {
void *context;
};

/* per-channel fifo management */
struct talitos_channel {
/* request fifo */
struct talitos_request *fifo;

/* number of requests pending in channel h/w fifo */
atomic_t submit_count ____cacheline_aligned;

/* request submission (head) lock */
spinlock_t head_lock ____cacheline_aligned;
/* index to next free descriptor request */
int head;

/* request release (tail) lock */
spinlock_t tail_lock ____cacheline_aligned;
/* index to next in-progress/done descriptor request */
int tail;
};

struct talitos_private {
struct device *dev;
struct of_device *ofdev;
Expand All @@ -101,31 +120,17 @@ struct talitos_private {
/* SEC Compatibility info */
unsigned long features;

/* next channel to be assigned next incoming descriptor */
atomic_t last_chan;

/* per-channel number of requests pending in channel h/w fifo */
atomic_t *submit_count;

/* per-channel request fifo */
struct talitos_request **fifo;

/*
* length of the request fifo
* fifo_len is chfifo_len rounded up to next power of 2
* so we can use bitwise ops to wrap
*/
unsigned int fifo_len;

/* per-channel index to next free descriptor request */
int *head;

/* per-channel index to next in-progress/done descriptor request */
int *tail;
struct talitos_channel *chan;

/* per-channel request submission (head) and release (tail) locks */
spinlock_t *head_lock;
spinlock_t *tail_lock;
/* next channel to be assigned next incoming descriptor */
atomic_t last_chan ____cacheline_aligned;

/* request callback tasklet */
struct tasklet_struct done_task;
Expand Down Expand Up @@ -282,16 +287,16 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc,
/* emulate SEC's round-robin channel fifo polling scheme */
ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);

spin_lock_irqsave(&priv->head_lock[ch], flags);
spin_lock_irqsave(&priv->chan[ch].head_lock, flags);

if (!atomic_inc_not_zero(&priv->submit_count[ch])) {
if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
/* h/w fifo is full */
spin_unlock_irqrestore(&priv->head_lock[ch], flags);
spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
return -EAGAIN;
}

head = priv->head[ch];
request = &priv->fifo[ch][head];
head = priv->chan[ch].head;
request = &priv->chan[ch].fifo[head];

/* map descriptor and save caller data */
request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
Expand All @@ -300,7 +305,7 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc,
request->context = context;

/* increment fifo head */
priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);

smp_wmb();
request->desc = desc;
Expand All @@ -309,7 +314,7 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc,
wmb();
out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);

spin_unlock_irqrestore(&priv->head_lock[ch], flags);
spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);

return -EINPROGRESS;
}
Expand All @@ -324,11 +329,11 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
unsigned long flags;
int tail, status;

spin_lock_irqsave(&priv->tail_lock[ch], flags);
spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);

tail = priv->tail[ch];
while (priv->fifo[ch][tail].desc) {
request = &priv->fifo[ch][tail];
tail = priv->chan[ch].tail;
while (priv->chan[ch].fifo[tail].desc) {
request = &priv->chan[ch].fifo[tail];

/* descriptors with their done bits set don't get the error */
rmb();
Expand All @@ -354,22 +359,22 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
request->desc = NULL;

/* increment fifo tail */
priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);

spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);

atomic_dec(&priv->submit_count[ch]);
atomic_dec(&priv->chan[ch].submit_count);

saved_req.callback(dev, saved_req.desc, saved_req.context,
status);
/* channel may resume processing in single desc error case */
if (error && !reset_ch && status == error)
return;
spin_lock_irqsave(&priv->tail_lock[ch], flags);
tail = priv->tail[ch];
spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
tail = priv->chan[ch].tail;
}

spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
}

/*
Expand Down Expand Up @@ -397,20 +402,20 @@ static void talitos_done(unsigned long data)
static struct talitos_desc *current_desc(struct device *dev, int ch)
{
struct talitos_private *priv = dev_get_drvdata(dev);
int tail = priv->tail[ch];
int tail = priv->chan[ch].tail;
dma_addr_t cur_desc;

cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));

while (priv->fifo[ch][tail].dma_desc != cur_desc) {
while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
tail = (tail + 1) & (priv->fifo_len - 1);
if (tail == priv->tail[ch]) {
if (tail == priv->chan[ch].tail) {
dev_err(dev, "couldn't locate current descriptor\n");
return NULL;
}
}

return priv->fifo[ch][tail].desc;
return priv->chan[ch].fifo[tail].desc;
}

/*
Expand Down Expand Up @@ -1740,17 +1745,11 @@ static int talitos_remove(struct of_device *ofdev)
if (hw_supports(dev, DESC_HDR_SEL0_RNG))
talitos_unregister_rng(dev);

kfree(priv->submit_count);
kfree(priv->tail);
kfree(priv->head);

if (priv->fifo)
for (i = 0; i < priv->num_channels; i++)
kfree(priv->fifo[i]);
for (i = 0; i < priv->num_channels; i++)
if (priv->chan[i].fifo)
kfree(priv->chan[i].fifo);

kfree(priv->fifo);
kfree(priv->head_lock);
kfree(priv->tail_lock);
kfree(priv->chan);

if (priv->irq != NO_IRQ) {
free_irq(priv->irq, dev);
Expand Down Expand Up @@ -1870,58 +1869,34 @@ static int talitos_probe(struct of_device *ofdev,
if (of_device_is_compatible(np, "fsl,sec2.1"))
priv->features |= TALITOS_FTR_HW_AUTH_CHECK;

priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
GFP_KERNEL);
priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
GFP_KERNEL);
if (!priv->head_lock || !priv->tail_lock) {
dev_err(dev, "failed to allocate fifo locks\n");
priv->chan = kzalloc(sizeof(struct talitos_channel) *
priv->num_channels, GFP_KERNEL);
if (!priv->chan) {
dev_err(dev, "failed to allocate channel management space\n");
err = -ENOMEM;
goto err_out;
}

for (i = 0; i < priv->num_channels; i++) {
spin_lock_init(&priv->head_lock[i]);
spin_lock_init(&priv->tail_lock[i]);
}

priv->fifo = kmalloc(sizeof(struct talitos_request *) *
priv->num_channels, GFP_KERNEL);
if (!priv->fifo) {
dev_err(dev, "failed to allocate request fifo\n");
err = -ENOMEM;
goto err_out;
spin_lock_init(&priv->chan[i].head_lock);
spin_lock_init(&priv->chan[i].tail_lock);
}

priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);

for (i = 0; i < priv->num_channels; i++) {
priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
priv->fifo_len, GFP_KERNEL);
if (!priv->fifo[i]) {
priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
priv->fifo_len, GFP_KERNEL);
if (!priv->chan[i].fifo) {
dev_err(dev, "failed to allocate request fifo %d\n", i);
err = -ENOMEM;
goto err_out;
}
}

priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels,
GFP_KERNEL);
if (!priv->submit_count) {
dev_err(dev, "failed to allocate fifo submit count space\n");
err = -ENOMEM;
goto err_out;
}
for (i = 0; i < priv->num_channels; i++)
atomic_set(&priv->submit_count[i], -(priv->chfifo_len - 1));

priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
if (!priv->head || !priv->tail) {
dev_err(dev, "failed to allocate request index space\n");
err = -ENOMEM;
goto err_out;
}
atomic_set(&priv->chan[i].submit_count,
-(priv->chfifo_len - 1));

/* reset and initialize the h/w */
err = init_device(dev);
Expand Down

0 comments on commit 4b99262

Please sign in to comment.