Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 204351
b: refs/heads/master
c: e15bacb
h: refs/heads/master
i:
  204349: df77725
  204347: e973d9f
  204343: da9d2b6
  204335: bcdae9a
  204319: f6921bc
  204287: 90ad073
v: v3
  • Loading branch information
Dan Kruchinin authored and Herbert Xu committed Jul 19, 2010
1 parent 7922a39 commit 6ac4f39
Show file tree
Hide file tree
Showing 4 changed files with 565 additions and 215 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 2197f9a16df9de94655992941d80953ba47042c2
refs/heads/master: e15bacbebb9dcc95f148f28dfc83a6d5e48b60b8
191 changes: 137 additions & 54 deletions trunk/crypto/pcrypt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,38 @@
#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/notifier.h>
#include <crypto/pcrypt.h>

static struct padata_instance *pcrypt_enc_padata;
static struct padata_instance *pcrypt_dec_padata;
static struct workqueue_struct *encwq;
static struct workqueue_struct *decwq;
struct pcrypt_instance {
struct padata_instance *pinst;
struct workqueue_struct *wq;

/*
* Cpumask for callback CPUs. It should be
* equal to serial cpumask of corresponding padata instance,
* so it is updated when padata notifies us about serial
* cpumask change.
*
* cb_cpumask is protected by RCU. This fact prevents us from
* using cpumask_var_t directly because the actual type of
* cpumsak_var_t depends on kernel configuration(particularly on
* CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
* cpumask_var_t may be either a pointer to the struct cpumask
* or a variable allocated on the stack. Thus we can not safely use
* cpumask_var_t with RCU operations such as rcu_assign_pointer or
* rcu_dereference. So cpumask_var_t is wrapped with struct
* pcrypt_cpumask which makes possible to use it with RCU.
*/
struct pcrypt_cpumask {
cpumask_var_t mask;
} *cb_cpumask;
struct notifier_block nblock;
};

static struct pcrypt_instance pencrypt;
static struct pcrypt_instance pdecrypt;


struct pcrypt_instance_ctx {
struct crypto_spawn spawn;
Expand All @@ -42,25 +68,29 @@ struct pcrypt_aead_ctx {
};

static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu,
struct padata_instance *pinst)
struct pcrypt_instance *pcrypt)
{
unsigned int cpu_index, cpu, i;
struct pcrypt_cpumask *cpumask;

cpu = *cb_cpu;

if (cpumask_test_cpu(cpu, cpu_active_mask))
rcu_read_lock_bh();
cpumask = rcu_dereference(pcrypt->cb_cpumask);
if (cpumask_test_cpu(cpu, cpumask->mask))
goto out;

cpu_index = cpu % cpumask_weight(cpu_active_mask);
cpu_index = cpu % cpumask_weight(cpumask->mask);

cpu = cpumask_first(cpu_active_mask);
cpu = cpumask_first(cpumask->mask);
for (i = 0; i < cpu_index; i++)
cpu = cpumask_next(cpu, cpu_active_mask);
cpu = cpumask_next(cpu, cpumask->mask);

*cb_cpu = cpu;

out:
return padata_do_parallel(pinst, padata, cpu);
rcu_read_unlock_bh();
return padata_do_parallel(pcrypt->pinst, padata, cpu);
}

static int pcrypt_aead_setkey(struct crypto_aead *parent,
Expand Down Expand Up @@ -142,7 +172,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req)
req->cryptlen, req->iv);
aead_request_set_assoc(creq, req->assoc, req->assoclen);

err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata);
err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
if (!err)
return -EINPROGRESS;

Expand Down Expand Up @@ -184,7 +214,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req)
req->cryptlen, req->iv);
aead_request_set_assoc(creq, req->assoc, req->assoclen);

err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_dec_padata);
err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt);
if (!err)
return -EINPROGRESS;

Expand Down Expand Up @@ -228,7 +258,7 @@ static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
aead_givcrypt_set_giv(creq, req->giv, req->seq);

err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata);
err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
if (!err)
return -EINPROGRESS;

Expand Down Expand Up @@ -370,6 +400,88 @@ static void pcrypt_free(struct crypto_instance *inst)
kfree(inst);
}

static int pcrypt_cpumask_change_notify(struct notifier_block *self,
unsigned long val, void *data)
{
struct pcrypt_instance *pcrypt;
struct pcrypt_cpumask *new_mask, *old_mask;

if (!(val & PADATA_CPU_SERIAL))
return 0;

pcrypt = container_of(self, struct pcrypt_instance, nblock);
new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL);
if (!new_mask)
return -ENOMEM;
if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) {
kfree(new_mask);
return -ENOMEM;
}

old_mask = pcrypt->cb_cpumask;

padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask);
rcu_assign_pointer(pcrypt->cb_cpumask, new_mask);
synchronize_rcu_bh();

free_cpumask_var(old_mask->mask);
kfree(old_mask);
return 0;
}

static int __pcrypt_init_instance(struct pcrypt_instance *pcrypt,
const char *name)
{
int ret = -ENOMEM;
struct pcrypt_cpumask *mask;

pcrypt->wq = create_workqueue(name);
if (!pcrypt->wq)
goto err;

pcrypt->pinst = padata_alloc(pcrypt->wq);
if (!pcrypt->pinst)
goto err_destroy_workqueue;

mask = kmalloc(sizeof(*mask), GFP_KERNEL);
if (!mask)
goto err_free_padata;
if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) {
kfree(mask);
goto err_free_padata;
}

padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask);
rcu_assign_pointer(pcrypt->cb_cpumask, mask);

pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify;
ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
if (ret)
goto err_free_cpumask;

return ret;
err_free_cpumask:
free_cpumask_var(mask->mask);
kfree(mask);
err_free_padata:
padata_free(pcrypt->pinst);
err_destroy_workqueue:
destroy_workqueue(pcrypt->wq);
err:
return ret;
}

static void __pcrypt_deinit_instance(struct pcrypt_instance *pcrypt)
{
free_cpumask_var(pcrypt->cb_cpumask->mask);
kfree(pcrypt->cb_cpumask);

padata_stop(pcrypt->pinst);
padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
destroy_workqueue(pcrypt->wq);
padata_free(pcrypt->pinst);
}

static struct crypto_template pcrypt_tmpl = {
.name = "pcrypt",
.alloc = pcrypt_alloc,
Expand All @@ -379,60 +491,31 @@ static struct crypto_template pcrypt_tmpl = {

static int __init pcrypt_init(void)
{
int err = -ENOMEM;
encwq = create_workqueue("pencrypt");
if (!encwq)
goto err;

decwq = create_workqueue("pdecrypt");
if (!decwq)
goto err_destroy_encwq;


pcrypt_enc_padata = padata_alloc(cpu_possible_mask, encwq);
if (!pcrypt_enc_padata)
goto err_destroy_decwq;

pcrypt_dec_padata = padata_alloc(cpu_possible_mask, decwq);
if (!pcrypt_dec_padata)
goto err_free_enc_padata;
int err;

err = padata_start(pcrypt_enc_padata);
err = __pcrypt_init_instance(&pencrypt, "pencrypt");
if (err)
goto err_free_dec_padata;
goto err;

err = padata_start(pcrypt_dec_padata);
err = __pcrypt_init_instance(&pdecrypt, "pdecrypt");
if (err)
goto err_free_dec_padata;

return crypto_register_template(&pcrypt_tmpl);

err_free_dec_padata:
padata_free(pcrypt_dec_padata);
goto err_deinit_pencrypt;

err_free_enc_padata:
padata_free(pcrypt_enc_padata);
padata_start(pencrypt.pinst);
padata_start(pdecrypt.pinst);

err_destroy_decwq:
destroy_workqueue(decwq);

err_destroy_encwq:
destroy_workqueue(encwq);
return crypto_register_template(&pcrypt_tmpl);

err_deinit_pencrypt:
__pcrypt_deinit_instance(&pencrypt);
err:
return err;
}

static void __exit pcrypt_exit(void)
{
padata_stop(pcrypt_enc_padata);
padata_stop(pcrypt_dec_padata);

destroy_workqueue(encwq);
destroy_workqueue(decwq);

padata_free(pcrypt_enc_padata);
padata_free(pcrypt_dec_padata);
__pcrypt_deinit_instance(&pencrypt);
__pcrypt_deinit_instance(&pdecrypt);

crypto_unregister_template(&pcrypt_tmpl);
}
Expand Down
Loading

0 comments on commit 6ac4f39

Please sign in to comment.