Skip to content

Commit

Permalink
crypto: x86/poly1305 - expose existing driver as poly1305 library
Browse files Browse the repository at this point in the history 
Implement the arch init/update/final Poly1305 library routines in the
accelerated SIMD driver for x86 so they are accessible to users of
the Poly1305 library interface as well.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  • Loading branch information
Ard Biesheuvel authored and Herbert Xu committed Nov 17, 2019
1 parent 1b2c6a5 commit f0e89bc
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 16 deletions.
57 changes: 41 additions & 16 deletions arch/x86/crypto/poly1305_glue.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include <crypto/internal/poly1305.h>
#include <crypto/internal/simd.h>
#include <linux/crypto.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/simd.h>
Expand All @@ -21,7 +22,8 @@ asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r,
asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
unsigned int blocks, const u32 *u);

static bool poly1305_use_avx2 __ro_after_init;
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);

static void poly1305_simd_mult(u32 *a, const u32 *b)
{
Expand Down Expand Up @@ -64,7 +66,7 @@ static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
}

if (IS_ENABLED(CONFIG_AS_AVX2) &&
poly1305_use_avx2 &&
static_branch_likely(&poly1305_use_avx2) &&
srclen >= POLY1305_BLOCK_SIZE * 4) {
if (unlikely(dctx->rset < 4)) {
if (dctx->rset < 2) {
Expand Down Expand Up @@ -103,10 +105,15 @@ static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
return srclen;
}

static int poly1305_simd_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key)
{
poly1305_init_generic(desc, key);
}
EXPORT_SYMBOL(poly1305_init_arch);

void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
unsigned int srclen)
{
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
unsigned int bytes;

if (unlikely(dctx->buflen)) {
Expand All @@ -117,7 +124,8 @@ static int poly1305_simd_update(struct shash_desc *desc,
dctx->buflen += bytes;

if (dctx->buflen == POLY1305_BLOCK_SIZE) {
if (likely(crypto_simd_usable())) {
if (static_branch_likely(&poly1305_use_simd) &&
likely(crypto_simd_usable())) {
kernel_fpu_begin();
poly1305_simd_blocks(dctx, dctx->buf,
POLY1305_BLOCK_SIZE);
Expand All @@ -131,7 +139,8 @@ static int poly1305_simd_update(struct shash_desc *desc,
}

if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
if (likely(crypto_simd_usable())) {
if (static_branch_likely(&poly1305_use_simd) &&
likely(crypto_simd_usable())) {
kernel_fpu_begin();
bytes = poly1305_simd_blocks(dctx, src, srclen);
kernel_fpu_end();
Expand All @@ -147,6 +156,13 @@ static int poly1305_simd_update(struct shash_desc *desc,
memcpy(dctx->buf, src, srclen);
}
}
EXPORT_SYMBOL(poly1305_update_arch);

void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest)
{
poly1305_final_generic(desc, digest);
}
EXPORT_SYMBOL(poly1305_final_arch);

static int crypto_poly1305_init(struct shash_desc *desc)
{
Expand All @@ -171,6 +187,15 @@ static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
return 0;
}

static int poly1305_simd_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
{
struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);

poly1305_update_arch(dctx, src, srclen);
return 0;
}

static struct shash_alg alg = {
.digestsize = POLY1305_DIGEST_SIZE,
.init = crypto_poly1305_init,
Expand All @@ -189,15 +214,15 @@ static struct shash_alg alg = {
static int __init poly1305_simd_mod_init(void)
{
if (!boot_cpu_has(X86_FEATURE_XMM2))
return -ENODEV;

poly1305_use_avx2 = IS_ENABLED(CONFIG_AS_AVX2) &&
boot_cpu_has(X86_FEATURE_AVX) &&
boot_cpu_has(X86_FEATURE_AVX2) &&
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
alg.descsize = sizeof(struct poly1305_desc_ctx) + 5 * sizeof(u32);
if (poly1305_use_avx2)
alg.descsize += 10 * sizeof(u32);
return 0;

static_branch_enable(&poly1305_use_simd);

if (IS_ENABLED(CONFIG_AS_AVX2) &&
boot_cpu_has(X86_FEATURE_AVX) &&
boot_cpu_has(X86_FEATURE_AVX2) &&
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
static_branch_enable(&poly1305_use_avx2);

return crypto_register_shash(&alg);
}
Expand Down
1 change: 1 addition & 0 deletions crypto/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,7 @@ config CRYPTO_POLY1305_X86_64
tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)"
depends on X86 && 64BIT
select CRYPTO_LIB_POLY1305_GENERIC
select CRYPTO_ARCH_HAVE_LIB_POLY1305
help
Poly1305 authenticator algorithm, RFC7539.

Expand Down
1 change: 1 addition & 0 deletions lib/crypto/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ config CRYPTO_LIB_DES

config CRYPTO_LIB_POLY1305_RSIZE
int
default 4 if X86_64
default 1

config CRYPTO_ARCH_HAVE_LIB_POLY1305
Expand Down

0 comments on commit f0e89bc

Please sign in to comment.