Skip to content

Commit

Permalink
crypto: arm64/chacha - expose arm64 ChaCha routine as library function
Browse files Browse the repository at this point in the history
Expose the accelerated NEON ChaCha routine directly as a symbol
export so that users of the ChaCha library API can use it directly.

Given that calls into the library API will always go through the
routines in this module if it is enabled, switch to static keys
to select the optimal implementation available (which may be none
at all, in which case we defer to the generic implementation for
all invocations).

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  • Loading branch information
Ard Biesheuvel authored and Herbert Xu committed Nov 17, 2019
1 parent c77da48 commit b3aad5b
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 11 deletions.
1 change: 1 addition & 0 deletions arch/arm64/crypto/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ config CRYPTO_CHACHA20_NEON
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_LIB_CHACHA_GENERIC
select CRYPTO_ARCH_HAVE_LIB_CHACHA

config CRYPTO_NHPOLY1305_NEON
tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)"
Expand Down
53 changes: 42 additions & 11 deletions arch/arm64/crypto/chacha-neon-glue.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <crypto/internal/chacha.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>

Expand All @@ -36,6 +37,8 @@ asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,
int nrounds, int bytes);
asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);

static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
int bytes, int nrounds)
{
Expand All @@ -59,6 +62,37 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
}
}

void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
{
if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
hchacha_block_generic(state, stream, nrounds);
} else {
kernel_neon_begin();
hchacha_block_neon(state, stream, nrounds);
kernel_neon_end();
}
}
EXPORT_SYMBOL(hchacha_block_arch);

void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
{
chacha_init_generic(state, key, iv);
}
EXPORT_SYMBOL(chacha_init_arch);

void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
int nrounds)
{
if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE ||
!crypto_simd_usable())
return chacha_crypt_generic(state, dst, src, bytes, nrounds);

kernel_neon_begin();
chacha_doneon(state, dst, src, bytes, nrounds);
kernel_neon_end();
}
EXPORT_SYMBOL(chacha_crypt_arch);

static int chacha_neon_stream_xor(struct skcipher_request *req,
const struct chacha_ctx *ctx, const u8 *iv)
{
Expand All @@ -76,7 +110,8 @@ static int chacha_neon_stream_xor(struct skcipher_request *req,
if (nbytes < walk.total)
nbytes = rounddown(nbytes, walk.stride);

if (!crypto_simd_usable()) {
if (!static_branch_likely(&have_neon) ||
!crypto_simd_usable()) {
chacha_crypt_generic(state, walk.dst.virt.addr,
walk.src.virt.addr, nbytes,
ctx->nrounds);
Expand Down Expand Up @@ -109,14 +144,7 @@ static int xchacha_neon(struct skcipher_request *req)
u8 real_iv[16];

chacha_init_generic(state, ctx->key, req->iv);

if (crypto_simd_usable()) {
kernel_neon_begin();
hchacha_block_neon(state, subctx.key, ctx->nrounds);
kernel_neon_end();
} else {
hchacha_block_generic(state, subctx.key, ctx->nrounds);
}
hchacha_block_arch(state, subctx.key, ctx->nrounds);
subctx.nrounds = ctx->nrounds;

memcpy(&real_iv[0], req->iv + 24, 8);
Expand Down Expand Up @@ -179,14 +207,17 @@ static struct skcipher_alg algs[] = {
static int __init chacha_simd_mod_init(void)
{
if (!cpu_have_named_feature(ASIMD))
return -ENODEV;
return 0;

static_branch_enable(&have_neon);

return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
}

static void __exit chacha_simd_mod_fini(void)
{
crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
if (cpu_have_named_feature(ASIMD))
crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
}

module_init(chacha_simd_mod_init);
Expand Down

0 comments on commit b3aad5b

Please sign in to comment.