Subject: [PATCH] arm64/crypto: use crypto instructions for generating AES key schedule

From: [email-masked] (Steve Capper)
Date: Mon, 3 Nov 2014 16:38:07 +0000

On Wed, Oct 22, 2014 at 09:15:32AM +0200, Ard Biesheuvel wrote:
> This patch implements the AES key schedule generation using ARMv8
> Crypto Instructions. It replaces the table based C implementation
> in aes_generic.ko, which means we can drop the dependency on that
> module.
> 
> Signed-off-by: Ard Biesheuvel <[email-masked]>

I've given this a test on Juno running 3.18-rc2.

I disabled CONFIG_CRYPTO_MANAGER_DISABLE_TESTS, and now /proc/crypto
indicates self-test results of "passed".

Also, I ran the tcrypt test module with the following parameters:
modprobe tcrypt mode=10
modprobe tcrypt mode=37
modprobe tcrypt mode=45

No failures were reported in dmesg.

For extra fun I applied dynamic ftrace probes to ce_aes_expandkey and
ce_aes_setkey; and for each test I ran, at least one of those probes
fired.

So for this patch:
Tested-by: Steve Capper <[email-masked]>

The patch looks reasonable to me (apart from some checkpatch warnings
regarding trailing whitespace), so if it helps things:
Acked-by: Steve Capper <[email-masked]>

> ---
>  arch/arm64/crypto/Kconfig           |   5 +-
>  arch/arm64/crypto/aes-ce-ccm-glue.c |   4 +-
>  arch/arm64/crypto/aes-ce-cipher.c   | 112 +++++++++++++++++++++++++++++++++++-
>  arch/arm64/crypto/aes-ce-setkey.h   |   5 ++
>  arch/arm64/crypto/aes-glue.c        |  18 ++++--
>  5 files changed, 133 insertions(+), 11 deletions(-)
>  create mode 100644 arch/arm64/crypto/aes-ce-setkey.h
> 
> diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
> index 5562652c5316..a38b02ce5f9a 100644
> --- a/arch/arm64/crypto/Kconfig
> +++ b/arch/arm64/crypto/Kconfig
> @@ -27,20 +27,19 @@ config CRYPTO_AES_ARM64_CE
>  	tristate "AES core cipher using ARMv8 Crypto Extensions"
>  	depends on ARM64 && KERNEL_MODE_NEON
>  	select CRYPTO_ALGAPI
> -	select CRYPTO_AES
>  
>  config CRYPTO_AES_ARM64_CE_CCM
>  	tristate "AES in CCM mode using ARMv8 Crypto Extensions"
>  	depends on ARM64 && KERNEL_MODE_NEON
>  	select CRYPTO_ALGAPI
> -	select CRYPTO_AES
> +	select CRYPTO_AES_ARM64_CE
>  	select CRYPTO_AEAD
>  
>  config CRYPTO_AES_ARM64_CE_BLK
>  	tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions"
>  	depends on ARM64 && KERNEL_MODE_NEON
>  	select CRYPTO_BLKCIPHER
> -	select CRYPTO_AES
> +	select CRYPTO_AES_ARM64_CE
>  	select CRYPTO_ABLK_HELPER
>  
>  config CRYPTO_AES_ARM64_NEON_BLK
> diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
> index 9e6cdde9b43d..0ac73b838fa3 100644
> --- a/arch/arm64/crypto/aes-ce-ccm-glue.c
> +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
> @@ -16,6 +16,8 @@
>  #include <linux/crypto.h>
>  #include <linux/module.h>
>  
> +#include "aes-ce-setkey.h"
> +
>  static int num_rounds(struct crypto_aes_ctx *ctx)
>  {
>  	/*
> @@ -48,7 +50,7 @@ static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
>  	struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
>  	int ret;
>  
> -	ret = crypto_aes_expand_key(ctx, in_key, key_len);
> +	ret = ce_aes_expandkey(ctx, in_key, key_len);
>  	if (!ret)
>  		return 0;
>  
> diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
> index 2075e1acae6b..4207c83389d3 100644
> --- a/arch/arm64/crypto/aes-ce-cipher.c
> +++ b/arch/arm64/crypto/aes-ce-cipher.c
> @@ -14,6 +14,8 @@
>  #include <linux/crypto.h>
>  #include <linux/module.h>
>  
> +#include "aes-ce-setkey.h"
> +
>  MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
>  MODULE_AUTHOR("Ard Biesheuvel <[email-masked]>");
>  MODULE_LICENSE("GPL v2");
> @@ -124,6 +126,114 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
>  	kernel_neon_end();
>  }
>  
> +/*
> + * aes_sub() - use the aese instruction to perform the AES sbox substitution
> + *             on each byte in 'input'
> + */
> +static u32 aes_sub(u32 input)
> +{
> +	u32 ret;
> +
> +	__asm__("dup	v1.4s, %w[in]		;"
> +		"movi	v0.16b, #0		;"
> +		"aese	v0.16b, v1.16b		;"
> +		"umov	%w[out], v0.4s[0]	;"
> +
> +	:	[out]	"=r"(ret)
> +	:	[in]	"r"(input)
> +	:		"v0","v1");
> +
> +	return ret;
> +}

I like this use of named arguments in the inline asm.

> +
> +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
> +		     unsigned int key_len)
> +{
> +	/*
> +	 * The AES key schedule round constants
> +	 */
> +	static u8 const rcon[] = {
> +		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
> +	};
> +
> +	u32 kwords = key_len / sizeof(u32);
> +	struct aes_block *key_enc, *key_dec;
> +	int i, j;
> +
> +	if (key_len != AES_KEYSIZE_128 &&
> +	    key_len != AES_KEYSIZE_192 &&
> +	    key_len != AES_KEYSIZE_256)
> +		return -EINVAL;
> +
> +	memcpy(ctx->key_enc, in_key, key_len);
> +	ctx->key_length = key_len;
> +
> +	kernel_neon_begin_partial(2);
> +	for (i = 0; i < sizeof(rcon); i++) {
> +		u32 *rki = ctx->key_enc + (i * kwords);
> +		u32 *rko = rki + kwords;
> +
> +		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
> +		rko[1] = rko[0] ^ rki[1];
> +		rko[2] = rko[1] ^ rki[2];
> +		rko[3] = rko[2] ^ rki[3];
> +
> +		if (key_len == AES_KEYSIZE_192) {
> +			if (i >= 7)
> +				break;
> +			rko[4] = rko[3] ^ rki[4];
> +			rko[5] = rko[4] ^ rki[5];
> +		} else if (key_len == AES_KEYSIZE_256) {
> +			if (i >= 6)
> +				break;
> +			rko[4] = aes_sub(rko[3]) ^ rki[4];
> +			rko[5] = rko[4] ^ rki[5];
> +			rko[6] = rko[5] ^ rki[6];
> +			rko[7] = rko[6] ^ rki[7];
> +		}
> +	}
> +
> +	/*
> +	 * Generate the decryption keys for the Inverse Equivalent Cipher.
> +	 * This involves reversing the order of the round keys, and applying
> +	 * the Inverse Mix Columns transformation on all but the first and
> +	 * the last one.
> +	 */
> +	key_enc = (struct aes_block *)ctx->key_enc;
> +	key_dec = (struct aes_block *)ctx->key_dec;
> +	j = num_rounds(ctx);
> +
> +	key_dec[0] = key_enc[j];
> +	for (i = 1, j--; j > 0; i++, j--)
> +		__asm__("ld1	{v0.16b}, %[in]		;"
> +			"aesimc	v1.16b, v0.16b		;"
> +			"st1	{v1.16b}, %[out]	;"
> +
> +		:	[out]	"=Q"(key_dec[i])
> +		:	[in]	"Q"(key_enc[j])
> +		:		"v0","v1");
> +	key_dec[i] = key_enc[0];
> +
> +	kernel_neon_end();
> +	return 0;
> +}
> +EXPORT_SYMBOL(ce_aes_expandkey);
> +
> +int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, 
> +		  unsigned int key_len)
> +{
> +	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
> +	int ret;
> +
> +	ret = ce_aes_expandkey(ctx, in_key, key_len);
> +	if (!ret)
> +		return 0;
> +
> +	tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
> +	return -EINVAL;
> +}
> +EXPORT_SYMBOL(ce_aes_setkey);
> +
>  static struct crypto_alg aes_alg = {
>  	.cra_name		= "aes",
>  	.cra_driver_name	= "aes-ce",
> @@ -135,7 +245,7 @@ static struct crypto_alg aes_alg = {
>  	.cra_cipher = {
>  		.cia_min_keysize	= AES_MIN_KEY_SIZE,
>  		.cia_max_keysize	= AES_MAX_KEY_SIZE,
> -		.cia_setkey		= crypto_aes_set_key,
> +		.cia_setkey		= ce_aes_setkey,
>  		.cia_encrypt		= aes_cipher_encrypt,
>  		.cia_decrypt		= aes_cipher_decrypt
>  	}
> diff --git a/arch/arm64/crypto/aes-ce-setkey.h b/arch/arm64/crypto/aes-ce-setkey.h
> new file mode 100644
> index 000000000000..9d61e98ae347
> --- /dev/null
> +++ b/arch/arm64/crypto/aes-ce-setkey.h
> @@ -0,0 +1,5 @@
> +
> +int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, 
> +		  unsigned int key_len);
> +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
> +		     unsigned int key_len);
> diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
> index 79cd911ef88c..801aae32841f 100644
> --- a/arch/arm64/crypto/aes-glue.c
> +++ b/arch/arm64/crypto/aes-glue.c
> @@ -16,9 +16,13 @@
>  #include <linux/module.h>
>  #include <linux/cpufeature.h>
>  
> +#include "aes-ce-setkey.h"
> +
>  #ifdef USE_V8_CRYPTO_EXTENSIONS
>  #define MODE			"ce"
>  #define PRIO			300
> +#define aes_setkey		ce_aes_setkey
> +#define aes_expandkey		ce_aes_expandkey
>  #define aes_ecb_encrypt		ce_aes_ecb_encrypt
>  #define aes_ecb_decrypt		ce_aes_ecb_decrypt
>  #define aes_cbc_encrypt		ce_aes_cbc_encrypt
> @@ -30,6 +34,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
>  #else
>  #define MODE			"neon"
>  #define PRIO			200
> +#define aes_setkey		crypto_aes_set_key
> +#define aes_expandkey		crypto_aes_expand_key
>  #define aes_ecb_encrypt		neon_aes_ecb_encrypt
>  #define aes_ecb_decrypt		neon_aes_ecb_decrypt
>  #define aes_cbc_encrypt		neon_aes_cbc_encrypt
> @@ -79,10 +85,10 @@ static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
>  	struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
>  	int ret;
>  
> -	ret = crypto_aes_expand_key(&ctx->key1, in_key, key_len / 2);
> +	ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
>  	if (!ret)
> -		ret = crypto_aes_expand_key(&ctx->key2, &in_key[key_len / 2],
> -					    key_len / 2);
> +		ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
> +				    key_len / 2);
>  	if (!ret)
>  		return 0;
>  
> @@ -288,7 +294,7 @@ static struct crypto_alg aes_algs[] = { {
>  		.min_keysize	= AES_MIN_KEY_SIZE,
>  		.max_keysize	= AES_MAX_KEY_SIZE,
>  		.ivsize		= AES_BLOCK_SIZE,
> -		.setkey		= crypto_aes_set_key,
> +		.setkey		= aes_setkey,
>  		.encrypt	= ecb_encrypt,
>  		.decrypt	= ecb_decrypt,
>  	},
> @@ -306,7 +312,7 @@ static struct crypto_alg aes_algs[] = { {
>  		.min_keysize	= AES_MIN_KEY_SIZE,
>  		.max_keysize	= AES_MAX_KEY_SIZE,
>  		.ivsize		= AES_BLOCK_SIZE,
> -		.setkey		= crypto_aes_set_key,
> +		.setkey		= aes_setkey,
>  		.encrypt	= cbc_encrypt,
>  		.decrypt	= cbc_decrypt,
>  	},
> @@ -324,7 +330,7 @@ static struct crypto_alg aes_algs[] = { {
>  		.min_keysize	= AES_MIN_KEY_SIZE,
>  		.max_keysize	= AES_MAX_KEY_SIZE,
>  		.ivsize		= AES_BLOCK_SIZE,
> -		.setkey		= crypto_aes_set_key,
> +		.setkey		= aes_setkey,
>  		.encrypt	= ctr_encrypt,
>  		.decrypt	= ctr_encrypt,
>  	},
> -- 
> 1.8.3.2
> 

Cheers,
-- 
Steve