From df87a12c8154afd3f90057037716dc590ebe9859 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 10 Jan 2023 18:17:15 +0800 Subject: [PATCH 1/8] Add GCM support Signed-off-by: Jerry Yu --- library/aesce.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ library/aesce.h | 18 ++++++++++++++ library/gcm.c | 24 +++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/library/aesce.c b/library/aesce.c index acfac23432..011c989979 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -276,6 +276,69 @@ int mbedtls_aesce_setkey_enc(unsigned char *rk, return 0; } +#if defined(MBEDTLS_GCM_C) + +static inline uint8x16_t pmull_low(uint8x16_t a, uint8x16_t b) +{ + return vreinterpretq_u8_p128( + vmull_p64( + (poly64_t) vget_low_p64(vreinterpretq_p64_u8(a)), + (poly64_t) vget_low_p64(vreinterpretq_p64_u8(b)))); +} + +static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b) +{ + return vreinterpretq_u8_p128( + vmull_high_p64(vreinterpretq_p64_u8(a), + vreinterpretq_p64_u8(b))); +} + +static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b) +{ + uint8x16x3_t ret; + uint8x16_t c = vextq_u8(b, b, 8); + ret.val[0] = pmull_high(a, b); /* a1*b1 */ + ret.val[1] = veorq_u8(pmull_high(a, c), /* a1*b0 + a0*b1 */ + pmull_low(a, c)); + ret.val[2] = pmull_low(a, b); /* a0*b0 */ + return ret; +} + +static inline uint8x16_t poly_mult_reduce(uint8x16x3_t a) +{ + uint8x16_t const Z = vdupq_n_u8(0); + /* use 'asm' as an optimisation barrier to prevent loading R from memory */ + uint64x2_t r = vreinterpretq_u64_u8(vdupq_n_u8(0x87)); + asm ("" : "+w" (r)); + uint8x16_t const R = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8)); + uint8x16_t d = a.val[0]; /* d3:d2:00:00 */ + uint8x16_t j = a.val[1]; /* j2:j1:00 */ + uint8x16_t g = a.val[2]; /* g1:g0 = a0*b0 */ + uint8x16_t h = pmull_high(d, R); /* h2:h1:00 = reduction of d3 */ + uint8x16_t i = pmull_low(d, R); /* i1:i0 = reduction of d2 */ + uint8x16_t k = veorq_u8(j, h); /* k2:k1:00 = j2:j1 + h2:h1 */ + uint8x16_t l = pmull_high(k, R); /* l1:l0 = reduction of k2 */ + uint8x16_t m = vextq_u8(Z, k, 8); /* m1:00 = k1:00 */ + uint8x16_t n = veorq_u8(g, i); /* n1:n0 = g1:g0 + i1:i0 */ + uint8x16_t o = veorq_u8(n, l); /* o1:o0 = l1:l0 + n1:n0 */ + return veorq_u8(o, m); /* = o1:o0 + m1:00 */ +} + +/* + * GCM multiplication: c = a times b in GF(2^128) + */ +void mbedtls_aesce_gcm_mult(unsigned char c[16], + const unsigned char a[16], + const unsigned char b[16]) +{ + uint8x16_t va, vb, vc; + va = vrbitq_u8(vld1q_u8(&a[0])); + vb = vrbitq_u8(vld1q_u8(&b[0])); + vc = vrbitq_u8(poly_mult_reduce(poly_mult_128(va, vb))); + vst1q_u8(&c[0], vc); +} + +#endif /* MBEDTLS_GCM_C */ #if defined(MBEDTLS_POP_TARGET_PRAGMA) #if defined(__clang__) diff --git a/library/aesce.h b/library/aesce.h index da42446997..1b3f81632c 100644 --- a/library/aesce.h +++ b/library/aesce.h @@ -64,6 +64,24 @@ int mbedtls_aesce_crypt_ecb(mbedtls_aes_context *ctx, const unsigned char input[16], unsigned char output[16]); +/** + * \brief Internal GCM multiplication: c = a * b in GF(2^128) + * + * \note This function is only for internal use by other library + * functions; you must not call it directly. + * + * \param c Result + * \param a First operand + * \param b Second operand + * + * \note Both operands and result are bit strings interpreted as + * elements of GF(2^128) as per the GCM spec. + */ +void mbedtls_aesce_gcm_mult(unsigned char c[16], + const unsigned char a[16], + const unsigned char b[16]); + + /** * \brief Internal round key inversion. This function computes * decryption round keys from the encryption round keys. diff --git a/library/gcm.c b/library/gcm.c index 6d4495fd39..0fa0008e38 100644 --- a/library/gcm.c +++ b/library/gcm.c @@ -42,6 +42,10 @@ #include "aesni.h" #endif +#if defined(MBEDTLS_AESCE_C) +#include "aesce.h" +#endif + #if !defined(MBEDTLS_GCM_ALT) /* @@ -93,6 +97,12 @@ static int gcm_gen_table(mbedtls_gcm_context *ctx) } #endif +#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64) + if (mbedtls_aesce_has_support()) { + return 0; + } +#endif + /* 0 corresponds to 0 in GF(2^128) */ ctx->HH[0] = 0; ctx->HL[0] = 0; @@ -197,6 +207,20 @@ static void gcm_mult(mbedtls_gcm_context *ctx, const unsigned char x[16], } #endif /* MBEDTLS_AESNI_C && MBEDTLS_HAVE_X86_64 */ +#if defined(MBEDTLS_AESCE_C) && defined(MBEDTLS_HAVE_ARM64) + if (mbedtls_aesce_has_support()) { + unsigned char h[16]; + + MBEDTLS_PUT_UINT32_BE(ctx->HH[8] >> 32, h, 0); + MBEDTLS_PUT_UINT32_BE(ctx->HH[8], h, 4); + MBEDTLS_PUT_UINT32_BE(ctx->HL[8] >> 32, h, 8); + MBEDTLS_PUT_UINT32_BE(ctx->HL[8], h, 12); + + mbedtls_aesce_gcm_mult(output, x, h); + return; + } +#endif + lo = x[15] & 0xf; zh = ctx->HH[lo]; From 2c26651938e33161c763968f7a2de5c3c2461737 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Wed, 1 Mar 2023 11:18:20 +0800 Subject: [PATCH 2/8] Improve comments for key expansion Signed-off-by: Jerry Yu --- library/aesce.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/library/aesce.c b/library/aesce.c index 011c989979..e236d865ad 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -212,14 +212,14 @@ static void aesce_setkey_enc(unsigned char *rk, 0x20, 0x40, 0x80, 0x1b, 0x36 }; /* See https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.197.pdf * - Section 5, Nr = Nk + 6 - * - Section 5.2, the key expansion size is Nb*(Nr+1) + * - Section 5.2, the length of round keys is Nb*(Nr+1) */ const uint32_t key_len_in_words = key_bit_length / 32; /* Nk */ const size_t round_key_len_in_words = 4; /* Nb */ - const size_t round_keys_needed = key_len_in_words + 6; /* Nr */ - const size_t key_expansion_size_in_words = - round_key_len_in_words * (round_keys_needed + 1); /* Nb*(Nr+1) */ - const uint32_t *rko_end = (uint32_t *) rk + key_expansion_size_in_words; + const size_t rounds_needed = key_len_in_words + 6; /* Nr */ + const size_t round_keys_len_in_words = + round_key_len_in_words * (rounds_needed + 1); /* Nb*(Nr+1) */ + const uint32_t *rko_end = (uint32_t *) rk + round_keys_len_in_words; memcpy(rk, key, key_len_in_words * 4); From 132d0cb74d6c2157d1d1d23a1c888541c3e54848 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Mar 2023 17:35:53 +0800 Subject: [PATCH 3/8] Add miss intrinsic by gcc-5.x Signed-off-by: Jerry Yu --- library/aesce.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/library/aesce.c b/library/aesce.c index e236d865ad..7c5bcca456 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -278,6 +278,18 @@ int mbedtls_aesce_setkey_enc(unsigned char *rk, #if defined(MBEDTLS_GCM_C) +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 5 +/* GCC 5.X miss some intrinsics, we add them here. */ +#define vreinterpretq_p64_u8(a) ((poly64x2_t) a) +#define vreinterpretq_u8_p128(a) ((uint8x16_t) a) +static inline poly64_t vget_low_p64(poly64x2_t __a) +{ + uint64x2_t tmp = (uint64x2_t) (__a); + uint64x1_t lo = vcreate_u64(vgetq_lane_u64(tmp, 0)); + return (poly64_t) (lo); +} +#endif /* !__clang__ && __GNUC__ && __GNUC__ == 5*/ + static inline uint8x16_t pmull_low(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128( From 1ac7f6b09de944c6d005d7f5b8a3346163a5524c Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 7 Mar 2023 15:44:59 +0800 Subject: [PATCH 4/8] Improve readability Signed-off-by: Jerry Yu --- library/aesce.c | 87 +++++++++++++++++++++++++++++++++++++------------ library/gcm.c | 2 ++ 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/library/aesce.c b/library/aesce.c index 7c5bcca456..22963c4f4b 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -279,7 +279,7 @@ int mbedtls_aesce_setkey_enc(unsigned char *rk, #if defined(MBEDTLS_GCM_C) #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 5 -/* GCC 5.X miss some intrinsics, we add them here. */ +/* Some intrinsics are not available for GCC 5.X. */ #define vreinterpretq_p64_u8(a) ((poly64x2_t) a) #define vreinterpretq_u8_p128(a) ((uint8x16_t) a) static inline poly64_t vget_low_p64(poly64x2_t __a) @@ -290,6 +290,11 @@ static inline poly64_t vget_low_p64(poly64x2_t __a) } #endif /* !__clang__ && __GNUC__ && __GNUC__ == 5*/ +/* vmull_p64/vmull_high_p64 wrappers. + * + * Older compilers miss some intrinsic functions for `poly*_t`. We use + * uint8x16_t and uint8x16x3_t as input/output parameters. + */ static inline uint8x16_t pmull_low(uint8x16_t a, uint8x16_t b) { return vreinterpretq_u8_p128( @@ -305,35 +310,75 @@ static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b) vreinterpretq_p64_u8(b))); } +/* GHASH do 128b karatsuba polynomial multiplication on block on GF(2^128) + * defined by `x^128 + x^7 + x^2 + x + 1`. + * + * Arm64 only has 64b->128b polynomial multipliers, we need to do 4 64b + * multiplies to generate a 128b. + * + * `poly_mult_128` executes polynomial multiplication and outputs 256b that + * represented by 3 128b due to code size optimization. + * + * Output layout: + * | | | | + * |------------|-------------|-------------| + * | ret.val[0] | h3:h2:00:00 | high 128b | + * | ret.val[1] | :m2:m1:00 | median 128b | + * | ret.val[2] | : :l1:l0 | low 128b | + */ static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b) { uint8x16x3_t ret; - uint8x16_t c = vextq_u8(b, b, 8); - ret.val[0] = pmull_high(a, b); /* a1*b1 */ - ret.val[1] = veorq_u8(pmull_high(a, c), /* a1*b0 + a0*b1 */ - pmull_low(a, c)); - ret.val[2] = pmull_low(a, b); /* a0*b0 */ + uint8x16_t h, m, l; /* retval high/median/low */ + uint8x16_t c, d, e; + + h = pmull_high(a, b); /* h3:h2:00:00 = a1*b1 */ + l = pmull_low(a, b); /* : :l1:l0 = a0*b0 */ + c = vextq_u8(b, b, 8); /* :c1:c0 = b0:b1 */ + d = pmull_high(a, c); /* :d2:d1:00 = a1*b0 */ + e = pmull_low(a, c); /* :e2:e1:00 = a0*b1 */ + m = veorq_u8(d, e); /* :m2:m1:00 = d + e */ + + ret.val[0] = h; + ret.val[1] = m; + ret.val[2] = l; return ret; } -static inline uint8x16_t poly_mult_reduce(uint8x16x3_t a) +/* + * Modulo reduction. + * + * See: https://www.researchgate.net/publication/285612706_Implementing_GCM_on_ARMv8 + * + * Section 4.3 + * + * Modular reduction is slightly more complex. Write the GCM modulus as f(z) = + * z^128 +r(z), where r(z) = z^7+z^2+z+ 1. The well known approach is to + * consider that z128 ≡r(z) (mod z128 +r(z)), allowing us to write the 256-bit + * operand to be reduced as a(z) = h(z)z128 +`(z)≡h(z)r(z) + `(z). That is, we + * simply multiply the higher part of the operand by r(z) and add it to `(z). If + * the result is still larger than 128 bits, we reduce again. + */ +static inline uint8x16_t poly_mult_reduce(uint8x16x3_t input) { - uint8x16_t const Z = vdupq_n_u8(0); - /* use 'asm' as an optimisation barrier to prevent loading R from memory */ + uint8x16_t const ZERO = vdupq_n_u8(0); + /* use 'asm' as an optimisation barrier to prevent loading MODULO from memory */ uint64x2_t r = vreinterpretq_u64_u8(vdupq_n_u8(0x87)); asm ("" : "+w" (r)); - uint8x16_t const R = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8)); - uint8x16_t d = a.val[0]; /* d3:d2:00:00 */ - uint8x16_t j = a.val[1]; /* j2:j1:00 */ - uint8x16_t g = a.val[2]; /* g1:g0 = a0*b0 */ - uint8x16_t h = pmull_high(d, R); /* h2:h1:00 = reduction of d3 */ - uint8x16_t i = pmull_low(d, R); /* i1:i0 = reduction of d2 */ - uint8x16_t k = veorq_u8(j, h); /* k2:k1:00 = j2:j1 + h2:h1 */ - uint8x16_t l = pmull_high(k, R); /* l1:l0 = reduction of k2 */ - uint8x16_t m = vextq_u8(Z, k, 8); /* m1:00 = k1:00 */ - uint8x16_t n = veorq_u8(g, i); /* n1:n0 = g1:g0 + i1:i0 */ - uint8x16_t o = veorq_u8(n, l); /* o1:o0 = l1:l0 + n1:n0 */ - return veorq_u8(o, m); /* = o1:o0 + m1:00 */ + uint8x16_t const MODULO = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8)); + uint8x16_t h, m, l; /* input high/median/low 128b */ + uint8x16_t c, d, e, f, g, n, o; + h = input.val[0]; /* h3:h2:00:00 */ + m = input.val[1]; /* :m2:m1:00 */ + l = input.val[2]; /* : :l1:l0 */ + c = pmull_high(h, MODULO); /* :c2:c1:00 = reduction of h3 */ + d = pmull_low(h, MODULO); /* : :d1:d0 = reduction of h2 */ + e = veorq_u8(c, m); /* :e2:e1:00 = m2:m1:00 + c2:c1:00 */ + f = pmull_high(e, MODULO); /* : :f1:f0 = reduction of e2 */ + g = vextq_u8(ZERO, e, 8); /* : :g1:00 = e1:00 */ + n = veorq_u8(d, l); /* : :n1:n0 = d1:d0 + l1:l0 */ + o = veorq_u8(n, f); /* o1:o0 = f1:f0 + n1:n0 */ + return veorq_u8(o, g); /* = o1:o0 + g1:00 */ } /* diff --git a/library/gcm.c b/library/gcm.c index 0fa0008e38..14886bd2e9 100644 --- a/library/gcm.c +++ b/library/gcm.c @@ -197,6 +197,7 @@ static void gcm_mult(mbedtls_gcm_context *ctx, const unsigned char x[16], if (mbedtls_aesni_has_support(MBEDTLS_AESNI_CLMUL)) { unsigned char h[16]; + /* mbedtls_aesni_gcm_mult needs big-endian input */ MBEDTLS_PUT_UINT32_BE(ctx->HH[8] >> 32, h, 0); MBEDTLS_PUT_UINT32_BE(ctx->HH[8], h, 4); MBEDTLS_PUT_UINT32_BE(ctx->HL[8] >> 32, h, 8); @@ -211,6 +212,7 @@ static void gcm_mult(mbedtls_gcm_context *ctx, const unsigned char x[16], if (mbedtls_aesce_has_support()) { unsigned char h[16]; + /* mbedtls_aesce_gcm_mult needs big-endian input */ MBEDTLS_PUT_UINT32_BE(ctx->HH[8] >> 32, h, 0); MBEDTLS_PUT_UINT32_BE(ctx->HH[8], h, 4); MBEDTLS_PUT_UINT32_BE(ctx->HL[8] >> 32, h, 8); From 49b4367eeed7ae7b285d0db2a5d11cca692985c5 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Mon, 13 Mar 2023 10:09:34 +0800 Subject: [PATCH 5/8] fix comment issue The algorithm is not karatsuba multiplication. Signed-off-by: Jerry Yu --- library/aesce.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/aesce.c b/library/aesce.c index 22963c4f4b..5833a5dd17 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -310,8 +310,8 @@ static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b) vreinterpretq_p64_u8(b))); } -/* GHASH do 128b karatsuba polynomial multiplication on block on GF(2^128) - * defined by `x^128 + x^7 + x^2 + x + 1`. +/* GHASH do 128b polynomial multiplication on block on GF(2^128) defined by + * `x^128 + x^7 + x^2 + x + 1`. * * Arm64 only has 64b->128b polynomial multipliers, we need to do 4 64b * multiplies to generate a 128b. From f0526a9ad0adfff8fac56d925c47a9f875d7bec3 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 14 Mar 2023 15:00:29 +0800 Subject: [PATCH 6/8] fix grammar issue in comment Signed-off-by: Jerry Yu --- library/aesce.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/aesce.c b/library/aesce.c index 5833a5dd17..653e28b29b 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -310,7 +310,7 @@ static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b) vreinterpretq_p64_u8(b))); } -/* GHASH do 128b polynomial multiplication on block on GF(2^128) defined by +/* GHASH does 128b polynomial multiplication on block in GF(2^128) defined by * `x^128 + x^7 + x^2 + x + 1`. * * Arm64 only has 64b->128b polynomial multipliers, we need to do 4 64b From 8f810605172910185fb4ef7926d7384d74b9c5ac Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 14 Mar 2023 17:28:52 +0800 Subject: [PATCH 7/8] Replace `median` with `middle Signed-off-by: Jerry Yu --- library/aesce.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/aesce.c b/library/aesce.c index 653e28b29b..047809a049 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -323,13 +323,13 @@ static inline uint8x16_t pmull_high(uint8x16_t a, uint8x16_t b) * | | | | * |------------|-------------|-------------| * | ret.val[0] | h3:h2:00:00 | high 128b | - * | ret.val[1] | :m2:m1:00 | median 128b | + * | ret.val[1] | :m2:m1:00 | middle 128b | * | ret.val[2] | : :l1:l0 | low 128b | */ static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b) { uint8x16x3_t ret; - uint8x16_t h, m, l; /* retval high/median/low */ + uint8x16_t h, m, l; /* retval high/middle/low */ uint8x16_t c, d, e; h = pmull_high(a, b); /* h3:h2:00:00 = a1*b1 */ @@ -366,7 +366,7 @@ static inline uint8x16_t poly_mult_reduce(uint8x16x3_t input) uint64x2_t r = vreinterpretq_u64_u8(vdupq_n_u8(0x87)); asm ("" : "+w" (r)); uint8x16_t const MODULO = vreinterpretq_u8_u64(vshrq_n_u64(r, 64 - 8)); - uint8x16_t h, m, l; /* input high/median/low 128b */ + uint8x16_t h, m, l; /* input high/middle/low 128b */ uint8x16_t c, d, e, f, g, n, o; h = input.val[0]; /* h3:h2:00:00 */ m = input.val[1]; /* :m2:m1:00 */ From be4fdef5139786749467e6515653c9a382e66be9 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Wed, 15 Mar 2023 14:50:42 +0800 Subject: [PATCH 8/8] fix comment issues Signed-off-by: Jerry Yu --- library/aesce.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/aesce.c b/library/aesce.c index 047809a049..70caebab42 100644 --- a/library/aesce.c +++ b/library/aesce.c @@ -354,9 +354,9 @@ static inline uint8x16x3_t poly_mult_128(uint8x16_t a, uint8x16_t b) * * Modular reduction is slightly more complex. Write the GCM modulus as f(z) = * z^128 +r(z), where r(z) = z^7+z^2+z+ 1. The well known approach is to - * consider that z128 ≡r(z) (mod z128 +r(z)), allowing us to write the 256-bit - * operand to be reduced as a(z) = h(z)z128 +`(z)≡h(z)r(z) + `(z). That is, we - * simply multiply the higher part of the operand by r(z) and add it to `(z). If + * consider that z^128 ≡r(z) (mod z^128 +r(z)), allowing us to write the 256-bit + * operand to be reduced as a(z) = h(z)z^128 +l(z)≡h(z)r(z) + l(z). That is, we + * simply multiply the higher part of the operand by r(z) and add it to l(z). If * the result is still larger than 128 bits, we reduce again. */ static inline uint8x16_t poly_mult_reduce(uint8x16x3_t input)