From 693d2a0085f4c1d289cb6d79c02ddd23f18d9478 Mon Sep 17 00:00:00 2001 From: Thomas Pornin Date: Sun, 15 Jan 2017 16:49:58 +0100 Subject: [PATCH] New implementation of Curve25519 (using multiplications of words of 15 bits or so, should be much faster on Cortex M0). --- Makefile | 5 +- inc/bearssl_ec.h | 16 +- src/ec/ec_c25519_i15.c | 52 ++ src/ec/ec_c25519_m15.c | 1451 ++++++++++++++++++++++++++++++++++++++++ test/test_crypto.c | 49 +- test/test_speed.c | 8 + 6 files changed, 1574 insertions(+), 7 deletions(-) create mode 100644 src/ec/ec_c25519_m15.c diff --git a/Makefile b/Makefile index ac5f241..98fefd5 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ TESTX509 = testx509 TESTMATH = testmath OBJCODEC = $(BUILD)/ccopy.o $(BUILD)/dec16be.o $(BUILD)/dec16le.o $(BUILD)/dec32be.o $(BUILD)/dec32le.o $(BUILD)/dec64be.o $(BUILD)/dec64le.o $(BUILD)/enc16be.o $(BUILD)/enc16le.o $(BUILD)/enc32be.o $(BUILD)/enc32le.o $(BUILD)/enc64be.o $(BUILD)/enc64le.o $(BUILD)/pemdec.o -OBJEC = $(BUILD)/ec_c25519_i15.o $(BUILD)/ec_curve25519.o $(BUILD)/ec_p256_m15.o $(BUILD)/ec_prime_i15.o $(BUILD)/ec_prime_i31.o $(BUILD)/ec_secp256r1.o $(BUILD)/ec_secp384r1.o $(BUILD)/ec_secp521r1.o $(BUILD)/ecdsa_atr.o $(BUILD)/ecdsa_i15_bits.o $(BUILD)/ecdsa_i15_sign_asn1.o $(BUILD)/ecdsa_i15_sign_raw.o $(BUILD)/ecdsa_i15_vrfy_asn1.o $(BUILD)/ecdsa_i15_vrfy_raw.o $(BUILD)/ecdsa_i31_bits.o $(BUILD)/ecdsa_i31_sign_asn1.o $(BUILD)/ecdsa_i31_sign_raw.o $(BUILD)/ecdsa_i31_vrfy_asn1.o $(BUILD)/ecdsa_i31_vrfy_raw.o $(BUILD)/ecdsa_rta.o +OBJEC = $(BUILD)/ec_c25519_i15.o $(BUILD)/ec_c25519_m15.o $(BUILD)/ec_curve25519.o $(BUILD)/ec_p256_m15.o $(BUILD)/ec_prime_i15.o $(BUILD)/ec_prime_i31.o $(BUILD)/ec_secp256r1.o $(BUILD)/ec_secp384r1.o $(BUILD)/ec_secp521r1.o $(BUILD)/ecdsa_atr.o $(BUILD)/ecdsa_i15_bits.o $(BUILD)/ecdsa_i15_sign_asn1.o $(BUILD)/ecdsa_i15_sign_raw.o $(BUILD)/ecdsa_i15_vrfy_asn1.o $(BUILD)/ecdsa_i15_vrfy_raw.o $(BUILD)/ecdsa_i31_bits.o $(BUILD)/ecdsa_i31_sign_asn1.o $(BUILD)/ecdsa_i31_sign_raw.o $(BUILD)/ecdsa_i31_vrfy_asn1.o $(BUILD)/ecdsa_i31_vrfy_raw.o $(BUILD)/ecdsa_rta.o # $(BUILD)/ec_prime_i31_secp256r1.o $(BUILD)/ec_prime_i31_secp384r1.o $(BUILD)/ec_prime_i31_secp521r1.o OBJHASH = $(BUILD)/dig_oid.o $(BUILD)/dig_size.o $(BUILD)/ghash_ctmul.o $(BUILD)/ghash_ctmul32.o $(BUILD)/ghash_ctmul64.o $(BUILD)/md5.o $(BUILD)/md5sha1.o $(BUILD)/multihash.o $(BUILD)/sha1.o $(BUILD)/sha2big.o $(BUILD)/sha2small.o OBJINT15 = $(BUILD)/i15_core.o $(BUILD)/i15_ext1.o $(BUILD)/i15_ext2.o @@ -165,6 +165,9 @@ $(BUILD)/ec_g_secp521r1.o: src/ec/ec_g_secp521r1.c $(HEADERS) $(BUILD)/ec_c25519_i15.o: src/ec/ec_c25519_i15.c $(HEADERS) $(CC) $(CFLAGS) -c -o $(BUILD)/ec_c25519_i15.o src/ec/ec_c25519_i15.c +$(BUILD)/ec_c25519_m15.o: src/ec/ec_c25519_m15.c $(HEADERS) + $(CC) $(CFLAGS) -c -o $(BUILD)/ec_c25519_m15.o src/ec/ec_c25519_m15.c + $(BUILD)/ec_curve25519.o: src/ec/ec_curve25519.c $(HEADERS) $(CC) $(CFLAGS) -c -o $(BUILD)/ec_curve25519.o src/ec/ec_curve25519.c diff --git a/inc/bearssl_ec.h b/inc/bearssl_ec.h index 2c84bad..1ecb4cd 100644 --- a/inc/bearssl_ec.h +++ b/inc/bearssl_ec.h @@ -410,12 +410,12 @@ extern const br_ec_impl br_ec_prime_i31; extern const br_ec_impl br_ec_prime_i15; /** - * \brief EC implementation "i15" for P-256. + * \brief EC implementation "m15" for P-256. * * This implementation uses specialised code for curve secp256r1 (also - * known as NIST P-256), with Karatsuba decomposition, and fast modular - * reduction thanks to the field modulus special format. Only 32-bit - * multiplications are used (with 32-bit results, not 64-bit). + * known as NIST P-256), with optional Karatsuba decomposition, and fast + * modular reduction thanks to the field modulus special format. Only + * 32-bit multiplications are used (with 32-bit results, not 64-bit). */ extern const br_ec_impl br_ec_p256_m15; @@ -428,6 +428,14 @@ extern const br_ec_impl br_ec_p256_m15; */ extern const br_ec_impl br_ec_c25519_i15; +/** + * \brief EC implementation "m15" (specialised code) for Curve25519. + * + * This implementation uses custom code relying on multiplication of + * integers up to 15 bits. The `muladd()` method is not implemented. + */ +extern const br_ec_impl br_ec_c25519_m15; + /** * \brief Convert a signature from "raw" to "asn1". * diff --git a/src/ec/ec_c25519_i15.c b/src/ec/ec_c25519_i15.c index 79560ae..6e4c4f8 100644 --- a/src/ec/ec_c25519_i15.c +++ b/src/ec/ec_c25519_i15.c @@ -46,6 +46,27 @@ static const uint16_t C255_R2[] = { 0x0000 }; +/* obsolete +#include +#include +static void +print_int_mont(const char *name, const uint16_t *x) +{ + uint16_t y[18]; + unsigned char tmp[32]; + size_t u; + + printf("%s = ", name); + memcpy(y, x, sizeof y); + br_i15_from_monty(y, C255_P, P0I); + br_i15_encode(tmp, sizeof tmp, y); + for (u = 0; u < sizeof tmp; u ++) { + printf("%02X", tmp[u]); + } + printf("\n"); +} +*/ + static const uint16_t C255_A24[] = { 0x0110, 0x45D3, 0x0046, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, @@ -194,6 +215,10 @@ api_mul(unsigned char *G, size_t Glen, k[31] &= 0x7F; k[31] |= 0x40; + /* obsolete + print_int_mont("x1", x1); + */ + swap = 0; for (i = 254; i >= 0; i --) { uint32_t kt; @@ -204,6 +229,13 @@ api_mul(unsigned char *G, size_t Glen, cswap(z2, z3, swap); swap = kt; + /* obsolete + print_int_mont("x2", x2); + print_int_mont("z2", z2); + print_int_mont("x3", x3); + print_int_mont("z3", z3); + */ + c255_add(a, x2, z2); c255_mul(aa, a, a); c255_sub(b, x2, z2); @@ -213,6 +245,19 @@ api_mul(unsigned char *G, size_t Glen, c255_sub(d, x3, z3); c255_mul(da, d, a); c255_mul(cb, c, b); + + /* obsolete + print_int_mont("a ", a); + print_int_mont("aa", aa); + print_int_mont("b ", b); + print_int_mont("bb", bb); + print_int_mont("e ", e); + print_int_mont("c ", c); + print_int_mont("d ", d); + print_int_mont("da", da); + print_int_mont("cb", cb); + */ + c255_add(x3, da, cb); c255_mul(x3, x3, x3); c255_sub(z3, da, cb); @@ -222,6 +267,13 @@ api_mul(unsigned char *G, size_t Glen, c255_mul(z2, C255_A24, e); c255_add(z2, z2, aa); c255_mul(z2, e, z2); + + /* obsolete + print_int_mont("x2", x2); + print_int_mont("z2", z2); + print_int_mont("x3", x3); + print_int_mont("z3", z3); + */ } cswap(x2, x3, swap); cswap(z2, z3, swap); diff --git a/src/ec/ec_c25519_m15.c b/src/ec/ec_c25519_m15.c new file mode 100644 index 0000000..5079d5e --- /dev/null +++ b/src/ec/ec_c25519_m15.c @@ -0,0 +1,1451 @@ +/* + * Copyright (c) 2017 Thomas Pornin + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "inner.h" + +/* obsolete +#include +#include +static void +print_int(const char *name, const uint32_t *x) +{ + size_t u; + unsigned char tmp[36]; + + printf("%s = ", name); + for (u = 0; u < 20; u ++) { + if (x[u] > 0x1FFF) { + printf("INVALID:"); + for (u = 0; u < 20; u ++) { + printf(" %04X", x[u]); + } + printf("\n"); + return; + } + } + memset(tmp, 0, sizeof tmp); + for (u = 0; u < 20; u ++) { + uint32_t w; + int j, k; + + w = x[u]; + j = 13 * (int)u; + k = j & 7; + if (k != 0) { + w <<= k; + j -= k; + } + k = j >> 3; + tmp[35 - k] |= (unsigned char)w; + tmp[34 - k] |= (unsigned char)(w >> 8); + tmp[33 - k] |= (unsigned char)(w >> 16); + tmp[32 - k] |= (unsigned char)(w >> 24); + } + for (u = 4; u < 36; u ++) { + printf("%02X", tmp[u]); + } + printf("\n"); +} +*/ + +/* + * If BR_NO_ARITH_SHIFT is undefined, or defined to 0, then we _assume_ + * that right-shifting a signed negative integer copies the sign bit + * (arithmetic right-shift). This is "implementation-defined behaviour", + * i.e. it is not undefined, but it may differ between compilers. Each + * compiler is supposed to document its behaviour in that respect. GCC + * explicitly defines that an arithmetic right shift is used. We expect + * all other compilers to do the same, because underlying CPU offer an + * arithmetic right shift opcode that could not be used otherwise. + */ +#if BR_NO_ARITH_SHIFT +#define ARSH(x, n) (((uint32_t)(x) >> (n)) \ + | ((-((uint32_t)(x) >> 31)) << (32 - (n)))) +#else +#define ARSH(x, n) ((*(int32_t *)&(x)) >> (n)) +#endif + +/* + * Convert an integer from unsigned little-endian encoding to a sequence of + * 13-bit words in little-endian order. The final "partial" word is + * returned. + */ +static uint32_t +le8_to_le13(uint32_t *dst, const unsigned char *src, size_t len) +{ + uint32_t acc; + int acc_len; + + acc = 0; + acc_len = 0; + while (len -- > 0) { + acc |= (uint32_t)(*src ++) << acc_len; + acc_len += 8; + if (acc_len >= 13) { + *dst ++ = acc & 0x1FFF; + acc >>= 13; + acc_len -= 13; + } + } + return acc; +} + +/* + * Convert an integer (13-bit words, little-endian) to unsigned + * little-endian encoding. The total encoding length is provided; all + * the destination bytes will be filled. + */ +static void +le13_to_le8(unsigned char *dst, size_t len, const uint32_t *src) +{ + uint32_t acc; + int acc_len; + + acc = 0; + acc_len = 0; + while (len -- > 0) { + if (acc_len < 8) { + acc |= (*src ++) << acc_len; + acc_len += 13; + } + *dst ++ = (unsigned char)acc; + acc >>= 8; + acc_len -= 8; + } +} + +/* + * Normalise an array of words to a strict 13 bits per word. Returned + * value is the resulting carry. The source (w) and destination (d) + * arrays may be identical, but shall not overlap partially. + */ +static inline uint32_t +norm13(uint32_t *d, const uint32_t *w, size_t len) +{ + size_t u; + uint32_t cc; + + cc = 0; + for (u = 0; u < len; u ++) { + int32_t z; + + z = w[u] + cc; + d[u] = z & 0x1FFF; + cc = ARSH(z, 13); + } + return cc; +} + +/* + * mul20() multiplies two 260-bit integers together. Each word must fit + * on 13 bits; source operands use 20 words, destination operand + * receives 40 words. All overlaps allowed. + * + * square20() computes the square of a 260-bit integer. Each word must + * fit on 13 bits; source operand uses 20 words, destination operand + * receives 40 words. All overlaps allowed. + */ + +#if BR_SLOW_MUL15 + +static void +mul20(uint32_t *d, const uint32_t *a, const uint32_t *b) +{ + /* + * Two-level Karatsuba: turns a 20x20 multiplication into + * nine 5x5 multiplications. We use 13-bit words but do not + * propagate carries immediately, so words may expand: + * + * - First Karatsuba decomposition turns the 20x20 mul on + * 13-bit words into three 10x10 muls, two on 13-bit words + * and one on 14-bit words. + * + * - Second Karatsuba decomposition further splits these into: + * + * * four 5x5 muls on 13-bit words + * * four 5x5 muls on 14-bit words + * * one 5x5 mul on 15-bit words + * + * Highest word value is 8191, 16382 or 32764, for 13-bit, 14-bit + * or 15-bit words, respectively. + */ + uint32_t u[45], v[45], w[90]; + uint32_t cc; + int i; + +#define ZADD(dw, d_off, s1w, s1_off, s2w, s2_off) do { \ + (dw)[5 * (d_off) + 0] = (s1w)[5 * (s1_off) + 0] \ + + (s2w)[5 * (s2_off) + 0]; \ + (dw)[5 * (d_off) + 1] = (s1w)[5 * (s1_off) + 1] \ + + (s2w)[5 * (s2_off) + 1]; \ + (dw)[5 * (d_off) + 2] = (s1w)[5 * (s1_off) + 2] \ + + (s2w)[5 * (s2_off) + 2]; \ + (dw)[5 * (d_off) + 3] = (s1w)[5 * (s1_off) + 3] \ + + (s2w)[5 * (s2_off) + 3]; \ + (dw)[5 * (d_off) + 4] = (s1w)[5 * (s1_off) + 4] \ + + (s2w)[5 * (s2_off) + 4]; \ + } while (0) + +#define ZADDT(dw, d_off, sw, s_off) do { \ + (dw)[5 * (d_off) + 0] += (sw)[5 * (s_off) + 0]; \ + (dw)[5 * (d_off) + 1] += (sw)[5 * (s_off) + 1]; \ + (dw)[5 * (d_off) + 2] += (sw)[5 * (s_off) + 2]; \ + (dw)[5 * (d_off) + 3] += (sw)[5 * (s_off) + 3]; \ + (dw)[5 * (d_off) + 4] += (sw)[5 * (s_off) + 4]; \ + } while (0) + +#define ZSUB2F(dw, d_off, s1w, s1_off, s2w, s2_off) do { \ + (dw)[5 * (d_off) + 0] -= (s1w)[5 * (s1_off) + 0] \ + + (s2w)[5 * (s2_off) + 0]; \ + (dw)[5 * (d_off) + 1] -= (s1w)[5 * (s1_off) + 1] \ + + (s2w)[5 * (s2_off) + 1]; \ + (dw)[5 * (d_off) + 2] -= (s1w)[5 * (s1_off) + 2] \ + + (s2w)[5 * (s2_off) + 2]; \ + (dw)[5 * (d_off) + 3] -= (s1w)[5 * (s1_off) + 3] \ + + (s2w)[5 * (s2_off) + 3]; \ + (dw)[5 * (d_off) + 4] -= (s1w)[5 * (s1_off) + 4] \ + + (s2w)[5 * (s2_off) + 4]; \ + } while (0) + +#define CPR1(w, cprcc) do { \ + uint32_t cprz = (w) + cprcc; \ + (w) = cprz & 0x1FFF; \ + cprcc = cprz >> 13; \ + } while (0) + +#define CPR(dw, d_off) do { \ + uint32_t cprcc; \ + cprcc = 0; \ + CPR1((dw)[(d_off) + 0], cprcc); \ + CPR1((dw)[(d_off) + 1], cprcc); \ + CPR1((dw)[(d_off) + 2], cprcc); \ + CPR1((dw)[(d_off) + 3], cprcc); \ + CPR1((dw)[(d_off) + 4], cprcc); \ + CPR1((dw)[(d_off) + 5], cprcc); \ + CPR1((dw)[(d_off) + 6], cprcc); \ + CPR1((dw)[(d_off) + 7], cprcc); \ + CPR1((dw)[(d_off) + 8], cprcc); \ + (dw)[(d_off) + 9] = cprcc; \ + } while (0) + + memcpy(u, a, 20 * sizeof *a); + ZADD(u, 4, a, 0, a, 1); + ZADD(u, 5, a, 2, a, 3); + ZADD(u, 6, a, 0, a, 2); + ZADD(u, 7, a, 1, a, 3); + ZADD(u, 8, u, 6, u, 7); + + memcpy(v, b, 20 * sizeof *b); + ZADD(v, 4, b, 0, b, 1); + ZADD(v, 5, b, 2, b, 3); + ZADD(v, 6, b, 0, b, 2); + ZADD(v, 7, b, 1, b, 3); + ZADD(v, 8, v, 6, v, 7); + + /* + * Do the eight first 8x8 muls. Source words are at most 16382 + * each, so we can add product results together "as is" in 32-bit + * words. + */ + for (i = 0; i < 40; i += 5) { + w[(i << 1) + 0] = MUL15(u[i + 0], v[i + 0]); + w[(i << 1) + 1] = MUL15(u[i + 0], v[i + 1]) + + MUL15(u[i + 1], v[i + 0]); + w[(i << 1) + 2] = MUL15(u[i + 0], v[i + 2]) + + MUL15(u[i + 1], v[i + 1]) + + MUL15(u[i + 2], v[i + 0]); + w[(i << 1) + 3] = MUL15(u[i + 0], v[i + 3]) + + MUL15(u[i + 1], v[i + 2]) + + MUL15(u[i + 2], v[i + 1]) + + MUL15(u[i + 3], v[i + 0]); + w[(i << 1) + 4] = MUL15(u[i + 0], v[i + 4]) + + MUL15(u[i + 1], v[i + 3]) + + MUL15(u[i + 2], v[i + 2]) + + MUL15(u[i + 3], v[i + 1]) + + MUL15(u[i + 4], v[i + 0]); + w[(i << 1) + 5] = MUL15(u[i + 1], v[i + 4]) + + MUL15(u[i + 2], v[i + 3]) + + MUL15(u[i + 3], v[i + 2]) + + MUL15(u[i + 4], v[i + 1]); + w[(i << 1) + 6] = MUL15(u[i + 2], v[i + 4]) + + MUL15(u[i + 3], v[i + 3]) + + MUL15(u[i + 4], v[i + 2]); + w[(i << 1) + 7] = MUL15(u[i + 3], v[i + 4]) + + MUL15(u[i + 4], v[i + 3]); + w[(i << 1) + 8] = MUL15(u[i + 4], v[i + 4]); + w[(i << 1) + 9] = 0; + } + + /* + * For the 9th multiplication, source words are up to 32764, + * so we must do some carry propagation. If we add up to + * 4 products and the carry is no more than 524224, then the + * result fits in 32 bits, and the next carry will be no more + * than 524224 (because 4*(32764^2)+524224 < 8192*524225). + * + * We thus just skip one of the products in the middle word, + * then do a carry propagation (this reduces words to 13 bits + * each, except possibly the last, which may use up to 17 bits + * or so), then add the missing product. + */ + w[80 + 0] = MUL15(u[40 + 0], v[40 + 0]); + w[80 + 1] = MUL15(u[40 + 0], v[40 + 1]) + + MUL15(u[40 + 1], v[40 + 0]); + w[80 + 2] = MUL15(u[40 + 0], v[40 + 2]) + + MUL15(u[40 + 1], v[40 + 1]) + + MUL15(u[40 + 2], v[40 + 0]); + w[80 + 3] = MUL15(u[40 + 0], v[40 + 3]) + + MUL15(u[40 + 1], v[40 + 2]) + + MUL15(u[40 + 2], v[40 + 1]) + + MUL15(u[40 + 3], v[40 + 0]); + w[80 + 4] = MUL15(u[40 + 0], v[40 + 4]) + + MUL15(u[40 + 1], v[40 + 3]) + + MUL15(u[40 + 2], v[40 + 2]) + + MUL15(u[40 + 3], v[40 + 1]); + /* + MUL15(u[40 + 4], v[40 + 0]) */ + w[80 + 5] = MUL15(u[40 + 1], v[40 + 4]) + + MUL15(u[40 + 2], v[40 + 3]) + + MUL15(u[40 + 3], v[40 + 2]) + + MUL15(u[40 + 4], v[40 + 1]); + w[80 + 6] = MUL15(u[40 + 2], v[40 + 4]) + + MUL15(u[40 + 3], v[40 + 3]) + + MUL15(u[40 + 4], v[40 + 2]); + w[80 + 7] = MUL15(u[40 + 3], v[40 + 4]) + + MUL15(u[40 + 4], v[40 + 3]); + w[80 + 8] = MUL15(u[40 + 4], v[40 + 4]); + + CPR(w, 80); + + w[80 + 4] += MUL15(u[40 + 4], v[40 + 0]); + + /* + * The products on 14-bit words in slots 6 and 7 yield values + * up to 5*(16382^2) each, and we need to subtract two such + * values from the higher word. We need the subtraction to fit + * in a _signed_ 32-bit integer, i.e. 31 bits + a sign bit. + * However, 10*(16382^2) does not fit. So we must perform a + * bit of reduction here. + */ + CPR(w, 60); + CPR(w, 70); + + /* + * Recompose results. + */ + + /* 0..1*0..1 into 0..3 */ + ZSUB2F(w, 8, w, 0, w, 2); + ZSUB2F(w, 9, w, 1, w, 3); + ZADDT(w, 1, w, 8); + ZADDT(w, 2, w, 9); + + /* 2..3*2..3 into 4..7 */ + ZSUB2F(w, 10, w, 4, w, 6); + ZSUB2F(w, 11, w, 5, w, 7); + ZADDT(w, 5, w, 10); + ZADDT(w, 6, w, 11); + + /* (0..1+2..3)*(0..1+2..3) into 12..15 */ + ZSUB2F(w, 16, w, 12, w, 14); + ZSUB2F(w, 17, w, 13, w, 15); + ZADDT(w, 13, w, 16); + ZADDT(w, 14, w, 17); + + /* first-level recomposition */ + ZSUB2F(w, 12, w, 0, w, 4); + ZSUB2F(w, 13, w, 1, w, 5); + ZSUB2F(w, 14, w, 2, w, 6); + ZSUB2F(w, 15, w, 3, w, 7); + ZADDT(w, 2, w, 12); + ZADDT(w, 3, w, 13); + ZADDT(w, 4, w, 14); + ZADDT(w, 5, w, 15); + + /* + * Perform carry propagation to bring all words down to 13 bits. + */ + cc = norm13(d, w, 40); + d[39] += (cc << 13); + +#undef ZADD +#undef ZADDT +#undef ZSUB2F +#undef CPR1 +#undef CPR +} + +static inline void +square20(uint32_t *d, const uint32_t *a) +{ + mul20(d, a, a); +} + +#else + +static void +mul20(uint32_t *d, const uint32_t *a, const uint32_t *b) +{ + uint32_t t[39]; + + t[ 0] = MUL15(a[ 0], b[ 0]); + t[ 1] = MUL15(a[ 0], b[ 1]) + + MUL15(a[ 1], b[ 0]); + t[ 2] = MUL15(a[ 0], b[ 2]) + + MUL15(a[ 1], b[ 1]) + + MUL15(a[ 2], b[ 0]); + t[ 3] = MUL15(a[ 0], b[ 3]) + + MUL15(a[ 1], b[ 2]) + + MUL15(a[ 2], b[ 1]) + + MUL15(a[ 3], b[ 0]); + t[ 4] = MUL15(a[ 0], b[ 4]) + + MUL15(a[ 1], b[ 3]) + + MUL15(a[ 2], b[ 2]) + + MUL15(a[ 3], b[ 1]) + + MUL15(a[ 4], b[ 0]); + t[ 5] = MUL15(a[ 0], b[ 5]) + + MUL15(a[ 1], b[ 4]) + + MUL15(a[ 2], b[ 3]) + + MUL15(a[ 3], b[ 2]) + + MUL15(a[ 4], b[ 1]) + + MUL15(a[ 5], b[ 0]); + t[ 6] = MUL15(a[ 0], b[ 6]) + + MUL15(a[ 1], b[ 5]) + + MUL15(a[ 2], b[ 4]) + + MUL15(a[ 3], b[ 3]) + + MUL15(a[ 4], b[ 2]) + + MUL15(a[ 5], b[ 1]) + + MUL15(a[ 6], b[ 0]); + t[ 7] = MUL15(a[ 0], b[ 7]) + + MUL15(a[ 1], b[ 6]) + + MUL15(a[ 2], b[ 5]) + + MUL15(a[ 3], b[ 4]) + + MUL15(a[ 4], b[ 3]) + + MUL15(a[ 5], b[ 2]) + + MUL15(a[ 6], b[ 1]) + + MUL15(a[ 7], b[ 0]); + t[ 8] = MUL15(a[ 0], b[ 8]) + + MUL15(a[ 1], b[ 7]) + + MUL15(a[ 2], b[ 6]) + + MUL15(a[ 3], b[ 5]) + + MUL15(a[ 4], b[ 4]) + + MUL15(a[ 5], b[ 3]) + + MUL15(a[ 6], b[ 2]) + + MUL15(a[ 7], b[ 1]) + + MUL15(a[ 8], b[ 0]); + t[ 9] = MUL15(a[ 0], b[ 9]) + + MUL15(a[ 1], b[ 8]) + + MUL15(a[ 2], b[ 7]) + + MUL15(a[ 3], b[ 6]) + + MUL15(a[ 4], b[ 5]) + + MUL15(a[ 5], b[ 4]) + + MUL15(a[ 6], b[ 3]) + + MUL15(a[ 7], b[ 2]) + + MUL15(a[ 8], b[ 1]) + + MUL15(a[ 9], b[ 0]); + t[10] = MUL15(a[ 0], b[10]) + + MUL15(a[ 1], b[ 9]) + + MUL15(a[ 2], b[ 8]) + + MUL15(a[ 3], b[ 7]) + + MUL15(a[ 4], b[ 6]) + + MUL15(a[ 5], b[ 5]) + + MUL15(a[ 6], b[ 4]) + + MUL15(a[ 7], b[ 3]) + + MUL15(a[ 8], b[ 2]) + + MUL15(a[ 9], b[ 1]) + + MUL15(a[10], b[ 0]); + t[11] = MUL15(a[ 0], b[11]) + + MUL15(a[ 1], b[10]) + + MUL15(a[ 2], b[ 9]) + + MUL15(a[ 3], b[ 8]) + + MUL15(a[ 4], b[ 7]) + + MUL15(a[ 5], b[ 6]) + + MUL15(a[ 6], b[ 5]) + + MUL15(a[ 7], b[ 4]) + + MUL15(a[ 8], b[ 3]) + + MUL15(a[ 9], b[ 2]) + + MUL15(a[10], b[ 1]) + + MUL15(a[11], b[ 0]); + t[12] = MUL15(a[ 0], b[12]) + + MUL15(a[ 1], b[11]) + + MUL15(a[ 2], b[10]) + + MUL15(a[ 3], b[ 9]) + + MUL15(a[ 4], b[ 8]) + + MUL15(a[ 5], b[ 7]) + + MUL15(a[ 6], b[ 6]) + + MUL15(a[ 7], b[ 5]) + + MUL15(a[ 8], b[ 4]) + + MUL15(a[ 9], b[ 3]) + + MUL15(a[10], b[ 2]) + + MUL15(a[11], b[ 1]) + + MUL15(a[12], b[ 0]); + t[13] = MUL15(a[ 0], b[13]) + + MUL15(a[ 1], b[12]) + + MUL15(a[ 2], b[11]) + + MUL15(a[ 3], b[10]) + + MUL15(a[ 4], b[ 9]) + + MUL15(a[ 5], b[ 8]) + + MUL15(a[ 6], b[ 7]) + + MUL15(a[ 7], b[ 6]) + + MUL15(a[ 8], b[ 5]) + + MUL15(a[ 9], b[ 4]) + + MUL15(a[10], b[ 3]) + + MUL15(a[11], b[ 2]) + + MUL15(a[12], b[ 1]) + + MUL15(a[13], b[ 0]); + t[14] = MUL15(a[ 0], b[14]) + + MUL15(a[ 1], b[13]) + + MUL15(a[ 2], b[12]) + + MUL15(a[ 3], b[11]) + + MUL15(a[ 4], b[10]) + + MUL15(a[ 5], b[ 9]) + + MUL15(a[ 6], b[ 8]) + + MUL15(a[ 7], b[ 7]) + + MUL15(a[ 8], b[ 6]) + + MUL15(a[ 9], b[ 5]) + + MUL15(a[10], b[ 4]) + + MUL15(a[11], b[ 3]) + + MUL15(a[12], b[ 2]) + + MUL15(a[13], b[ 1]) + + MUL15(a[14], b[ 0]); + t[15] = MUL15(a[ 0], b[15]) + + MUL15(a[ 1], b[14]) + + MUL15(a[ 2], b[13]) + + MUL15(a[ 3], b[12]) + + MUL15(a[ 4], b[11]) + + MUL15(a[ 5], b[10]) + + MUL15(a[ 6], b[ 9]) + + MUL15(a[ 7], b[ 8]) + + MUL15(a[ 8], b[ 7]) + + MUL15(a[ 9], b[ 6]) + + MUL15(a[10], b[ 5]) + + MUL15(a[11], b[ 4]) + + MUL15(a[12], b[ 3]) + + MUL15(a[13], b[ 2]) + + MUL15(a[14], b[ 1]) + + MUL15(a[15], b[ 0]); + t[16] = MUL15(a[ 0], b[16]) + + MUL15(a[ 1], b[15]) + + MUL15(a[ 2], b[14]) + + MUL15(a[ 3], b[13]) + + MUL15(a[ 4], b[12]) + + MUL15(a[ 5], b[11]) + + MUL15(a[ 6], b[10]) + + MUL15(a[ 7], b[ 9]) + + MUL15(a[ 8], b[ 8]) + + MUL15(a[ 9], b[ 7]) + + MUL15(a[10], b[ 6]) + + MUL15(a[11], b[ 5]) + + MUL15(a[12], b[ 4]) + + MUL15(a[13], b[ 3]) + + MUL15(a[14], b[ 2]) + + MUL15(a[15], b[ 1]) + + MUL15(a[16], b[ 0]); + t[17] = MUL15(a[ 0], b[17]) + + MUL15(a[ 1], b[16]) + + MUL15(a[ 2], b[15]) + + MUL15(a[ 3], b[14]) + + MUL15(a[ 4], b[13]) + + MUL15(a[ 5], b[12]) + + MUL15(a[ 6], b[11]) + + MUL15(a[ 7], b[10]) + + MUL15(a[ 8], b[ 9]) + + MUL15(a[ 9], b[ 8]) + + MUL15(a[10], b[ 7]) + + MUL15(a[11], b[ 6]) + + MUL15(a[12], b[ 5]) + + MUL15(a[13], b[ 4]) + + MUL15(a[14], b[ 3]) + + MUL15(a[15], b[ 2]) + + MUL15(a[16], b[ 1]) + + MUL15(a[17], b[ 0]); + t[18] = MUL15(a[ 0], b[18]) + + MUL15(a[ 1], b[17]) + + MUL15(a[ 2], b[16]) + + MUL15(a[ 3], b[15]) + + MUL15(a[ 4], b[14]) + + MUL15(a[ 5], b[13]) + + MUL15(a[ 6], b[12]) + + MUL15(a[ 7], b[11]) + + MUL15(a[ 8], b[10]) + + MUL15(a[ 9], b[ 9]) + + MUL15(a[10], b[ 8]) + + MUL15(a[11], b[ 7]) + + MUL15(a[12], b[ 6]) + + MUL15(a[13], b[ 5]) + + MUL15(a[14], b[ 4]) + + MUL15(a[15], b[ 3]) + + MUL15(a[16], b[ 2]) + + MUL15(a[17], b[ 1]) + + MUL15(a[18], b[ 0]); + t[19] = MUL15(a[ 0], b[19]) + + MUL15(a[ 1], b[18]) + + MUL15(a[ 2], b[17]) + + MUL15(a[ 3], b[16]) + + MUL15(a[ 4], b[15]) + + MUL15(a[ 5], b[14]) + + MUL15(a[ 6], b[13]) + + MUL15(a[ 7], b[12]) + + MUL15(a[ 8], b[11]) + + MUL15(a[ 9], b[10]) + + MUL15(a[10], b[ 9]) + + MUL15(a[11], b[ 8]) + + MUL15(a[12], b[ 7]) + + MUL15(a[13], b[ 6]) + + MUL15(a[14], b[ 5]) + + MUL15(a[15], b[ 4]) + + MUL15(a[16], b[ 3]) + + MUL15(a[17], b[ 2]) + + MUL15(a[18], b[ 1]) + + MUL15(a[19], b[ 0]); + t[20] = MUL15(a[ 1], b[19]) + + MUL15(a[ 2], b[18]) + + MUL15(a[ 3], b[17]) + + MUL15(a[ 4], b[16]) + + MUL15(a[ 5], b[15]) + + MUL15(a[ 6], b[14]) + + MUL15(a[ 7], b[13]) + + MUL15(a[ 8], b[12]) + + MUL15(a[ 9], b[11]) + + MUL15(a[10], b[10]) + + MUL15(a[11], b[ 9]) + + MUL15(a[12], b[ 8]) + + MUL15(a[13], b[ 7]) + + MUL15(a[14], b[ 6]) + + MUL15(a[15], b[ 5]) + + MUL15(a[16], b[ 4]) + + MUL15(a[17], b[ 3]) + + MUL15(a[18], b[ 2]) + + MUL15(a[19], b[ 1]); + t[21] = MUL15(a[ 2], b[19]) + + MUL15(a[ 3], b[18]) + + MUL15(a[ 4], b[17]) + + MUL15(a[ 5], b[16]) + + MUL15(a[ 6], b[15]) + + MUL15(a[ 7], b[14]) + + MUL15(a[ 8], b[13]) + + MUL15(a[ 9], b[12]) + + MUL15(a[10], b[11]) + + MUL15(a[11], b[10]) + + MUL15(a[12], b[ 9]) + + MUL15(a[13], b[ 8]) + + MUL15(a[14], b[ 7]) + + MUL15(a[15], b[ 6]) + + MUL15(a[16], b[ 5]) + + MUL15(a[17], b[ 4]) + + MUL15(a[18], b[ 3]) + + MUL15(a[19], b[ 2]); + t[22] = MUL15(a[ 3], b[19]) + + MUL15(a[ 4], b[18]) + + MUL15(a[ 5], b[17]) + + MUL15(a[ 6], b[16]) + + MUL15(a[ 7], b[15]) + + MUL15(a[ 8], b[14]) + + MUL15(a[ 9], b[13]) + + MUL15(a[10], b[12]) + + MUL15(a[11], b[11]) + + MUL15(a[12], b[10]) + + MUL15(a[13], b[ 9]) + + MUL15(a[14], b[ 8]) + + MUL15(a[15], b[ 7]) + + MUL15(a[16], b[ 6]) + + MUL15(a[17], b[ 5]) + + MUL15(a[18], b[ 4]) + + MUL15(a[19], b[ 3]); + t[23] = MUL15(a[ 4], b[19]) + + MUL15(a[ 5], b[18]) + + MUL15(a[ 6], b[17]) + + MUL15(a[ 7], b[16]) + + MUL15(a[ 8], b[15]) + + MUL15(a[ 9], b[14]) + + MUL15(a[10], b[13]) + + MUL15(a[11], b[12]) + + MUL15(a[12], b[11]) + + MUL15(a[13], b[10]) + + MUL15(a[14], b[ 9]) + + MUL15(a[15], b[ 8]) + + MUL15(a[16], b[ 7]) + + MUL15(a[17], b[ 6]) + + MUL15(a[18], b[ 5]) + + MUL15(a[19], b[ 4]); + t[24] = MUL15(a[ 5], b[19]) + + MUL15(a[ 6], b[18]) + + MUL15(a[ 7], b[17]) + + MUL15(a[ 8], b[16]) + + MUL15(a[ 9], b[15]) + + MUL15(a[10], b[14]) + + MUL15(a[11], b[13]) + + MUL15(a[12], b[12]) + + MUL15(a[13], b[11]) + + MUL15(a[14], b[10]) + + MUL15(a[15], b[ 9]) + + MUL15(a[16], b[ 8]) + + MUL15(a[17], b[ 7]) + + MUL15(a[18], b[ 6]) + + MUL15(a[19], b[ 5]); + t[25] = MUL15(a[ 6], b[19]) + + MUL15(a[ 7], b[18]) + + MUL15(a[ 8], b[17]) + + MUL15(a[ 9], b[16]) + + MUL15(a[10], b[15]) + + MUL15(a[11], b[14]) + + MUL15(a[12], b[13]) + + MUL15(a[13], b[12]) + + MUL15(a[14], b[11]) + + MUL15(a[15], b[10]) + + MUL15(a[16], b[ 9]) + + MUL15(a[17], b[ 8]) + + MUL15(a[18], b[ 7]) + + MUL15(a[19], b[ 6]); + t[26] = MUL15(a[ 7], b[19]) + + MUL15(a[ 8], b[18]) + + MUL15(a[ 9], b[17]) + + MUL15(a[10], b[16]) + + MUL15(a[11], b[15]) + + MUL15(a[12], b[14]) + + MUL15(a[13], b[13]) + + MUL15(a[14], b[12]) + + MUL15(a[15], b[11]) + + MUL15(a[16], b[10]) + + MUL15(a[17], b[ 9]) + + MUL15(a[18], b[ 8]) + + MUL15(a[19], b[ 7]); + t[27] = MUL15(a[ 8], b[19]) + + MUL15(a[ 9], b[18]) + + MUL15(a[10], b[17]) + + MUL15(a[11], b[16]) + + MUL15(a[12], b[15]) + + MUL15(a[13], b[14]) + + MUL15(a[14], b[13]) + + MUL15(a[15], b[12]) + + MUL15(a[16], b[11]) + + MUL15(a[17], b[10]) + + MUL15(a[18], b[ 9]) + + MUL15(a[19], b[ 8]); + t[28] = MUL15(a[ 9], b[19]) + + MUL15(a[10], b[18]) + + MUL15(a[11], b[17]) + + MUL15(a[12], b[16]) + + MUL15(a[13], b[15]) + + MUL15(a[14], b[14]) + + MUL15(a[15], b[13]) + + MUL15(a[16], b[12]) + + MUL15(a[17], b[11]) + + MUL15(a[18], b[10]) + + MUL15(a[19], b[ 9]); + t[29] = MUL15(a[10], b[19]) + + MUL15(a[11], b[18]) + + MUL15(a[12], b[17]) + + MUL15(a[13], b[16]) + + MUL15(a[14], b[15]) + + MUL15(a[15], b[14]) + + MUL15(a[16], b[13]) + + MUL15(a[17], b[12]) + + MUL15(a[18], b[11]) + + MUL15(a[19], b[10]); + t[30] = MUL15(a[11], b[19]) + + MUL15(a[12], b[18]) + + MUL15(a[13], b[17]) + + MUL15(a[14], b[16]) + + MUL15(a[15], b[15]) + + MUL15(a[16], b[14]) + + MUL15(a[17], b[13]) + + MUL15(a[18], b[12]) + + MUL15(a[19], b[11]); + t[31] = MUL15(a[12], b[19]) + + MUL15(a[13], b[18]) + + MUL15(a[14], b[17]) + + MUL15(a[15], b[16]) + + MUL15(a[16], b[15]) + + MUL15(a[17], b[14]) + + MUL15(a[18], b[13]) + + MUL15(a[19], b[12]); + t[32] = MUL15(a[13], b[19]) + + MUL15(a[14], b[18]) + + MUL15(a[15], b[17]) + + MUL15(a[16], b[16]) + + MUL15(a[17], b[15]) + + MUL15(a[18], b[14]) + + MUL15(a[19], b[13]); + t[33] = MUL15(a[14], b[19]) + + MUL15(a[15], b[18]) + + MUL15(a[16], b[17]) + + MUL15(a[17], b[16]) + + MUL15(a[18], b[15]) + + MUL15(a[19], b[14]); + t[34] = MUL15(a[15], b[19]) + + MUL15(a[16], b[18]) + + MUL15(a[17], b[17]) + + MUL15(a[18], b[16]) + + MUL15(a[19], b[15]); + t[35] = MUL15(a[16], b[19]) + + MUL15(a[17], b[18]) + + MUL15(a[18], b[17]) + + MUL15(a[19], b[16]); + t[36] = MUL15(a[17], b[19]) + + MUL15(a[18], b[18]) + + MUL15(a[19], b[17]); + t[37] = MUL15(a[18], b[19]) + + MUL15(a[19], b[18]); + t[38] = MUL15(a[19], b[19]); + d[39] = norm13(d, t, 39); +} + +static void +square20(uint32_t *d, const uint32_t *a) +{ + uint32_t t[39]; + + t[ 0] = MUL15(a[ 0], a[ 0]); + t[ 1] = ((MUL15(a[ 0], a[ 1])) << 1); + t[ 2] = MUL15(a[ 1], a[ 1]) + + ((MUL15(a[ 0], a[ 2])) << 1); + t[ 3] = ((MUL15(a[ 0], a[ 3]) + + MUL15(a[ 1], a[ 2])) << 1); + t[ 4] = MUL15(a[ 2], a[ 2]) + + ((MUL15(a[ 0], a[ 4]) + + MUL15(a[ 1], a[ 3])) << 1); + t[ 5] = ((MUL15(a[ 0], a[ 5]) + + MUL15(a[ 1], a[ 4]) + + MUL15(a[ 2], a[ 3])) << 1); + t[ 6] = MUL15(a[ 3], a[ 3]) + + ((MUL15(a[ 0], a[ 6]) + + MUL15(a[ 1], a[ 5]) + + MUL15(a[ 2], a[ 4])) << 1); + t[ 7] = ((MUL15(a[ 0], a[ 7]) + + MUL15(a[ 1], a[ 6]) + + MUL15(a[ 2], a[ 5]) + + MUL15(a[ 3], a[ 4])) << 1); + t[ 8] = MUL15(a[ 4], a[ 4]) + + ((MUL15(a[ 0], a[ 8]) + + MUL15(a[ 1], a[ 7]) + + MUL15(a[ 2], a[ 6]) + + MUL15(a[ 3], a[ 5])) << 1); + t[ 9] = ((MUL15(a[ 0], a[ 9]) + + MUL15(a[ 1], a[ 8]) + + MUL15(a[ 2], a[ 7]) + + MUL15(a[ 3], a[ 6]) + + MUL15(a[ 4], a[ 5])) << 1); + t[10] = MUL15(a[ 5], a[ 5]) + + ((MUL15(a[ 0], a[10]) + + MUL15(a[ 1], a[ 9]) + + MUL15(a[ 2], a[ 8]) + + MUL15(a[ 3], a[ 7]) + + MUL15(a[ 4], a[ 6])) << 1); + t[11] = ((MUL15(a[ 0], a[11]) + + MUL15(a[ 1], a[10]) + + MUL15(a[ 2], a[ 9]) + + MUL15(a[ 3], a[ 8]) + + MUL15(a[ 4], a[ 7]) + + MUL15(a[ 5], a[ 6])) << 1); + t[12] = MUL15(a[ 6], a[ 6]) + + ((MUL15(a[ 0], a[12]) + + MUL15(a[ 1], a[11]) + + MUL15(a[ 2], a[10]) + + MUL15(a[ 3], a[ 9]) + + MUL15(a[ 4], a[ 8]) + + MUL15(a[ 5], a[ 7])) << 1); + t[13] = ((MUL15(a[ 0], a[13]) + + MUL15(a[ 1], a[12]) + + MUL15(a[ 2], a[11]) + + MUL15(a[ 3], a[10]) + + MUL15(a[ 4], a[ 9]) + + MUL15(a[ 5], a[ 8]) + + MUL15(a[ 6], a[ 7])) << 1); + t[14] = MUL15(a[ 7], a[ 7]) + + ((MUL15(a[ 0], a[14]) + + MUL15(a[ 1], a[13]) + + MUL15(a[ 2], a[12]) + + MUL15(a[ 3], a[11]) + + MUL15(a[ 4], a[10]) + + MUL15(a[ 5], a[ 9]) + + MUL15(a[ 6], a[ 8])) << 1); + t[15] = ((MUL15(a[ 0], a[15]) + + MUL15(a[ 1], a[14]) + + MUL15(a[ 2], a[13]) + + MUL15(a[ 3], a[12]) + + MUL15(a[ 4], a[11]) + + MUL15(a[ 5], a[10]) + + MUL15(a[ 6], a[ 9]) + + MUL15(a[ 7], a[ 8])) << 1); + t[16] = MUL15(a[ 8], a[ 8]) + + ((MUL15(a[ 0], a[16]) + + MUL15(a[ 1], a[15]) + + MUL15(a[ 2], a[14]) + + MUL15(a[ 3], a[13]) + + MUL15(a[ 4], a[12]) + + MUL15(a[ 5], a[11]) + + MUL15(a[ 6], a[10]) + + MUL15(a[ 7], a[ 9])) << 1); + t[17] = ((MUL15(a[ 0], a[17]) + + MUL15(a[ 1], a[16]) + + MUL15(a[ 2], a[15]) + + MUL15(a[ 3], a[14]) + + MUL15(a[ 4], a[13]) + + MUL15(a[ 5], a[12]) + + MUL15(a[ 6], a[11]) + + MUL15(a[ 7], a[10]) + + MUL15(a[ 8], a[ 9])) << 1); + t[18] = MUL15(a[ 9], a[ 9]) + + ((MUL15(a[ 0], a[18]) + + MUL15(a[ 1], a[17]) + + MUL15(a[ 2], a[16]) + + MUL15(a[ 3], a[15]) + + MUL15(a[ 4], a[14]) + + MUL15(a[ 5], a[13]) + + MUL15(a[ 6], a[12]) + + MUL15(a[ 7], a[11]) + + MUL15(a[ 8], a[10])) << 1); + t[19] = ((MUL15(a[ 0], a[19]) + + MUL15(a[ 1], a[18]) + + MUL15(a[ 2], a[17]) + + MUL15(a[ 3], a[16]) + + MUL15(a[ 4], a[15]) + + MUL15(a[ 5], a[14]) + + MUL15(a[ 6], a[13]) + + MUL15(a[ 7], a[12]) + + MUL15(a[ 8], a[11]) + + MUL15(a[ 9], a[10])) << 1); + t[20] = MUL15(a[10], a[10]) + + ((MUL15(a[ 1], a[19]) + + MUL15(a[ 2], a[18]) + + MUL15(a[ 3], a[17]) + + MUL15(a[ 4], a[16]) + + MUL15(a[ 5], a[15]) + + MUL15(a[ 6], a[14]) + + MUL15(a[ 7], a[13]) + + MUL15(a[ 8], a[12]) + + MUL15(a[ 9], a[11])) << 1); + t[21] = ((MUL15(a[ 2], a[19]) + + MUL15(a[ 3], a[18]) + + MUL15(a[ 4], a[17]) + + MUL15(a[ 5], a[16]) + + MUL15(a[ 6], a[15]) + + MUL15(a[ 7], a[14]) + + MUL15(a[ 8], a[13]) + + MUL15(a[ 9], a[12]) + + MUL15(a[10], a[11])) << 1); + t[22] = MUL15(a[11], a[11]) + + ((MUL15(a[ 3], a[19]) + + MUL15(a[ 4], a[18]) + + MUL15(a[ 5], a[17]) + + MUL15(a[ 6], a[16]) + + MUL15(a[ 7], a[15]) + + MUL15(a[ 8], a[14]) + + MUL15(a[ 9], a[13]) + + MUL15(a[10], a[12])) << 1); + t[23] = ((MUL15(a[ 4], a[19]) + + MUL15(a[ 5], a[18]) + + MUL15(a[ 6], a[17]) + + MUL15(a[ 7], a[16]) + + MUL15(a[ 8], a[15]) + + MUL15(a[ 9], a[14]) + + MUL15(a[10], a[13]) + + MUL15(a[11], a[12])) << 1); + t[24] = MUL15(a[12], a[12]) + + ((MUL15(a[ 5], a[19]) + + MUL15(a[ 6], a[18]) + + MUL15(a[ 7], a[17]) + + MUL15(a[ 8], a[16]) + + MUL15(a[ 9], a[15]) + + MUL15(a[10], a[14]) + + MUL15(a[11], a[13])) << 1); + t[25] = ((MUL15(a[ 6], a[19]) + + MUL15(a[ 7], a[18]) + + MUL15(a[ 8], a[17]) + + MUL15(a[ 9], a[16]) + + MUL15(a[10], a[15]) + + MUL15(a[11], a[14]) + + MUL15(a[12], a[13])) << 1); + t[26] = MUL15(a[13], a[13]) + + ((MUL15(a[ 7], a[19]) + + MUL15(a[ 8], a[18]) + + MUL15(a[ 9], a[17]) + + MUL15(a[10], a[16]) + + MUL15(a[11], a[15]) + + MUL15(a[12], a[14])) << 1); + t[27] = ((MUL15(a[ 8], a[19]) + + MUL15(a[ 9], a[18]) + + MUL15(a[10], a[17]) + + MUL15(a[11], a[16]) + + MUL15(a[12], a[15]) + + MUL15(a[13], a[14])) << 1); + t[28] = MUL15(a[14], a[14]) + + ((MUL15(a[ 9], a[19]) + + MUL15(a[10], a[18]) + + MUL15(a[11], a[17]) + + MUL15(a[12], a[16]) + + MUL15(a[13], a[15])) << 1); + t[29] = ((MUL15(a[10], a[19]) + + MUL15(a[11], a[18]) + + MUL15(a[12], a[17]) + + MUL15(a[13], a[16]) + + MUL15(a[14], a[15])) << 1); + t[30] = MUL15(a[15], a[15]) + + ((MUL15(a[11], a[19]) + + MUL15(a[12], a[18]) + + MUL15(a[13], a[17]) + + MUL15(a[14], a[16])) << 1); + t[31] = ((MUL15(a[12], a[19]) + + MUL15(a[13], a[18]) + + MUL15(a[14], a[17]) + + MUL15(a[15], a[16])) << 1); + t[32] = MUL15(a[16], a[16]) + + ((MUL15(a[13], a[19]) + + MUL15(a[14], a[18]) + + MUL15(a[15], a[17])) << 1); + t[33] = ((MUL15(a[14], a[19]) + + MUL15(a[15], a[18]) + + MUL15(a[16], a[17])) << 1); + t[34] = MUL15(a[17], a[17]) + + ((MUL15(a[15], a[19]) + + MUL15(a[16], a[18])) << 1); + t[35] = ((MUL15(a[16], a[19]) + + MUL15(a[17], a[18])) << 1); + t[36] = MUL15(a[18], a[18]) + + ((MUL15(a[17], a[19])) << 1); + t[37] = ((MUL15(a[18], a[19])) << 1); + t[38] = MUL15(a[19], a[19]); + d[39] = norm13(d, t, 39); +} + +#endif + +/* + * Perform a "final reduction" in field F255 (field for Curve25519) + * The source value must be less than twice the modulus. If the value + * is not lower than the modulus, then the modulus is subtracted and + * this function returns 1; otherwise, it leaves it untouched and it + * returns 0. + */ +static uint32_t +reduce_final_f255(uint32_t *d) +{ + uint32_t t[20]; + uint32_t cc; + int i; + + memcpy(t, d, sizeof t); + cc = 19; + for (i = 0; i < 20; i ++) { + uint32_t w; + + w = t[i] + cc; + cc = w >> 13; + t[i] = w & 0x1FFF; + } + cc = t[19] >> 8; + t[19] &= 0xFF; + CCOPY(cc, d, t, sizeof t); + return cc; +} + +/* + * Perform a multiplication of two integers modulo 2^255-19. + * Operands are arrays of 20 words, each containing 13 bits of data, in + * little-endian order. Input value may be up to 2^256-1; on output, value + * fits on 256 bits and is lower than twice the modulus. + */ +static void +f255_mul(uint32_t *d, const uint32_t *a, const uint32_t *b) +{ + uint32_t t[40], cc, w; + int i; + + /* + * Compute raw multiplication. All result words fit in 13 bits + * each; upper word (t[39]) must fit on 5 bits, since the product + * of two 256-bit integers must fit on 512 bits. + */ + mul20(t, a, b); + + /* + * Modular reduction: each high word is added where necessary. + * Since the modulus is 2^255-19 and word 20 corresponds to + * offset 20*13 = 260, word 20+k must be added to word k with + * a factor of 19*2^5 = 608. The extra bits in word 19 are also + * added that way. + */ + cc = MUL15(t[19] >> 8, 19); + t[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = t[i] + cc + MUL15(t[i + 20], 608); + t[i] = w & 0x1FFF; + cc = w >> 13; + } + cc = MUL15(w >> 8, 19); + t[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = t[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } +} + +/* + * Square an integer modulo 2^255-19. + * Operand is an array of 20 words, each containing 13 bits of data, in + * little-endian order. Input value may be up to 2^256-1; on output, value + * fits on 256 bits and is lower than twice the modulus. + */ +static void +f255_square(uint32_t *d, const uint32_t *a) +{ + uint32_t t[40], cc, w; + int i; + + /* + * Compute raw multiplication. All result words fit in 13 bits + * each; upper word (t[39]) must fit on 5 bits, since the product + * of two 256-bit integers must fit on 512 bits. + */ + square20(t, a); + + /* + * Modular reduction: each high word is added where necessary. + * Since the modulus is 2^255-19 and word 20 corresponds to + * offset 20*13 = 260, word 20+k must be added to word k with + * a factor of 19*2^5 = 608. The extra bits in word 19 are also + * added that way. + */ + cc = MUL15(t[19] >> 8, 19); + t[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = t[i] + cc + MUL15(t[i + 20], 608); + t[i] = w & 0x1FFF; + cc = w >> 13; + } + cc = MUL15(w >> 8, 19); + t[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = t[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } +} + +/* + * Add two values in F255. Partial reduction is performed (down to less + * than twice the modulus). + */ +static void +f255_add(uint32_t *d, const uint32_t *a, const uint32_t *b) +{ + int i; + uint32_t cc, w; + + cc = 0; + for (i = 0; i < 20; i ++) { + w = a[i] + b[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } + cc = MUL15(w >> 8, 19); + d[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = d[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } +} + +/* + * Subtract one value from another in F255. Partial reduction is + * performed (down to less than twice the modulus). + */ +static void +f255_sub(uint32_t *d, const uint32_t *a, const uint32_t *b) +{ + /* + * We actually compute a - b + 2*p, so that the final value is + * necessarily positive. + */ + int i; + uint32_t cc, w; + + cc = (uint32_t)-38; + for (i = 0; i < 20; i ++) { + w = a[i] - b[i] + cc; + d[i] = w & 0x1FFF; + cc = ARSH(w, 13); + } + cc = MUL15((w + 0x200) >> 8, 19); + d[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = d[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } +} + +/* + * Multiply an integer by the 'A24' constant (121665). Partial reduction + * is performed (down to less than twice the modulus). + */ +static void +f255_mul_a24(uint32_t *d, const uint32_t *a) +{ + int i; + uint32_t cc, w; + + cc = 0; + for (i = 0; i < 20; i ++) { + w = MUL15(a[i], 121665) + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } + cc = MUL15(w >> 8, 19); + d[19] &= 0xFF; + for (i = 0; i < 20; i ++) { + w = d[i] + cc; + d[i] = w & 0x1FFF; + cc = w >> 13; + } +} + +static const unsigned char GEN[] = { + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const unsigned char ORDER[] = { + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x14, 0xDE, 0xF9, 0xDE, 0xA2, 0xF7, 0x9C, 0xD6, + 0x58, 0x12, 0x63, 0x1A, 0x5C, 0xF5, 0xD3, 0xED +}; + +static const unsigned char * +api_generator(int curve, size_t *len) +{ + (void)curve; + *len = 32; + return GEN; +} + +static const unsigned char * +api_order(int curve, size_t *len) +{ + (void)curve; + *len = 32; + return ORDER; +} + +static void +cswap(uint32_t *a, uint32_t *b, uint32_t ctl) +{ + int i; + + ctl = -ctl; + for (i = 0; i < 20; i ++) { + uint32_t aw, bw, tw; + + aw = a[i]; + bw = b[i]; + tw = ctl & (aw ^ bw); + a[i] = aw ^ tw; + b[i] = bw ^ tw; + } +} + +static uint32_t +api_mul(unsigned char *G, size_t Glen, + const unsigned char *kb, size_t kblen, int curve) +{ + uint32_t x1[20], x2[20], x3[20], z2[20], z3[20]; + uint32_t a[20], aa[20], b[20], bb[20]; + uint32_t c[20], d[20], e[20], da[20], cb[20]; + unsigned char k[32]; + uint32_t swap; + int i; + + (void)curve; + + /* + * Points are encoded over exactly 32 bytes. Multipliers must fit + * in 32 bytes as well. + * RFC 7748 mandates that the high bit of the last point byte must + * be ignored/cleared. + */ + if (Glen != 32 || kblen > 32) { + return 0; + } + G[31] &= 0x7F; + + /* + * Initialise variables x1, x2, z2, x3 and z3. We set all of them + * into Montgomery representation. + */ + x1[19] = le8_to_le13(x1, G, 32); + memcpy(x3, x1, sizeof x1); + memset(z2, 0, sizeof z2); + memset(x2, 0, sizeof x2); + x2[0] = 1; + memset(z3, 0, sizeof z3); + z3[0] = 1; + + memcpy(k, kb, kblen); + memset(k + kblen, 0, (sizeof k) - kblen); + k[0] &= 0xF8; + k[31] &= 0x7F; + k[31] |= 0x40; + + /* obsolete + print_int("x1", x1); + */ + + swap = 0; + for (i = 254; i >= 0; i --) { + uint32_t kt; + + kt = (k[i >> 3] >> (i & 7)) & 1; + swap ^= kt; + cswap(x2, x3, swap); + cswap(z2, z3, swap); + swap = kt; + + /* obsolete + print_int("x2", x2); + print_int("z2", z2); + print_int("x3", x3); + print_int("z3", z3); + */ + + f255_add(a, x2, z2); + f255_square(aa, a); + f255_sub(b, x2, z2); + f255_square(bb, b); + f255_sub(e, aa, bb); + f255_add(c, x3, z3); + f255_sub(d, x3, z3); + f255_mul(da, d, a); + f255_mul(cb, c, b); + + /* obsolete + print_int("a ", a); + print_int("aa", aa); + print_int("b ", b); + print_int("bb", bb); + print_int("e ", e); + print_int("c ", c); + print_int("d ", d); + print_int("da", da); + print_int("cb", cb); + */ + + f255_add(x3, da, cb); + f255_square(x3, x3); + f255_sub(z3, da, cb); + f255_square(z3, z3); + f255_mul(z3, z3, x1); + f255_mul(x2, aa, bb); + f255_mul_a24(z2, e); + f255_add(z2, z2, aa); + f255_mul(z2, e, z2); + + /* obsolete + print_int("x2", x2); + print_int("z2", z2); + print_int("x3", x3); + print_int("z3", z3); + */ + } + cswap(x2, x3, swap); + cswap(z2, z3, swap); + + /* + * Inverse z2 with a modular exponentiation. This is a simple + * square-and-multiply algorithm; we mutualise most non-squarings + * since the exponent contains almost only ones. + */ + memcpy(a, z2, sizeof z2); + for (i = 0; i < 15; i ++) { + f255_square(a, a); + f255_mul(a, a, z2); + } + memcpy(b, a, sizeof a); + for (i = 0; i < 14; i ++) { + int j; + + for (j = 0; j < 16; j ++) { + f255_square(b, b); + } + f255_mul(b, b, a); + } + for (i = 14; i >= 0; i --) { + f255_square(b, b); + if ((0xFFEB >> i) & 1) { + f255_mul(b, z2, b); + } + } + f255_mul(x2, x2, b); + reduce_final_f255(x2); + le13_to_le8(G, 32, x2); + return 1; +} + +static size_t +api_mulgen(unsigned char *R, + const unsigned char *x, size_t xlen, int curve) +{ + const unsigned char *G; + size_t Glen; + + G = api_generator(curve, &Glen); + memcpy(R, G, Glen); + api_mul(R, Glen, x, xlen, curve); + return Glen; +} + +static uint32_t +api_muladd(unsigned char *A, const unsigned char *B, size_t len, + const unsigned char *x, size_t xlen, + const unsigned char *y, size_t ylen, int curve) +{ + /* + * We don't implement this method, since it is used for ECDSA + * only, and there is no ECDSA over Curve25519 (which instead + * uses EdDSA). + */ + (void)A; + (void)B; + (void)len; + (void)x; + (void)xlen; + (void)y; + (void)ylen; + (void)curve; + return 0; +} + +/* see bearssl_ec.h */ +const br_ec_impl br_ec_c25519_m15 = { + (uint32_t)0x20000000, + &api_generator, + &api_order, + &api_mul, + &api_mulgen, + &api_muladd +}; diff --git a/test/test_crypto.c b/test/test_crypto.c index ddba9ac..46c8fa2 100644 --- a/test/test_crypto.c +++ b/test/test_crypto.c @@ -4935,13 +4935,13 @@ const struct { static void test_EC_c25519(const char *name, const br_ec_impl *iec) { + unsigned char bu[32], bk[32], br[32]; size_t v; + int i; printf("Test %s: ", name); fflush(stdout); for (v = 0; C25519_KAT[v].scalar; v ++) { - unsigned char bu[32], bk[32], br[32]; - hextobin(bk, C25519_KAT[v].scalar); hextobin(bu, C25519_KAT[v].u_in); hextobin(br, C25519_KAT[v].u_out); @@ -4956,6 +4956,44 @@ test_EC_c25519(const char *name, const br_ec_impl *iec) printf("."); fflush(stdout); } + printf(" "); + fflush(stdout); + + memset(bu, 0, sizeof bu); + bu[0] = 0x09; + memcpy(bk, bu, sizeof bu); + for (i = 1; i <= 1000; i ++) { + if (!iec->mul(bu, sizeof bu, bk, sizeof bk, BR_EC_curve25519)) { + fprintf(stderr, "Curve25519 multiplication failed" + " (iter=%d)\n", i); + exit(EXIT_FAILURE); + } + for (v = 0; v < sizeof bu; v ++) { + unsigned t; + + t = bu[v]; + bu[v] = bk[v]; + bk[v] = t; + } + if (i == 1 || i == 1000) { + const char *sref; + + sref = (i == 1) + ? "422C8E7A6227D7BCA1350B3E2BB7279F7897B87BB6854B783C60E80311AE3079" + : "684CF59BA83309552800EF566F2F4D3C1C3887C49360E3875F2EB94D99532C51"; + hextobin(br, sref); + if (memcmp(bk, br, sizeof bk) != 0) { + fprintf(stderr, + "Curve25519 failed KAT (iter=%d)\n", i); + exit(EXIT_FAILURE); + } + } + if (i % 100 == 0) { + printf("."); + fflush(stdout); + } + } + printf(" done.\n"); fflush(stdout); } @@ -4966,6 +5004,12 @@ test_EC_c25519_i15(void) test_EC_c25519("EC_c25519_i15", &br_ec_c25519_i15); } +static void +test_EC_c25519_m15(void) +{ + test_EC_c25519("EC_c25519_m15", &br_ec_c25519_m15); +} + static const unsigned char EC_P256_PUB_POINT[] = { 0x04, 0x60, 0xFE, 0xD4, 0xBA, 0x25, 0x5A, 0x9D, 0x31, 0xC9, 0x61, 0xEB, 0x74, 0xC6, 0x35, 0x6D, @@ -5513,6 +5557,7 @@ static const struct { STU(EC_p256_m15), /* STU(EC_prime_i32), */ STU(EC_c25519_i15), + STU(EC_c25519_m15), STU(ECDSA_i15), STU(ECDSA_i31), { 0, 0 } diff --git a/test/test_speed.c b/test/test_speed.c index 48f5fe3..48c2fdb 100644 --- a/test/test_speed.c +++ b/test/test_speed.c @@ -714,6 +714,13 @@ test_speed_ec_c25519_i15(void) &br_ec_c25519_i15, &br_curve25519); } +static void +test_speed_ec_c25519_m15(void) +{ + test_speed_ec_inner("EC m15 C25519", + &br_ec_c25519_m15, &br_curve25519); +} + static void test_speed_ecdsa_inner(const char *name, const br_ec_impl *impl, const br_ec_curve_def *cd, @@ -1256,6 +1263,7 @@ static const struct { STU(ec_prime_i15), STU(ec_prime_i31), STU(ec_c25519_i15), + STU(ec_c25519_m15), STU(ecdsa_p256_m15), STU(ecdsa_i15), STU(ecdsa_i31), -- 2.17.1