--- /dev/null
+/*
+ * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define BR_ENABLE_INTRINSICS 1
+#include "inner.h"
+
+#if BR_AES_X86NI
+
+/* see bearssl_block.h */
+const br_block_ctrcbc_class *
+br_aes_x86ni_ctrcbc_get_vtable(void)
+{
+ return br_aes_x86ni_supported() ? &br_aes_x86ni_ctrcbc_vtable : NULL;
+}
+
+/* see bearssl_block.h */
+void
+br_aes_x86ni_ctrcbc_init(br_aes_x86ni_ctrcbc_keys *ctx,
+ const void *key, size_t len)
+{
+ ctx->vtable = &br_aes_x86ni_ctrcbc_vtable;
+ ctx->num_rounds = br_aes_x86ni_keysched_enc(ctx->skey.skni, key, len);
+}
+
+BR_TARGETS_X86_UP
+
+/* see bearssl_block.h */
+BR_TARGET("sse2,sse4.1,aes")
+void
+br_aes_x86ni_ctrcbc_ctr(const br_aes_x86ni_ctrcbc_keys *ctx,
+ void *ctr, void *data, size_t len)
+{
+ unsigned char *buf;
+ unsigned num_rounds;
+ __m128i sk[15];
+ __m128i ivx0, ivx1, ivx2, ivx3;
+ __m128i erev, zero, one, four, notthree;
+ unsigned u;
+
+ buf = data;
+ num_rounds = ctx->num_rounds;
+ for (u = 0; u <= num_rounds; u ++) {
+ sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
+ }
+
+ /*
+ * Some SSE2 constants.
+ */
+ erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15);
+ zero = _mm_setzero_si128();
+ one = _mm_set_epi64x(0, 1);
+ four = _mm_set_epi64x(0, 4);
+ notthree = _mm_sub_epi64(zero, four);
+
+ /*
+ * Decode the counter in big-endian and pre-increment the other
+ * three counters.
+ */
+ ivx0 = _mm_shuffle_epi8(_mm_loadu_si128((void *)ctr), erev);
+ ivx1 = _mm_add_epi64(ivx0, one);
+ ivx1 = _mm_sub_epi64(ivx1,
+ _mm_slli_si128(_mm_cmpeq_epi64(ivx1, zero), 8));
+ ivx2 = _mm_add_epi64(ivx1, one);
+ ivx2 = _mm_sub_epi64(ivx2,
+ _mm_slli_si128(_mm_cmpeq_epi64(ivx2, zero), 8));
+ ivx3 = _mm_add_epi64(ivx2, one);
+ ivx3 = _mm_sub_epi64(ivx3,
+ _mm_slli_si128(_mm_cmpeq_epi64(ivx3, zero), 8));
+ while (len > 0) {
+ __m128i x0, x1, x2, x3;
+
+ /*
+ * Load counter values; we need to byteswap them because
+ * the specification says that they use big-endian.
+ */
+ x0 = _mm_shuffle_epi8(ivx0, erev);
+ x1 = _mm_shuffle_epi8(ivx1, erev);
+ x2 = _mm_shuffle_epi8(ivx2, erev);
+ x3 = _mm_shuffle_epi8(ivx3, erev);
+
+ x0 = _mm_xor_si128(x0, sk[0]);
+ x1 = _mm_xor_si128(x1, sk[0]);
+ x2 = _mm_xor_si128(x2, sk[0]);
+ x3 = _mm_xor_si128(x3, sk[0]);
+ x0 = _mm_aesenc_si128(x0, sk[1]);
+ x1 = _mm_aesenc_si128(x1, sk[1]);
+ x2 = _mm_aesenc_si128(x2, sk[1]);
+ x3 = _mm_aesenc_si128(x3, sk[1]);
+ x0 = _mm_aesenc_si128(x0, sk[2]);
+ x1 = _mm_aesenc_si128(x1, sk[2]);
+ x2 = _mm_aesenc_si128(x2, sk[2]);
+ x3 = _mm_aesenc_si128(x3, sk[2]);
+ x0 = _mm_aesenc_si128(x0, sk[3]);
+ x1 = _mm_aesenc_si128(x1, sk[3]);
+ x2 = _mm_aesenc_si128(x2, sk[3]);
+ x3 = _mm_aesenc_si128(x3, sk[3]);
+ x0 = _mm_aesenc_si128(x0, sk[4]);
+ x1 = _mm_aesenc_si128(x1, sk[4]);
+ x2 = _mm_aesenc_si128(x2, sk[4]);
+ x3 = _mm_aesenc_si128(x3, sk[4]);
+ x0 = _mm_aesenc_si128(x0, sk[5]);
+ x1 = _mm_aesenc_si128(x1, sk[5]);
+ x2 = _mm_aesenc_si128(x2, sk[5]);
+ x3 = _mm_aesenc_si128(x3, sk[5]);
+ x0 = _mm_aesenc_si128(x0, sk[6]);
+ x1 = _mm_aesenc_si128(x1, sk[6]);
+ x2 = _mm_aesenc_si128(x2, sk[6]);
+ x3 = _mm_aesenc_si128(x3, sk[6]);
+ x0 = _mm_aesenc_si128(x0, sk[7]);
+ x1 = _mm_aesenc_si128(x1, sk[7]);
+ x2 = _mm_aesenc_si128(x2, sk[7]);
+ x3 = _mm_aesenc_si128(x3, sk[7]);
+ x0 = _mm_aesenc_si128(x0, sk[8]);
+ x1 = _mm_aesenc_si128(x1, sk[8]);
+ x2 = _mm_aesenc_si128(x2, sk[8]);
+ x3 = _mm_aesenc_si128(x3, sk[8]);
+ x0 = _mm_aesenc_si128(x0, sk[9]);
+ x1 = _mm_aesenc_si128(x1, sk[9]);
+ x2 = _mm_aesenc_si128(x2, sk[9]);
+ x3 = _mm_aesenc_si128(x3, sk[9]);
+ if (num_rounds == 10) {
+ x0 = _mm_aesenclast_si128(x0, sk[10]);
+ x1 = _mm_aesenclast_si128(x1, sk[10]);
+ x2 = _mm_aesenclast_si128(x2, sk[10]);
+ x3 = _mm_aesenclast_si128(x3, sk[10]);
+ } else if (num_rounds == 12) {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x2 = _mm_aesenc_si128(x2, sk[10]);
+ x3 = _mm_aesenc_si128(x3, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x2 = _mm_aesenc_si128(x2, sk[11]);
+ x3 = _mm_aesenc_si128(x3, sk[11]);
+ x0 = _mm_aesenclast_si128(x0, sk[12]);
+ x1 = _mm_aesenclast_si128(x1, sk[12]);
+ x2 = _mm_aesenclast_si128(x2, sk[12]);
+ x3 = _mm_aesenclast_si128(x3, sk[12]);
+ } else {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x2 = _mm_aesenc_si128(x2, sk[10]);
+ x3 = _mm_aesenc_si128(x3, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x2 = _mm_aesenc_si128(x2, sk[11]);
+ x3 = _mm_aesenc_si128(x3, sk[11]);
+ x0 = _mm_aesenc_si128(x0, sk[12]);
+ x1 = _mm_aesenc_si128(x1, sk[12]);
+ x2 = _mm_aesenc_si128(x2, sk[12]);
+ x3 = _mm_aesenc_si128(x3, sk[12]);
+ x0 = _mm_aesenc_si128(x0, sk[13]);
+ x1 = _mm_aesenc_si128(x1, sk[13]);
+ x2 = _mm_aesenc_si128(x2, sk[13]);
+ x3 = _mm_aesenc_si128(x3, sk[13]);
+ x0 = _mm_aesenclast_si128(x0, sk[14]);
+ x1 = _mm_aesenclast_si128(x1, sk[14]);
+ x2 = _mm_aesenclast_si128(x2, sk[14]);
+ x3 = _mm_aesenclast_si128(x3, sk[14]);
+ }
+ if (len >= 64) {
+ x0 = _mm_xor_si128(x0,
+ _mm_loadu_si128((void *)(buf + 0)));
+ x1 = _mm_xor_si128(x1,
+ _mm_loadu_si128((void *)(buf + 16)));
+ x2 = _mm_xor_si128(x2,
+ _mm_loadu_si128((void *)(buf + 32)));
+ x3 = _mm_xor_si128(x3,
+ _mm_loadu_si128((void *)(buf + 48)));
+ _mm_storeu_si128((void *)(buf + 0), x0);
+ _mm_storeu_si128((void *)(buf + 16), x1);
+ _mm_storeu_si128((void *)(buf + 32), x2);
+ _mm_storeu_si128((void *)(buf + 48), x3);
+ buf += 64;
+ len -= 64;
+ } else {
+ unsigned char tmp[64];
+
+ _mm_storeu_si128((void *)(tmp + 0), x0);
+ _mm_storeu_si128((void *)(tmp + 16), x1);
+ _mm_storeu_si128((void *)(tmp + 32), x2);
+ _mm_storeu_si128((void *)(tmp + 48), x3);
+ for (u = 0; u < len; u ++) {
+ buf[u] ^= tmp[u];
+ }
+ switch (len) {
+ case 16:
+ ivx0 = ivx1;
+ break;
+ case 32:
+ ivx0 = ivx2;
+ break;
+ case 48:
+ ivx0 = ivx3;
+ break;
+ }
+ break;
+ }
+
+ /*
+ * Add 4 to each counter value. For carry propagation
+ * into the upper 64-bit words, we would need to compare
+ * the results with 4, but SSE2+ has only _signed_
+ * comparisons. Instead, we mask out the low two bits,
+ * and check whether the remaining bits are zero.
+ */
+ ivx0 = _mm_add_epi64(ivx0, four);
+ ivx1 = _mm_add_epi64(ivx1, four);
+ ivx2 = _mm_add_epi64(ivx2, four);
+ ivx3 = _mm_add_epi64(ivx3, four);
+ ivx0 = _mm_sub_epi64(ivx0,
+ _mm_slli_si128(_mm_cmpeq_epi64(
+ _mm_and_si128(ivx0, notthree), zero), 8));
+ ivx1 = _mm_sub_epi64(ivx1,
+ _mm_slli_si128(_mm_cmpeq_epi64(
+ _mm_and_si128(ivx1, notthree), zero), 8));
+ ivx2 = _mm_sub_epi64(ivx2,
+ _mm_slli_si128(_mm_cmpeq_epi64(
+ _mm_and_si128(ivx2, notthree), zero), 8));
+ ivx3 = _mm_sub_epi64(ivx3,
+ _mm_slli_si128(_mm_cmpeq_epi64(
+ _mm_and_si128(ivx3, notthree), zero), 8));
+ }
+
+ /*
+ * Write back new counter value. The loop took care to put the
+ * right counter value in ivx0.
+ */
+ _mm_storeu_si128((void *)ctr, _mm_shuffle_epi8(ivx0, erev));
+}
+
+/* see bearssl_block.h */
+BR_TARGET("sse2,sse4.1,aes")
+void
+br_aes_x86ni_ctrcbc_mac(const br_aes_x86ni_ctrcbc_keys *ctx,
+ void *cbcmac, const void *data, size_t len)
+{
+ const unsigned char *buf;
+ unsigned num_rounds;
+ __m128i sk[15], ivx;
+ unsigned u;
+
+ buf = data;
+ ivx = _mm_loadu_si128(cbcmac);
+ num_rounds = ctx->num_rounds;
+ for (u = 0; u <= num_rounds; u ++) {
+ sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
+ }
+ while (len > 0) {
+ __m128i x;
+
+ x = _mm_xor_si128(_mm_loadu_si128((void *)buf), ivx);
+ x = _mm_xor_si128(x, sk[0]);
+ x = _mm_aesenc_si128(x, sk[1]);
+ x = _mm_aesenc_si128(x, sk[2]);
+ x = _mm_aesenc_si128(x, sk[3]);
+ x = _mm_aesenc_si128(x, sk[4]);
+ x = _mm_aesenc_si128(x, sk[5]);
+ x = _mm_aesenc_si128(x, sk[6]);
+ x = _mm_aesenc_si128(x, sk[7]);
+ x = _mm_aesenc_si128(x, sk[8]);
+ x = _mm_aesenc_si128(x, sk[9]);
+ if (num_rounds == 10) {
+ x = _mm_aesenclast_si128(x, sk[10]);
+ } else if (num_rounds == 12) {
+ x = _mm_aesenc_si128(x, sk[10]);
+ x = _mm_aesenc_si128(x, sk[11]);
+ x = _mm_aesenclast_si128(x, sk[12]);
+ } else {
+ x = _mm_aesenc_si128(x, sk[10]);
+ x = _mm_aesenc_si128(x, sk[11]);
+ x = _mm_aesenc_si128(x, sk[12]);
+ x = _mm_aesenc_si128(x, sk[13]);
+ x = _mm_aesenclast_si128(x, sk[14]);
+ }
+ ivx = x;
+ buf += 16;
+ len -= 16;
+ }
+ _mm_storeu_si128(cbcmac, ivx);
+}
+
+/* see bearssl_block.h */
+BR_TARGET("sse2,sse4.1,aes")
+void
+br_aes_x86ni_ctrcbc_encrypt(const br_aes_x86ni_ctrcbc_keys *ctx,
+ void *ctr, void *cbcmac, void *data, size_t len)
+{
+ unsigned char *buf;
+ unsigned num_rounds;
+ __m128i sk[15];
+ __m128i ivx, cmx;
+ __m128i erev, zero, one;
+ unsigned u;
+ int first_iter;
+
+ num_rounds = ctx->num_rounds;
+ for (u = 0; u <= num_rounds; u ++) {
+ sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
+ }
+
+ /*
+ * Some SSE2 constants.
+ */
+ erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15);
+ zero = _mm_setzero_si128();
+ one = _mm_set_epi64x(0, 1);
+
+ /*
+ * Decode the counter in big-endian.
+ */
+ ivx = _mm_shuffle_epi8(_mm_loadu_si128(ctr), erev);
+ cmx = _mm_loadu_si128(cbcmac);
+
+ buf = data;
+ first_iter = 1;
+ while (len > 0) {
+ __m128i dx, x0, x1;
+
+ /*
+ * Load initial values:
+ * dx encrypted block of data
+ * x0 counter (for CTR encryption)
+ * x1 input for CBC-MAC
+ */
+ dx = _mm_loadu_si128((void *)buf);
+ x0 = _mm_shuffle_epi8(ivx, erev);
+ x1 = cmx;
+
+ x0 = _mm_xor_si128(x0, sk[0]);
+ x1 = _mm_xor_si128(x1, sk[0]);
+ x0 = _mm_aesenc_si128(x0, sk[1]);
+ x1 = _mm_aesenc_si128(x1, sk[1]);
+ x0 = _mm_aesenc_si128(x0, sk[2]);
+ x1 = _mm_aesenc_si128(x1, sk[2]);
+ x0 = _mm_aesenc_si128(x0, sk[3]);
+ x1 = _mm_aesenc_si128(x1, sk[3]);
+ x0 = _mm_aesenc_si128(x0, sk[4]);
+ x1 = _mm_aesenc_si128(x1, sk[4]);
+ x0 = _mm_aesenc_si128(x0, sk[5]);
+ x1 = _mm_aesenc_si128(x1, sk[5]);
+ x0 = _mm_aesenc_si128(x0, sk[6]);
+ x1 = _mm_aesenc_si128(x1, sk[6]);
+ x0 = _mm_aesenc_si128(x0, sk[7]);
+ x1 = _mm_aesenc_si128(x1, sk[7]);
+ x0 = _mm_aesenc_si128(x0, sk[8]);
+ x1 = _mm_aesenc_si128(x1, sk[8]);
+ x0 = _mm_aesenc_si128(x0, sk[9]);
+ x1 = _mm_aesenc_si128(x1, sk[9]);
+ if (num_rounds == 10) {
+ x0 = _mm_aesenclast_si128(x0, sk[10]);
+ x1 = _mm_aesenclast_si128(x1, sk[10]);
+ } else if (num_rounds == 12) {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x0 = _mm_aesenclast_si128(x0, sk[12]);
+ x1 = _mm_aesenclast_si128(x1, sk[12]);
+ } else {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x0 = _mm_aesenc_si128(x0, sk[12]);
+ x1 = _mm_aesenc_si128(x1, sk[12]);
+ x0 = _mm_aesenc_si128(x0, sk[13]);
+ x1 = _mm_aesenc_si128(x1, sk[13]);
+ x0 = _mm_aesenclast_si128(x0, sk[14]);
+ x1 = _mm_aesenclast_si128(x1, sk[14]);
+ }
+
+ x0 = _mm_xor_si128(x0, dx);
+ if (first_iter) {
+ cmx = _mm_xor_si128(cmx, x0);
+ first_iter = 0;
+ } else {
+ cmx = _mm_xor_si128(x1, x0);
+ }
+ _mm_storeu_si128((void *)buf, x0);
+
+ buf += 16;
+ len -= 16;
+
+ /*
+ * Increment the counter value.
+ */
+ ivx = _mm_add_epi64(ivx, one);
+ ivx = _mm_sub_epi64(ivx,
+ _mm_slli_si128(_mm_cmpeq_epi64(ivx, zero), 8));
+
+ /*
+ * If this was the last iteration, then compute the
+ * extra block encryption to complete CBC-MAC.
+ */
+ if (len == 0) {
+ cmx = _mm_xor_si128(cmx, sk[0]);
+ cmx = _mm_aesenc_si128(cmx, sk[1]);
+ cmx = _mm_aesenc_si128(cmx, sk[2]);
+ cmx = _mm_aesenc_si128(cmx, sk[3]);
+ cmx = _mm_aesenc_si128(cmx, sk[4]);
+ cmx = _mm_aesenc_si128(cmx, sk[5]);
+ cmx = _mm_aesenc_si128(cmx, sk[6]);
+ cmx = _mm_aesenc_si128(cmx, sk[7]);
+ cmx = _mm_aesenc_si128(cmx, sk[8]);
+ cmx = _mm_aesenc_si128(cmx, sk[9]);
+ if (num_rounds == 10) {
+ cmx = _mm_aesenclast_si128(cmx, sk[10]);
+ } else if (num_rounds == 12) {
+ cmx = _mm_aesenc_si128(cmx, sk[10]);
+ cmx = _mm_aesenc_si128(cmx, sk[11]);
+ cmx = _mm_aesenclast_si128(cmx, sk[12]);
+ } else {
+ cmx = _mm_aesenc_si128(cmx, sk[10]);
+ cmx = _mm_aesenc_si128(cmx, sk[11]);
+ cmx = _mm_aesenc_si128(cmx, sk[12]);
+ cmx = _mm_aesenc_si128(cmx, sk[13]);
+ cmx = _mm_aesenclast_si128(cmx, sk[14]);
+ }
+ break;
+ }
+ }
+
+ /*
+ * Write back new counter value and CBC-MAC value.
+ */
+ _mm_storeu_si128(ctr, _mm_shuffle_epi8(ivx, erev));
+ _mm_storeu_si128(cbcmac, cmx);
+}
+
+/* see bearssl_block.h */
+BR_TARGET("sse2,sse4.1,aes")
+void
+br_aes_x86ni_ctrcbc_decrypt(const br_aes_x86ni_ctrcbc_keys *ctx,
+ void *ctr, void *cbcmac, void *data, size_t len)
+{
+ unsigned char *buf;
+ unsigned num_rounds;
+ __m128i sk[15];
+ __m128i ivx, cmx;
+ __m128i erev, zero, one;
+ unsigned u;
+
+ num_rounds = ctx->num_rounds;
+ for (u = 0; u <= num_rounds; u ++) {
+ sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
+ }
+
+ /*
+ * Some SSE2 constants.
+ */
+ erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
+ 8, 9, 10, 11, 12, 13, 14, 15);
+ zero = _mm_setzero_si128();
+ one = _mm_set_epi64x(0, 1);
+
+ /*
+ * Decode the counter in big-endian.
+ */
+ ivx = _mm_shuffle_epi8(_mm_loadu_si128(ctr), erev);
+ cmx = _mm_loadu_si128(cbcmac);
+
+ buf = data;
+ while (len > 0) {
+ __m128i dx, x0, x1;
+
+ /*
+ * Load initial values:
+ * dx encrypted block of data
+ * x0 counter (for CTR encryption)
+ * x1 input for CBC-MAC
+ */
+ dx = _mm_loadu_si128((void *)buf);
+ x0 = _mm_shuffle_epi8(ivx, erev);
+ x1 = _mm_xor_si128(cmx, dx);
+
+ x0 = _mm_xor_si128(x0, sk[0]);
+ x1 = _mm_xor_si128(x1, sk[0]);
+ x0 = _mm_aesenc_si128(x0, sk[1]);
+ x1 = _mm_aesenc_si128(x1, sk[1]);
+ x0 = _mm_aesenc_si128(x0, sk[2]);
+ x1 = _mm_aesenc_si128(x1, sk[2]);
+ x0 = _mm_aesenc_si128(x0, sk[3]);
+ x1 = _mm_aesenc_si128(x1, sk[3]);
+ x0 = _mm_aesenc_si128(x0, sk[4]);
+ x1 = _mm_aesenc_si128(x1, sk[4]);
+ x0 = _mm_aesenc_si128(x0, sk[5]);
+ x1 = _mm_aesenc_si128(x1, sk[5]);
+ x0 = _mm_aesenc_si128(x0, sk[6]);
+ x1 = _mm_aesenc_si128(x1, sk[6]);
+ x0 = _mm_aesenc_si128(x0, sk[7]);
+ x1 = _mm_aesenc_si128(x1, sk[7]);
+ x0 = _mm_aesenc_si128(x0, sk[8]);
+ x1 = _mm_aesenc_si128(x1, sk[8]);
+ x0 = _mm_aesenc_si128(x0, sk[9]);
+ x1 = _mm_aesenc_si128(x1, sk[9]);
+ if (num_rounds == 10) {
+ x0 = _mm_aesenclast_si128(x0, sk[10]);
+ x1 = _mm_aesenclast_si128(x1, sk[10]);
+ } else if (num_rounds == 12) {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x0 = _mm_aesenclast_si128(x0, sk[12]);
+ x1 = _mm_aesenclast_si128(x1, sk[12]);
+ } else {
+ x0 = _mm_aesenc_si128(x0, sk[10]);
+ x1 = _mm_aesenc_si128(x1, sk[10]);
+ x0 = _mm_aesenc_si128(x0, sk[11]);
+ x1 = _mm_aesenc_si128(x1, sk[11]);
+ x0 = _mm_aesenc_si128(x0, sk[12]);
+ x1 = _mm_aesenc_si128(x1, sk[12]);
+ x0 = _mm_aesenc_si128(x0, sk[13]);
+ x1 = _mm_aesenc_si128(x1, sk[13]);
+ x0 = _mm_aesenclast_si128(x0, sk[14]);
+ x1 = _mm_aesenclast_si128(x1, sk[14]);
+ }
+ x0 = _mm_xor_si128(x0, dx);
+ cmx = x1;
+ _mm_storeu_si128((void *)buf, x0);
+
+ buf += 16;
+ len -= 16;
+
+ /*
+ * Increment the counter value.
+ */
+ ivx = _mm_add_epi64(ivx, one);
+ ivx = _mm_sub_epi64(ivx,
+ _mm_slli_si128(_mm_cmpeq_epi64(ivx, zero), 8));
+ }
+
+ /*
+ * Write back new counter value and CBC-MAC value.
+ */
+ _mm_storeu_si128(ctr, _mm_shuffle_epi8(ivx, erev));
+ _mm_storeu_si128(cbcmac, cmx);
+}
+
+BR_TARGETS_X86_DOWN
+
+/* see bearssl_block.h */
+const br_block_ctrcbc_class br_aes_x86ni_ctrcbc_vtable = {
+ sizeof(br_aes_x86ni_ctrcbc_keys),
+ 16,
+ 4,
+ (void (*)(const br_block_ctrcbc_class **, const void *, size_t))
+ &br_aes_x86ni_ctrcbc_init,
+ (void (*)(const br_block_ctrcbc_class *const *,
+ void *, void *, void *, size_t))
+ &br_aes_x86ni_ctrcbc_encrypt,
+ (void (*)(const br_block_ctrcbc_class *const *,
+ void *, void *, void *, size_t))
+ &br_aes_x86ni_ctrcbc_decrypt,
+ (void (*)(const br_block_ctrcbc_class *const *,
+ void *, void *, size_t))
+ &br_aes_x86ni_ctrcbc_ctr,
+ (void (*)(const br_block_ctrcbc_class *const *,
+ void *, const void *, size_t))
+ &br_aes_x86ni_ctrcbc_mac
+};
+
+#else
+
+/* see bearssl_block.h */
+const br_block_ctrcbc_class *
+br_aes_x86ni_ctrcbc_get_vtable(void)
+{
+ return NULL;
+}
+
+#endif
projects / BearSSL / blobdiff