_ Git - BearSSL/blob - src/symcipher/aes_x86ni.c

   1 /*
   2  * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining
   5  * a copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sublicense, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be
  13  * included in all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  */
  24
  25 #include "inner.h"
  26
  27 /*
  28  * This code contains the AES key schedule implementation using the
  29  * AES-NI opcodes.
  30  */
  31
  32 #if BR_AES_X86NI
  33
  34 #if BR_AES_X86NI_GCC
  35 #include <wmmintrin.h>
  36 #include <cpuid.h>
  37 #endif
  38
  39 #if BR_AES_X86NI_MSC
  40 #include <intrin.h>
  41 #endif
  42
  43 /* see inner.h */
  44 int
  45 br_aes_x86ni_supported(void)
  46 {
  47         /*
  48          * Bit mask for features in ECX:
  49          *   19   SSE4.1 (used for _mm_insert_epi32(), for AES-CTR)
  50          *   25   AES-NI
  51          */
  52 #define MASK   0x02080000
  53
  54 #if BR_AES_X86NI_GCC
  55         unsigned eax, ebx, ecx, edx;
  56
  57         if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
  58                 return (ecx & MASK) == MASK;
  59         } else {
  60                 return 0;
  61         }
  62 #elif BR_AES_X86NI_MSC
  63         int info[4];
  64
  65         __cpuid(info, 1);
  66         return ((uint32_t)info[2] & MASK) == MASK;
  67 #else
  68         return 0;
  69 #endif
  70
  71 #undef MASK
  72 }
  73
  74 BR_TARGET("sse2,aes")
  75 static inline __m128i
  76 expand_step128(__m128i k, __m128i k2)
  77 {
  78         k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
  79         k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
  80         k = _mm_xor_si128(k, _mm_slli_si128(k, 4));
  81         k2 = _mm_shuffle_epi32(k2, 0xFF);
  82         return _mm_xor_si128(k, k2);
  83 }
  84
  85 BR_TARGET("sse2,aes")
  86 static inline void
  87 expand_step192(__m128i *t1, __m128i *t2, __m128i *t3)
  88 {
  89         __m128i t4;
  90
  91         *t2 = _mm_shuffle_epi32(*t2, 0x55);
  92         t4 = _mm_slli_si128(*t1, 0x4);
  93         *t1 = _mm_xor_si128(*t1, t4);
  94         t4 = _mm_slli_si128(t4, 0x4);
  95         *t1 = _mm_xor_si128(*t1, t4);
  96         t4 = _mm_slli_si128(t4, 0x4);
  97         *t1 = _mm_xor_si128(*t1, t4);
  98         *t1 = _mm_xor_si128(*t1, *t2);
  99         *t2 = _mm_shuffle_epi32(*t1, 0xFF);
 100         t4 = _mm_slli_si128(*t3, 0x4);
 101         *t3 = _mm_xor_si128(*t3, t4);
 102         *t3 = _mm_xor_si128(*t3, *t2);
 103 }
 104
 105 BR_TARGET("sse2,aes")
 106 static inline void
 107 expand_step256_1(__m128i *t1, __m128i *t2)
 108 {
 109         __m128i t4;
 110
 111         *t2 = _mm_shuffle_epi32(*t2, 0xFF);
 112         t4 = _mm_slli_si128(*t1, 0x4);
 113         *t1 = _mm_xor_si128(*t1, t4);
 114         t4 = _mm_slli_si128(t4, 0x4);
 115         *t1 = _mm_xor_si128(*t1, t4);
 116         t4 = _mm_slli_si128(t4, 0x4);
 117         *t1 = _mm_xor_si128(*t1, t4);
 118         *t1 = _mm_xor_si128(*t1, *t2);
 119 }
 120
 121 BR_TARGET("sse2,aes")
 122 static inline void
 123 expand_step256_2(__m128i *t1, __m128i *t3)
 124 {
 125         __m128i t2, t4;
 126
 127         t4 = _mm_aeskeygenassist_si128(*t1, 0x0);
 128         t2 = _mm_shuffle_epi32(t4, 0xAA);
 129         t4 = _mm_slli_si128(*t3, 0x4);
 130         *t3 = _mm_xor_si128(*t3, t4);
 131         t4 = _mm_slli_si128(t4, 0x4);
 132         *t3 = _mm_xor_si128(*t3, t4);
 133         t4 = _mm_slli_si128(t4, 0x4);
 134         *t3 = _mm_xor_si128(*t3, t4);
 135         *t3 = _mm_xor_si128(*t3, t2);
 136 }
 137
 138 /*
 139  * Perform key schedule for AES, encryption direction. Subkeys are written
 140  * in sk[], and the number of rounds is returned. Key length MUST be 16,
 141  * 24 or 32 bytes.
 142  */
 143 BR_TARGET("sse2,aes")
 144 static unsigned
 145 x86ni_keysched(__m128i *sk, const void *key, size_t len)
 146 {
 147         const unsigned char *kb;
 148
 149 #define KEXP128(k, i, rcon)   do { \
 150                 k = expand_step128(k, _mm_aeskeygenassist_si128(k, rcon)); \
 151                 sk[i] = k; \
 152         } while (0)
 153
 154 #define KEXP192(i, rcon1, rcon2)   do { \
 155                 sk[(i) + 0] = t1; \
 156                 sk[(i) + 1] = t3; \
 157                 t2 = _mm_aeskeygenassist_si128(t3, rcon1); \
 158                 expand_step192(&t1, &t2, &t3); \
 159                 sk[(i) + 1] = _mm_castpd_si128(_mm_shuffle_pd( \
 160                         _mm_castsi128_pd(sk[(i) + 1]), \
 161                         _mm_castsi128_pd(t1), 0)); \
 162                 sk[(i) + 2] = _mm_castpd_si128(_mm_shuffle_pd( \
 163                         _mm_castsi128_pd(t1), \
 164                         _mm_castsi128_pd(t3), 1)); \
 165                 t2 = _mm_aeskeygenassist_si128(t3, rcon2); \
 166                 expand_step192(&t1, &t2, &t3); \
 167         } while (0)
 168
 169 #define KEXP256(i, rcon)   do { \
 170                 sk[(i) + 0] = t3; \
 171                 t2 = _mm_aeskeygenassist_si128(t3, rcon); \
 172                 expand_step256_1(&t1, &t2); \
 173                 sk[(i) + 1] = t1; \
 174                 expand_step256_2(&t1, &t3); \
 175         } while (0)
 176
 177         kb = key;
 178         switch (len) {
 179                 __m128i t1, t2, t3;
 180
 181         case 16:
 182                 t1 = _mm_loadu_si128((const void *)kb);
 183                 sk[0] = t1;
 184                 KEXP128(t1,  1, 0x01);
 185                 KEXP128(t1,  2, 0x02);
 186                 KEXP128(t1,  3, 0x04);
 187                 KEXP128(t1,  4, 0x08);
 188                 KEXP128(t1,  5, 0x10);
 189                 KEXP128(t1,  6, 0x20);
 190                 KEXP128(t1,  7, 0x40);
 191                 KEXP128(t1,  8, 0x80);
 192                 KEXP128(t1,  9, 0x1B);
 193                 KEXP128(t1, 10, 0x36);
 194                 return 10;
 195
 196         case 24:
 197                 t1 = _mm_loadu_si128((const void *)kb);
 198                 t3 = _mm_loadu_si128((const void *)(kb + 8));
 199                 t3 = _mm_shuffle_epi32(t3, 0x4E);
 200                 KEXP192(0, 0x01, 0x02);
 201                 KEXP192(3, 0x04, 0x08);
 202                 KEXP192(6, 0x10, 0x20);
 203                 KEXP192(9, 0x40, 0x80);
 204                 sk[12] = t1;
 205                 return 12;
 206
 207         case 32:
 208                 t1 = _mm_loadu_si128((const void *)kb);
 209                 t3 = _mm_loadu_si128((const void *)(kb + 16));
 210                 sk[0] = t1;
 211                 KEXP256( 1, 0x01);
 212                 KEXP256( 3, 0x02);
 213                 KEXP256( 5, 0x04);
 214                 KEXP256( 7, 0x08);
 215                 KEXP256( 9, 0x10);
 216                 KEXP256(11, 0x20);
 217                 sk[13] = t3;
 218                 t2 = _mm_aeskeygenassist_si128(t3, 0x40);
 219                 expand_step256_1(&t1, &t2);
 220                 sk[14] = t1;
 221                 return 14;
 222
 223         default:
 224                 return 0;
 225         }
 226
 227 #undef KEXP128
 228 #undef KEXP192
 229 #undef KEXP256
 230 }
 231
 232 /* see inner.h */
 233 BR_TARGET("sse2,aes")
 234 unsigned
 235 br_aes_x86ni_keysched_enc(unsigned char *skni, const void *key, size_t len)
 236 {
 237         __m128i sk[15];
 238         unsigned num_rounds;
 239
 240         num_rounds = x86ni_keysched(sk, key, len);
 241         memcpy(skni, sk, (num_rounds + 1) << 4);
 242         return num_rounds;
 243 }
 244
 245 /* see inner.h */
 246 BR_TARGET("sse2,aes")
 247 unsigned
 248 br_aes_x86ni_keysched_dec(unsigned char *skni, const void *key, size_t len)
 249 {
 250         __m128i sk[15];
 251         unsigned u, num_rounds;
 252
 253         num_rounds = x86ni_keysched(sk, key, len);
 254         _mm_storeu_si128((void *)skni, sk[num_rounds]);
 255         for (u = 1; u < num_rounds; u ++) {
 256                 _mm_storeu_si128((void *)(skni + (u << 4)),
 257                         _mm_aesimc_si128(sk[num_rounds - u]));
 258         }
 259         _mm_storeu_si128((void *)(skni + (num_rounds << 4)), sk[0]);
 260         return num_rounds;
 261 }
 262
 263 #endif