Crypto++  7.0
Free C++ class library of cryptographic schemes
rijndael-simd.cpp
1 // rijndael-simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 // AES-NI code originally written by Wei Dai.
4 //
5 // This source file uses intrinsics and built-ins to gain access to
6 // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7 // file is needed because additional CXXFLAGS are required to enable
8 // the appropriate instructions sets in some build configurations.
9 //
10 // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11 // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12 // mbedTLS under a debugger was helped for us to determine problems
13 // with our subkey generation and scheduling.
14 //
15 // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16 // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17 // For Power8 do not remove the casts, even when const-ness is cast away. It causes
18 // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19 // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20 // questions and filling the gaps in the IBM documentation.
21 //
22 
23 #include "pch.h"
24 #include "config.h"
25 #include "misc.h"
26 #include "adv-simd.h"
27 
28 // We set CRYPTOPP_POWER8_CRYPTO_AVAILABLE based on compiler version.
29 // If the crypto is not available, then we have to disable it here.
30 #if !(defined(__CRYPTO) || defined(_ARCH_PWR8) || defined(_ARCH_PWR9))
31 # undef CRYPTOPP_POWER8_CRYPTO_AVAILABLE
32 # undef CRYPTOPP_POWER8_AES_AVAILABLE
33 #endif
34 
35 #if (CRYPTOPP_AESNI_AVAILABLE)
36 # include <smmintrin.h>
37 # include <wmmintrin.h>
38 #endif
39 
40 // Use ARMv8 rather than NEON due to compiler inconsistencies
41 #if (CRYPTOPP_ARM_AES_AVAILABLE)
42 # include <arm_neon.h>
43 #endif
44 
45 // Can't use CRYPTOPP_ARM_XXX_AVAILABLE because too many
46 // compilers don't follow ACLE conventions for the include.
47 #if defined(CRYPTOPP_ARM_ACLE_AVAILABLE)
48 # include <stdint.h>
49 # include <arm_acle.h>
50 #endif
51 
52 #if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
53 # include "ppc-simd.h"
54 #endif
55 
56 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
57 # include <signal.h>
58 # include <setjmp.h>
59 #endif
60 
61 #ifndef EXCEPTION_EXECUTE_HANDLER
62 # define EXCEPTION_EXECUTE_HANDLER 1
63 #endif
64 
65 // Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
66 #define M128_CAST(x) ((__m128i *)(void *)(x))
67 #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
68 
69 NAMESPACE_BEGIN(CryptoPP)
70 
71 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
72 extern "C" {
73  typedef void (*SigHandler)(int);
74 
75  static jmp_buf s_jmpSIGILL;
76  static void SigIllHandler(int)
77  {
78  longjmp(s_jmpSIGILL, 1);
79  }
80 }
81 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
82 
83 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64)
84 bool CPU_ProbeAES()
85 {
86 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
87  return false;
88 #elif (CRYPTOPP_ARM_AES_AVAILABLE)
89 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
90  volatile bool result = true;
91  __try
92  {
93  // AES encrypt and decrypt
94  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
95  uint8x16_t r1 = vaeseq_u8(data, key);
96  uint8x16_t r2 = vaesdq_u8(data, key);
97  r1 = vaesmcq_u8(r1);
98  r2 = vaesimcq_u8(r2);
99 
100  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
101  }
102  __except (EXCEPTION_EXECUTE_HANDLER)
103  {
104  return false;
105  }
106  return result;
107 # else
108 
109  // longjmp and clobber warnings. Volatile is required.
110  // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
111  volatile bool result = true;
112 
113  volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
114  if (oldHandler == SIG_ERR)
115  return false;
116 
117  volatile sigset_t oldMask;
118  if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
119  return false;
120 
121  if (setjmp(s_jmpSIGILL))
122  result = false;
123  else
124  {
125  uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
126  uint8x16_t r1 = vaeseq_u8(data, key);
127  uint8x16_t r2 = vaesdq_u8(data, key);
128  r1 = vaesmcq_u8(r1);
129  r2 = vaesimcq_u8(r2);
130 
131  // Hack... GCC optimizes away the code and returns true
132  result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
133  }
134 
135  sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
136  signal(SIGILL, oldHandler);
137  return result;
138 # endif
139 #else
140  return false;
141 #endif // CRYPTOPP_ARM_AES_AVAILABLE
142 }
143 #endif // ARM32 or ARM64
144 
145 // ***************************** ARMv8 ***************************** //
146 
147 #if (CRYPTOPP_ARM_AES_AVAILABLE)
148 
149 ANONYMOUS_NAMESPACE_BEGIN
150 
151 static inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
152 {
153  CRYPTOPP_ASSERT(subkeys);
154  const byte *keys = reinterpret_cast<const byte*>(subkeys);
155  uint8x16_t block = vreinterpretq_u8_u64(data);
156 
157  // AES single round encryption
158  block = vaeseq_u8(block, vld1q_u8(keys+0*16));
159  // AES mix columns
160  block = vaesmcq_u8(block);
161 
162  for (unsigned int i=1; i<rounds-1; i+=2)
163  {
164  // AES single round encryption
165  block = vaeseq_u8(block, vld1q_u8(keys+i*16));
166  // AES mix columns
167  block = vaesmcq_u8(block);
168  // AES single round encryption
169  block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
170  // AES mix columns
171  block = vaesmcq_u8(block);
172  }
173 
174  // AES single round encryption
175  block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
176  // Final Add (bitwise Xor)
177  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
178 
179  data = vreinterpretq_u64_u8(block);
180 }
181 
182 static inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
183  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
184  const word32 *subkeys, unsigned int rounds)
185 {
186  CRYPTOPP_ASSERT(subkeys);
187  const byte *keys = reinterpret_cast<const byte*>(subkeys);
188 
189  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
190  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
191  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
192  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
193  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
194  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
195 
196  uint8x16_t key;
197  for (unsigned int i=0; i<rounds-1; ++i)
198  {
199  uint8x16_t key = vld1q_u8(keys+i*16);
200  // AES single round encryption
201  block0 = vaeseq_u8(block0, key);
202  // AES mix columns
203  block0 = vaesmcq_u8(block0);
204  // AES single round encryption
205  block1 = vaeseq_u8(block1, key);
206  // AES mix columns
207  block1 = vaesmcq_u8(block1);
208  // AES single round encryption
209  block2 = vaeseq_u8(block2, key);
210  // AES mix columns
211  block2 = vaesmcq_u8(block2);
212  // AES single round encryption
213  block3 = vaeseq_u8(block3, key);
214  // AES mix columns
215  block3 = vaesmcq_u8(block3);
216  // AES single round encryption
217  block4 = vaeseq_u8(block4, key);
218  // AES mix columns
219  block4 = vaesmcq_u8(block4);
220  // AES single round encryption
221  block5 = vaeseq_u8(block5, key);
222  // AES mix columns
223  block5 = vaesmcq_u8(block5);
224  }
225 
226  // AES single round encryption
227  key = vld1q_u8(keys+(rounds-1)*16);
228  block0 = vaeseq_u8(block0, key);
229  block1 = vaeseq_u8(block1, key);
230  block2 = vaeseq_u8(block2, key);
231  block3 = vaeseq_u8(block3, key);
232  block4 = vaeseq_u8(block4, key);
233  block5 = vaeseq_u8(block5, key);
234 
235  // Final Add (bitwise Xor)
236  key = vld1q_u8(keys+rounds*16);
237  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
238  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
239  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
240  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
241  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
242  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
243 }
244 
245 static inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
246 {
247  CRYPTOPP_ASSERT(subkeys);
248  const byte *keys = reinterpret_cast<const byte*>(subkeys);
249  uint8x16_t block = vreinterpretq_u8_u64(data);
250 
251  // AES single round decryption
252  block = vaesdq_u8(block, vld1q_u8(keys+0*16));
253  // AES inverse mix columns
254  block = vaesimcq_u8(block);
255 
256  for (unsigned int i=1; i<rounds-1; i+=2)
257  {
258  // AES single round decryption
259  block = vaesdq_u8(block, vld1q_u8(keys+i*16));
260  // AES inverse mix columns
261  block = vaesimcq_u8(block);
262  // AES single round decryption
263  block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
264  // AES inverse mix columns
265  block = vaesimcq_u8(block);
266  }
267 
268  // AES single round decryption
269  block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
270  // Final Add (bitwise Xor)
271  block = veorq_u8(block, vld1q_u8(keys+rounds*16));
272 
273  data = vreinterpretq_u64_u8(block);
274 }
275 
276 static inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
277  uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
278  const word32 *subkeys, unsigned int rounds)
279 {
280  CRYPTOPP_ASSERT(subkeys);
281  const byte *keys = reinterpret_cast<const byte*>(subkeys);
282 
283  uint8x16_t block0 = vreinterpretq_u8_u64(data0);
284  uint8x16_t block1 = vreinterpretq_u8_u64(data1);
285  uint8x16_t block2 = vreinterpretq_u8_u64(data2);
286  uint8x16_t block3 = vreinterpretq_u8_u64(data3);
287  uint8x16_t block4 = vreinterpretq_u8_u64(data4);
288  uint8x16_t block5 = vreinterpretq_u8_u64(data5);
289 
290  uint8x16_t key;
291  for (unsigned int i=0; i<rounds-1; ++i)
292  {
293  key = vld1q_u8(keys+i*16);
294  // AES single round decryption
295  block0 = vaesdq_u8(block0, key);
296  // AES inverse mix columns
297  block0 = vaesimcq_u8(block0);
298  // AES single round decryption
299  block1 = vaesdq_u8(block1, key);
300  // AES inverse mix columns
301  block1 = vaesimcq_u8(block1);
302  // AES single round decryption
303  block2 = vaesdq_u8(block2, key);
304  // AES inverse mix columns
305  block2 = vaesimcq_u8(block2);
306  // AES single round decryption
307  block3 = vaesdq_u8(block3, key);
308  // AES inverse mix columns
309  block3 = vaesimcq_u8(block3);
310  // AES single round decryption
311  block4 = vaesdq_u8(block4, key);
312  // AES inverse mix columns
313  block4 = vaesimcq_u8(block4);
314  // AES single round decryption
315  block5 = vaesdq_u8(block5, key);
316  // AES inverse mix columns
317  block5 = vaesimcq_u8(block5);
318  }
319 
320  // AES single round decryption
321  key = vld1q_u8(keys+(rounds-1)*16);
322  block0 = vaesdq_u8(block0, key);
323  block1 = vaesdq_u8(block1, key);
324  block2 = vaesdq_u8(block2, key);
325  block3 = vaesdq_u8(block3, key);
326  block4 = vaesdq_u8(block4, key);
327  block5 = vaesdq_u8(block5, key);
328 
329  // Final Add (bitwise Xor)
330  key = vld1q_u8(keys+rounds*16);
331  data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
332  data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
333  data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
334  data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
335  data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
336  data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
337 }
338 
339 ANONYMOUS_NAMESPACE_END
340 
341 size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
342  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
343 {
344  return AdvancedProcessBlocks128_NEON1x6(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
345  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
346 }
347 
348 size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
349  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
350 {
351  return AdvancedProcessBlocks128_NEON1x6(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
352  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
353 }
354 
355 #endif // CRYPTOPP_ARM_AES_AVAILABLE
356 
357 // ***************************** AES-NI ***************************** //
358 
359 #if (CRYPTOPP_AESNI_AVAILABLE)
360 
361 ANONYMOUS_NAMESPACE_BEGIN
362 
363 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
364 CRYPTOPP_ALIGN_DATA(16)
365 const word32 s_rconLE[] = {
366  0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
367 };
368 
369 static inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
370 {
371  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
372 
373  block = _mm_xor_si128(block, skeys[0]);
374  for (unsigned int i=1; i<rounds-1; i+=2)
375  {
376  block = _mm_aesenc_si128(block, skeys[i]);
377  block = _mm_aesenc_si128(block, skeys[i+1]);
378  }
379  block = _mm_aesenc_si128(block, skeys[rounds-1]);
380  block = _mm_aesenclast_si128(block, skeys[rounds]);
381 }
382 
383 static inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
384  MAYBE_CONST word32 *subkeys, unsigned int rounds)
385 {
386  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
387 
388  __m128i rk = skeys[0];
389  block0 = _mm_xor_si128(block0, rk);
390  block1 = _mm_xor_si128(block1, rk);
391  block2 = _mm_xor_si128(block2, rk);
392  block3 = _mm_xor_si128(block3, rk);
393  for (unsigned int i=1; i<rounds; i++)
394  {
395  rk = skeys[i];
396  block0 = _mm_aesenc_si128(block0, rk);
397  block1 = _mm_aesenc_si128(block1, rk);
398  block2 = _mm_aesenc_si128(block2, rk);
399  block3 = _mm_aesenc_si128(block3, rk);
400  }
401  rk = skeys[rounds];
402  block0 = _mm_aesenclast_si128(block0, rk);
403  block1 = _mm_aesenclast_si128(block1, rk);
404  block2 = _mm_aesenclast_si128(block2, rk);
405  block3 = _mm_aesenclast_si128(block3, rk);
406 }
407 
408 static inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
409 {
410  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
411 
412  block = _mm_xor_si128(block, skeys[0]);
413  for (unsigned int i=1; i<rounds-1; i+=2)
414  {
415  block = _mm_aesdec_si128(block, skeys[i]);
416  block = _mm_aesdec_si128(block, skeys[i+1]);
417  }
418  block = _mm_aesdec_si128(block, skeys[rounds-1]);
419  block = _mm_aesdeclast_si128(block, skeys[rounds]);
420 }
421 
422 static inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
423  MAYBE_CONST word32 *subkeys, unsigned int rounds)
424 {
425  const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
426 
427  __m128i rk = skeys[0];
428  block0 = _mm_xor_si128(block0, rk);
429  block1 = _mm_xor_si128(block1, rk);
430  block2 = _mm_xor_si128(block2, rk);
431  block3 = _mm_xor_si128(block3, rk);
432  for (unsigned int i=1; i<rounds; i++)
433  {
434  rk = skeys[i];
435  block0 = _mm_aesdec_si128(block0, rk);
436  block1 = _mm_aesdec_si128(block1, rk);
437  block2 = _mm_aesdec_si128(block2, rk);
438  block3 = _mm_aesdec_si128(block3, rk);
439  }
440  rk = skeys[rounds];
441  block0 = _mm_aesdeclast_si128(block0, rk);
442  block1 = _mm_aesdeclast_si128(block1, rk);
443  block2 = _mm_aesdeclast_si128(block2, rk);
444  block3 = _mm_aesdeclast_si128(block3, rk);
445 }
446 
447 ANONYMOUS_NAMESPACE_END
448 
449 void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
450 {
451  const size_t rounds = keyLen / 4 + 6;
452  const word32 *rc = s_rconLE;
453 
454  __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
455  std::memcpy(rk, userKey, keyLen);
456 
457  // keySize: m_key allocates 4*(rounds+1) word32's.
458  const size_t keySize = 4*(rounds+1);
459  const word32* end = rk + keySize;
460 
461  while (true)
462  {
463  rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
464  rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
465  rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
466  rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
467 
468  if (rk + keyLen/4 + 4 == end)
469  break;
470 
471  if (keyLen == 24)
472  {
473  rk[10] = rk[ 4] ^ rk[ 9];
474  rk[11] = rk[ 5] ^ rk[10];
475  temp = _mm_insert_epi32(temp, rk[11], 3);
476  }
477  else if (keyLen == 32)
478  {
479  temp = _mm_insert_epi32(temp, rk[11], 3);
480  rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
481  rk[13] = rk[ 5] ^ rk[12];
482  rk[14] = rk[ 6] ^ rk[13];
483  rk[15] = rk[ 7] ^ rk[14];
484  temp = _mm_insert_epi32(temp, rk[15], 3);
485  }
486  else
487  {
488  temp = _mm_insert_epi32(temp, rk[7], 3);
489  }
490 
491  rk += keyLen/4;
492  }
493 }
494 
495 void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
496 {
497  unsigned int i, j;
498  __m128i temp;
499 
500  vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
501 
502  for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
503  {
504  temp = _mm_aesimc_si128(*M128_CAST(key+i));
505  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
506  *M128_CAST(key+j) = temp;
507  }
508 
509  *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
510 }
511 
512 size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
513  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
514 {
515  // SunCC workaround
516  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
517  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
518  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
519 
520  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
521  sk, rounds, ib, xb, outBlocks, length, flags);
522 }
523 
524 size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
525  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
526 {
527  MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
528  MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
529  MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
530 
531  return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
532  sk, rounds, ib, xb, outBlocks, length, flags);
533 }
534 
535 #endif // CRYPTOPP_AESNI_AVAILABLE
536 
537 // ***************************** Power 8 ***************************** //
538 
539 #if (CRYPTOPP_POWER8_AES_AVAILABLE)
540 
541 ANONYMOUS_NAMESPACE_BEGIN
542 
543 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
544 CRYPTOPP_ALIGN_DATA(16)
545 static const uint32_t s_rconBE[] = {
546  0x01000000, 0x02000000, 0x04000000, 0x08000000,
547  0x10000000, 0x20000000, 0x40000000, 0x80000000,
548  0x1B000000, 0x36000000
549 };
550 
551 static inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
552 {
553  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
554  const byte *keys = reinterpret_cast<const byte*>(subkeys);
555 
556  uint32x4_p k = VectorLoadKey(keys);
557  block = VectorXor(block, k);
558 
559  for (size_t i=1; i<rounds-1; i+=2)
560  {
561  block = VectorEncrypt(block, VectorLoadKey( i*16, keys));
562  block = VectorEncrypt(block, VectorLoadKey((i+1)*16, keys));
563  }
564 
565  block = VectorEncrypt(block, VectorLoadKey((rounds-1)*16, keys));
566  block = VectorEncryptLast(block, VectorLoadKey(rounds*16, keys));
567 }
568 
569 static inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
570  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
571  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
572 {
573  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
574  const byte *keys = reinterpret_cast<const byte*>(subkeys);
575 
576  uint32x4_p k = VectorLoadKey(keys);
577  block0 = VectorXor(block0, k);
578  block1 = VectorXor(block1, k);
579  block2 = VectorXor(block2, k);
580  block3 = VectorXor(block3, k);
581  block4 = VectorXor(block4, k);
582  block5 = VectorXor(block5, k);
583 
584  for (size_t i=1; i<rounds; ++i)
585  {
586  k = VectorLoadKey(i*16, keys);
587  block0 = VectorEncrypt(block0, k);
588  block1 = VectorEncrypt(block1, k);
589  block2 = VectorEncrypt(block2, k);
590  block3 = VectorEncrypt(block3, k);
591  block4 = VectorEncrypt(block4, k);
592  block5 = VectorEncrypt(block5, k);
593  }
594 
595  k = VectorLoadKey(rounds*16, keys);
596  block0 = VectorEncryptLast(block0, k);
597  block1 = VectorEncryptLast(block1, k);
598  block2 = VectorEncryptLast(block2, k);
599  block3 = VectorEncryptLast(block3, k);
600  block4 = VectorEncryptLast(block4, k);
601  block5 = VectorEncryptLast(block5, k);
602 }
603 
604 static inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
605 {
606  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
607  const byte *keys = reinterpret_cast<const byte*>(subkeys);
608 
609  uint32x4_p k = VectorLoadKey(rounds*16, keys);
610  block = VectorXor(block, k);
611 
612  for (size_t i=rounds-1; i>1; i-=2)
613  {
614  block = VectorDecrypt(block, VectorLoadKey( i*16, keys));
615  block = VectorDecrypt(block, VectorLoadKey((i-1)*16, keys));
616  }
617 
618  block = VectorDecrypt(block, VectorLoadKey(16, keys));
619  block = VectorDecryptLast(block, VectorLoadKey(0, keys));
620 }
621 
622 static inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
623  uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
624  uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
625 {
626  CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
627  const byte *keys = reinterpret_cast<const byte*>(subkeys);
628 
629  uint32x4_p k = VectorLoadKey(rounds*16, keys);
630  block0 = VectorXor(block0, k);
631  block1 = VectorXor(block1, k);
632  block2 = VectorXor(block2, k);
633  block3 = VectorXor(block3, k);
634  block4 = VectorXor(block4, k);
635  block5 = VectorXor(block5, k);
636 
637  for (size_t i=rounds-1; i>0; --i)
638  {
639  k = VectorLoadKey(i*16, keys);
640  block0 = VectorDecrypt(block0, k);
641  block1 = VectorDecrypt(block1, k);
642  block2 = VectorDecrypt(block2, k);
643  block3 = VectorDecrypt(block3, k);
644  block4 = VectorDecrypt(block4, k);
645  block5 = VectorDecrypt(block5, k);
646  }
647 
648  k = VectorLoadKey(0, keys);
649  block0 = VectorDecryptLast(block0, k);
650  block1 = VectorDecryptLast(block1, k);
651  block2 = VectorDecryptLast(block2, k);
652  block3 = VectorDecryptLast(block3, k);
653  block4 = VectorDecryptLast(block4, k);
654  block5 = VectorDecryptLast(block5, k);
655 }
656 
657 ANONYMOUS_NAMESPACE_END
658 
659 void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
660 {
661  const size_t rounds = keyLen / 4 + 6;
662  const word32 *rc = s_rconBE;
663 
664  GetUserKey(BIG_ENDIAN_ORDER, rk, keyLen/4, userKey, keyLen);
665  word32 *rk_saved = rk, temp; // unused in big-endian
666  CRYPTOPP_UNUSED(rk_saved);
667 
668  // keySize: m_key allocates 4*(rounds+1) word32's.
669  const size_t keySize = 4*(rounds+1);
670  const word32* end = rk + keySize;
671 
672  while (true)
673  {
674  temp = rk[keyLen/4-1];
675  word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
676  (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
677  rk[keyLen/4] = rk[0] ^ x ^ *(rc++);
678  rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
679  rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
680  rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
681 
682  if (rk + keyLen/4 + 4 == end)
683  break;
684 
685  if (keyLen == 24)
686  {
687  rk[10] = rk[ 4] ^ rk[ 9];
688  rk[11] = rk[ 5] ^ rk[10];
689  }
690  else if (keyLen == 32)
691  {
692  temp = rk[11];
693  rk[12] = rk[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
694  rk[13] = rk[ 5] ^ rk[12];
695  rk[14] = rk[ 6] ^ rk[13];
696  rk[15] = rk[ 7] ^ rk[14];
697  }
698  rk += keyLen/4;
699  }
700 
701 #if defined(CRYPTOPP_LITTLE_ENDIAN)
702  rk = rk_saved;
703  const uint8x16_p mask = ((uint8x16_p){12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3});
704  const uint8x16_p zero = {0};
705 
706  unsigned int i=0;
707  for (i=0; i<rounds; i+=2, rk+=8)
708  {
709  const uint8x16_p d1 = vec_vsx_ld( 0, (uint8_t*)rk);
710  const uint8x16_p d2 = vec_vsx_ld(16, (uint8_t*)rk);
711  vec_vsx_st(vec_perm(d1, zero, mask), 0, (uint8_t*)rk);
712  vec_vsx_st(vec_perm(d2, zero, mask), 16, (uint8_t*)rk);
713  }
714 
715  for ( ; i<rounds+1; i++, rk+=4)
716  vec_vsx_st(vec_perm(vec_vsx_ld(0, (uint8_t*)rk), zero, mask), 0, (uint8_t*)rk);
717 #endif
718 }
719 
720 size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
721  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
722 {
723  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
724  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
725 }
726 
727 size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
728  const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
729 {
730  return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
731  subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
732 }
733 
734 #endif // CRYPTOPP_POWER8_AES_AVAILABLE
735 NAMESPACE_END
T1 VectorEncrypt(const T1 &state, const T2 &key)
One round of AES encryption.
Definition: ppc-simd.h:438
Utility functions for the Crypto++ library.
T1 VectorEncryptLast(const T1 &state, const T2 &key)
Final round of AES encryption.
Definition: ppc-simd.h:458
Library configuration file.
T1 VectorDecryptLast(const T1 &state, const T2 &key)
Final round of AES decryption.
Definition: ppc-simd.h:498
Support functions for PowerPC and vector operations.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1030
Precompiled header file.
uint32x4_p VectorLoadKey(const byte src[16])
Loads a vector from a byte array.
Definition: ppc-simd.h:213
byte order is big-endian
Definition: cryptlib.h:144
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:60
T1 VectorXor(const T1 &vec1, const T2 &vec2)
XOR two vectors.
Definition: ppc-simd.h:373
T1 VectorDecrypt(const T1 &state, const T2 &key)
One round of AES decryption.
Definition: ppc-simd.h:478
Crypto++ library namespace.
void vec_swap(T &a, T &b)
Swaps two variables which are arrays.
Definition: misc.h:471