Crypto++ 8.2
Free C&
rijndael_simd.cpp
1// rijndael_simd.cpp - written and placed in the public domain by
2// Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3// AES-NI code originally written by Wei Dai.
4//
5// This source file uses intrinsics and built-ins to gain access to
6// AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7// file is needed because additional CXXFLAGS are required to enable
8// the appropriate instructions sets in some build configurations.
9//
10// ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11// Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12// mbedTLS under a debugger was helped for us to determine problems
13// with our subkey generation and scheduling.
14//
15// AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16// http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17// For Power8 do not remove the casts, even when const-ness is cast away. It causes
18// failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19// absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20// questions and filling the gaps in the IBM documentation.
21//
22
23#include "pch.h"
24#include "config.h"
25#include "misc.h"
26
27#if (CRYPTOPP_AESNI_AVAILABLE)
28# include "adv_simd.h"
29# include <emmintrin.h>
30# include <smmintrin.h>
31# include <wmmintrin.h>
32#endif
33
34// C1189: error: This header is specific to ARM targets
35#if (CRYPTOPP_ARM_NEON_AVAILABLE)
36# include "adv_simd.h"
37# ifndef _M_ARM64
38# include <arm_neon.h>
39# endif
40#endif
41
42#if (CRYPTOPP_ARM_ACLE_AVAILABLE)
43# include <stdint.h>
44# include <arm_acle.h>
45#endif
46
47#if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
48# include "adv_simd.h"
49# include "ppc_simd.h"
50#endif
51
52#ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
53# include <signal.h>
54# include <setjmp.h>
55#endif
56
57#ifndef EXCEPTION_EXECUTE_HANDLER
58# define EXCEPTION_EXECUTE_HANDLER 1
59#endif
60
61// Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
62#define M128_CAST(x) ((__m128i *)(void *)(x))
63#define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
64
65// Squash MS LNK4221 and libtool warnings
66extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__;
67
68NAMESPACE_BEGIN(CryptoPP)
69
70// ************************* Feature Probes ************************* //
71
72#ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
73extern "C" {
74 typedef void (*SigHandler)(int);
75
76 static jmp_buf s_jmpSIGILL;
77 static void SigIllHandler(int)
78 {
79 longjmp(s_jmpSIGILL, 1);
80 }
81}
82#endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
83
84#if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
85bool CPU_ProbeAES()
86{
87#if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
88 return false;
89#elif (CRYPTOPP_ARM_AES_AVAILABLE)
90# if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
91 volatile bool result = true;
92 __try
93 {
94 // AES encrypt and decrypt
95 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
96 uint8x16_t r1 = vaeseq_u8(data, key);
97 uint8x16_t r2 = vaesdq_u8(data, key);
98 r1 = vaesmcq_u8(r1);
99 r2 = vaesimcq_u8(r2);
100
101 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
102 }
103 __except (EXCEPTION_EXECUTE_HANDLER)
104 {
105 return false;
106 }
107 return result;
108# else
109
110 // longjmp and clobber warnings. Volatile is required.
111 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
112 volatile bool result = true;
113
114 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
115 if (oldHandler == SIG_ERR)
116 return false;
117
118 volatile sigset_t oldMask;
119 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
120 return false;
121
122 if (setjmp(s_jmpSIGILL))
123 result = false;
124 else
125 {
126 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
127 uint8x16_t r1 = vaeseq_u8(data, key);
128 uint8x16_t r2 = vaesdq_u8(data, key);
129 r1 = vaesmcq_u8(r1);
130 r2 = vaesimcq_u8(r2);
131
132 // Hack... GCC optimizes away the code and returns true
133 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
134 }
135
136 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
137 signal(SIGILL, oldHandler);
138 return result;
139# endif
140#else
141 return false;
142#endif // CRYPTOPP_ARM_AES_AVAILABLE
143}
144#endif // ARM32 or ARM64
145
146// ***************************** ARMv8 ***************************** //
147
148#if (CRYPTOPP_ARM_AES_AVAILABLE)
149
150ANONYMOUS_NAMESPACE_BEGIN
151
152static inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
153{
154 CRYPTOPP_ASSERT(subkeys);
155 const byte *keys = reinterpret_cast<const byte*>(subkeys);
156 uint8x16_t block = vreinterpretq_u8_u64(data);
157
158 // AES single round encryption
159 block = vaeseq_u8(block, vld1q_u8(keys+0*16));
160 // AES mix columns
161 block = vaesmcq_u8(block);
162
163 for (unsigned int i=1; i<rounds-1; i+=2)
164 {
165 // AES single round encryption
166 block = vaeseq_u8(block, vld1q_u8(keys+i*16));
167 // AES mix columns
168 block = vaesmcq_u8(block);
169 // AES single round encryption
170 block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
171 // AES mix columns
172 block = vaesmcq_u8(block);
173 }
174
175 // AES single round encryption
176 block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
177 // Final Add (bitwise Xor)
178 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
179
180 data = vreinterpretq_u64_u8(block);
181}
182
183static inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
184 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
185 const word32 *subkeys, unsigned int rounds)
186{
187 CRYPTOPP_ASSERT(subkeys);
188 const byte *keys = reinterpret_cast<const byte*>(subkeys);
189
190 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
191 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
192 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
193 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
194 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
195 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
196
197 uint8x16_t key;
198 for (unsigned int i=0; i<rounds-1; ++i)
199 {
200 key = vld1q_u8(keys+i*16);
201 // AES single round encryption
202 block0 = vaeseq_u8(block0, key);
203 // AES mix columns
204 block0 = vaesmcq_u8(block0);
205 // AES single round encryption
206 block1 = vaeseq_u8(block1, key);
207 // AES mix columns
208 block1 = vaesmcq_u8(block1);
209 // AES single round encryption
210 block2 = vaeseq_u8(block2, key);
211 // AES mix columns
212 block2 = vaesmcq_u8(block2);
213 // AES single round encryption
214 block3 = vaeseq_u8(block3, key);
215 // AES mix columns
216 block3 = vaesmcq_u8(block3);
217 // AES single round encryption
218 block4 = vaeseq_u8(block4, key);
219 // AES mix columns
220 block4 = vaesmcq_u8(block4);
221 // AES single round encryption
222 block5 = vaeseq_u8(block5, key);
223 // AES mix columns
224 block5 = vaesmcq_u8(block5);
225 }
226
227 // AES single round encryption
228 key = vld1q_u8(keys+(rounds-1)*16);
229 block0 = vaeseq_u8(block0, key);
230 block1 = vaeseq_u8(block1, key);
231 block2 = vaeseq_u8(block2, key);
232 block3 = vaeseq_u8(block3, key);
233 block4 = vaeseq_u8(block4, key);
234 block5 = vaeseq_u8(block5, key);
235
236 // Final Add (bitwise Xor)
237 key = vld1q_u8(keys+rounds*16);
238 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
239 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
240 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
241 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
242 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
243 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
244}
245
246static inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
247{
248 CRYPTOPP_ASSERT(subkeys);
249 const byte *keys = reinterpret_cast<const byte*>(subkeys);
250 uint8x16_t block = vreinterpretq_u8_u64(data);
251
252 // AES single round decryption
253 block = vaesdq_u8(block, vld1q_u8(keys+0*16));
254 // AES inverse mix columns
255 block = vaesimcq_u8(block);
256
257 for (unsigned int i=1; i<rounds-1; i+=2)
258 {
259 // AES single round decryption
260 block = vaesdq_u8(block, vld1q_u8(keys+i*16));
261 // AES inverse mix columns
262 block = vaesimcq_u8(block);
263 // AES single round decryption
264 block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
265 // AES inverse mix columns
266 block = vaesimcq_u8(block);
267 }
268
269 // AES single round decryption
270 block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
271 // Final Add (bitwise Xor)
272 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
273
274 data = vreinterpretq_u64_u8(block);
275}
276
277static inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
278 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
279 const word32 *subkeys, unsigned int rounds)
280{
281 CRYPTOPP_ASSERT(subkeys);
282 const byte *keys = reinterpret_cast<const byte*>(subkeys);
283
284 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
285 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
286 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
287 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
288 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
289 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
290
291 uint8x16_t key;
292 for (unsigned int i=0; i<rounds-1; ++i)
293 {
294 key = vld1q_u8(keys+i*16);
295 // AES single round decryption
296 block0 = vaesdq_u8(block0, key);
297 // AES inverse mix columns
298 block0 = vaesimcq_u8(block0);
299 // AES single round decryption
300 block1 = vaesdq_u8(block1, key);
301 // AES inverse mix columns
302 block1 = vaesimcq_u8(block1);
303 // AES single round decryption
304 block2 = vaesdq_u8(block2, key);
305 // AES inverse mix columns
306 block2 = vaesimcq_u8(block2);
307 // AES single round decryption
308 block3 = vaesdq_u8(block3, key);
309 // AES inverse mix columns
310 block3 = vaesimcq_u8(block3);
311 // AES single round decryption
312 block4 = vaesdq_u8(block4, key);
313 // AES inverse mix columns
314 block4 = vaesimcq_u8(block4);
315 // AES single round decryption
316 block5 = vaesdq_u8(block5, key);
317 // AES inverse mix columns
318 block5 = vaesimcq_u8(block5);
319 }
320
321 // AES single round decryption
322 key = vld1q_u8(keys+(rounds-1)*16);
323 block0 = vaesdq_u8(block0, key);
324 block1 = vaesdq_u8(block1, key);
325 block2 = vaesdq_u8(block2, key);
326 block3 = vaesdq_u8(block3, key);
327 block4 = vaesdq_u8(block4, key);
328 block5 = vaesdq_u8(block5, key);
329
330 // Final Add (bitwise Xor)
331 key = vld1q_u8(keys+rounds*16);
332 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
333 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
334 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
335 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
336 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
337 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
338}
339
340ANONYMOUS_NAMESPACE_END
341
342size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
343 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
344{
345 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
346 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
347}
348
349size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
350 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
351{
352 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
353 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
354}
355
356#endif // CRYPTOPP_ARM_AES_AVAILABLE
357
358// ***************************** AES-NI ***************************** //
359
360#if (CRYPTOPP_AESNI_AVAILABLE)
361
362ANONYMOUS_NAMESPACE_BEGIN
363
364/* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
365CRYPTOPP_ALIGN_DATA(16)
366const word32 s_rconLE[] = {
367 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
368};
369
370static inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
371{
372 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
373
374 block = _mm_xor_si128(block, skeys[0]);
375 for (unsigned int i=1; i<rounds-1; i+=2)
376 {
377 block = _mm_aesenc_si128(block, skeys[i]);
378 block = _mm_aesenc_si128(block, skeys[i+1]);
379 }
380 block = _mm_aesenc_si128(block, skeys[rounds-1]);
381 block = _mm_aesenclast_si128(block, skeys[rounds]);
382}
383
384static inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
385 MAYBE_CONST word32 *subkeys, unsigned int rounds)
386{
387 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
388
389 __m128i rk = skeys[0];
390 block0 = _mm_xor_si128(block0, rk);
391 block1 = _mm_xor_si128(block1, rk);
392 block2 = _mm_xor_si128(block2, rk);
393 block3 = _mm_xor_si128(block3, rk);
394 for (unsigned int i=1; i<rounds; i++)
395 {
396 rk = skeys[i];
397 block0 = _mm_aesenc_si128(block0, rk);
398 block1 = _mm_aesenc_si128(block1, rk);
399 block2 = _mm_aesenc_si128(block2, rk);
400 block3 = _mm_aesenc_si128(block3, rk);
401 }
402 rk = skeys[rounds];
403 block0 = _mm_aesenclast_si128(block0, rk);
404 block1 = _mm_aesenclast_si128(block1, rk);
405 block2 = _mm_aesenclast_si128(block2, rk);
406 block3 = _mm_aesenclast_si128(block3, rk);
407}
408
409static inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
410{
411 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
412
413 block = _mm_xor_si128(block, skeys[0]);
414 for (unsigned int i=1; i<rounds-1; i+=2)
415 {
416 block = _mm_aesdec_si128(block, skeys[i]);
417 block = _mm_aesdec_si128(block, skeys[i+1]);
418 }
419 block = _mm_aesdec_si128(block, skeys[rounds-1]);
420 block = _mm_aesdeclast_si128(block, skeys[rounds]);
421}
422
423static inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
424 MAYBE_CONST word32 *subkeys, unsigned int rounds)
425{
426 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
427
428 __m128i rk = skeys[0];
429 block0 = _mm_xor_si128(block0, rk);
430 block1 = _mm_xor_si128(block1, rk);
431 block2 = _mm_xor_si128(block2, rk);
432 block3 = _mm_xor_si128(block3, rk);
433 for (unsigned int i=1; i<rounds; i++)
434 {
435 rk = skeys[i];
436 block0 = _mm_aesdec_si128(block0, rk);
437 block1 = _mm_aesdec_si128(block1, rk);
438 block2 = _mm_aesdec_si128(block2, rk);
439 block3 = _mm_aesdec_si128(block3, rk);
440 }
441 rk = skeys[rounds];
442 block0 = _mm_aesdeclast_si128(block0, rk);
443 block1 = _mm_aesdeclast_si128(block1, rk);
444 block2 = _mm_aesdeclast_si128(block2, rk);
445 block3 = _mm_aesdeclast_si128(block3, rk);
446}
447
448ANONYMOUS_NAMESPACE_END
449
450void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
451{
452 const size_t rounds = keyLen / 4 + 6;
453 const word32 *rc = s_rconLE;
454
455 __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
456 std::memcpy(rk, userKey, keyLen);
457
458 // keySize: m_key allocates 4*(rounds+1) word32's.
459 const size_t keySize = 4*(rounds+1);
460 const word32* end = rk + keySize;
461
462 while (true)
463 {
464 rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
465 rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
466 rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
467 rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
468
469 if (rk + keyLen/4 + 4 == end)
470 break;
471
472 if (keyLen == 24)
473 {
474 rk[10] = rk[ 4] ^ rk[ 9];
475 rk[11] = rk[ 5] ^ rk[10];
476 temp = _mm_insert_epi32(temp, rk[11], 3);
477 }
478 else if (keyLen == 32)
479 {
480 temp = _mm_insert_epi32(temp, rk[11], 3);
481 rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
482 rk[13] = rk[ 5] ^ rk[12];
483 rk[14] = rk[ 6] ^ rk[13];
484 rk[15] = rk[ 7] ^ rk[14];
485 temp = _mm_insert_epi32(temp, rk[15], 3);
486 }
487 else
488 {
489 temp = _mm_insert_epi32(temp, rk[7], 3);
490 }
491
492 rk += keyLen/4;
493 }
494}
495
496void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
497{
498 unsigned int i, j;
499 __m128i temp;
500
501 vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
502
503 for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
504 {
505 temp = _mm_aesimc_si128(*M128_CAST(key+i));
506 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
507 *M128_CAST(key+j) = temp;
508 }
509
510 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
511}
512
513size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
514 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
515{
516 // SunCC workaround
517 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
518 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
519 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
520
521 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
522 sk, rounds, ib, xb, outBlocks, length, flags);
523}
524
525size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
526 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
527{
528 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
529 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
530 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
531
532 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
533 sk, rounds, ib, xb, outBlocks, length, flags);
534}
535
536#endif // CRYPTOPP_AESNI_AVAILABLE
537
538// ************************** Power 8 Crypto ************************** //
539
540#if (CRYPTOPP_POWER8_AES_AVAILABLE)
541
542ANONYMOUS_NAMESPACE_BEGIN
543
544/* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
545CRYPTOPP_ALIGN_DATA(16)
546static const uint32_t s_rconBE[] = {
547 0x01000000, 0x02000000, 0x04000000, 0x08000000,
548 0x10000000, 0x20000000, 0x40000000, 0x80000000,
549 0x1B000000, 0x36000000
550};
551
552static inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
553{
554 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
555 const byte *keys = reinterpret_cast<const byte*>(subkeys);
556
557 uint32x4_p k = VecLoad(keys);
558 block = VecXor(block, k);
559
560 for (size_t i=1; i<rounds-1; i+=2)
561 {
562 block = VecEncrypt(block, VecLoad( i*16, keys));
563 block = VecEncrypt(block, VecLoad((i+1)*16, keys));
564 }
565
566 block = VecEncrypt(block, VecLoad((rounds-1)*16, keys));
567 block = VecEncryptLast(block, VecLoad(rounds*16, keys));
568}
569
570static inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
571 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
572 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
573{
574 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
575 const byte *keys = reinterpret_cast<const byte*>(subkeys);
576
577 uint32x4_p k = VecLoad(keys);
578 block0 = VecXor(block0, k);
579 block1 = VecXor(block1, k);
580 block2 = VecXor(block2, k);
581 block3 = VecXor(block3, k);
582 block4 = VecXor(block4, k);
583 block5 = VecXor(block5, k);
584
585 for (size_t i=1; i<rounds; ++i)
586 {
587 k = VecLoad(i*16, keys);
588 block0 = VecEncrypt(block0, k);
589 block1 = VecEncrypt(block1, k);
590 block2 = VecEncrypt(block2, k);
591 block3 = VecEncrypt(block3, k);
592 block4 = VecEncrypt(block4, k);
593 block5 = VecEncrypt(block5, k);
594 }
595
596 k = VecLoad(rounds*16, keys);
597 block0 = VecEncryptLast(block0, k);
598 block1 = VecEncryptLast(block1, k);
599 block2 = VecEncryptLast(block2, k);
600 block3 = VecEncryptLast(block3, k);
601 block4 = VecEncryptLast(block4, k);
602 block5 = VecEncryptLast(block5, k);
603}
604
605static inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
606{
607 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
608 const byte *keys = reinterpret_cast<const byte*>(subkeys);
609
610 uint32x4_p k = VecLoad(rounds*16, keys);
611 block = VecXor(block, k);
612
613 for (size_t i=rounds-1; i>1; i-=2)
614 {
615 block = VecDecrypt(block, VecLoad( i*16, keys));
616 block = VecDecrypt(block, VecLoad((i-1)*16, keys));
617 }
618
619 block = VecDecrypt(block, VecLoad(16, keys));
620 block = VecDecryptLast(block, VecLoad(0, keys));
621}
622
623static inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
624 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
625 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
626{
627 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
628 const byte *keys = reinterpret_cast<const byte*>(subkeys);
629
630 uint32x4_p k = VecLoad(rounds*16, keys);
631 block0 = VecXor(block0, k);
632 block1 = VecXor(block1, k);
633 block2 = VecXor(block2, k);
634 block3 = VecXor(block3, k);
635 block4 = VecXor(block4, k);
636 block5 = VecXor(block5, k);
637
638 for (size_t i=rounds-1; i>0; --i)
639 {
640 k = VecLoad(i*16, keys);
641 block0 = VecDecrypt(block0, k);
642 block1 = VecDecrypt(block1, k);
643 block2 = VecDecrypt(block2, k);
644 block3 = VecDecrypt(block3, k);
645 block4 = VecDecrypt(block4, k);
646 block5 = VecDecrypt(block5, k);
647 }
648
649 k = VecLoad(0, keys);
650 block0 = VecDecryptLast(block0, k);
651 block1 = VecDecryptLast(block1, k);
652 block2 = VecDecryptLast(block2, k);
653 block3 = VecDecryptLast(block3, k);
654 block4 = VecDecryptLast(block4, k);
655 block5 = VecDecryptLast(block5, k);
656}
657
658ANONYMOUS_NAMESPACE_END
659
660void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
661{
662 const size_t rounds = keyLen / 4 + 6;
663 const word32 *rc = s_rconBE;
664 word32 *rkey = rk, temp;
665
666 GetUserKey(BIG_ENDIAN_ORDER, rkey, keyLen/4, userKey, keyLen);
667
668 // keySize: m_key allocates 4*(rounds+1) word32's.
669 const size_t keySize = 4*(rounds+1);
670 const word32* end = rkey + keySize;
671
672 while (true)
673 {
674 temp = rkey[keyLen/4-1];
675 word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
676 (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
677 rkey[keyLen/4] = rkey[0] ^ x ^ *(rc++);
678 rkey[keyLen/4+1] = rkey[1] ^ rkey[keyLen/4];
679 rkey[keyLen/4+2] = rkey[2] ^ rkey[keyLen/4+1];
680 rkey[keyLen/4+3] = rkey[3] ^ rkey[keyLen/4+2];
681
682 if (rkey + keyLen/4 + 4 == end)
683 break;
684
685 if (keyLen == 24)
686 {
687 rkey[10] = rkey[ 4] ^ rkey[ 9];
688 rkey[11] = rkey[ 5] ^ rkey[10];
689 }
690 else if (keyLen == 32)
691 {
692 temp = rkey[11];
693 rkey[12] = rkey[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
694 rkey[13] = rkey[ 5] ^ rkey[12];
695 rkey[14] = rkey[ 6] ^ rkey[13];
696 rkey[15] = rkey[ 7] ^ rkey[14];
697 }
698 rkey += keyLen/4;
699 }
700
701#if (CRYPTOPP_LITTLE_ENDIAN)
702 rkey = rk;
703 const uint8x16_p mask = ((uint8x16_p){12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3});
704 const uint8x16_p zero = {0};
705
706 unsigned int i=0;
707 for (i=0; i<rounds; i+=2, rkey+=8)
708 {
709 const uint8x16_p d1 = vec_vsx_ld( 0, (uint8_t*)rkey);
710 const uint8x16_p d2 = vec_vsx_ld(16, (uint8_t*)rkey);
711 vec_vsx_st(VecPermute(d1, zero, mask), 0, (uint8_t*)rkey);
712 vec_vsx_st(VecPermute(d2, zero, mask), 16, (uint8_t*)rkey);
713 }
714
715 for ( ; i<rounds+1; i++, rkey+=4)
716 {
717 const uint8x16_p d = vec_vsx_ld( 0, (uint8_t*)rkey);
718 vec_vsx_st(VecPermute(d, zero, mask), 0, (uint8_t*)rkey);
719 }
720#endif
721}
722
723size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
724 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
725{
726 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
727 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
728}
729
730size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
731 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
732{
733 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
734 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
735}
736
737#endif // CRYPTOPP_POWER8_AES_AVAILABLE
738NAMESPACE_END
Template for AdvancedProcessBlocks and SIMD processing.
Library configuration file.
@ BIG_ENDIAN_ORDER
byte order is big-endian
Definition: cryptlib.h:147
Utility functions for the Crypto++ library.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1143
void vec_swap(T &a, T &b)
Swaps two variables which are arrays.
Definition: misc.h:531
Crypto++ library namespace.
Precompiled header file.
Support functions for PowerPC and vector operations.
__vector unsigned int uint32x4_p
Vector of 32-bit elements.
Definition: ppc_simd.h:129
T1 VecPermute(const T1 vec, const T2 mask)
Permutes a vector.
Definition: ppc_simd.h:1010
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
Definition: ppc_simd.h:119
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
Definition: ppc_simd.h:916
T1 VecEncryptLast(const T1 state, const T2 key)
Final round of AES encryption.
Definition: ppc_simd.h:1618
T1 VecEncrypt(const T1 state, const T2 key)
One round of AES encryption.
Definition: ppc_simd.h:1593
T1 VecDecryptLast(const T1 state, const T2 key)
Final round of AES decryption.
Definition: ppc_simd.h:1668
T1 VecDecrypt(const T1 state, const T2 key)
One round of AES decryption.
Definition: ppc_simd.h:1643
uint32x4_p VecLoad(const byte src[16])
Loads a vector from a byte array.
Definition: ppc_simd.h:253
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:69