mirror of
https://github.com/SerenityOS/serenity
synced 2026-05-12 09:57:00 +02:00
779 lines
30 KiB
C++
779 lines
30 KiB
C++
/*
|
|
* Copyright (c) 2020, Ali Mohammad Pur <mpfard@serenityos.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <AK/CPUFeatures.h>
|
|
#include <AK/Platform.h>
|
|
#include <AK/SIMD.h>
|
|
#include <AK/SIMDExtras.h>
|
|
#include <AK/StringBuilder.h>
|
|
#include <LibCrypto/Cipher/AES.h>
|
|
#include <LibCrypto/Cipher/AESTables.h>
|
|
|
|
namespace Crypto::Cipher {
|
|
|
|
template<typename T>
|
|
constexpr u32 get_key(T pt)
|
|
{
|
|
return ((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] << 8) ^ ((u32)(pt)[3]);
|
|
}
|
|
|
|
constexpr void swap_keys(u32* keys, size_t i, size_t j)
|
|
{
|
|
u32 temp = keys[i];
|
|
keys[i] = keys[j];
|
|
keys[j] = temp;
|
|
}
|
|
|
|
#ifndef KERNEL
|
|
ByteString AESCipherBlock::to_byte_string() const
|
|
{
|
|
StringBuilder builder;
|
|
for (auto value : m_data)
|
|
builder.appendff("{:02x}", value);
|
|
return builder.to_byte_string();
|
|
}
|
|
|
|
ByteString AESCipherKey::to_byte_string() const
|
|
{
|
|
StringBuilder builder;
|
|
for (size_t i = 0; i < (rounds() + 1) * 4; ++i)
|
|
builder.appendff("{:02x}", m_rd_keys[i]);
|
|
return builder.to_byte_string();
|
|
}
|
|
#endif
|
|
|
|
template<>
|
|
void AESCipherKey::expand_encrypt_key_impl<CPUFeatures::None>(ReadonlyBytes user_key, size_t bits)
|
|
{
|
|
u32* round_key;
|
|
u32 temp;
|
|
size_t i { 0 };
|
|
|
|
VERIFY(!user_key.is_null());
|
|
VERIFY(is_valid_key_size(bits));
|
|
VERIFY(user_key.size() == bits / 8);
|
|
|
|
round_key = round_keys();
|
|
|
|
if (bits == 128) {
|
|
m_rounds = 10;
|
|
} else if (bits == 192) {
|
|
m_rounds = 12;
|
|
} else {
|
|
m_rounds = 14;
|
|
}
|
|
|
|
round_key[0] = get_key(user_key.data());
|
|
round_key[1] = get_key(user_key.data() + 4);
|
|
round_key[2] = get_key(user_key.data() + 8);
|
|
round_key[3] = get_key(user_key.data() + 12);
|
|
if (bits == 128) {
|
|
for (;;) {
|
|
temp = round_key[3];
|
|
// clang-format off
|
|
round_key[4] = round_key[0] ^
|
|
(AESTables::Encode2[(temp >> 16) & 0xff] & 0xff000000) ^
|
|
(AESTables::Encode3[(temp >> 8) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(temp ) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(temp >> 24) ] & 0x000000ff) ^ AESTables::RCON[i];
|
|
// clang-format on
|
|
round_key[5] = round_key[1] ^ round_key[4];
|
|
round_key[6] = round_key[2] ^ round_key[5];
|
|
round_key[7] = round_key[3] ^ round_key[6];
|
|
++i;
|
|
if (i == 10)
|
|
break;
|
|
round_key += 4;
|
|
}
|
|
return;
|
|
}
|
|
|
|
round_key[4] = get_key(user_key.data() + 16);
|
|
round_key[5] = get_key(user_key.data() + 20);
|
|
if (bits == 192) {
|
|
for (;;) {
|
|
temp = round_key[5];
|
|
// clang-format off
|
|
round_key[6] = round_key[0] ^
|
|
(AESTables::Encode2[(temp >> 16) & 0xff] & 0xff000000) ^
|
|
(AESTables::Encode3[(temp >> 8) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(temp ) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(temp >> 24) ] & 0x000000ff) ^ AESTables::RCON[i];
|
|
// clang-format on
|
|
round_key[7] = round_key[1] ^ round_key[6];
|
|
round_key[8] = round_key[2] ^ round_key[7];
|
|
round_key[9] = round_key[3] ^ round_key[8];
|
|
|
|
++i;
|
|
if (i == 8)
|
|
break;
|
|
|
|
round_key[10] = round_key[4] ^ round_key[9];
|
|
round_key[11] = round_key[5] ^ round_key[10];
|
|
|
|
round_key += 6;
|
|
}
|
|
return;
|
|
}
|
|
|
|
round_key[6] = get_key(user_key.data() + 24);
|
|
round_key[7] = get_key(user_key.data() + 28);
|
|
if (true) { // bits == 256
|
|
for (;;) {
|
|
temp = round_key[7];
|
|
// clang-format off
|
|
round_key[8] = round_key[0] ^
|
|
(AESTables::Encode2[(temp >> 16) & 0xff] & 0xff000000) ^
|
|
(AESTables::Encode3[(temp >> 8) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(temp ) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(temp >> 24) ] & 0x000000ff) ^ AESTables::RCON[i];
|
|
// clang-format on
|
|
round_key[9] = round_key[1] ^ round_key[8];
|
|
round_key[10] = round_key[2] ^ round_key[9];
|
|
round_key[11] = round_key[3] ^ round_key[10];
|
|
|
|
++i;
|
|
if (i == 7)
|
|
break;
|
|
|
|
temp = round_key[11];
|
|
// clang-format off
|
|
round_key[12] = round_key[4] ^
|
|
(AESTables::Encode2[(temp >> 24) ] & 0xff000000) ^
|
|
(AESTables::Encode3[(temp >> 16) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(temp >> 8) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(temp ) & 0xff] & 0x000000ff) ;
|
|
// clang-format on
|
|
round_key[13] = round_key[5] ^ round_key[12];
|
|
round_key[14] = round_key[6] ^ round_key[13];
|
|
round_key[15] = round_key[7] ^ round_key[14];
|
|
|
|
round_key += 8;
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
#if AK_CAN_CODEGEN_FOR_X86_AES
|
|
[[gnu::target("aes")]] static auto AESCipherKey_expand_encrypt_key_128_assist(auto const& a, auto const& b)
|
|
{
|
|
AK::SIMD::i32x4 ta {};
|
|
AK::SIMD::i32x4 tb {};
|
|
AK::SIMD::i32x4 tc {};
|
|
|
|
static_assert(sizeof(a) == 16);
|
|
static_assert(sizeof(b) == 16);
|
|
static_assert(IsSame<decltype(a), decltype(b)>);
|
|
|
|
ta = bit_cast<AK::SIMD::i32x4>(a);
|
|
tb = bit_cast<AK::SIMD::i32x4>(b);
|
|
tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] };
|
|
tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
|
|
ta ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
|
|
ta ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
|
|
ta ^= tc;
|
|
ta ^= tb;
|
|
return bit_cast<RemoveReference<decltype(a)>>(ta);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_128(u32* round_key, ReadonlyBytes user_key)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
auto const* user_ptr = user_key.data();
|
|
illx2 ta {};
|
|
illx2 tb {};
|
|
|
|
ta = AK::SIMD::load_unaligned<illx2>(&user_ptr[0 * 16]);
|
|
AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x01);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[1 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x02);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[2 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x04);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[3 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x08);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[4 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x10);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[5 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x20);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[6 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x40);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[7 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x80);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[8 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x1b);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[9 * 4], ta);
|
|
|
|
tb = __builtin_ia32_aeskeygenassist128(ta, 0x36);
|
|
ta = AESCipherKey_expand_encrypt_key_128_assist(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[10 * 4], ta);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192_assist(AK::SIMD::i32x4& a, AK::SIMD::i32x4& b, AK::SIMD::i32x4& c)
|
|
{
|
|
AK::SIMD::i32x4 ta {};
|
|
AK::SIMD::i32x4 tb {};
|
|
AK::SIMD::i32x4 tc {};
|
|
AK::SIMD::i32x4 td {};
|
|
|
|
ta = bit_cast<AK::SIMD::i32x4>(a);
|
|
tb = bit_cast<AK::SIMD::i32x4>(b);
|
|
tc = bit_cast<AK::SIMD::i32x4>(c);
|
|
tb = AK::SIMD::i32x4 { tb[1], tb[1], tb[1], tb[1] };
|
|
td = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
|
|
ta ^= td;
|
|
td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] };
|
|
ta ^= td;
|
|
td = AK::SIMD::i32x4 { 0, td[0], td[1], td[2] };
|
|
ta ^= td;
|
|
ta ^= tb;
|
|
tb = AK::SIMD::i32x4 { ta[3], ta[3], ta[3], ta[3] };
|
|
td = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
|
|
tc ^= td;
|
|
tc ^= tb;
|
|
a = bit_cast<AK::SIMD::i32x4>(ta);
|
|
b = bit_cast<AK::SIMD::i32x4>(tb);
|
|
c = bit_cast<AK::SIMD::i32x4>(tc);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_192(u32* round_key, ReadonlyBytes user_key)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
auto const* user_ptr = user_key.data();
|
|
AK::SIMD::i32x4 ta {};
|
|
AK::SIMD::i32x4 tb {};
|
|
AK::SIMD::i32x4 tc {};
|
|
AK::SIMD::i32x4 td {};
|
|
|
|
ta = AK::SIMD::load_unaligned<AK::SIMD::i32x4>(&user_ptr[0 * 16]);
|
|
tb = bit_cast<AK::SIMD::i32x4>(AK::SIMD::i64x2 { (i64)ByteReader::load64(&user_ptr[1 * 16]), 0 });
|
|
AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
|
|
# define ROUND(i1, i2, i3, k1, k2) \
|
|
td = tb; \
|
|
tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), k1)); \
|
|
AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb); \
|
|
AK::SIMD::store_unaligned(&round_key[i1 * 4], AK::SIMD::i32x4 { td[0], td[1], ta[0], ta[1] }); \
|
|
AK::SIMD::store_unaligned(&round_key[i2 * 4], AK::SIMD::i32x4 { ta[2], ta[3], tb[0], tb[1] }); \
|
|
tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), k2)); \
|
|
AESCipherKey_expand_encrypt_key_192_assist(ta, tc, tb); \
|
|
AK::SIMD::store_unaligned(&round_key[i3 * 4], ta);
|
|
|
|
ROUND(1, 2, 3, 0x01, 0x02);
|
|
ROUND(4, 5, 6, 0x04, 0x08);
|
|
ROUND(7, 8, 9, 0x10, 0x20);
|
|
ROUND(10, 11, 12, 0x40, 0x80);
|
|
AK::SIMD::store_unaligned(&round_key[12 * 4], ta);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_a(AK::SIMD::i64x2& a, AK::SIMD::i64x2& b)
|
|
{
|
|
AK::SIMD::i32x4 ta {};
|
|
AK::SIMD::i32x4 tb {};
|
|
AK::SIMD::i32x4 tc {};
|
|
|
|
ta = bit_cast<AK::SIMD::i32x4>(a);
|
|
tb = bit_cast<AK::SIMD::i32x4>(b);
|
|
tb = AK::SIMD::i32x4 { tb[3], tb[3], tb[3], tb[3] };
|
|
tc = AK::SIMD::i32x4 { 0, ta[0], ta[1], ta[2] };
|
|
ta ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
|
|
ta ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
|
|
ta ^= tc;
|
|
ta ^= tb;
|
|
a = bit_cast<AK::SIMD::i64x2>(ta);
|
|
b = bit_cast<AK::SIMD::i64x2>(tb);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256_assist_b(AK::SIMD::i64x2 const& a, AK::SIMD::i64x2& b)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
AK::SIMD::i32x4 tb {};
|
|
AK::SIMD::i32x4 tc {};
|
|
AK::SIMD::i32x4 td {};
|
|
|
|
tb = bit_cast<AK::SIMD::i32x4>(b);
|
|
tc = bit_cast<AK::SIMD::i32x4>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(a), 0x00));
|
|
td = AK::SIMD::i32x4 { tc[2], tc[2], tc[2], tc[2] };
|
|
tc = AK::SIMD::i32x4 { 0, tb[0], tb[1], tb[2] };
|
|
tb ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
|
|
tb ^= tc;
|
|
tc = AK::SIMD::i32x4 { 0, tc[0], tc[1], tc[2] };
|
|
tb ^= tc;
|
|
tb ^= td;
|
|
b = bit_cast<AK::SIMD::i64x2>(tb);
|
|
}
|
|
|
|
[[gnu::target("aes")]] static void AESCipherKey_expand_encrypt_key_256(u32* round_key, ReadonlyBytes user_key)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
auto const* user_ptr = user_key.data();
|
|
AK::SIMD::i64x2 ta {};
|
|
AK::SIMD::i64x2 tb {};
|
|
AK::SIMD::i64x2 tc {};
|
|
|
|
ta = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&user_ptr[0 * 16]);
|
|
tb = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&user_ptr[1 * 16]);
|
|
AK::SIMD::store_unaligned(&round_key[0 * 4], ta);
|
|
AK::SIMD::store_unaligned(&round_key[1 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x01));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[2 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[3 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x02));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[4 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[5 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x04));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[6 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[7 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x08));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[8 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[9 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x10));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[10 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[11 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x20));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[12 * 4], ta);
|
|
|
|
AESCipherKey_expand_encrypt_key_256_assist_b(ta, tb);
|
|
AK::SIMD::store_unaligned(&round_key[13 * 4], tb);
|
|
|
|
tc = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aeskeygenassist128(bit_cast<illx2>(tb), 0x40));
|
|
AESCipherKey_expand_encrypt_key_256_assist_a(ta, tc);
|
|
AK::SIMD::store_unaligned(&round_key[14 * 4], ta);
|
|
}
|
|
|
|
template<>
|
|
[[gnu::target("aes")]] void AESCipherKey::expand_encrypt_key_impl<CPUFeatures::X86_AES>(ReadonlyBytes user_key, size_t bits)
|
|
{
|
|
switch (bits) {
|
|
case 128:
|
|
m_rounds = 10;
|
|
AESCipherKey_expand_encrypt_key_128(m_rd_keys, user_key);
|
|
break;
|
|
case 192:
|
|
m_rounds = 12;
|
|
AESCipherKey_expand_encrypt_key_192(m_rd_keys, user_key);
|
|
break;
|
|
case 256:
|
|
m_rounds = 14;
|
|
AESCipherKey_expand_encrypt_key_256(m_rd_keys, user_key);
|
|
break;
|
|
}
|
|
}
|
|
|
|
template<>
|
|
[[gnu::target("aes")]] void AESCipherKey::expand_decrypt_key_impl<CPUFeatures::X86_AES>(ReadonlyBytes user_key, size_t bits)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
u32* round_key = m_rd_keys;
|
|
expand_encrypt_key_impl<CPUFeatures::X86_AES>(user_key, bits);
|
|
for (size_t i_round = 1; i_round != m_rounds; ++i_round) {
|
|
AK::SIMD::i64x2 ta = AK::SIMD::load_unaligned<AK::SIMD::i64x2>(&round_key[i_round * 4]);
|
|
ta = bit_cast<AK::SIMD::i64x2>(__builtin_ia32_aesimc128(bit_cast<illx2>(ta)));
|
|
AK::SIMD::store_unaligned(&round_key[i_round * 4], ta);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
template<>
|
|
void AESCipherKey::expand_decrypt_key_impl<CPUFeatures::None>(ReadonlyBytes user_key, size_t bits)
|
|
{
|
|
u32* round_key;
|
|
|
|
expand_encrypt_key(user_key, bits);
|
|
|
|
round_key = round_keys();
|
|
|
|
// reorder round keys
|
|
for (size_t i = 0, j = 4 * rounds(); i < j; i += 4, j -= 4) {
|
|
swap_keys(round_key, i, j);
|
|
swap_keys(round_key, i + 1, j + 1);
|
|
swap_keys(round_key, i + 2, j + 2);
|
|
swap_keys(round_key, i + 3, j + 3);
|
|
}
|
|
|
|
// apply inverse mix-column to middle rounds
|
|
for (size_t i = 1; i < rounds(); ++i) {
|
|
round_key += 4;
|
|
// clang-format off
|
|
round_key[0] =
|
|
AESTables::Decode0[AESTables::Encode1[(round_key[0] >> 24) ] & 0xff] ^
|
|
AESTables::Decode1[AESTables::Encode1[(round_key[0] >> 16) & 0xff] & 0xff] ^
|
|
AESTables::Decode2[AESTables::Encode1[(round_key[0] >> 8) & 0xff] & 0xff] ^
|
|
AESTables::Decode3[AESTables::Encode1[(round_key[0] ) & 0xff] & 0xff] ;
|
|
round_key[1] =
|
|
AESTables::Decode0[AESTables::Encode1[(round_key[1] >> 24) ] & 0xff] ^
|
|
AESTables::Decode1[AESTables::Encode1[(round_key[1] >> 16) & 0xff] & 0xff] ^
|
|
AESTables::Decode2[AESTables::Encode1[(round_key[1] >> 8) & 0xff] & 0xff] ^
|
|
AESTables::Decode3[AESTables::Encode1[(round_key[1] ) & 0xff] & 0xff] ;
|
|
round_key[2] =
|
|
AESTables::Decode0[AESTables::Encode1[(round_key[2] >> 24) ] & 0xff] ^
|
|
AESTables::Decode1[AESTables::Encode1[(round_key[2] >> 16) & 0xff] & 0xff] ^
|
|
AESTables::Decode2[AESTables::Encode1[(round_key[2] >> 8) & 0xff] & 0xff] ^
|
|
AESTables::Decode3[AESTables::Encode1[(round_key[2] ) & 0xff] & 0xff] ;
|
|
round_key[3] =
|
|
AESTables::Decode0[AESTables::Encode1[(round_key[3] >> 24) ] & 0xff] ^
|
|
AESTables::Decode1[AESTables::Encode1[(round_key[3] >> 16) & 0xff] & 0xff] ^
|
|
AESTables::Decode2[AESTables::Encode1[(round_key[3] >> 8) & 0xff] & 0xff] ^
|
|
AESTables::Decode3[AESTables::Encode1[(round_key[3] ) & 0xff] & 0xff] ;
|
|
// clang-format on
|
|
}
|
|
}
|
|
|
|
decltype(AESCipherKey::expand_encrypt_key_dispatched) AESCipherKey::expand_encrypt_key_dispatched = [] {
|
|
CPUFeatures features = detect_cpu_features();
|
|
|
|
if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
|
|
if (has_flag(features, CPUFeatures::X86_AES))
|
|
return &AESCipherKey::expand_encrypt_key_impl<CPUFeatures::X86_AES>;
|
|
}
|
|
|
|
return &AESCipherKey::expand_encrypt_key_impl<CPUFeatures::None>;
|
|
}();
|
|
|
|
decltype(AESCipherKey::expand_decrypt_key_dispatched) AESCipherKey::expand_decrypt_key_dispatched = [] {
|
|
CPUFeatures features = detect_cpu_features();
|
|
|
|
if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
|
|
if (has_flag(features, CPUFeatures::X86_AES))
|
|
return &AESCipherKey::expand_decrypt_key_impl<CPUFeatures::X86_AES>;
|
|
}
|
|
|
|
return &AESCipherKey::expand_decrypt_key_impl<CPUFeatures::None>;
|
|
}();
|
|
|
|
template<>
|
|
void AESCipher::encrypt_block_impl<CPUFeatures::None>(AESCipherBlock const& in, AESCipherBlock& out)
|
|
{
|
|
u32 s0, s1, s2, s3, t0, t1, t2, t3;
|
|
size_t r { 0 };
|
|
|
|
auto const& dec_key = key();
|
|
auto const* round_keys = dec_key.round_keys();
|
|
|
|
s0 = get_key(in.bytes().offset_pointer(0)) ^ round_keys[0];
|
|
s1 = get_key(in.bytes().offset_pointer(4)) ^ round_keys[1];
|
|
s2 = get_key(in.bytes().offset_pointer(8)) ^ round_keys[2];
|
|
s3 = get_key(in.bytes().offset_pointer(12)) ^ round_keys[3];
|
|
|
|
r = dec_key.rounds() >> 1;
|
|
|
|
// apply the first |r - 1| rounds
|
|
for (;;) {
|
|
// clang-format off
|
|
t0 = AESTables::Encode0[(s0 >> 24) ] ^
|
|
AESTables::Encode1[(s1 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(s2 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(s3 ) & 0xff] ^ round_keys[4];
|
|
t1 = AESTables::Encode0[(s1 >> 24) ] ^
|
|
AESTables::Encode1[(s2 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(s3 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(s0 ) & 0xff] ^ round_keys[5];
|
|
t2 = AESTables::Encode0[(s2 >> 24) ] ^
|
|
AESTables::Encode1[(s3 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(s0 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(s1 ) & 0xff] ^ round_keys[6];
|
|
t3 = AESTables::Encode0[(s3 >> 24) ] ^
|
|
AESTables::Encode1[(s0 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(s1 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(s2 ) & 0xff] ^ round_keys[7];
|
|
// clang-format on
|
|
|
|
round_keys += 8;
|
|
--r;
|
|
if (r == 0)
|
|
break;
|
|
|
|
// clang-format off
|
|
s0 = AESTables::Encode0[(t0 >> 24) ] ^
|
|
AESTables::Encode1[(t1 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(t2 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(t3 ) & 0xff] ^ round_keys[0];
|
|
s1 = AESTables::Encode0[(t1 >> 24) ] ^
|
|
AESTables::Encode1[(t2 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(t3 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(t0 ) & 0xff] ^ round_keys[1];
|
|
s2 = AESTables::Encode0[(t2 >> 24) ] ^
|
|
AESTables::Encode1[(t3 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(t0 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(t1 ) & 0xff] ^ round_keys[2];
|
|
s3 = AESTables::Encode0[(t3 >> 24) ] ^
|
|
AESTables::Encode1[(t0 >> 16) & 0xff] ^
|
|
AESTables::Encode2[(t1 >> 8) & 0xff] ^
|
|
AESTables::Encode3[(t2 ) & 0xff] ^ round_keys[3];
|
|
// clang-format on
|
|
}
|
|
|
|
// apply the last round and put the encrypted data into out
|
|
// clang-format off
|
|
s0 = (AESTables::Encode2[(t0 >> 24) ] & 0xff000000) ^
|
|
(AESTables::Encode3[(t1 >> 16) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(t2 >> 8) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(t3 ) & 0xff] & 0x000000ff) ^ round_keys[0];
|
|
out.put(0, s0);
|
|
|
|
s1 = (AESTables::Encode2[(t1 >> 24) ] & 0xff000000) ^
|
|
(AESTables::Encode3[(t2 >> 16) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(t3 >> 8) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(t0 ) & 0xff] & 0x000000ff) ^ round_keys[1];
|
|
out.put(4, s1);
|
|
|
|
s2 = (AESTables::Encode2[(t2 >> 24) ] & 0xff000000) ^
|
|
(AESTables::Encode3[(t3 >> 16) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(t0 >> 8) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(t1 ) & 0xff] & 0x000000ff) ^ round_keys[2];
|
|
out.put(8, s2);
|
|
|
|
s3 = (AESTables::Encode2[(t3 >> 24) ] & 0xff000000) ^
|
|
(AESTables::Encode3[(t0 >> 16) & 0xff] & 0x00ff0000) ^
|
|
(AESTables::Encode0[(t1 >> 8) & 0xff] & 0x0000ff00) ^
|
|
(AESTables::Encode1[(t2 ) & 0xff] & 0x000000ff) ^ round_keys[3];
|
|
out.put(12, s3);
|
|
// clang-format on
|
|
}
|
|
|
|
#if AK_CAN_CODEGEN_FOR_X86_AES
|
|
template<>
|
|
[[gnu::target("aes")]] void AESCipher::encrypt_block_impl<CPUFeatures::X86_AES>(AESCipherBlock const& in, AESCipherBlock& out)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
AESCipherKey const& key = m_key;
|
|
auto round_keys = key.round_keys();
|
|
auto n_rounds = static_cast<int>(key.rounds());
|
|
auto input_ptr = in.bytes().data();
|
|
auto output_ptr = out.bytes().data();
|
|
|
|
auto value = AK::SIMD::load_unaligned<illx2>(input_ptr);
|
|
auto round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(0 + 0) * 4]);
|
|
value ^= round_key;
|
|
for (int i_round = 0; i_round != n_rounds - 1; ++i_round) {
|
|
round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(1 + i_round) * 4]);
|
|
value = __builtin_ia32_aesenc128(value, round_key);
|
|
}
|
|
round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[(1 + (n_rounds - 1)) * 4]);
|
|
value = __builtin_ia32_aesenclast128(value, round_key);
|
|
AK::SIMD::store_unaligned(output_ptr, value);
|
|
}
|
|
#endif
|
|
|
|
template<>
|
|
void AESCipher::decrypt_block_impl<CPUFeatures::None>(AESCipherBlock const& in, AESCipherBlock& out)
|
|
{
|
|
u32 s0, s1, s2, s3, t0, t1, t2, t3;
|
|
size_t r { 0 };
|
|
|
|
auto const& dec_key = key();
|
|
auto const* round_keys = dec_key.round_keys();
|
|
|
|
s0 = get_key(in.bytes().offset_pointer(0)) ^ round_keys[0];
|
|
s1 = get_key(in.bytes().offset_pointer(4)) ^ round_keys[1];
|
|
s2 = get_key(in.bytes().offset_pointer(8)) ^ round_keys[2];
|
|
s3 = get_key(in.bytes().offset_pointer(12)) ^ round_keys[3];
|
|
|
|
r = dec_key.rounds() >> 1;
|
|
|
|
// apply the first |r - 1| rounds
|
|
for (;;) {
|
|
// clang-format off
|
|
t0 = AESTables::Decode0[(s0 >> 24) ] ^
|
|
AESTables::Decode1[(s3 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(s2 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(s1 ) & 0xff] ^ round_keys[4];
|
|
t1 = AESTables::Decode0[(s1 >> 24) ] ^
|
|
AESTables::Decode1[(s0 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(s3 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(s2 ) & 0xff] ^ round_keys[5];
|
|
t2 = AESTables::Decode0[(s2 >> 24) ] ^
|
|
AESTables::Decode1[(s1 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(s0 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(s3 ) & 0xff] ^ round_keys[6];
|
|
t3 = AESTables::Decode0[(s3 >> 24) ] ^
|
|
AESTables::Decode1[(s2 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(s1 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(s0 ) & 0xff] ^ round_keys[7];
|
|
// clang-format on
|
|
|
|
round_keys += 8;
|
|
--r;
|
|
if (r == 0)
|
|
break;
|
|
|
|
// clang-format off
|
|
s0 = AESTables::Decode0[(t0 >> 24) ] ^
|
|
AESTables::Decode1[(t3 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(t2 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(t1 ) & 0xff] ^ round_keys[0];
|
|
s1 = AESTables::Decode0[(t1 >> 24) ] ^
|
|
AESTables::Decode1[(t0 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(t3 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(t2 ) & 0xff] ^ round_keys[1];
|
|
s2 = AESTables::Decode0[(t2 >> 24) ] ^
|
|
AESTables::Decode1[(t1 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(t0 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(t3 ) & 0xff] ^ round_keys[2];
|
|
s3 = AESTables::Decode0[(t3 >> 24) ] ^
|
|
AESTables::Decode1[(t2 >> 16) & 0xff] ^
|
|
AESTables::Decode2[(t1 >> 8) & 0xff] ^
|
|
AESTables::Decode3[(t0 ) & 0xff] ^ round_keys[3];
|
|
// clang-format on
|
|
}
|
|
|
|
// apply the last round and put the decrypted data into out
|
|
// clang-format off
|
|
s0 = ((u32)AESTables::Decode4[(t0 >> 24) ] << 24) ^
|
|
((u32)AESTables::Decode4[(t3 >> 16) & 0xff] << 16) ^
|
|
((u32)AESTables::Decode4[(t2 >> 8) & 0xff] << 8) ^
|
|
((u32)AESTables::Decode4[(t1 ) & 0xff] ) ^ round_keys[0];
|
|
out.put(0, s0);
|
|
|
|
s1 = ((u32)AESTables::Decode4[(t1 >> 24) ] << 24) ^
|
|
((u32)AESTables::Decode4[(t0 >> 16) & 0xff] << 16) ^
|
|
((u32)AESTables::Decode4[(t3 >> 8) & 0xff] << 8) ^
|
|
((u32)AESTables::Decode4[(t2 ) & 0xff] ) ^ round_keys[1];
|
|
out.put(4, s1);
|
|
|
|
s2 = ((u32)AESTables::Decode4[(t2 >> 24) ] << 24) ^
|
|
((u32)AESTables::Decode4[(t1 >> 16) & 0xff] << 16) ^
|
|
((u32)AESTables::Decode4[(t0 >> 8) & 0xff] << 8) ^
|
|
((u32)AESTables::Decode4[(t3 ) & 0xff] ) ^ round_keys[2];
|
|
out.put(8, s2);
|
|
|
|
s3 = ((u32)AESTables::Decode4[(t3 >> 24) ] << 24) ^
|
|
((u32)AESTables::Decode4[(t2 >> 16) & 0xff] << 16) ^
|
|
((u32)AESTables::Decode4[(t1 >> 8) & 0xff] << 8) ^
|
|
((u32)AESTables::Decode4[(t0 ) & 0xff] ) ^ round_keys[3];
|
|
out.put(12, s3);
|
|
// clang-format on
|
|
}
|
|
|
|
#if AK_CAN_CODEGEN_FOR_X86_AES
|
|
template<>
|
|
[[gnu::target("aes")]] void AESCipher::decrypt_block_impl<CPUFeatures::X86_AES>(AESCipherBlock const& in, AESCipherBlock& out)
|
|
{
|
|
using illx2 = signed long long int __attribute__((vector_size(16)));
|
|
|
|
AESCipherKey const& key = m_key;
|
|
auto round_keys = key.round_keys();
|
|
auto n_rounds = static_cast<int>(key.rounds());
|
|
auto input_ptr = in.bytes().data();
|
|
auto output_ptr = out.bytes().data();
|
|
|
|
auto value = AK::SIMD::load_unaligned<illx2>(input_ptr);
|
|
auto round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[n_rounds * 4]);
|
|
value ^= round_key;
|
|
for (int i_round = 0; i_round != n_rounds - 1; ++i_round) {
|
|
int const idx = (n_rounds - 1) - i_round;
|
|
round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[idx * 4]);
|
|
value = __builtin_ia32_aesdec128(value, round_key);
|
|
}
|
|
round_key = AK::SIMD::load_unaligned<illx2>(&round_keys[0]);
|
|
value = __builtin_ia32_aesdeclast128(value, round_key);
|
|
AK::SIMD::store_unaligned(output_ptr, value);
|
|
}
|
|
#endif
|
|
|
|
decltype(AESCipher::encrypt_block_dispatched) AESCipher::encrypt_block_dispatched = [] {
|
|
CPUFeatures features = detect_cpu_features();
|
|
|
|
if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
|
|
if (has_flag(features, CPUFeatures::X86_AES))
|
|
return &AESCipher::encrypt_block_impl<CPUFeatures::X86_AES>;
|
|
}
|
|
|
|
return &AESCipher::encrypt_block_impl<CPUFeatures::None>;
|
|
}();
|
|
|
|
decltype(AESCipher::decrypt_block_dispatched) AESCipher::decrypt_block_dispatched = [] {
|
|
CPUFeatures features = detect_cpu_features();
|
|
|
|
if constexpr (is_valid_feature(CPUFeatures::X86_AES)) {
|
|
if (has_flag(features, CPUFeatures::X86_AES))
|
|
return &AESCipher::decrypt_block_impl<CPUFeatures::X86_AES>;
|
|
}
|
|
|
|
return &AESCipher::decrypt_block_impl<CPUFeatures::None>;
|
|
}();
|
|
|
|
void AESCipherBlock::overwrite(ReadonlyBytes bytes)
|
|
{
|
|
auto data = bytes.data();
|
|
auto length = bytes.size();
|
|
|
|
VERIFY(length <= this->data_size());
|
|
this->bytes().overwrite(0, data, length);
|
|
if (length < this->data_size()) {
|
|
switch (padding_mode()) {
|
|
case PaddingMode::Null:
|
|
// fill with zeros
|
|
__builtin_memset(m_data + length, 0, this->data_size() - length);
|
|
break;
|
|
case PaddingMode::CMS:
|
|
// fill with the length of the padding bytes
|
|
__builtin_memset(m_data + length, this->data_size() - length, this->data_size() - length);
|
|
break;
|
|
case PaddingMode::RFC5246:
|
|
// fill with the length of the padding bytes minus one
|
|
__builtin_memset(m_data + length, this->data_size() - length - 1, this->data_size() - length);
|
|
break;
|
|
default:
|
|
// FIXME: We should handle the rest of the common padding modes
|
|
VERIFY_NOT_REACHED();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|