use super::{
aes::{self, Counter},
gcm, shift, Aad, Block, Direction, Nonce, Tag, BLOCK_LEN,
};
use crate::{aead, cpu, endian::*, error, polyfill};
pub static AES_128_GCM: aead::Algorithm = aead::Algorithm {
key_len: 16,
init: init_128,
seal: aes_gcm_seal,
open: aes_gcm_open,
id: aead::AlgorithmID::AES_128_GCM,
max_input_len: AES_GCM_MAX_INPUT_LEN,
};
pub static AES_256_GCM: aead::Algorithm = aead::Algorithm {
key_len: 32,
init: init_256,
seal: aes_gcm_seal,
open: aes_gcm_open,
id: aead::AlgorithmID::AES_256_GCM,
max_input_len: AES_GCM_MAX_INPUT_LEN,
};
pub struct Key {
gcm_key: gcm::Key, aes_key: aes::Key,
}
fn init_128(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
init(key, aes::Variant::AES_128, cpu_features)
}
fn init_256(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
init(key, aes::Variant::AES_256, cpu_features)
}
fn init(
key: &[u8],
variant: aes::Variant,
cpu_features: cpu::Features,
) -> Result<aead::KeyInner, error::Unspecified> {
let aes_key = aes::Key::new(key, variant, cpu_features)?;
let gcm_key = gcm::Key::new(aes_key.encrypt_block(Block::zero()), cpu_features);
Ok(aead::KeyInner::AesGcm(Key { aes_key, gcm_key }))
}
const CHUNK_BLOCKS: usize = 3 * 1024 / 16;
fn aes_gcm_seal(
key: &aead::KeyInner,
nonce: Nonce,
aad: Aad<&[u8]>,
in_out: &mut [u8],
cpu_features: cpu::Features,
) -> Tag {
aead(key, nonce, aad, in_out, Direction::Sealing, cpu_features)
}
fn aes_gcm_open(
key: &aead::KeyInner,
nonce: Nonce,
aad: Aad<&[u8]>,
in_prefix_len: usize,
in_out: &mut [u8],
cpu_features: cpu::Features,
) -> Tag {
aead(
key,
nonce,
aad,
in_out,
Direction::Opening { in_prefix_len },
cpu_features,
)
}
#[inline(always)] fn aead(
key: &aead::KeyInner,
nonce: Nonce,
aad: Aad<&[u8]>,
in_out: &mut [u8],
direction: Direction,
cpu_features: cpu::Features,
) -> Tag {
let Key { aes_key, gcm_key } = match key {
aead::KeyInner::AesGcm(key) => key,
_ => unreachable!(),
};
let mut ctr = Counter::one(nonce);
let tag_iv = ctr.increment();
let aad_len = aad.0.len();
let mut gcm_ctx = gcm::Context::new(gcm_key, aad, cpu_features);
let in_prefix_len = match direction {
Direction::Opening { in_prefix_len } => in_prefix_len,
Direction::Sealing => 0,
};
let total_in_out_len = in_out.len() - in_prefix_len;
let in_out = integrated_aes_gcm(
aes_key,
&mut gcm_ctx,
in_out,
&mut ctr,
direction,
cpu_features,
);
let in_out_len = in_out.len() - in_prefix_len;
let whole_len = in_out_len - (in_out_len % BLOCK_LEN);
{
let mut chunk_len = CHUNK_BLOCKS * BLOCK_LEN;
let mut output = 0;
let mut input = in_prefix_len;
loop {
if whole_len - output < chunk_len {
chunk_len = whole_len - output;
}
if chunk_len == 0 {
break;
}
if let Direction::Opening { .. } = direction {
gcm_ctx.update_blocks(&in_out[input..][..chunk_len]);
}
aes_key.ctr32_encrypt_blocks(
&mut in_out[output..][..(chunk_len + in_prefix_len)],
direction,
&mut ctr,
);
if let Direction::Sealing = direction {
gcm_ctx.update_blocks(&in_out[output..][..chunk_len]);
}
output += chunk_len;
input += chunk_len;
}
}
let remainder = &mut in_out[whole_len..];
shift::shift_partial((in_prefix_len, remainder), |remainder| {
let mut input = Block::zero();
input.overwrite_part_at(0, remainder);
if let Direction::Opening { .. } = direction {
gcm_ctx.update_block(input);
}
let mut output = aes_key.encrypt_iv_xor_block(ctr.into(), input);
if let Direction::Sealing = direction {
output.zero_from(remainder.len());
gcm_ctx.update_block(output);
}
output
});
let aad_bits = polyfill::u64_from_usize(aad_len) << 3;
let ciphertext_bits = polyfill::u64_from_usize(total_in_out_len) << 3;
gcm_ctx.update_block(Block::from_u64_be(
BigEndian::from(aad_bits),
BigEndian::from(ciphertext_bits),
));
gcm_ctx.pre_finish(|pre_tag| {
let bytes = tag_iv.into_bytes_less_safe();
let mut tag = aes_key.encrypt_block(Block::from(&bytes));
tag.bitxor_assign(pre_tag.into());
Tag(*tag.as_ref())
})
}
#[cfg(target_arch = "x86_64")]
#[inline] fn integrated_aes_gcm<'a>(
aes_key: &aes::Key,
gcm_ctx: &mut gcm::Context,
in_out: &'a mut [u8],
ctr: &mut Counter,
direction: Direction,
cpu_features: cpu::Features,
) -> &'a mut [u8] {
use crate::c;
if !aes_key.is_aes_hw() || !gcm_ctx.is_avx2(cpu_features) {
return in_out;
}
let processed = match direction {
Direction::Opening { in_prefix_len } => {
extern "C" {
fn GFp_aesni_gcm_decrypt(
input: *const u8,
output: *mut u8,
len: c::size_t,
key: &aes::AES_KEY,
ivec: &mut Counter,
gcm: &mut gcm::ContextInner,
) -> c::size_t;
}
unsafe {
GFp_aesni_gcm_decrypt(
in_out[in_prefix_len..].as_ptr(),
in_out.as_mut_ptr(),
in_out.len() - in_prefix_len,
aes_key.inner_less_safe(),
ctr,
gcm_ctx.inner(),
)
}
}
Direction::Sealing => {
extern "C" {
fn GFp_aesni_gcm_encrypt(
input: *const u8,
output: *mut u8,
len: c::size_t,
key: &aes::AES_KEY,
ivec: &mut Counter,
gcm: &mut gcm::ContextInner,
) -> c::size_t;
}
unsafe {
GFp_aesni_gcm_encrypt(
in_out.as_ptr(),
in_out.as_mut_ptr(),
in_out.len(),
aes_key.inner_less_safe(),
ctr,
gcm_ctx.inner(),
)
}
}
};
&mut in_out[processed..]
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
fn integrated_aes_gcm<'a>(
_: &aes::Key,
_: &mut gcm::Context,
in_out: &'a mut [u8],
_: &mut Counter,
_: Direction,
_: cpu::Features,
) -> &'a mut [u8] {
in_out }
const AES_GCM_MAX_INPUT_LEN: u64 = super::max_input_len(BLOCK_LEN, 2);
#[cfg(test)]
mod tests {
#[test]
fn max_input_len_test() {
const NIST_SP800_38D_MAX_BITS: u64 = (1u64 << 39) - 256;
assert_eq!(NIST_SP800_38D_MAX_BITS, 549_755_813_632u64);
assert_eq!(
super::AES_128_GCM.max_input_len * 8,
NIST_SP800_38D_MAX_BITS
);
assert_eq!(
super::AES_256_GCM.max_input_len * 8,
NIST_SP800_38D_MAX_BITS
);
}
}