1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
/// A trait for describing vector operations used by vectorized searchers.
///
/// The trait is highly constrained to low level vector operations needed. In
/// general, it was invented mostly to be generic over x86's __m128i and
/// __m256i types. It's likely that once std::simd becomes a thing, we can
/// migrate to that since the operations required are quite simple.
///
/// TODO: Consider moving this trait up a level and using it to implement
/// memchr as well. The trait might need to grow one or two methods, but
/// otherwise should be close to sufficient already.
///
/// # Safety
///
/// All methods are not safe since they are intended to be implemented using
/// vendor intrinsics, which are also not safe. Callers must ensure that the
/// appropriate target features are enabled in the calling function, and that
/// the current CPU supports them. All implementations should avoid marking the
/// routines with #[target_feature] and instead mark them as #[inline(always)]
/// to ensure they get appropriately inlined. (inline(always) cannot be used
/// with target_feature.)
pub(crate) trait Vector: Copy + core::fmt::Debug {
/// _mm_set1_epi8 or _mm256_set1_epi8
unsafe fn splat(byte: u8) -> Self;
/// _mm_loadu_si128 or _mm256_loadu_si256
unsafe fn load_unaligned(data: *const u8) -> Self;
/// _mm_movemask_epi8 or _mm256_movemask_epi8
unsafe fn movemask(self) -> u32;
/// _mm_cmpeq_epi8 or _mm256_cmpeq_epi8
unsafe fn cmpeq(self, vector2: Self) -> Self;
/// _mm_and_si128 or _mm256_and_si256
unsafe fn and(self, vector2: Self) -> Self;
}
#[cfg(target_arch = "x86_64")]
mod x86sse {
use super::Vector;
use core::arch::x86_64::*;
impl Vector for __m128i {
#[inline(always)]
unsafe fn splat(byte: u8) -> __m128i {
_mm_set1_epi8(byte as i8)
}
#[inline(always)]
unsafe fn load_unaligned(data: *const u8) -> __m128i {
_mm_loadu_si128(data as *const __m128i)
}
#[inline(always)]
unsafe fn movemask(self) -> u32 {
_mm_movemask_epi8(self) as u32
}
#[inline(always)]
unsafe fn cmpeq(self, vector2: Self) -> __m128i {
_mm_cmpeq_epi8(self, vector2)
}
#[inline(always)]
unsafe fn and(self, vector2: Self) -> __m128i {
_mm_and_si128(self, vector2)
}
}
}
#[cfg(all(feature = "std", target_arch = "x86_64"))]
mod x86avx {
use super::Vector;
use core::arch::x86_64::*;
impl Vector for __m256i {
#[inline(always)]
unsafe fn splat(byte: u8) -> __m256i {
_mm256_set1_epi8(byte as i8)
}
#[inline(always)]
unsafe fn load_unaligned(data: *const u8) -> __m256i {
_mm256_loadu_si256(data as *const __m256i)
}
#[inline(always)]
unsafe fn movemask(self) -> u32 {
_mm256_movemask_epi8(self) as u32
}
#[inline(always)]
unsafe fn cmpeq(self, vector2: Self) -> __m256i {
_mm256_cmpeq_epi8(self, vector2)
}
#[inline(always)]
unsafe fn and(self, vector2: Self) -> __m256i {
_mm256_and_si256(self, vector2)
}
}
}
#[cfg(target_arch = "wasm32")]
mod wasm_simd128 {
use super::Vector;
use core::arch::wasm32::*;
impl Vector for v128 {
#[inline(always)]
unsafe fn splat(byte: u8) -> v128 {
u8x16_splat(byte)
}
#[inline(always)]
unsafe fn load_unaligned(data: *const u8) -> v128 {
v128_load(data.cast())
}
#[inline(always)]
unsafe fn movemask(self) -> u32 {
u8x16_bitmask(self).into()
}
#[inline(always)]
unsafe fn cmpeq(self, vector2: Self) -> v128 {
u8x16_eq(self, vector2)
}
#[inline(always)]
unsafe fn and(self, vector2: Self) -> v128 {
v128_and(self, vector2)
}
}
}