cpufeatures/
x86.rs

1//! x86/x86-64 CPU feature detection support.
2//!
3//! Portable, `no_std`-friendly implementation that relies on the x86 `CPUID`
4//! instruction for feature detection.
5
6/// Evaluate the given `$body` expression any of the supplied target features
7/// are not enabled. Otherwise returns true.
8///
9/// The `$body` expression is not evaluated on SGX targets, and returns false
10/// on these targets unless *all* supplied target features are enabled.
11#[macro_export]
12#[doc(hidden)]
13macro_rules! __unless_target_features {
14    ($($tf:tt),+ => $body:expr ) => {{
15        #[cfg(not(all($(target_feature=$tf,)*)))]
16        {
17            #[cfg(not(any(target_env = "sgx", target_os = "none", target_os = "uefi")))]
18            $body
19
20            // CPUID is not available on SGX. Freestanding and UEFI targets
21            // do not support SIMD features with default compilation flags.
22            #[cfg(any(target_env = "sgx", target_os = "none", target_os = "uefi"))]
23            false
24        }
25
26        #[cfg(all($(target_feature=$tf,)*))]
27        true
28    }};
29}
30
31/// Use CPUID to detect the presence of all supplied target features.
32#[macro_export]
33#[doc(hidden)]
34macro_rules! __detect_target_features {
35    ($($tf:tt),+) => {{
36        #[cfg(target_arch = "x86")]
37        use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult};
38        #[cfg(target_arch = "x86_64")]
39        use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult};
40
41        unsafe fn cpuid(leaf: u32) -> CpuidResult {
42            __cpuid(leaf)
43        }
44
45        unsafe fn cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult {
46            __cpuid_count(leaf, sub_leaf)
47        }
48
49        let cr = unsafe {
50            [cpuid(1), cpuid_count(7, 0), cpuid_count(7, 1)]
51        };
52
53        $($crate::check!(cr, $tf) & )+ true
54    }};
55}
56
57/// Check that OS supports required SIMD registers
58#[macro_export]
59#[doc(hidden)]
60macro_rules! __xgetbv {
61    ($cr:expr, $mask:expr) => {{
62        #[cfg(target_arch = "x86")]
63        use core::arch::x86 as arch;
64        #[cfg(target_arch = "x86_64")]
65        use core::arch::x86_64 as arch;
66
67        // Check bits 26 and 27
68        let xmask = 0b11 << 26;
69        let xsave = $cr[0].ecx & xmask == xmask;
70        if xsave {
71            let xcr0 = unsafe { arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK) };
72            (xcr0 & $mask) == $mask
73        } else {
74            false
75        }
76    }};
77}
78
79macro_rules! __expand_check_macro {
80    ($(($name:tt, $reg_cap:tt $(, $i:expr, $reg:ident, $offset:expr)*)),* $(,)?) => {
81        #[macro_export]
82        #[doc(hidden)]
83        macro_rules! check {
84            $(
85                ($cr:expr, $name) => {{
86                    // Register bits are listed here:
87                    // https://wiki.osdev.org/CPU_Registers_x86#Extended_Control_Registers
88                    let reg_cap = match $reg_cap {
89                        // Bit 1
90                        "xmm" => $crate::__xgetbv!($cr, 0b10),
91                        // Bits 1 and 2
92                        "ymm" => $crate::__xgetbv!($cr, 0b110),
93                        // Bits 1, 2, 5, 6, and 7
94                        "zmm" => $crate::__xgetbv!($cr, 0b1110_0110),
95                        _ => true,
96                    };
97                    reg_cap
98                    $(
99                        & ($cr[$i].$reg & (1 << $offset) != 0)
100                    )*
101                }};
102            )*
103        }
104    };
105}
106
107__expand_check_macro! {
108    ("sse3", "", 0, ecx, 0),
109    ("pclmulqdq", "", 0, ecx, 1),
110    ("ssse3", "", 0, ecx, 9),
111    ("fma", "ymm", 0, ecx, 12, 0, ecx, 28),
112    ("sse4.1", "", 0, ecx, 19),
113    ("sse4.2", "", 0, ecx, 20),
114    ("popcnt", "", 0, ecx, 23),
115    ("aes", "", 0, ecx, 25),
116    ("avx", "xmm", 0, ecx, 28),
117    ("rdrand", "", 0, ecx, 30),
118
119    ("mmx", "", 0, edx, 23),
120    ("sse", "", 0, edx, 25),
121    ("sse2", "", 0, edx, 26),
122
123    ("sgx", "", 1, ebx, 2),
124    ("bmi1", "", 1, ebx, 3),
125    ("bmi2", "", 1, ebx, 8),
126    ("avx2", "ymm", 1, ebx, 5, 0, ecx, 28),
127    ("avx512f", "zmm", 1, ebx, 16),
128    ("avx512dq", "zmm", 1, ebx, 17),
129    ("rdseed", "", 1, ebx, 18),
130    ("adx", "", 1, ebx, 19),
131    ("avx512ifma", "zmm", 1, ebx, 21),
132    ("avx512pf", "zmm", 1, ebx, 26),
133    ("avx512er", "zmm", 1, ebx, 27),
134    ("avx512cd", "zmm", 1, ebx, 28),
135    ("sha", "", 1, ebx, 29),
136    ("avx512bw", "zmm", 1, ebx, 30),
137    ("avx512vl", "zmm", 1, ebx, 31),
138    ("avx512vbmi", "zmm", 1, ecx, 1),
139    ("avx512vbmi2", "zmm", 1, ecx, 6),
140    ("gfni", "zmm", 1, ecx, 8),
141    ("vaes", "zmm", 1, ecx, 9),
142    ("vpclmulqdq", "zmm", 1, ecx, 10),
143    ("avx512bitalg", "zmm", 1, ecx, 12),
144    ("avx512vpopcntdq", "zmm", 1, ecx, 14),
145
146    ("sha512", "ymm", 2, eax, 0),
147    ("sm3", "xmm", 2, eax, 1),
148    ("sm4", "ymm", 2, eax, 2),
149}
cpufeatures/x86.rs

cpufeatures/
x86.rs
