pulp/core_arch/
mod.rs

1mod arch {
2	#[cfg(target_arch = "aarch64")]
3	pub use core::arch::aarch64::*;
4	#[cfg(target_arch = "wasm32")]
5	pub use core::arch::wasm32::*;
6	#[cfg(target_arch = "x86")]
7	pub use core::arch::x86::*;
8	#[cfg(target_arch = "x86_64")]
9	pub use core::arch::x86_64::*;
10
11	#[cfg(all(feature = "nightly", target_arch = "arm"))]
12	pub use core::arch::arm::*;
13	#[cfg(all(feature = "nightly", target_arch = "mips"))]
14	pub use core::arch::mips::*;
15	#[cfg(all(feature = "nightly", target_arch = "mips64"))]
16	pub use core::arch::mips64::*;
17	#[cfg(all(feature = "nightly", target_arch = "powerpc"))]
18	pub use core::arch::powerpc::*;
19	#[cfg(all(feature = "nightly", target_arch = "powerpc64"))]
20	pub use core::arch::powerpc64::*;
21	#[cfg(all(feature = "nightly", target_arch = "riscv32"))]
22	pub use core::arch::riscv32::*;
23	#[cfg(all(feature = "nightly", target_arch = "riscv64"))]
24	pub use core::arch::riscv64::*;
25	#[cfg(all(feature = "nightly", target_arch = "wasm64"))]
26	pub use core::arch::wasm64::*;
27}
28
29#[allow(unused_macros)]
30macro_rules! delegate {
31    ({$(
32        $(#[$attr: meta])*
33        $(const $($const: lifetime)?)?
34        $(unsafe $($unsafe: lifetime)?)?
35        fn $func: ident $(<$(const $generic: ident: $generic_ty: ty),* $(,)?>)?(
36            $($arg: ident: $ty: ty),* $(,)?
37        ) $(-> $ret: ty)?;
38    )*}) => {
39        $(
40            $(#[$attr])*
41            #[allow(clippy::missing_safety_doc)]
42            #[inline(always)]
43            pub $(const $($const)?)? $(unsafe $($unsafe)?)? fn $func $(<$(const $generic: $generic_ty,)*>)?(self, $($arg: $ty,)*) $(-> $ret)? {
44                #[allow(unused_unsafe)]
45                unsafe { arch::$func $(::<$($generic,)*>)?($($arg,)*) }
46            }
47        )*
48    };
49}
50
51#[cfg(all(feature = "std", any(target_arch = "x86", target_arch = "x86_64")))]
52#[macro_export]
53macro_rules! feature_detected {
54	($feature: tt) => {
55		::std::is_x86_feature_detected!($feature)
56	};
57}
58
59#[cfg(all(feature = "std", target_arch = "aarch64"))]
60#[macro_export]
61macro_rules! feature_detected {
62	($feature: tt) => {
63		::std::arch::is_aarch64_feature_detected!($feature)
64	};
65}
66
67#[cfg(target_arch = "wasm32")]
68#[macro_export]
69macro_rules! feature_detected {
70	("simd128") => {
71		$crate::wasm::is_simd128_enabled()
72	};
73	("relaxed-simd") => {{
74		#[cfg(feature = "relaxed-simd")]
75		{
76			$crate::wasm::is_relaxed_simd_enabled()
77		}
78		#[cfg(not(feature = "relaxed-simd"))]
79		{
80			false
81		}
82	}};
83}
84
85#[cfg(all(
86	not(feature = "std"),
87	any(
88		all(target_arch = "x86", not(target_env = "sgx"), target_feature = "sse"),
89		all(target_arch = "x86_64", not(target_env = "sgx"))
90	)
91))]
92#[doc(hidden)]
93pub mod raw_cpuid_detect;
94
95#[cfg(all(
96	not(feature = "std"),
97	any(
98		all(target_arch = "x86", not(target_env = "sgx"), target_feature = "sse"),
99		all(target_arch = "x86_64", not(target_env = "sgx")),
100	)
101))]
102#[macro_export]
103macro_rules! feature_detected {
104	($feature: tt) => {
105		cfg!(target_feature = $feature)
106			|| $crate::core_arch::raw_cpuid_detect::feature($crate::feature_idx!($feature))
107	};
108}
109
110#[cfg(not(any(
111	target_arch = "wasm32",
112	all(
113		feature = "std",
114		any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")
115	),
116	all(any(
117		all(target_arch = "x86", not(target_env = "sgx"), target_feature = "sse"),
118		all(target_arch = "x86_64", not(target_env = "sgx"))
119	)),
120)))]
121#[macro_export]
122macro_rules! feature_detected {
123	($feature: tt) => {
124		cfg!(target_feature = $feature)
125	};
126}
127
128#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
129#[doc(hidden)]
130#[rustfmt::skip]
131#[macro_export]
132macro_rules! __impl_type {
133    ("aes") => { $crate::core_arch::x86::Aes };
134    ("pclmulqdq") => { $crate::core_arch::x86::Pclmulqdq };
135    ("rdrand") => { $crate::core_arch::x86::Rdrand };
136    ("rdseed") => { $crate::core_arch::x86::Rdseed };
137    ("tsc") => { $crate::core_arch::x86::Tsc };
138    ("mmx") => { $crate::core_arch::x86::Mmx };
139    ("sse") => { $crate::core_arch::x86::Sse };
140    ("sse2") => { $crate::core_arch::x86::Sse2 };
141    ("sse3") => { $crate::core_arch::x86::Sse3 };
142    ("ssse3") => { $crate::core_arch::x86::Ssse3 };
143    ("sse4.1") => { $crate::core_arch::x86::Sse4_1 };
144    ("sse4.2") => { $crate::core_arch::x86::Sse4_2 };
145    ("sse4a") => { $crate::core_arch::x86::Sse4a };
146    ("sha") => { $crate::core_arch::x86::Sha };
147    ("avx") => { $crate::core_arch::x86::Avx };
148    ("avx2") => { $crate::core_arch::x86::Avx2 };
149    ("gfni") => { $crate::core_arch::x86::Gfni };
150    ("vaes") => { $crate::core_arch::x86::Vaes };
151    ("vpclmulqdq") => { $crate::core_arch::x86::Vpclmulqdq };
152    ("avx512f") => { $crate::core_arch::x86::Avx512f };
153    ("avx512cd") => { $crate::core_arch::x86::Avx512cd };
154    ("avx512er") => { $crate::core_arch::x86::Avx512er };
155    ("avx512pf") => { $crate::core_arch::x86::Avx512pf };
156    ("avx512bw") => { $crate::core_arch::x86::Avx512bw };
157    ("avx512dq") => { $crate::core_arch::x86::Avx512dq };
158    ("avx512vl") => { $crate::core_arch::x86::Avx512f };
159    ("avx512ifma") => { $crate::core_arch::x86::Avx512ifma };
160    ("avx512vbmi") => { $crate::core_arch::x86::Avx512vbmi };
161    ("avx512vpopcntdq") => { $crate::core_arch::x86::Avx512vpopcntdq };
162    ("avx512vbmi2") => { $crate::core_arch::x86::Avx512vbmi2 };
163    ("avx512gfni") => { $crate::core_arch::x86::Avx512gfni };
164    ("avx512vaes") => { $crate::core_arch::x86::Avx512vaes };
165    ("avx512vpclmulqdq") => { $crate::core_arch::x86::Avx512vpclmulqdq };
166    ("avx512vnni") => { $crate::core_arch::x86::Avx512vnni };
167    ("avx512bitalg") => { $crate::core_arch::x86::Avx512bitalg };
168    ("avx512bf16") => { $crate::core_arch::x86::Avx512bf16 };
169    ("avx512vp2intersect") => { $crate::core_arch::x86::Avx512vp2intersect };
170    ("f16c") => { $crate::core_arch::x86::F16c };
171    ("fma") => { $crate::core_arch::x86::Fma };
172    ("bmi1") => { $crate::core_arch::x86::Bmi1 };
173    ("bmi2") => { $crate::core_arch::x86::Bmi2 };
174    ("lzcnt") => { $crate::core_arch::x86::Lzcnt };
175    ("tbm") => { $crate::core_arch::x86::Tbm };
176    ("popcnt") => { $crate::core_arch::x86::Popcnt };
177    ("fxsr") => { $crate::core_arch::x86::Fxsr };
178    ("xsave") => { $crate::core_arch::x86::Xsave };
179    ("xsaveopt") => { $crate::core_arch::x86::Xsaveopt };
180    ("xsaves") => { $crate::core_arch::x86::Xsaves };
181    ("xsavec") => { $crate::core_arch::x86::Xsavec };
182    ("cmpxchg16b") => { $crate::core_arch::x86::Cmpxchg16b };
183    ("adx") => { $crate::core_arch::x86::Adx };
184    ("rtm") => { $crate::core_arch::x86::Rtm };
185    ("abm") => { $crate::core_arch::x86::Abm };
186}
187
188#[cfg(target_arch = "aarch64")]
189#[doc(hidden)]
190#[rustfmt::skip]
191#[macro_export]
192macro_rules! __impl_type {
193    ("neon") => { $crate::core_arch::aarch64::Neon };
194    ("pmull") => { $crate::core_arch::aarch64::Pmull };
195    ("fp") => { $crate::core_arch::aarch64::Fp };
196    ("fp16") => { $crate::core_arch::aarch64::Fp16 };
197    ("sve") => { $crate::core_arch::aarch64::Sve };
198    ("crc") => { $crate::core_arch::aarch64::Crc };
199    ("lse") => { $crate::core_arch::aarch64::Lse };
200    ("lse2") => { $crate::core_arch::aarch64::Lse2 };
201    ("rdm") => { $crate::core_arch::aarch64::Rdm };
202    ("rcpc") => { $crate::core_arch::aarch64::Rcpc };
203    ("rcpc2") => { $crate::core_arch::aarch64::Rcpc2 };
204    ("dotprod") => { $crate::core_arch::aarch64::Dotprod };
205    ("tme") => { $crate::core_arch::aarch64::Tme };
206    ("fhm") => { $crate::core_arch::aarch64::Fhm };
207    ("dit") => { $crate::core_arch::aarch64::Dit };
208    ("flagm") => { $crate::core_arch::aarch64::Flagm };
209    ("ssbs") => { $crate::core_arch::aarch64::Ssbs };
210    ("sb") => { $crate::core_arch::aarch64::Sb };
211    ("paca") => { $crate::core_arch::aarch64::Paca };
212    ("pacg") => { $crate::core_arch::aarch64::Pacg };
213    ("dpb") => { $crate::core_arch::aarch64::Dpb };
214    ("dpb2") => { $crate::core_arch::aarch64::Dpb2 };
215    ("sve2") => { $crate::core_arch::aarch64::Sve2 };
216    ("sve2-aes") => { $crate::core_arch::aarch64::Sve2Aes };
217    ("sve2-sm4") => { $crate::core_arch::aarch64::Sve2Sm4 };
218    ("sve2-sha3") => { $crate::core_arch::aarch64::Sve2Sha3 };
219    ("sve2-bitperm") => { $crate::core_arch::aarch64::Sve2Bitperm };
220    ("frintts") => { $crate::core_arch::aarch64::Frintts };
221    ("i8mm") => { $crate::core_arch::aarch64::I8mm };
222    ("f32mm") => { $crate::core_arch::aarch64::F32mm };
223    ("f64mm") => { $crate::core_arch::aarch64::F64mm };
224    ("bf16") => { $crate::core_arch::aarch64::Bf16 };
225    ("rand") => { $crate::core_arch::aarch64::Rand };
226    ("bti") => { $crate::core_arch::aarch64::Bti };
227    ("mte") => { $crate::core_arch::aarch64::Mte };
228    ("jsconv") => { $crate::core_arch::aarch64::Jsconv };
229    ("fcma") => { $crate::core_arch::aarch64::Fcma };
230    ("aes") => { $crate::core_arch::aarch64::Aes };
231    ("sha2") => { $crate::core_arch::aarch64::Sha2 };
232    ("sha3") => { $crate::core_arch::aarch64::Sha3 };
233    ("sm4") => { $crate::core_arch::aarch64::Sm4 };
234    ("asimd") => { $crate::core_arch::aarch64::Asimd };
235}
236
237#[doc(hidden)]
238pub struct Unavailable<const N: usize>;
239
240#[cfg(all(target_arch = "wasm32", feature = "relaxed-simd"))]
241#[doc(hidden)]
242#[rustfmt::skip]
243#[macro_export]
244macro_rules! __impl_type {
245    ("simd128") => { $crate::core_arch::wasm::Simd128 };
246    ("relaxed-simd") => { $crate::core_arch::wasm::RelaxedSimd };
247}
248
249#[cfg(all(target_arch = "wasm32", not(feature = "relaxed-simd")))]
250#[doc(hidden)]
251#[rustfmt::skip]
252#[macro_export]
253macro_rules! __impl_type {
254    ("simd128") => { $crate::core_arch::wasm::Simd128 };
255    ("relaxed-simd") => { $crate::core_arch::Unavailable::<{compile_error!("enable the `relaxed-simd` feature to use wasm relaxed-simd")}> };
256}
257
258#[cfg(not(any(target_arch = "x86_64", target_arch = "x86", target_arch = "aarch64", target_arch = "wasm32")))]
259#[doc(hidden)]
260#[rustfmt::skip]
261#[macro_export]
262macro_rules! __impl_type {
263    ($tt: tt) => { $crate::core_arch::Unavailable::<{compile_error!("unsupported arch")}> }
264}
265
266#[macro_export]
267macro_rules! simd_type {
268    (
269        $(
270            $(#[$attr: meta])*
271            $vis: vis struct $name: ident {
272                $($feature_vis: vis $ident: ident: $feature: tt),* $(,)?
273            }
274        )*
275    ) => {
276        $crate::simd_type!({$(
277            $(#[$attr])*
278            $vis struct $name {
279                $($feature_vis $ident: target_feature!($feature),)*
280            }
281        )*});
282    };
283
284    ({
285        $(
286            $(#[$attr: meta])*
287            $vis: vis struct $name: ident {
288                $($feature_vis: vis $ident: ident: $_: ident!($feature: tt)),* $(,)?
289            }
290        )*
291    }) => {
292        $(
293            #[allow(dead_code)]
294            $(#[$attr])*
295            #[derive(Clone, Copy, Debug)]
296            $vis struct $name{
297                $($feature_vis $ident : $crate::core_arch::__impl_type!($feature),)*
298            }
299
300            #[allow(dead_code)]
301            $(#[$attr])*
302            impl $name {
303                /// Returns a SIMD token type without checking if the required CPU features for
304                /// this type are available.
305                ///
306                /// # Safety
307                /// - the required CPU features must be available.
308                #[inline]
309                pub const unsafe fn new_unchecked() -> Self {
310                    #[allow(unused_unsafe)]
311                    unsafe {
312                        Self{
313                            $($ident: <$crate::core_arch::__impl_type!($feature)>::new_unchecked(),)*
314                        }
315                    }
316                }
317
318                /// Returns a SIMD token type if the required CPU features for this type are
319                /// available, otherwise returns `None`.
320                #[inline]
321                pub fn try_new() -> Option<Self> {
322                    #[allow(unused_unsafe)]
323                    if Self::is_available() {
324                        Some(unsafe{Self{
325                            $($ident: <$crate::core_arch::__impl_type!($feature)>::new_unchecked(),)*
326                        }})
327                    } else {
328                        None
329                    }
330                }
331
332                #[inline(always)]
333                fn __static_available() -> &'static ::core::sync::atomic::AtomicU8 {
334                    static AVAILABLE: ::core::sync::atomic::AtomicU8 = ::core::sync::atomic::AtomicU8::new(u8::MAX);
335                    &AVAILABLE
336                }
337
338                /// Returns `true` if the required CPU features for this type are available,
339                /// otherwise returns `false`.
340                #[inline]
341                pub fn is_available() -> bool {
342                    let mut available = Self::__static_available().load(::core::sync::atomic::Ordering::Relaxed);
343                    if available == u8::MAX {
344                        available = Self::__detect_is_available() as u8;
345                    }
346
347                    available != 0
348                }
349
350                #[inline(never)]
351                fn __detect_is_available() -> bool {
352                    let out = true $(&& <$crate::core_arch::__impl_type!($feature)>::is_available())*;
353                    Self::__static_available().store(out as u8, ::core::sync::atomic::Ordering::Relaxed);
354                    out
355                }
356
357                /// Vectorizes the given function as if the CPU features for this type were applied
358                /// to it.
359                ///
360                /// # Note
361                /// For the vectorization to work properly, the given function must be inlined.
362                /// Consider marking it as `#[inline(always)]`
363                #[inline(always)]
364                pub fn vectorize<F: $crate::NullaryFnOnce>(self, f: F) -> F::Output {
365                    $(#[target_feature(enable = $feature)])*
366                    #[inline]
367                	#[allow(clippy::too_many_arguments)]
368                    unsafe fn imp_fastcall<F: $crate::NullaryFnOnce>(
369                        f0: ::core::mem::MaybeUninit<::core::primitive::usize>,
370                        f1: ::core::mem::MaybeUninit<::core::primitive::usize>,
371                        f2: ::core::mem::MaybeUninit<::core::primitive::usize>,
372                        f3: ::core::mem::MaybeUninit<::core::primitive::usize>,
373                        f4: ::core::mem::MaybeUninit<::core::primitive::usize>,
374                        f5: ::core::mem::MaybeUninit<::core::primitive::usize>,
375                        f6: ::core::mem::MaybeUninit<::core::primitive::usize>,
376                        f7: ::core::mem::MaybeUninit<::core::primitive::usize>,
377                    ) -> F::Output {
378                        let f: F = core::mem::transmute_copy(&[f0, f1, f2, f3, f4, f5, f6, f7]);
379                        f.call()
380                    }
381
382                    $(#[target_feature(enable = $feature)])*
383                    #[inline]
384                    unsafe fn imp<F: $crate::NullaryFnOnce>(f: F) -> F::Output {
385                        f.call()
386                    }
387
388                    if const { ::core::mem::size_of::<F>() <= 8 * ::core::mem::size_of::<::core::primitive::usize>() } {
389                        union Pad<T> {
390                            t: ::core::mem::ManuallyDrop<T>,
391                            __u: ::core::mem::MaybeUninit<[usize; 8]>,
392                        }
393
394                        let f = Pad {
395                            t: ::core::mem::ManuallyDrop::new(f),
396                        };
397                        let p = (&f) as *const _ as *const ::core::mem::MaybeUninit<usize>;
398
399                        unsafe {
400                            imp_fastcall::<F>(
401                                *p.add(0),
402                                *p.add(1),
403                                *p.add(2),
404                                *p.add(3),
405                                *p.add(4),
406                                *p.add(5),
407                                *p.add(6),
408                                *p.add(7),
409                            )
410                        }
411                    } else {
412                        unsafe { imp(f) }
413                    }
414                }
415
416                /// Takes a proof of the existence of this SIMD token (`self`), and returns a
417                /// persistent reference to it.
418                #[inline]
419                pub fn to_ref(self) -> &'static Self {
420                    const __ASSERT_ZST: () = {
421                        assert!(::core::mem::size_of::<$name>() == 0);
422                    };
423                    unsafe { &*::core::ptr::NonNull::dangling().as_ptr() }
424                }
425            }
426        )*
427    };
428}
429
430#[rustfmt::skip]
431#[doc(hidden)]
432pub use __impl_type as __impl_type;
433
434#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
435#[cfg_attr(docsrs, doc(cfg(any(target_arch = "x86", target_arch = "x86_64"))))]
436pub mod x86;
437
438#[cfg(target_arch = "aarch64")]
439#[cfg_attr(docsrs, doc(cfg(target_arch = "aarch64")))]
440pub mod aarch64;
441
442#[cfg(target_arch = "wasm32")]
443#[cfg_attr(docsrs, doc(cfg(target_arch = "wasm32")))]
444pub mod wasm;
pulp/core_arch/mod.rs

pulp/core_arch/
mod.rs
