1use super::*;
2#[cfg(target_arch = "x86_64")]
3use crate::core_arch::x86::Avx2;
4
5#[cfg(target_arch = "x86")]
6use core::arch::x86::*;
7#[cfg(target_arch = "x86_64")]
8use core::arch::x86_64::*;
9
10macro_rules! x86_call_128 {
11 ($ext: expr, $func: ident, f32, $($arg: expr),*) => {
12 paste!($ext.[<_mm_ $func _ ps>]($($arg),*))
13 };
14 ($ext: expr, $func: ident, f64, $($arg: expr),*) => {
15 paste!($ext.[<_mm_ $func _ pd>]($($arg),*))
16 };
17 ($ext: expr, $func: ident, $ty: ty, $($arg: expr),*) => {
18 paste!($ext.[<_mm_ $func _ep $ty>]($($arg),*))
19 }
20}
21
22macro_rules! x86_call_256 {
23 ($ext: expr, $func: ident, f32, $($arg: expr),*) => {
24 paste!($ext.[<_mm256_ $func _ ps>]($($arg),*))
25 };
26 ($ext: expr, $func: ident, f64, $($arg: expr),*) => {
27 paste!($ext.[<_mm256_ $func _ pd>]($($arg),*))
28 };
29 ($ext: expr, $func: ident, $ty: ty, $($arg: expr),*) => {
30 paste!($ext.[<_mm256_ $func _ep $ty>]($($arg),*))
31 }
32}
33
34macro_rules! x86_call_128_nosign {
35 ($ext: expr, $func: ident, u8, $($arg: expr),*) => {
36 x86_call_128!($ext, $func, i8, $($arg),*)
37 };
38 ($ext: expr, $func: ident, u16, $($arg: expr),*) => {
39 x86_call_128!($ext, $func, i16, $($arg),*)
40 };
41 ($ext: expr, $func: ident, u32, $($arg: expr),*) => {
42 x86_call_128!($ext, $func, i32, $($arg),*)
43 };
44 ($ext: expr, $func: ident, u64, $($arg: expr),*) => {
45 x86_call_128!($ext, $func, i64, $($arg),*)
46 };
47 ($ext: expr, $func: ident, m8, $($arg: expr),*) => {
48 x86_call_128!($ext, $func, i8, $($arg),*)
49 };
50 ($ext: expr, $func: ident, m16, $($arg: expr),*) => {
51 x86_call_128!($ext, $func, i16, $($arg),*)
52 };
53 ($ext: expr, $func: ident, m32, $($arg: expr),*) => {
54 x86_call_128!($ext, $func, i32, $($arg),*)
55 };
56 ($ext: expr, $func: ident, m64, $($arg: expr),*) => {
57 x86_call_128!($ext, $func, i64, $($arg),*)
58 };
59 ($ext: expr, $func: ident, $ty: ident, $($arg: expr),*) => {
60 x86_call_128!($ext, $func, $ty, $($arg),*)
61 };
62}
63
64macro_rules! x86_call_256_nosign {
65 ($ext: expr, $func: ident, u8, $($arg: expr),*) => {
66 x86_call_256!($ext, $func, i8, $($arg),*)
67 };
68 ($ext: expr, $func: ident, u16, $($arg: expr),*) => {
69 x86_call_256!($ext, $func, i16, $($arg),*)
70 };
71 ($ext: expr, $func: ident, u32, $($arg: expr),*) => {
72 x86_call_256!($ext, $func, i32, $($arg),*)
73 };
74 ($ext: expr, $func: ident, u64, $($arg: expr),*) => {
75 x86_call_256!($ext, $func, i64, $($arg),*)
76 };
77 ($ext: expr, $func: ident, m8, $($arg: expr),*) => {
78 x86_call_256!($ext, $func, i8, $($arg),*)
79 };
80 ($ext: expr, $func: ident, m16, $($arg: expr),*) => {
81 x86_call_256!($ext, $func, i16, $($arg),*)
82 };
83 ($ext: expr, $func: ident, m32, $($arg: expr),*) => {
84 x86_call_256!($ext, $func, i32, $($arg),*)
85 };
86 ($ext: expr, $func: ident, m64, $($arg: expr),*) => {
87 x86_call_256!($ext, $func, i64, $($arg),*)
88 };
89 ($ext: expr, $func: ident, $ty: ident, $($arg: expr),*) => {
90 x86_call_256!($ext, $func, $ty, $($arg),*)
91 };
92}
93
94macro_rules! binop_128 {
95 ($func: ident, $op: ident, $doc: literal, $ty: ident, $out: ident, $factor: literal, $ext: ident) => {
96 paste! {
97 #[inline(always)]
98 #[doc = $doc]
99 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$out x $factor>] {
100 cast!(x86_call_128!(self.$ext, $op, $ty, cast!(a), cast!(b)))
101 }
102 }
103 };
104 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal => $out: ident),*) => {
105 $(binop_128!($func, $op, $doc, $ty, $out, $factor, $ext);)*
106 };
107 ($ext: ident: $op: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
108 $(binop_128!($op, $op, $doc, $ty, $ty, $factor, $ext);)*
109 };
110 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
111 $(binop_128!($func, $op, $doc, $ty, $ty, $factor, $ext);)*
112 };
113}
114
115macro_rules! binop_256 {
116 ($func: ident, $op: ident, $doc: literal, $ty: ident, $out: ident, $factor: literal, $ext: ident) => {
117 paste! {
118 #[inline(always)]
119 #[doc = $doc]
120 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$out x $factor>] {
121 cast!(x86_call_256!(self.$ext, $op, $ty, cast!(a), cast!(b)))
122 }
123 }
124 };
125 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal => $out: ident),*) => {
126 $(binop_256!($func, $op, $doc, $ty, $out, $factor, $ext);)*
127 };
128 ($ext: ident: $op: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
129 $(binop_256!($op, $op, $doc, $ty, $ty, $factor, $ext);)*
130 };
131 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
132 $(binop_256!($func, $op, $doc, $ty, $ty, $factor, $ext);)*
133 };
134}
135
136macro_rules! unop_128 {
137 ($func: ident, $op: ident, $doc: literal, $ty: ident, $factor: literal, $ext: ident) => {
138 paste! {
139 #[inline(always)]
140 #[doc = $doc]
141 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>]) -> [<$ty x $factor>] {
142 cast!(x86_call_128!(self.$ext, $op, $ty, cast!(a)))
143 }
144 }
145 };
146 ($ext: ident: $op: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
147 $(unop_128!($op, $op, $doc, $ty, $factor, $ext);)*
148 };
149 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
150 $(unop_128!($func, $op, $doc, $ty, $factor, $ext);)*
151 };
152}
153
154macro_rules! unop_256 {
155 ($func: ident, $op: ident, $doc: literal, $ty: ident, $factor: literal, $ext: ident) => {
156 paste! {
157 #[inline(always)]
158 #[doc = $doc]
159 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>]) -> [<$ty x $factor>] {
160 cast!(x86_call_256!(self.$ext, $op, $ty, cast!(a)))
161 }
162 }
163 };
164 ($ext: ident: $op: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
165 $(unop_256!($op, $op, $doc, $ty, $factor, $ext);)*
166 };
167 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
168 $(unop_256!($func, $op, $doc, $ty, $factor, $ext);)*
169 };
170}
171
172macro_rules! binop_128_nosign {
173 ($func: ident, $op: ident, $doc: literal, $ty: ident, $out: ident, $factor: literal, $ext: ident) => {
174 paste! {
175 #[inline(always)]
176 #[doc = $doc]
177 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$out x $factor>] {
178 cast!(x86_call_128_nosign!(self.$ext, $op, $ty, cast!(a), cast!(b)))
179 }
180 }
181 };
182 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal => $out: ident),*) => {
183 $(binop_128_nosign!($func, $op, $doc, $ty, $out, $factor, $ext);)*
184 };
185 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
186 $(binop_128_nosign!($func, $op, $doc, $ty, $ty, $factor, $ext);)*
187 };
188 ($ext: ident: $func: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
189 $(binop_128_nosign!($func, $func, $doc, $ty, $ty, $factor, $ext);)*
190 };
191}
192
193macro_rules! binop_256_nosign {
194 ($func: ident, $op: ident, $doc: literal, $ty: ident, $out: ident, $factor: literal, $ext: ident) => {
195 paste! {
196 #[inline(always)]
197 #[doc = $doc]
198 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$out x $factor>] {
199 cast!(x86_call_256_nosign!(self.$ext, $op, $ty, cast!(a), cast!(b)))
200 }
201 }
202 };
203 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal => $out: ident),*) => {
204 $(binop_256_nosign!($func, $op, $doc, $ty, $out, $factor, $ext);)*
205 };
206 ($ext: ident: $op: ident, $doc: literal, $func: ident, $($ty: ident x $factor: literal),*) => {
207 $(binop_256_nosign!($func, $op, $doc, $ty, $ty, $factor, $ext);)*
208 };
209 ($ext: ident: $func: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
210 $(binop_256_nosign!($func, $func, $doc, $ty, $ty, $factor, $ext);)*
211 };
212}
213
214macro_rules! binop_128_full {
215 ($func: ident, $doc: literal, $ty: ident, $factor: literal, $ext: ident) => {
216 paste! {
217 #[inline(always)]
218 #[doc = $doc]
219 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$ty x $factor>] {
220 cast!(self.$ext.[<_mm_ $func _si128>](cast!(a), cast!(b)))
221 }
222 }
223 };
224 ($ext: ident: $func: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
225 $(binop_128_full!($func, $doc, $ty, $factor, $ext);)*
226 };
227}
228
229macro_rules! binop_256_full {
230 ($func: ident, $doc: literal, $ty: ident, $factor: literal, $ext: ident) => {
231 paste! {
232 #[inline(always)]
233 #[doc = $doc]
234 pub fn [<$func _ $ty x $factor>](self, a: [<$ty x $factor>], b: [<$ty x $factor>]) -> [<$ty x $factor>] {
235 cast!(self.$ext.[<_mm256_ $func _si256>](cast!(a), cast!(b)))
236 }
237 }
238 };
239 ($ext: ident: $func: ident, $doc: literal, $($ty: ident x $factor: literal),*) => {
240 $(binop_256_full!($func, $doc, $ty, $factor, $ext);)*
241 };
242}
243
244mod v1;
245mod v2;
246mod v3;
247#[cfg(feature = "x86-v4")]
248#[cfg_attr(docsrs, doc(cfg(feature = "x86-v4")))]
249mod v4;
250
251pub use v1::*;
252pub use v2::*;
253pub use v3::*;
254#[cfg(feature = "x86-v4")]
255#[cfg_attr(docsrs, doc(cfg(feature = "x86-v4")))]
256pub use v4::*;
257
258#[cfg(target_arch = "x86_64")]
259#[target_feature(enable = "avx,avx2")]
260#[inline]
261unsafe fn avx_ld_u32s(ptr: *const u32, f: unsafe extern "C" fn()) -> u32x8 {
262 let ret: __m256;
263 core::arch::asm! {
264 "lea rcx, [rip + 2f]",
265 "jmp {f}",
266 "2:",
267 f = in(reg) f,
268 in("rax") ptr,
269 out("rcx") _,
270 out("ymm0") ret,
271 out("ymm1") _,
272 };
273
274 cast!(ret)
275}
276
277#[cfg(target_arch = "x86_64")]
278#[target_feature(enable = "avx,avx2")]
279#[inline]
280unsafe fn avx_st_u32s(ptr: *mut u32, value: u32x8, f: unsafe extern "C" fn()) {
281 core::arch::asm! {
282 "lea rcx, [rip + 2f]",
283 "jmp {f}",
284 "2:",
285 f = in(reg) f,
286
287 in("rax") ptr,
288 out("rcx") _,
289 inout("ymm0") cast::<_, __m256>(value) => _,
290 out("ymm1") _,
291 };
292}
293
294#[derive(Debug, Clone, Copy)]
296#[non_exhaustive]
297#[repr(u8)]
298pub enum Arch {
299 Scalar = 0,
300
301 #[cfg(feature = "x86-v3")]
302 #[cfg_attr(docsrs, doc(cfg(feature = "x86-v3")))]
303 V3(V3) = 1,
304
305 #[cfg(feature = "x86-v4")]
306 #[cfg_attr(docsrs, doc(cfg(feature = "x86-v4")))]
307 V4(V4) = 2,
308}
309
310impl Arch {
311 #[inline]
313 pub fn new() -> Self {
314 #[cfg(feature = "x86-v4")]
315 if let Some(simd) = V4::try_new() {
316 return Self::V4(simd);
317 }
318 #[cfg(feature = "x86-v3")]
319 if let Some(simd) = V3::try_new() {
320 return Self::V3(simd);
321 }
322 Self::Scalar
323 }
324
325 #[inline(always)]
327 pub fn dispatch<Op: WithSimd>(self, op: Op) -> Op::Output {
328 match self {
329 #[cfg(feature = "x86-v4")]
330 Arch::V4(simd) => Simd::vectorize(simd, op),
331 #[cfg(feature = "x86-v3")]
332 Arch::V3(simd) => Simd::vectorize(simd, op),
333
334 Arch::Scalar => Simd::vectorize(Scalar, op),
335 }
336 }
337}
338
339impl Default for Arch {
340 #[inline]
341 fn default() -> Self {
342 Self::new()
343 }
344}
345
346#[cfg(target_arch = "x86_64")]
347include!(concat!(env!("OUT_DIR"), "/x86_64_asm.rs"));
348
349#[cfg(test)]
350mod tests {
351 extern crate alloc;
352
353 use super::*;
354 use alloc::vec;
355 use alloc::vec::Vec;
356 use assert_approx_eq::assert_approx_eq;
357 use core::iter::zip;
358 use rand::random;
359
360 #[allow(unused_macros)]
361 macro_rules! dbgx {
362 () => {
363 ::std::eprintln!("[{}:{}]", ::std::file!(), ::std::line!())
364 };
365 ($val:expr $(,)?) => {
366 match $val {
367 tmp => {
368 ::std::eprintln!("[{}:{}] {} = {:#X?}",
369 ::std::file!(), ::std::line!(), ::std::stringify!($val), &tmp);
370 tmp
371 }
372 }
373 };
374 ($($val:expr),+ $(,)?) => {
375 ($(dbgx!($val)),+,)
376 };
377 }
378
379 #[test]
380 fn times_two() {
381 let n = 1312;
382 let mut v = (0..n).map(|i| i as f64).collect::<Vec<_>>();
383 let arch = Arch::new();
384
385 struct TimesThree<'a>(&'a mut [f64]);
386 impl WithSimd for TimesThree<'_> {
387 type Output = ();
388
389 #[inline(always)]
390 fn with_simd<S: Simd>(self, simd: S) -> Self::Output {
391 let v = self.0;
392 let (head, tail) = S::as_mut_simd_f64s(v);
393
394 let three = simd.splat_f64s(3.0);
395 for x in head {
396 *x = simd.mul_f64s(three, *x);
397 }
398
399 for x in tail {
400 *x *= 3.0;
401 }
402 }
403 }
404
405 arch.dispatch(|| {
406 for x in &mut v {
407 *x *= 2.0;
408 }
409 });
410
411 arch.dispatch(TimesThree(&mut v));
412
413 for (i, x) in v.into_iter().enumerate() {
414 assert_eq!(x, 6.0 * i as f64);
415 }
416 }
417
418 #[test]
419 fn cplx_ops() {
420 let n = 16;
421 let a = (0..n)
422 .map(|_| c32 {
423 re: random(),
424 im: random(),
425 })
426 .collect::<Vec<_>>();
427 let b = (0..n)
428 .map(|_| c32 {
429 re: random(),
430 im: random(),
431 })
432 .collect::<Vec<_>>();
433 let c = (0..n)
434 .map(|_| c32 {
435 re: random(),
436 im: random(),
437 })
438 .collect::<Vec<_>>();
439
440 let axb_target = zip(&a, &b).map(|(a, b)| a * b).collect::<Vec<_>>();
441 let conjaxb_target = zip(&a, &b).map(|(a, b)| a.conj() * b).collect::<Vec<_>>();
442 let axbpc_target = zip(zip(&a, &b), &c)
443 .map(|((a, b), c)| a * b + c)
444 .collect::<Vec<_>>();
445 let conjaxbpc_target = zip(zip(&a, &b), &c)
446 .map(|((a, b), c)| a.conj() * b + c)
447 .collect::<Vec<_>>();
448
449 if let Some(simd) = V3::try_new() {
450 let mut axb = vec![c32::new(0.0, 0.0); n];
451 let mut conjaxb = vec![c32::new(0.0, 0.0); n];
452 let mut axbpc = vec![c32::new(0.0, 0.0); n];
453 let mut conjaxbpc = vec![c32::new(0.0, 0.0); n];
454
455 {
456 let a = V3::as_simd_c32s(&a).0;
457 let b = V3::as_simd_c32s(&b).0;
458 let c = V3::as_simd_c32s(&c).0;
459 let axb = V3::as_mut_simd_c32s(&mut axb).0;
460 let conjaxb = V3::as_mut_simd_c32s(&mut conjaxb).0;
461 let axbpc = V3::as_mut_simd_c32s(&mut axbpc).0;
462 let conjaxbpc = V3::as_mut_simd_c32s(&mut conjaxbpc).0;
463
464 for (axb, (a, b)) in zip(axb, zip(a, b)) {
465 *axb = simd.mul_e_c32s(*a, *b);
466 }
467 for (conjaxb, (a, b)) in zip(conjaxb, zip(a, b)) {
468 *conjaxb = simd.conj_mul_e_c32s(*a, *b);
469 }
470 for (axbpc, ((a, b), c)) in zip(axbpc, zip(zip(a, b), c)) {
471 *axbpc = simd.mul_add_e_c32s(*a, *b, *c);
472 }
473 for (conjaxbpc, ((a, b), c)) in zip(conjaxbpc, zip(zip(a, b), c)) {
474 *conjaxbpc = simd.conj_mul_add_e_c32s(*a, *b, *c);
475 }
476 }
477
478 for (target, actual) in zip(&axb_target, &axb) {
479 assert_approx_eq!(target.re, actual.re);
480 assert_approx_eq!(target.im, actual.im);
481 }
482 for (target, actual) in zip(&conjaxb_target, &conjaxb) {
483 assert_approx_eq!(target.re, actual.re);
484 assert_approx_eq!(target.im, actual.im);
485 }
486 for (target, actual) in zip(&axbpc_target, &axbpc) {
487 assert_approx_eq!(target.re, actual.re);
488 assert_approx_eq!(target.im, actual.im);
489 }
490 for (target, actual) in zip(&conjaxbpc_target, &conjaxbpc) {
491 assert_approx_eq!(target.re, actual.re);
492 assert_approx_eq!(target.im, actual.im);
493 }
494 }
495
496 #[cfg(feature = "x86-v4")]
497 if let Some(simd) = V4::try_new() {
498 let mut axb = vec![c32::new(0.0, 0.0); n];
499 let mut conjaxb = vec![c32::new(0.0, 0.0); n];
500 let mut axbpc = vec![c32::new(0.0, 0.0); n];
501 let mut conjaxbpc = vec![c32::new(0.0, 0.0); n];
502
503 {
504 let a = V4::as_simd_c32s(&a).0;
505 let b = V4::as_simd_c32s(&b).0;
506 let c = V4::as_simd_c32s(&c).0;
507 let axb = V4::as_mut_simd_c32s(&mut axb).0;
508 let conjaxb = V4::as_mut_simd_c32s(&mut conjaxb).0;
509 let axbpc = V4::as_mut_simd_c32s(&mut axbpc).0;
510 let conjaxbpc = V4::as_mut_simd_c32s(&mut conjaxbpc).0;
511
512 for (axb, (a, b)) in zip(axb, zip(a, b)) {
513 *axb = simd.mul_e_c32s(*a, *b);
514 }
515 for (conjaxb, (a, b)) in zip(conjaxb, zip(a, b)) {
516 *conjaxb = simd.conj_mul_e_c32s(*a, *b);
517 }
518 for (axbpc, ((a, b), c)) in zip(axbpc, zip(zip(a, b), c)) {
519 *axbpc = simd.mul_add_e_c32s(*a, *b, *c);
520 }
521 for (conjaxbpc, ((a, b), c)) in zip(conjaxbpc, zip(zip(a, b), c)) {
522 *conjaxbpc = simd.conj_mul_add_e_c32s(*a, *b, *c);
523 }
524 }
525
526 for (target, actual) in zip(&axb_target, &axb) {
527 assert_approx_eq!(target.re, actual.re);
528 assert_approx_eq!(target.im, actual.im);
529 }
530 for (target, actual) in zip(&conjaxb_target, &conjaxb) {
531 assert_approx_eq!(target.re, actual.re);
532 assert_approx_eq!(target.im, actual.im);
533 }
534 for (target, actual) in zip(&axbpc_target, &axbpc) {
535 assert_approx_eq!(target.re, actual.re);
536 assert_approx_eq!(target.im, actual.im);
537 }
538 for (target, actual) in zip(&conjaxbpc_target, &conjaxbpc) {
539 assert_approx_eq!(target.re, actual.re);
540 assert_approx_eq!(target.im, actual.im);
541 }
542 }
543 }
544
545 #[test]
546 fn test_to_ref() {
547 let simd_ref = unsafe { V2::new_unchecked() }.to_ref();
548 let _ = *simd_ref;
549 }
550
551 #[test]
552 fn test_widening_mul_u32x4() {
553 if let Some(simd) = V2::try_new() {
554 const N: usize = 4;
555 let a = u32x4(2298413717, 568259975, 2905436181, 175547995);
556 let b = u32x4(2022374205, 1446824162, 3165580604, 3011091403);
557 let a_array: [u32; N] = cast!(a);
558 let b_array: [u32; N] = cast!(b);
559 let mut lo_array = [0u32; N];
560 let mut hi_array = [0u32; N];
561
562 for i in 0..N {
563 let prod = a_array[i] as u64 * b_array[i] as u64;
564 let lo = prod as u32;
565 let hi = (prod >> 32) as u32;
566 lo_array[i] = lo;
567 hi_array[i] = hi;
568 }
569
570 let (lo, hi) = simd.widening_mul_u32x4(a, b);
571 assert_eq!(lo, cast!(lo_array));
572 assert_eq!(hi, cast!(hi_array));
573 }
574 if let Some(simd) = V3::try_new() {
575 const N: usize = 8;
576 let a = u32x8(
577 2298413717, 568259975, 2905436181, 175547995, 2298413717, 568259975, 2905436181,
578 175547995,
579 );
580 let b = u32x8(
581 2022374205, 1446824162, 3165580604, 3011091403, 2022374205, 1446824162, 3165580604,
582 3011091403,
583 );
584 let a_array: [u32; N] = cast!(a);
585 let b_array: [u32; N] = cast!(b);
586 let mut lo_array = [0u32; N];
587 let mut hi_array = [0u32; N];
588
589 for i in 0..N {
590 let prod = a_array[i] as u64 * b_array[i] as u64;
591 let lo = prod as u32;
592 let hi = (prod >> 32) as u32;
593 lo_array[i] = lo;
594 hi_array[i] = hi;
595 }
596
597 let (lo, hi) = simd.widening_mul_u32x8(a, b);
598 assert_eq!(lo, cast!(lo_array));
599 assert_eq!(hi, cast!(hi_array));
600 }
601 }
602
603 #[test]
604 fn test_widening_mul_i32() {
605 if let Some(simd) = V2::try_new() {
606 const N: usize = 4;
607 let a = cast!(u32x4(2298413717, 568259975, 2905436181, 175547995));
608 let b = cast!(u32x4(2022374205, 1446824162, 3165580604, 3011091403));
609
610 let a_array: [i32; N] = cast!(a);
611 let b_array: [i32; N] = cast!(b);
612 let mut lo_array = [0i32; N];
613 let mut hi_array = [0i32; N];
614
615 for i in 0..N {
616 let prod = a_array[i] as i64 * b_array[i] as i64;
617 let lo = prod as i32;
618 let hi = (prod >> 32) as i32;
619 lo_array[i] = lo;
620 hi_array[i] = hi;
621 }
622
623 let (lo, hi) = simd.widening_mul_i32x4(a, b);
624 assert_eq!(lo, cast!(lo_array));
625 assert_eq!(hi, cast!(hi_array));
626 }
627 if let Some(simd) = V3::try_new() {
628 const N: usize = 8;
629 let a = cast!(u32x8(
630 2298413717, 568259975, 2905436181, 175547995, 2298413717, 568259975, 2905436181,
631 175547995,
632 ));
633 let b = cast!(u32x8(
634 2022374205, 1446824162, 3165580604, 3011091403, 2022374205, 1446824162, 3165580604,
635 3011091403,
636 ));
637
638 let a_array: [i32; N] = cast!(a);
639 let b_array: [i32; N] = cast!(b);
640 let mut lo_array = [0i32; N];
641 let mut hi_array = [0i32; N];
642
643 for i in 0..N {
644 let prod = a_array[i] as i64 * b_array[i] as i64;
645 let lo = prod as i32;
646 let hi = (prod >> 32) as i32;
647 lo_array[i] = lo;
648 hi_array[i] = hi;
649 }
650
651 let (lo, hi) = simd.widening_mul_i32x8(a, b);
652 assert_eq!(lo, cast!(lo_array));
653 assert_eq!(hi, cast!(hi_array));
654 }
655 }
656
657 #[test]
658 fn test_shift() {
659 if let Some(simd) = V2::try_new() {
660 let a = u16x8(54911, 46958, 49991, 22366, 46365, 39572, 22704, 60060);
661 assert_eq!(simd.shl_const_u16x8::<16>(a), simd.splat_u16x8(0));
662 assert_eq!(simd.shl_u16x8(a, simd.splat_u64x2(!0)), simd.splat_u16x8(0),);
663 }
664 }
665
666 #[test]
667 fn test_abs() {
668 if let Some(simd) = V2::try_new() {
669 let a = f32x4(1.0, -2.0, -1.0, 2.0);
670 assert_eq!(simd.abs_f32x4(a), f32x4(1.0, 2.0, 1.0, 2.0));
671 let a = f64x2(1.0, -2.0);
672 assert_eq!(simd.abs_f64x2(a), f64x2(1.0, 2.0));
673 }
674 }
675
676 #[test]
677 fn test_subadd() {
678 if let Some(simd) = V2::try_new() {
679 let a = f32x4(1.0, -2.0, -1.0, 2.0);
680 assert_eq!(simd.subadd_f32x4(a, a), f32x4(0.0, -4.0, 0.0, 4.0));
681 }
682 }
683
684 #[test]
685 fn test_signed_to_unsigned() {
686 if let Some(simd) = V2::try_new() {
687 let a = i8x16(1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
688 assert_eq!(simd.convert_i8x16_to_u64x2(a), u64x2(1, !0));
689 }
690 }
691
692 #[test]
693 fn test_int_cmp() {
694 if let Some(simd) = V2::try_new() {
695 {
696 const N: usize = 16;
697
698 let a = u8x16(
699 174, 191, 248, 232, 11, 186, 42, 236, 3, 59, 223, 72, 161, 146, 98, 69,
700 );
701 let b = u8x16(
702 97, 239, 164, 173, 208, 0, 121, 247, 218, 58, 119, 131, 213, 133, 22, 128,
703 );
704 let lt = simd.cmp_lt_u8x16(a, b);
705
706 let a_array: [u8; N] = cast!(a);
707 let b_array: [u8; N] = cast!(b);
708 let mut lt_array = [m8::new(false); N];
709
710 for i in 0..N {
711 lt_array[i] = m8::new(a_array[i] < b_array[i]);
712 }
713
714 assert_eq!(lt, cast!(lt_array));
715 }
716 {
717 const N: usize = 8;
718
719 let a = u16x8(174, 191, 248, 232, 11, 186, 42, 236);
720 let b = u16x8(97, 239, 164, 173, 208, 0, 121, 247);
721 let lt = simd.cmp_lt_u16x8(a, b);
722
723 let a_array: [u16; N] = cast!(a);
724 let b_array: [u16; N] = cast!(b);
725 let mut lt_array = [m16::new(false); N];
726
727 for i in 0..N {
728 lt_array[i] = m16::new(a_array[i] < b_array[i]);
729 }
730
731 assert_eq!(lt, cast!(lt_array));
732 }
733 {
734 const N: usize = 4;
735
736 let a = u32x4(174, 191, 248, 232);
737 let b = u32x4(97, 239, 164, 173);
738 let lt = simd.cmp_lt_u32x4(a, b);
739
740 let a_array: [u32; N] = cast!(a);
741 let b_array: [u32; N] = cast!(b);
742 let mut lt_array = [m32::new(false); N];
743
744 for i in 0..N {
745 lt_array[i] = m32::new(a_array[i] < b_array[i]);
746 }
747
748 assert_eq!(lt, cast!(lt_array));
749 }
750 {
751 const N: usize = 2;
752
753 let a = u64x2(174, 191);
754 let b = u64x2(97, 239);
755 let lt = simd.cmp_lt_u64x2(a, b);
756
757 let a_array: [u64; N] = cast!(a);
758 let b_array: [u64; N] = cast!(b);
759 let mut lt_array = [m64::new(false); N];
760
761 for i in 0..N {
762 lt_array[i] = m64::new(a_array[i] < b_array[i]);
763 }
764
765 assert_eq!(lt, cast!(lt_array));
766 }
767 }
768 }
769
770 #[test]
771 fn test_is_nan() {
772 if let Some(simd) = V2::try_new() {
773 assert_eq!(
774 simd.is_nan_f32x4(f32x4(0.0, f32::NAN, f32::INFINITY, -f32::NAN)),
775 m32x4(
776 m32::new(false),
777 m32::new(true),
778 m32::new(false),
779 m32::new(true),
780 ),
781 );
782 assert_eq!(
783 simd.is_nan_f64x2(f64x2(0.0, f64::NAN)),
784 m64x2(m64::new(false), m64::new(true)),
785 );
786 }
787 }
788
789 #[test]
790 fn test_rotate() {
791 if let Some(simd) = V3::try_new() {
792 for amount in 0..128 {
793 let mut array = [0u32; 8];
794 for (i, dst) in array.iter_mut().enumerate() {
795 *dst = 1000 + i as u32;
796 }
797
798 let rot: [u32; 8] = cast!(simd.rotate_right_u32s(cast!(array), amount));
799 for i in 0..8 {
800 assert_eq!(rot[(i + amount) % 8], array[i]);
801 }
802 }
803 for amount in 0..128 {
804 let mut array = [0u64; 4];
805 for (i, dst) in array.iter_mut().enumerate() {
806 *dst = 1000 + i as u64;
807 }
808
809 let rot: [u64; 4] = cast!(simd.rotate_right_u64s(cast!(array), amount));
810 for i in 0..4 {
811 assert_eq!(rot[(i + amount) % 4], array[i]);
812 }
813 }
814 }
815
816 #[cfg(feature = "x86-v4")]
817 if let Some(simd) = V4::try_new() {
818 for amount in 0..128 {
819 let mut array = [0u32; 16];
820 for (i, dst) in array.iter_mut().enumerate() {
821 *dst = 1000 + i as u32;
822 }
823
824 let rot: [u32; 16] = cast!(simd.rotate_right_u32s(cast!(array), amount));
825 for i in 0..16 {
826 assert_eq!(rot[(i + amount) % 16], array[i]);
827 }
828 }
829 for amount in 0..128 {
830 let mut array = [0u64; 8];
831 for (i, dst) in array.iter_mut().enumerate() {
832 *dst = 1000 + i as u64;
833 }
834
835 let rot: [u64; 8] = cast!(simd.rotate_right_u64s(cast!(array), amount));
836 for i in 0..8 {
837 assert_eq!(rot[(i + amount) % 8], array[i]);
838 }
839 }
840 }
841 }
842
843 #[test]
844 fn test_partial() {
845 if let Some(simd) = V3::try_new() {
846 for n in 0..=8 {
847 let src = core::array::from_fn::<f32, 8, _>(|i| i as _);
848 let mut dst = [0.0f32; 8];
849 let zero = dst;
850
851 assert_eq!(simd.partial_load_f32s(&src[..n]), unsafe {
852 simd.mask_load_ptr_f32s(simd.mask_between_m32s(0, n as u32), src.as_ptr())
853 });
854 {
855 let src = &src[..n];
856 let dst = &mut dst[..n];
857
858 simd.partial_store_f32s(dst, simd.partial_load_f32s(src));
859
860 assert_eq!(src, dst);
861 }
862 assert_eq!(dst[n..], zero[n..]);
863 }
864 }
865
866 #[cfg(feature = "x86-v4")]
867 if let Some(simd) = V4::try_new() {
868 for n in 0..=16 {
869 let src = core::array::from_fn::<f32, 16, _>(|i| i as _);
870 let mut dst = [0.0f32; 16];
871 let zero = dst;
872
873 assert_eq!(simd.partial_load_f32s(&src[..n]), unsafe {
874 simd.mask_load_ptr_f32s(simd.mask_between_m32s(0, n as u32), src.as_ptr())
875 });
876
877 {
878 let src = &src[..n];
879 let dst = &mut dst[..n];
880
881 simd.partial_store_f32s(dst, simd.partial_load_f32s(src));
882
883 assert_eq!(src, dst);
884 }
885 assert_eq!(dst[n..], zero[n..]);
886 }
887 }
888 }
889
890 #[test]
891 fn test_interleave() {
892 if let Some(simd) = V3::try_new() {
893 {
894 let src = [f64x4(0.0, 0.1, 1.0, 1.1), f64x4(2.0, 2.1, 3.0, 3.1)];
895 let dst = simd.deinterleave_shfl_f64s(src);
896 assert_eq!(dst[1], simd.add_f64x4(dst[0], simd.splat_f64x4(0.1)));
897 assert_eq!(src, simd.interleave_shfl_f64s(dst));
898 }
899 {
900 let src = [
901 f64x4(0.0, 0.1, 0.2, 0.3),
902 f64x4(1.0, 1.1, 1.2, 1.3),
903 f64x4(2.0, 2.1, 2.2, 2.3),
904 f64x4(3.0, 3.1, 3.2, 3.3),
905 ];
906 let dst = simd.deinterleave_shfl_f64s(src);
907 assert_eq!(dst[1], simd.add_f64x4(dst[0], simd.splat_f64x4(0.1)));
908 assert_eq!(dst[2], simd.add_f64x4(dst[0], simd.splat_f64x4(0.2)));
909 assert_eq!(dst[3], simd.add_f64x4(dst[0], simd.splat_f64x4(0.3)));
910 assert_eq!(src, simd.interleave_shfl_f64s(dst));
911 }
912 {
913 let src = [
914 f32x8(0.0, 0.1, 1.0, 1.1, 2.0, 2.1, 3.0, 3.1),
915 f32x8(4.0, 4.1, 5.0, 5.1, 6.0, 6.1, 7.0, 7.1),
916 ];
917 let dst = simd.deinterleave_shfl_f32s(src);
918 assert_eq!(dst[1], simd.add_f32x8(dst[0], simd.splat_f32x8(0.1)));
919 assert_eq!(src, simd.interleave_shfl_f32s(dst));
920 }
921 {
922 let src = [
923 f32x8(0.0, 0.1, 0.2, 0.3, 1.0, 1.1, 1.2, 1.3),
924 f32x8(2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3),
925 f32x8(4.0, 4.1, 4.2, 4.3, 5.0, 5.1, 5.2, 5.3),
926 f32x8(6.0, 6.1, 6.2, 6.3, 7.0, 7.1, 7.2, 7.3),
927 ];
928 let dst = simd.deinterleave_shfl_f32s(src);
929 assert_eq!(dst[1], simd.add_f32x8(dst[0], simd.splat_f32x8(0.1)));
930 assert_eq!(dst[2], simd.add_f32x8(dst[0], simd.splat_f32x8(0.2)));
931 assert_eq!(dst[3], simd.add_f32x8(dst[0], simd.splat_f32x8(0.3)));
932 assert_eq!(src, simd.interleave_shfl_f32s(dst));
933 }
934 }
935 #[cfg(feature = "x86-v4")]
936 if let Some(simd) = V4::try_new() {
937 {
938 let src = [
939 f64x8(0.0, 0.1, 1.0, 1.1, 2.0, 2.1, 3.0, 3.1),
940 f64x8(4.0, 4.1, 5.0, 5.1, 6.0, 6.1, 7.0, 7.1),
941 ];
942 let dst = simd.deinterleave_shfl_f64s(src);
943 assert_eq!(dst[1], simd.add_f64x8(dst[0], simd.splat_f64x8(0.1)));
944 assert_eq!(src, simd.interleave_shfl_f64s(dst));
945 }
946
947 {
948 let src = [
949 f64x8(0.0, 0.1, 0.2, 1.0, 1.1, 1.2, 2.0, 2.1),
950 f64x8(2.2, 3.0, 3.1, 3.2, 4.0, 4.1, 4.2, 5.0),
951 f64x8(5.1, 5.2, 6.0, 6.1, 6.2, 7.0, 7.1, 7.2),
952 ];
953 let dst = simd.deinterleave_shfl_f64s(src);
954 assert_eq!(dst[1], simd.add_f64x8(dst[0], simd.splat_f64x8(0.1)));
955 assert_eq!(dst[2], simd.add_f64x8(dst[0], simd.splat_f64x8(0.2)));
956 assert_eq!(src, simd.interleave_shfl_f64s(dst));
957 }
958 {
959 let src = [
960 f64x8(0.0, 0.1, 0.2, 0.3, 1.0, 1.1, 1.2, 1.3),
961 f64x8(2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3),
962 f64x8(4.0, 4.1, 4.2, 4.3, 5.0, 5.1, 5.2, 5.3),
963 f64x8(6.0, 6.1, 6.2, 6.3, 7.0, 7.1, 7.2, 7.3),
964 ];
965 let dst = simd.deinterleave_shfl_f64s(src);
966 assert_eq!(dst[1], simd.add_f64x8(dst[0], simd.splat_f64x8(0.1)));
967 assert_eq!(dst[2], simd.add_f64x8(dst[0], simd.splat_f64x8(0.2)));
968 assert_eq!(dst[3], simd.add_f64x8(dst[0], simd.splat_f64x8(0.3)));
969 assert_eq!(src, simd.interleave_shfl_f64s(dst));
970 }
971
972 {
973 let src = [
974 f32x16(
975 0.0, 0.1, 1.0, 1.1, 2.0, 2.1, 3.0, 3.1, 4.0, 4.1, 5.0, 5.1, 6.0, 6.1, 7.0,
976 7.1,
977 ),
978 f32x16(
979 8.0, 8.1, 9.0, 9.1, 10.0, 10.1, 11.0, 11.1, 12.0, 12.1, 13.0, 13.1, 14.0,
980 14.1, 15.0, 15.1,
981 ),
982 ];
983 let dst = simd.deinterleave_shfl_f32s(src);
984 assert_eq!(dst[1], simd.add_f32x16(dst[0], simd.splat_f32x16(0.1)));
985 assert_eq!(src, simd.interleave_shfl_f32s(dst));
986 }
987 {
988 let src = [
989 f32x16(
990 0.0, 0.1, 0.2, 0.3, 1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2,
991 3.3,
992 ),
993 f32x16(
994 4.0, 4.1, 4.2, 4.3, 5.0, 5.1, 5.2, 5.3, 6.0, 6.1, 6.2, 6.3, 7.0, 7.1, 7.2,
995 7.3,
996 ),
997 f32x16(
998 8.0, 8.1, 8.2, 8.3, 9.0, 9.1, 9.2, 9.3, 10.0, 10.1, 10.2, 10.3, 11.0, 11.1,
999 11.2, 11.3,
1000 ),
1001 f32x16(
1002 12.0, 12.1, 12.2, 12.3, 13.0, 13.1, 13.2, 13.3, 14.0, 14.1, 14.2, 14.3,
1003 15.0, 15.1, 15.2, 15.3,
1004 ),
1005 ];
1006 let dst = simd.deinterleave_shfl_f32s(src);
1007 assert_eq!(dst[1], simd.add_f32x16(dst[0], simd.splat_f32x16(0.1)));
1008 assert_eq!(dst[2], simd.add_f32x16(dst[0], simd.splat_f32x16(0.2)));
1009 assert_eq!(dst[3], simd.add_f32x16(dst[0], simd.splat_f32x16(0.3)));
1010 assert_eq!(src, simd.interleave_shfl_f32s(dst));
1011 }
1012 }
1013 }
1014
1015 #[test]
1016 fn test_mul_add() {
1017 if let Some(simd) = V2::try_new() {
1018 let a = f32x4(-1.0, 1.0, -2.0, 2.0);
1019 let b = f32x4(-5.0, 5.0, -6.0, 6.0);
1020 let c = f32x4(2.0, -2.0, -3.0, 3.0);
1021
1022 let d = simd.add_f32s(c, simd.mul_f32s(a, b));
1023
1024 assert_eq!(simd.mul_add_f32s(a, b, c), d);
1025
1026 let a = f64x2(-1.0, 1.0);
1027 let b = f64x2(-5.0, 5.0);
1028 let c = f64x2(2.0, -2.0);
1029
1030 let d = simd.add_f64s(c, simd.mul_f64s(a, b));
1031
1032 assert_eq!(simd.mul_add_f64s(a, b, c), d);
1033 }
1034 if let Some(simd) = V3::try_new() {
1035 let a = f32x8(-1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0);
1036 let b = f32x8(-5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0);
1037 let c = f32x8(2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0);
1038
1039 let d = simd.add_f32s(c, simd.mul_f32s(a, b));
1040
1041 assert_eq!(simd.mul_add_f32s(a, b, c), d);
1042
1043 let a = f64x4(-1.0, 1.0, -2.0, 2.0);
1044 let b = f64x4(-5.0, 5.0, -6.0, 6.0);
1045 let c = f64x4(2.0, -2.0, -3.0, 3.0);
1046
1047 let d = simd.add_f64s(c, simd.mul_f64s(a, b));
1048
1049 assert_eq!(simd.mul_add_f64s(a, b, c), d);
1050 }
1051 #[cfg(feature = "x86-v4")]
1052 if let Some(simd) = V4::try_new() {
1053 let a = f32x16(
1054 -1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0, 1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0,
1055 -4.0,
1056 );
1057 let b = f32x16(
1058 -5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0, -5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0,
1059 8.0,
1060 );
1061 let c = f32x16(
1062 2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0, 2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0,
1063 5.0,
1064 );
1065
1066 let d = simd.add_f32s(c, simd.mul_f32s(a, b));
1067
1068 assert_eq!(simd.mul_add_f32s(a, b, c), d);
1069
1070 let a = f64x8(-1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0);
1071 let b = f64x8(-5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0);
1072 let c = f64x8(2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0);
1073
1074 let d = simd.add_f64s(c, simd.mul_f64s(a, b));
1075
1076 assert_eq!(simd.mul_add_f64s(a, b, c), d);
1077 }
1078 }
1079
1080 #[test]
1081 fn test_negate_mul_add() {
1082 if let Some(simd) = V2::try_new() {
1083 let a = f32x4(-1.0, 1.0, -2.0, 2.0);
1084 let b = f32x4(-5.0, 5.0, -6.0, 6.0);
1085 let c = f32x4(2.0, -2.0, -3.0, 3.0);
1086
1087 let d = simd.sub_f32s(c, simd.mul_f32s(a, b));
1088
1089 assert_eq!(simd.negate_mul_add_f32s(a, b, c), d);
1090
1091 let a = f64x2(-1.0, 1.0);
1092 let b = f64x2(-5.0, 5.0);
1093 let c = f64x2(2.0, -2.0);
1094
1095 let d = simd.sub_f64s(c, simd.mul_f64s(a, b));
1096
1097 assert_eq!(simd.negate_mul_add_f64s(a, b, c), d);
1098 }
1099 if let Some(simd) = V3::try_new() {
1100 let a = f32x8(-1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0);
1101 let b = f32x8(-5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0);
1102 let c = f32x8(2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0);
1103
1104 let d = simd.sub_f32s(c, simd.mul_f32s(a, b));
1105
1106 assert_eq!(simd.negate_mul_add_f32s(a, b, c), d);
1107
1108 let a = f64x4(-1.0, 1.0, -2.0, 2.0);
1109 let b = f64x4(-5.0, 5.0, -6.0, 6.0);
1110 let c = f64x4(2.0, -2.0, -3.0, 3.0);
1111
1112 let d = simd.sub_f64s(c, simd.mul_f64s(a, b));
1113
1114 assert_eq!(simd.negate_mul_add_f64s(a, b, c), d);
1115 }
1116 #[cfg(feature = "x86-v4")]
1117 if let Some(simd) = V4::try_new() {
1118 let a = f32x16(
1119 -1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0, 1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0,
1120 -4.0,
1121 );
1122 let b = f32x16(
1123 -5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0, -5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0,
1124 8.0,
1125 );
1126 let c = f32x16(
1127 2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0, 2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0,
1128 5.0,
1129 );
1130
1131 let d = simd.sub_f32s(c, simd.mul_f32s(a, b));
1132
1133 assert_eq!(simd.negate_mul_add_f32s(a, b, c), d);
1134
1135 let a = f64x8(-1.0, 1.0, -2.0, 2.0, -3.0, 3.0, -4.0, 4.0);
1136 let b = f64x8(-5.0, 5.0, -6.0, 6.0, -7.0, 7.0, -8.0, 8.0);
1137 let c = f64x8(2.0, -2.0, -3.0, 3.0, -4.0, 4.0, -5.0, 5.0);
1138
1139 let d = simd.sub_f64s(c, simd.mul_f64s(a, b));
1140
1141 assert_eq!(simd.negate_mul_add_f64s(a, b, c), d);
1142 }
1143 }
1144}