pulp/x86/

v3.rs

use super::*;

// https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
simd_type!({
	/// AVX instruction set.
	///
	/// Notable additions over [`V2`] include:
	///  - Instructions operating on 256-bit SIMD vectors.
	///  - Shift functions with a separate shift per lane, such as [`V3::shl_dyn_u32x4`].
	///  - Fused multiply-accumulate instructions, such as [`V3::mul_add_f32x4`].
	#[allow(missing_docs)]
	pub struct V3 {
		pub sse: f!("sse"),
		pub sse2: f!("sse2"),
		pub fxsr: f!("fxsr"),
		pub sse3: f!("sse3"),
		pub ssse3: f!("ssse3"),
		pub sse4_1: f!("sse4.1"),
		pub sse4_2: f!("sse4.2"),
		pub popcnt: f!("popcnt"),
		pub avx: f!("avx"),
		pub avx2: f!("avx2"),
		pub bmi1: f!("bmi1"),
		pub bmi2: f!("bmi2"),
		pub fma: f!("fma"),
		pub lzcnt: f!("lzcnt"),
	}
});

// copied from the standard library
#[inline(always)]
fn avx2_pshufb(simd: V3, bytes: __m256i, idxs: __m256i) -> __m256i {
	let mid = simd.avx._mm256_set1_epi8(16i8);
	let high = simd.avx._mm256_set1_epi8(32i8);
	// This is ordering sensitive, and LLVM will order these how you put them.
	// Most AVX2 impls use ~5 "ports", and only 1 or 2 are capable of permutes.
	// But the "compose" step will lower to ops that can also use at least 1 other port.
	// So this tries to break up permutes so composition flows through "open" ports.
	// Comparative benches should be done on multiple AVX2 CPUs before reordering this

	let hihi = simd.avx2._mm256_permute2x128_si256::<0x11>(bytes, bytes);
	let hi_shuf = simd.avx2._mm256_shuffle_epi8(
		hihi, // duplicate the vector's top half
		idxs, // so that using only 4 bits of an index still picks bytes 16-31
	);
	// A zero-fill during the compose step gives the "all-Neon-like" OOB-is-0 semantics
	let compose = simd.avx2._mm256_blendv_epi8(
		simd.avx._mm256_set1_epi8(0),
		hi_shuf,
		simd.avx2._mm256_cmpgt_epi8(high, idxs),
	);
	let lolo = simd.avx2._mm256_permute2x128_si256::<0x00>(bytes, bytes);
	let lo_shuf = simd.avx2._mm256_shuffle_epi8(lolo, idxs);
	// Repeat, then pick indices < 16, overwriting indices 0-15 from previous compose step
	simd.avx2
		._mm256_blendv_epi8(compose, lo_shuf, simd.avx2._mm256_cmpgt_epi8(mid, idxs))
}

static AVX2_ROTATE_IDX: [u8x32; 32] = [
	u8x32(
		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
		25, 26, 27, 28, 29, 30, 31,
	),
	u8x32(
		31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
		24, 25, 26, 27, 28, 29, 30,
	),
	u8x32(
		30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
		23, 24, 25, 26, 27, 28, 29,
	),
	u8x32(
		29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
		22, 23, 24, 25, 26, 27, 28,
	),
	u8x32(
		28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
		21, 22, 23, 24, 25, 26, 27,
	),
	u8x32(
		27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
		20, 21, 22, 23, 24, 25, 26,
	),
	u8x32(
		26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
		19, 20, 21, 22, 23, 24, 25,
	),
	u8x32(
		25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
		18, 19, 20, 21, 22, 23, 24,
	),
	u8x32(
		24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
		17, 18, 19, 20, 21, 22, 23,
	),
	u8x32(
		23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
		16, 17, 18, 19, 20, 21, 22,
	),
	u8x32(
		22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
		15, 16, 17, 18, 19, 20, 21,
	),
	u8x32(
		21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
		14, 15, 16, 17, 18, 19, 20,
	),
	u8x32(
		20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
		13, 14, 15, 16, 17, 18, 19,
	),
	u8x32(
		19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
		12, 13, 14, 15, 16, 17, 18,
	),
	u8x32(
		18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
		11, 12, 13, 14, 15, 16, 17,
	),
	u8x32(
		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
		10, 11, 12, 13, 14, 15, 16,
	),
	u8x32(
		16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8,
		9, 10, 11, 12, 13, 14, 15,
	),
	u8x32(
		15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7,
		8, 9, 10, 11, 12, 13, 14,
	),
	u8x32(
		14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4, 5,
		6, 7, 8, 9, 10, 11, 12, 13,
	),
	u8x32(
		13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3, 4,
		5, 6, 7, 8, 9, 10, 11, 12,
	),
	u8x32(
		12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1, 2, 3,
		4, 5, 6, 7, 8, 9, 10, 11,
	),
	u8x32(
		11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0, 1,
		2, 3, 4, 5, 6, 7, 8, 9, 10,
	),
	u8x32(
		10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 0,
		1, 2, 3, 4, 5, 6, 7, 8, 9,
	),
	u8x32(
		9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
		0, 1, 2, 3, 4, 5, 6, 7, 8,
	),
	u8x32(
		8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
		31, 0, 1, 2, 3, 4, 5, 6, 7,
	),
	u8x32(
		7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
		30, 31, 0, 1, 2, 3, 4, 5, 6,
	),
	u8x32(
		6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
		30, 31, 0, 1, 2, 3, 4, 5,
	),
	u8x32(
		5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
		29, 30, 31, 0, 1, 2, 3, 4,
	),
	u8x32(
		4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
		28, 29, 30, 31, 0, 1, 2, 3,
	),
	u8x32(
		3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
		27, 28, 29, 30, 31, 0, 1, 2,
	),
	u8x32(
		2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
		27, 28, 29, 30, 31, 0, 1,
	),
	u8x32(
		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
		26, 27, 28, 29, 30, 31, 0,
	),
];

static AVX2_128_ROTATE_IDX: [u8x16; 16] = [
	u8x16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
	u8x16(15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14),
	u8x16(14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
	u8x16(13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12),
	u8x16(12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
	u8x16(11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10),
	u8x16(10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
	u8x16(9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8),
	u8x16(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7),
	u8x16(7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6),
	u8x16(6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5),
	u8x16(5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4),
	u8x16(4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3),
	u8x16(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2),
	u8x16(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1),
	u8x16(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0),
];

static V3_U32_MASKS: [u32x8; 9] = [
	u32x8(0, 0, 0, 0, 0, 0, 0, 0),
	u32x8(!0, 0, 0, 0, 0, 0, 0, 0),
	u32x8(!0, !0, 0, 0, 0, 0, 0, 0),
	u32x8(!0, !0, !0, 0, 0, 0, 0, 0),
	u32x8(!0, !0, !0, !0, 0, 0, 0, 0),
	u32x8(!0, !0, !0, !0, !0, 0, 0, 0),
	u32x8(!0, !0, !0, !0, !0, !0, 0, 0),
	u32x8(!0, !0, !0, !0, !0, !0, !0, 0),
	u32x8(!0, !0, !0, !0, !0, !0, !0, !0),
];
static V3_U32_LAST_MASKS: [u32x8; 9] = [
	u32x8(0, 0, 0, 0, 0, 0, 0, 0),
	u32x8(0, 0, 0, 0, 0, 0, 0, !0),
	u32x8(0, 0, 0, 0, 0, 0, !0, !0),
	u32x8(0, 0, 0, 0, 0, !0, !0, !0),
	u32x8(0, 0, 0, 0, !0, !0, !0, !0),
	u32x8(0, 0, 0, !0, !0, !0, !0, !0),
	u32x8(0, 0, !0, !0, !0, !0, !0, !0),
	u32x8(0, !0, !0, !0, !0, !0, !0, !0),
	u32x8(!0, !0, !0, !0, !0, !0, !0, !0),
];

impl Seal for V3 {}
impl Seal for V3_Scalar {}

#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct V3_Scalar(pub V3);

#[cfg(target_arch = "x86_64")]
#[inline(always)]
pub(super) fn avx_load_u32s(simd: Avx2, slice: &[u32]) -> u32x8 {
	_ = simd;
	unsafe { avx_ld_u32s(slice.as_ptr(), LD_ST[2 * (16 * slice.len().min(8))]) }
}

#[cfg(target_arch = "x86_64")]
#[inline(always)]
pub(super) fn avx_store_u32s(simd: Avx2, slice: &mut [u32], value: u32x8) {
	_ = simd;
	unsafe {
		avx_st_u32s(
			slice.as_mut_ptr(),
			value,
			LD_ST[2 * (16 * slice.len().min(8)) + 1],
		);
	}
}

impl core::ops::Deref for V3 {
	type Target = V2;

	#[inline(always)]
	fn deref(&self) -> &Self::Target {
		V2 {
			sse: self.sse,
			sse2: self.sse2,
			fxsr: self.fxsr,
			sse3: self.sse3,
			ssse3: self.ssse3,
			sse4_1: self.sse4_1,
			sse4_2: self.sse4_2,
			popcnt: self.popcnt,
		}
		.to_ref()
	}
}

macro_rules! splat {
	($ty: ty, $factor: literal) => {
		paste! {
			#[inline(always)]
			fn [<splat_ $ty s>](self, value: $ty) -> Self::[<$ty s>] {
				self.[<splat_ $ty x $factor>](value)
			}
		}
	};
	($($ty: ident x $factor: literal),*) => {
		$(splat!($ty, $factor);)*
	};
}

macro_rules! impl_simd_binop {
	($func: ident, $op: ident, $ty: ident, $out: ty, $factor: literal) => {
		paste! {
			#[inline(always)]
			fn [<$func _ $ty s>](self, a: Self::[<$ty s>], b: Self::[<$ty s>]) -> Self::[<$out s>] {
				self.[<$op _ $ty x $factor>](a, b)
			}
		}
	};
	($func: ident, $op: ident, $($ty: ident x $factor: literal => $out: ty),*) => {
		$(impl_simd_binop!($func, $op, $ty, $out, $factor);)*
	};
	($func: ident, $op: ident, $($ty: ident x $factor: literal),*) => {
		$(impl_simd_binop!($func, $op, $ty, $ty, $factor);)*
	};
	($func: ident, $($ty: ident x $factor: literal => $out: ty),*) => {
		$(impl_simd_binop!($func, $func, $ty, $out, $factor);)*
	};
	($func: ident, $($ty: ident x $factor: literal),*) => {
		$(impl_simd_binop!($func, $func, $ty, $ty, $factor);)*
	};
}

macro_rules! impl_simd_unop {
	($func: ident, $op: ident, $ty: ident, $out: ty, $factor: literal) => {
		paste! {
			#[inline(always)]
			fn [<$func _ $ty s>](self, a: Self::[<$ty s>]) -> Self::[<$out s>] {
				self.[<$op _ $ty x $factor>](a)
			}
		}
	};
	($func: ident, $($ty: ident x $factor: literal),*) => {
		$(impl_simd_unop!($func, $func, $ty, $ty, $factor);)*
	};
}

macro_rules! impl_scalar_binop {
	($func: ident, $ty: ident, $out: ty, impl) => {
		paste! {
			#[inline(always)]
			fn [<$func _ $ty s>](self, a: Self::[<$ty s>], b: Self::[<$ty s>]) -> Self::[<$out s>] {
				Scalar256b.[<$func _ $ty s>](a, b)
			}
		}
	};
	($func: ident, $($ty: ident => $out: ty),*) => {
		$(impl_scalar_binop!($func, $ty, $out, impl);)*
	};
	($func: ident, $($ty: ident),*) => {
		$(impl_scalar_binop!($func, $ty, $ty, impl);)*
	};
}

impl Simd for V3 {
	type c32s = f32x8;
	type c64s = f64x4;
	type f32s = f32x8;
	type f64s = f64x4;
	type i16s = i16x16;
	type i32s = i32x8;
	type i64s = i64x4;
	type i8s = i8x32;
	type m16s = m16x16;
	type m32s = m32x8;
	type m64s = m64x4;
	type m8s = m8x32;
	type u16s = u16x16;
	type u32s = u32x8;
	type u64s = u64x4;
	type u8s = u8x32;

	const REGISTER_COUNT: usize = 16;

	impl_simd_binop!(add, f32 x 8, f64 x 4);

	impl_simd_binop!(add, wrapping_add, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	impl_simd_binop!(sub, f32 x 8, f64 x 4);

	impl_simd_binop!(sub, wrapping_sub, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	impl_simd_binop!(mul, f32 x 8, f64 x 4);

	impl_simd_binop!(mul, wrapping_mul, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	impl_scalar_binop!(mul, u64, i64);

	impl_simd_binop!(and, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(or, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(xor, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(div, f32 x 8, f64 x 4);

	impl_simd_binop!(equal, cmp_eq, u8 x 32 => m8, u16 x 16 => m16, u32 x 8 => m32, u64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(greater_than, cmp_gt, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(greater_than_or_equal, cmp_ge, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(less_than, cmp_lt, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(less_than_or_equal, cmp_le, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	splat!(u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(max, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, f32 x 8, f64 x 4);

	impl_scalar_binop!(max, u64, i64);

	impl_simd_binop!(min, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, f32 x 8, f64 x 4);

	impl_scalar_binop!(min, u64, i64);

	impl_simd_unop!(not, m8 x 32, u8 x 32, m16 x 16, u16 x 16, m32 x 8, u32 x 8, m64 x 4, u64 x 4);

	#[inline(always)]
	fn abs2_c32s(self, a: Self::c32s) -> Self::c32s {
		let sqr = self.mul_f32s(a, a);
		let sqr_rev = self
			.avx
			._mm256_shuffle_ps::<0b10_11_00_01>(cast!(sqr), cast!(sqr));
		self.add_f32s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn abs2_c64s(self, a: Self::c64s) -> Self::c64s {
		let sqr = self.mul_f64s(a, a);
		let sqr_rev = self.avx._mm256_shuffle_pd::<0b0101>(cast!(sqr), cast!(sqr));
		self.add_f64s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn abs_max_c32s(self, a: Self::c32s) -> Self::c32s {
		let max = self.abs_f32s(a);
		let max_rev = self
			.avx
			._mm256_shuffle_ps::<0b10_11_00_01>(cast!(a), cast!(a));
		self.max_f32s(max, cast!(max_rev))
	}

	#[inline(always)]
	fn abs_max_c64s(self, a: Self::c64s) -> Self::c64s {
		let max = self.abs_f64s(a);
		let max_rev = self.avx._mm256_shuffle_pd::<0b0101>(cast!(max), cast!(max));
		self.max_f64s(max, cast!(max_rev))
	}

	#[inline(always)]
	fn add_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		self.add_f32s(a, b)
	}

	#[inline(always)]
	fn add_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		self.add_f64s(a, b)
	}

	#[inline(always)]
	fn equal_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::m32s {
		self.equal_f32s(a, b)
	}

	#[inline(always)]
	fn equal_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::m64s {
		self.equal_f64s(a, b)
	}

	#[inline(always)]
	fn conj_c32s(self, a: Self::c32s) -> Self::c32s {
		self.xor_f32s(a, self.splat_c32s(c32 { re: 0.0, im: -0.0 }))
	}

	#[inline(always)]
	fn conj_c64s(self, a: Self::c64s) -> Self::c64s {
		self.xor_f64s(a, self.splat_c64s(c64 { re: 0.0, im: -0.0 }))
	}

	#[inline(always)]
	fn conj_mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.avx._mm256_moveldup_ps(ab);
		let bb = self.avx._mm256_movehdup_ps(ab);

		cast!(
			self.fma
				._mm256_fmsubadd_ps(aa, xy, self.fma._mm256_fmsubadd_ps(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn conj_mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_pd::<0b0101>(xy);
		let aa = self.avx._mm256_unpacklo_pd(ab, ab);
		let bb = self.avx._mm256_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm256_fmsubadd_pd(aa, xy, self.fma._mm256_fmsubadd_pd(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn conj_mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.avx._mm256_moveldup_ps(ab);
		let bb = self.avx._mm256_movehdup_ps(ab);

		cast!(
			self.fma
				._mm256_fmsubadd_ps(aa, xy, self.avx._mm256_mul_ps(bb, yx))
		)
	}

	#[inline(always)]
	fn conj_mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_pd::<0b0101>(xy);
		let aa = self.avx._mm256_unpacklo_pd(ab, ab);
		let bb = self.avx._mm256_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm256_fmsubadd_pd(aa, xy, self.avx._mm256_mul_pd(bb, yx))
		)
	}

	#[inline(always)]
	fn deinterleave_shfl_f32s<T: Interleave>(self, values: T) -> T {
		let avx = self.avx;

		if const { core::mem::size_of::<T>() == 2 * core::mem::size_of::<Self::f32s>() } {
			let values: [__m256d; 2] = unsafe { core::mem::transmute_copy(&values) };
			// a0 b0 a1 b1 a2 b2 a3 b3
			// a4 b4 a5 b5 a6 b6 a7 b7

			// a0 a4 b0 b4 a2 a6 b2 b6
			// a1 a5 b1 b5 a3 a7 b3 b7
			let values = [
				cast!(avx._mm256_unpacklo_ps(cast!(values[0]), cast!(values[1]))),
				cast!(avx._mm256_unpackhi_ps(cast!(values[0]), cast!(values[1]))),
			];

			// a0 a4 a1 a5 a2 a6 a3 a7
			// b0 b4 b1 b5 b2 b6 b3 b7
			let values = [
				avx._mm256_unpacklo_pd(values[0], values[1]),
				avx._mm256_unpackhi_pd(values[0], values[1]),
			];

			unsafe { core::mem::transmute_copy(&values) }
		} else if const { core::mem::size_of::<T>() == 4 * core::mem::size_of::<Self::f32s>() } {
			// a0 b0 c0 d0 a1 b1 c1 d1
			// a2 b2 c2 d2 a3 b3 c3 d3
			// a4 b4 c4 d4 a5 b5 c5 d5
			// a6 b6 c6 d6 a7 b7 c7 d7
			let values: [__m256d; 4] = unsafe { core::mem::transmute_copy(&values) };

			// a0 a2 c0 c2 a1 a3 c1 c3
			// b0 b2 d0 d2 b1 b3 d1 d3
			// a4 a6 c4 c6 a5 a7 c5 c7
			// b4 b6 d4 d6 b5 b7 d5 d7
			let values = [
				cast!(avx._mm256_unpacklo_ps(cast!(values[0]), cast!(values[1]))),
				cast!(avx._mm256_unpackhi_ps(cast!(values[0]), cast!(values[1]))),
				cast!(avx._mm256_unpacklo_ps(cast!(values[2]), cast!(values[3]))),
				cast!(avx._mm256_unpackhi_ps(cast!(values[2]), cast!(values[3]))),
			];

			let values = [
				avx._mm256_unpacklo_pd(values[0], values[1]),
				avx._mm256_unpackhi_pd(values[0], values[1]),
				avx._mm256_unpacklo_pd(values[2], values[3]),
				avx._mm256_unpackhi_pd(values[2], values[3]),
			];

			// a0 a2 a4 a6 a1 a3 a5 a7
			// b0 b2 b4 b6 b1 b3 b5 b7
			// c0 c2 c4 c6 c1 c3 c5 c7
			// d0 d2 d4 d6 d1 d3 d5 d7
			let values = [
				avx._mm256_unpacklo_pd(values[0], values[2]),
				avx._mm256_unpacklo_pd(values[1], values[3]),
				avx._mm256_unpackhi_pd(values[0], values[2]),
				avx._mm256_unpackhi_pd(values[1], values[3]),
			];

			unsafe { core::mem::transmute_copy(&values) }
		} else {
			unsafe { deinterleave_fallback::<f32, Self::f32s, T>(values) }
		}
	}

	#[inline(always)]
	fn deinterleave_shfl_f64s<T: Interleave>(self, values: T) -> T {
		let avx = self.avx;

		if const { core::mem::size_of::<T>() == 2 * core::mem::size_of::<Self::f64s>() } {
			let values: [__m256d; 2] = unsafe { core::mem::transmute_copy(&values) };
			let values = [
				avx._mm256_unpacklo_pd(values[0], values[1]),
				avx._mm256_unpackhi_pd(values[0], values[1]),
			];
			unsafe { core::mem::transmute_copy(&values) }
		} else if const { core::mem::size_of::<T>() == 4 * core::mem::size_of::<Self::f64s>() } {
			let values: [__m256d; 4] = unsafe { core::mem::transmute_copy(&values) };

			// a0 b0 c0 d0
			// a1 b1 c1 d1
			// a2 b2 c2 d2
			// a3 b3 c3 d3

			// a0 a1 c0 c1
			// b0 b1 d0 d1
			// a2 a3 c2 c3
			// b2 b3 d2 d3
			let values: [__m256d; 4] = [
				avx._mm256_unpacklo_pd(values[0], values[1]),
				avx._mm256_unpackhi_pd(values[0], values[1]),
				avx._mm256_unpacklo_pd(values[2], values[3]),
				avx._mm256_unpackhi_pd(values[2], values[3]),
			];

			// a0 a1 a2 a3
			// b0 b1 b2 b3
			// c0 c1 c2 c3
			// d0 d1 d2 d3
			let values = [
				avx._mm256_permute2f128_pd::<0b0010_0000>(values[0], values[2]),
				avx._mm256_permute2f128_pd::<0b0010_0000>(values[1], values[3]),
				avx._mm256_permute2f128_pd::<0b0011_0001>(values[0], values[2]),
				avx._mm256_permute2f128_pd::<0b0011_0001>(values[1], values[3]),
			];

			unsafe { core::mem::transmute_copy(&values) }
		} else {
			unsafe { deinterleave_fallback::<f64, Self::f64s, T>(values) }
		}
	}

	#[inline(always)]
	fn interleave_shfl_f32s<T: Interleave>(self, values: T) -> T {
		if const {
			(core::mem::size_of::<T>() == 2 * core::mem::size_of::<Self::f32s>())
				|| (core::mem::size_of::<T>() == 4 * core::mem::size_of::<Self::f32s>())
		} {
			// permutation is inverse of itself in this case
			self.deinterleave_shfl_f32s(values)
		} else {
			unsafe { interleave_fallback::<f32, Self::f32s, T>(values) }
		}
	}

	#[inline(always)]
	fn interleave_shfl_f64s<T: Interleave>(self, values: T) -> T {
		if const {
			(core::mem::size_of::<T>() == 2 * core::mem::size_of::<Self::f64s>())
				|| (core::mem::size_of::<T>() == 4 * core::mem::size_of::<Self::f64s>())
		} {
			// permutation is inverse of itself in this case
			self.deinterleave_shfl_f64s(values)
		} else {
			unsafe { interleave_fallback::<f64, Self::f64s, T>(values) }
		}
	}

	#[inline(always)]
	fn mask_between_m32s(self, start: u32, end: u32) -> MemMask<Self::m32s> {
		let start = start.min(8) as usize;
		let end = end.min(8) as usize;
		MemMask {
			mask: self.and_m32s(
				cast!(V3_U32_LAST_MASKS[8 - start]),
				cast!(V3_U32_MASKS[end]),
			),
			#[cfg(target_arch = "x86_64")]
			load: Some(LD_ST[2 * (16 * end + start) + 0]),
			#[cfg(target_arch = "x86_64")]
			store: Some(LD_ST[2 * (16 * end + start) + 1]),
		}
	}

	#[inline(always)]
	fn mask_between_m64s(self, start: u64, end: u64) -> MemMask<Self::m64s> {
		let start = (2 * start.min(4)) as usize;
		let end = (2 * end.min(4)) as usize;
		MemMask {
			mask: self.and_m64s(
				cast!(V3_U32_LAST_MASKS[8 - start]),
				cast!(V3_U32_MASKS[end]),
			),
			#[cfg(target_arch = "x86_64")]
			load: Some(LD_ST[2 * (16 * end + start) + 0]),
			#[cfg(target_arch = "x86_64")]
			store: Some(LD_ST[2 * (16 * end + start) + 1]),
		}
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_c32s(self, mask: MemMask<Self::m32s>, ptr: *const c32) -> Self::c32s {
		cast!(self.mask_load_ptr_u32s(mask, ptr as _))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_c64s(self, mask: MemMask<Self::m64s>, ptr: *const c64) -> Self::c64s {
		cast!(self.mask_load_ptr_u64s(mask, ptr as _))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *const u8) -> Self::u8s {
		Scalar256b.mask_load_ptr_u8s(mask, ptr)
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u16s(self, mask: MemMask<Self::m16s>, ptr: *const u16) -> Self::u16s {
		Scalar256b.mask_load_ptr_u16s(mask, ptr)
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u32s(self, mask: MemMask<Self::m32s>, ptr: *const u32) -> Self::u32s {
		#[cfg(target_arch = "x86_64")]
		if let Some(load) = mask.load {
			return avx_ld_u32s(ptr, load);
		}
		cast!(self.avx2._mm256_maskload_epi32(ptr as _, cast!(mask.mask)))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u64s(self, mask: MemMask<Self::m64s>, ptr: *const u64) -> Self::u64s {
		#[cfg(target_arch = "x86_64")]
		if let Some(load) = mask.load {
			return cast!(avx_ld_u32s(ptr as _, load));
		}
		cast!(self.avx2._mm256_maskload_epi64(ptr as _, cast!(mask.mask)))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_c32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut c32,
		values: Self::c32s,
	) {
		self.mask_store_ptr_u32s(mask, ptr as _, cast!(values))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_c64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut c64,
		values: Self::c64s,
	) {
		self.mask_store_ptr_u64s(mask, ptr as _, cast!(values))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *mut u8, values: Self::u8s) {
		Scalar256b.mask_store_ptr_u8s(mask, ptr, values);
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u16s(
		self,
		mask: MemMask<Self::m16s>,
		ptr: *mut u16,
		values: Self::u16s,
	) {
		Scalar256b.mask_store_ptr_u16s(mask, ptr, values);
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut u32,
		values: Self::u32s,
	) {
		#[cfg(target_arch = "x86_64")]
		if let Some(store) = mask.store {
			return avx_st_u32s(ptr, values, store);
		}
		_mm256_maskstore_epi32(ptr as *mut i32, cast!(mask.mask), cast!(values))
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut u64,
		values: Self::u64s,
	) {
		self.mask_store_ptr_u32s(
			MemMask {
				mask: cast!(mask.mask),
				#[cfg(target_arch = "x86_64")]
				load: mask.load,
				#[cfg(target_arch = "x86_64")]
				store: mask.store,
			},
			ptr as _,
			cast!(values),
		)
	}

	#[inline(always)]
	fn mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.avx._mm256_moveldup_ps(ab);
		let bb = self.avx._mm256_movehdup_ps(ab);

		cast!(
			self.fma
				._mm256_fmaddsub_ps(aa, xy, self.fma._mm256_fmaddsub_ps(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_pd::<0b0101>(xy);
		let aa = self.avx._mm256_unpacklo_pd(ab, ab);
		let bb = self.avx._mm256_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm256_fmaddsub_pd(aa, xy, self.fma._mm256_fmaddsub_pd(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.mul_add_f32s(a, b, c)
	}

	#[inline(always)]
	fn mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.mul_add_f64s(a, b, c)
	}

	#[inline(always)]
	fn mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		cast!(self.fma._mm256_fmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	#[inline(always)]
	fn mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		cast!(self.fma._mm256_fmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	#[inline(always)]
	fn negate_mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.negate_mul_add_f32s(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.negate_mul_add_f64s(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		cast!(self.fma._mm256_fnmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	#[inline(always)]
	fn negate_mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		cast!(self.fma._mm256_fnmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	#[inline(always)]
	fn mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.avx._mm256_moveldup_ps(ab);
		let bb = self.avx._mm256_movehdup_ps(ab);

		cast!(
			self.fma
				._mm256_fmaddsub_ps(aa, xy, self.avx._mm256_mul_ps(bb, yx))
		)
	}

	#[inline(always)]
	fn mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm256_permute_pd::<0b0101>(xy);
		let aa = self.avx._mm256_unpacklo_pd(ab, ab);
		let bb = self.avx._mm256_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm256_fmaddsub_pd(aa, xy, self.avx._mm256_mul_pd(bb, yx))
		)
	}

	#[inline(always)]
	fn neg_c32s(self, a: Self::c32s) -> Self::c32s {
		self.xor_f32s(a, self.splat_f32s(-0.0))
	}

	#[inline(always)]
	fn neg_c64s(self, a: Self::c64s) -> Self::c64s {
		self.xor_f64s(a, self.splat_f64s(-0.0))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_load_u32s(self, slice: &[u32]) -> Self::u32s {
		avx_load_u32s(self.avx2, slice)
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_load_u64s(self, slice: &[u64]) -> Self::u64s {
		cast!(self.partial_load_u32s(bytemuck::cast_slice(slice)))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_store_u32s(self, slice: &mut [u32], values: Self::u32s) {
		avx_store_u32s(self.avx2, slice, values)
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_store_u64s(self, slice: &mut [u64], values: Self::u64s) {
		self.partial_store_u32s(bytemuck::cast_slice_mut(slice), cast!(values))
	}

	#[inline(always)]
	fn reduce_max_c32s(self, a: Self::c32s) -> c32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_max_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_max_c32x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_max_c64s(self, a: Self::c64s) -> c64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_max_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_max_c64x1(cast!(r))
	}

	#[inline(always)]
	fn reduce_max_f32s(self, a: Self::f32s) -> f32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_max_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_max_f32x4(cast!(r))
	}

	#[inline(always)]
	fn reduce_max_f64s(self, a: Self::f64s) -> f64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_max_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_max_f64x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_min_c32s(self, a: Self::c32s) -> c32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_min_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_min_c32x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_min_c64s(self, a: Self::c64s) -> c64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_min_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_min_c64x1(cast!(r))
	}

	#[inline(always)]
	fn reduce_min_f32s(self, a: Self::f32s) -> f32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_min_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_min_f32x4(cast!(r))
	}

	#[inline(always)]
	fn reduce_min_f64s(self, a: Self::f64s) -> f64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_min_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_min_f64x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_product_f32s(self, a: Self::f32s) -> f32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_mul_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_product_f32x4(cast!(r))
	}

	#[inline(always)]
	fn reduce_product_f64s(self, a: Self::f64s) -> f64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_mul_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_product_f64x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_sum_c32s(self, a: Self::c32s) -> c32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_add_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_sum_c32x2(cast!(r))
	}

	#[inline(always)]
	fn reduce_sum_c64s(self, a: Self::c64s) -> c64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_add_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_sum_c64x1(cast!(r))
	}

	#[inline(always)]
	fn reduce_sum_f32s(self, a: Self::f32s) -> f32 {
		let a: __m256 = cast!(a);
		let r = self.sse._mm_add_ps(
			self.avx._mm256_castps256_ps128(a),
			self.avx._mm256_extractf128_ps::<1>(a),
		);
		(*self).reduce_sum_f32x4(cast!(r))
	}

	#[inline(always)]
	fn reduce_sum_f64s(self, a: Self::f64s) -> f64 {
		let a: __m256d = cast!(a);
		let r = self.sse2._mm_add_pd(
			self.avx._mm256_castpd256_pd128(a),
			self.avx._mm256_extractf128_pd::<1>(a),
		);
		(*self).reduce_sum_f64x2(cast!(r))
	}

	#[inline(always)]
	fn rotate_right_c32s(self, a: Self::c32s, amount: usize) -> Self::c32s {
		cast!(avx2_pshufb(
			self,
			cast!(a),
			cast!(AVX2_ROTATE_IDX[8 * (amount % 4)]),
		))
	}

	#[inline(always)]
	fn rotate_right_c64s(self, a: Self::c64s, amount: usize) -> Self::c64s {
		cast!(avx2_pshufb(
			self,
			cast!(a),
			cast!(AVX2_ROTATE_IDX[16 * (amount % 2)]),
		))
	}

	#[inline(always)]
	fn rotate_right_u32s(self, a: Self::u32s, amount: usize) -> Self::u32s {
		cast!(avx2_pshufb(
			self,
			cast!(a),
			cast!(AVX2_ROTATE_IDX[4 * (amount % 8)]),
		))
	}

	#[inline(always)]
	fn rotate_right_u64s(self, a: Self::u64s, amount: usize) -> Self::u64s {
		cast!(avx2_pshufb(
			self,
			cast!(a),
			cast!(AVX2_ROTATE_IDX[8 * (amount % 4)]),
		))
	}

	#[inline(always)]
	fn select_u32s(
		self,
		mask: Self::m32s,
		if_true: Self::u32s,
		if_false: Self::u32s,
	) -> Self::u32s {
		let mask: __m256 = cast!(mask);
		let if_true: __m256 = cast!(if_true);
		let if_false: __m256 = cast!(if_false);

		cast!(self.avx._mm256_blendv_ps(if_false, if_true, mask))
	}

	#[inline(always)]
	fn select_u64s(
		self,
		mask: Self::m64s,
		if_true: Self::u64s,
		if_false: Self::u64s,
	) -> Self::u64s {
		let mask: __m256d = cast!(mask);
		let if_true: __m256d = cast!(if_true);
		let if_false: __m256d = cast!(if_false);

		cast!(self.avx._mm256_blendv_pd(if_false, if_true, mask))
	}

	#[inline(always)]
	fn splat_c32s(self, value: c32) -> Self::c32s {
		cast!(self.splat_f64s(cast!(value)))
	}

	#[inline(always)]
	fn splat_c64s(self, value: c64) -> Self::c64s {
		cast!(self.avx._mm256_broadcast_pd(&cast!(value)))
	}

	#[inline(always)]
	fn sub_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		self.sub_f32s(a, b)
	}

	#[inline(always)]
	fn sub_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		self.sub_f64s(a, b)
	}

	#[inline(always)]
	fn swap_re_im_c32s(self, a: Self::c32s) -> Self::c32s {
		cast!(self.avx._mm256_permute_ps::<0b10_11_00_01>(cast!(a)))
	}

	#[inline(always)]
	fn swap_re_im_c64s(self, a: Self::c64s) -> Self::c64s {
		cast!(self.avx._mm256_permute_pd::<0b0101>(cast!(a)))
	}

	#[inline(always)]
	fn vectorize<Op: WithSimd>(self, op: Op) -> Op::Output {
		struct Impl<Op> {
			this: V3,
			op: Op,
		}
		impl<Op: WithSimd> crate::NullaryFnOnce for Impl<Op> {
			type Output = Op::Output;

			#[inline(always)]
			fn call(self) -> Self::Output {
				self.op.with_simd(self.this)
			}
		}
		self.vectorize(Impl { this: self, op })
	}

	#[inline(always)]
	fn widening_mul_u32s(self, a: Self::u32s, b: Self::u32s) -> (Self::u32s, Self::u32s) {
		self.widening_mul_u32x8(a, b)
	}

	#[inline(always)]
	fn wrapping_dyn_shl_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s {
		self.shl_dyn_u32x8(a, self.and_u32x8(amount, self.splat_u32x8(32 - 1)))
	}

	#[inline(always)]
	fn wrapping_dyn_shr_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s {
		self.shr_dyn_u32x8(a, self.and_u32x8(amount, self.splat_u32x8(32 - 1)))
	}

	#[inline(always)]
	fn sqrt_f32s(self, a: Self::f32s) -> Self::f32s {
		self.sqrt_f32x8(a)
	}

	#[inline(always)]
	fn sqrt_f64s(self, a: Self::f64s) -> Self::f64s {
		self.sqrt_f64x4(a)
	}
}

#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct V3_128b(pub V3);

#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct V3_256b(pub V3);

#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct V3_512b(pub V3);

impl core::ops::Deref for V3_128b {
	type Target = V3;

	#[inline]
	fn deref(&self) -> &Self::Target {
		&self.0
	}
}

impl core::ops::Deref for V3_256b {
	type Target = V3;

	#[inline]
	fn deref(&self) -> &Self::Target {
		&self.0
	}
}

impl core::ops::Deref for V3_512b {
	type Target = V3;

	#[inline]
	fn deref(&self) -> &Self::Target {
		&self.0
	}
}

impl Seal for V3_128b {}
impl Seal for V3_256b {}
impl Seal for V3_512b {}

macro_rules! impl_scalar_binop {
	($func: ident, $ty: ident, $out: ty, impl) => {
		paste! {
			#[inline(always)]
			fn [<$func _ $ty s>](self, a: Self::[<$ty s>], b: Self::[<$ty s>]) -> Self::[<$out s>] {
				Scalar128b.[<$func _ $ty s>](a, b)
			}
		}
	};
	($func: ident, $($ty: ident => $out: ty),*) => {
		$(impl_scalar_binop!($func, $ty, $out, impl);)*
	};
	($func: ident, $($ty: ident),*) => {
		$(impl_scalar_binop!($func, $ty, $ty, impl);)*
	};
}

impl Simd for V3_128b {
	type c32s = f32x4;
	type c64s = f64x2;
	type f32s = f32x4;
	type f64s = f64x2;
	type i16s = i16x8;
	type i32s = i32x4;
	type i64s = i64x2;
	type i8s = i8x16;
	type m16s = m16x8;
	type m32s = m32x4;
	type m64s = m64x2;
	type m8s = m8x16;
	type u16s = u16x8;
	type u32s = u32x4;
	type u64s = u64x2;
	type u8s = u8x16;

	const REGISTER_COUNT: usize = 16;

	impl_simd_binop!(add, f32 x 4, f64 x 2);

	impl_simd_binop!(add, wrapping_add, u8 x 16, i8 x 16, u16 x 8, i16 x 8, u32 x 4, i32 x 4, u64 x 2, i64 x 2);

	impl_simd_binop!(sub, f32 x 4, f64 x 2);

	impl_simd_binop!(sub, wrapping_sub, u8 x 16, i8 x 16, u16 x 8, i16 x 8, u32 x 4, i32 x 4, u64 x 2, i64 x 2);

	impl_simd_binop!(mul, f32 x 4, f64 x 2);

	impl_simd_binop!(mul, wrapping_mul, u16 x 8, i16 x 8, u32 x 4, i32 x 4);

	impl_scalar_binop!(mul, u64, i64);

	impl_simd_binop!(and, m8 x 16, u8 x 16, i8 x 16, m16 x 8, u16 x 8, i16 x 8, m32 x 4, u32 x 4, i32 x 4, m64 x 2, u64 x 2, i64 x 2, f32 x 4, f64 x 2);

	impl_simd_binop!(or, m8 x 16, u8 x 16, i8 x 16, m16 x 8, u16 x 8, i16 x 8, m32 x 4, u32 x 4, i32 x 4, m64 x 2, u64 x 2, i64 x 2, f32 x 4, f64 x 2);

	impl_simd_binop!(xor, m8 x 16, u8 x 16, i8 x 16, m16 x 8, u16 x 8, i16 x 8, m32 x 4, u32 x 4, i32 x 4, m64 x 2, u64 x 2, i64 x 2, f32 x 4, f64 x 2);

	impl_simd_binop!(div, f32 x 4, f64 x 2);

	impl_simd_binop!(equal, cmp_eq, u8 x 16 => m8, u16 x 8 => m16, u32 x 4 => m32, u64 x 2 => m64, f32 x 4 => m32, f64 x 2 => m64);

	impl_simd_binop!(greater_than, cmp_gt, u8 x 16 => m8, i8 x 16 => m8, u16 x 8 => m16, i16 x 8 => m16, u32 x 4 => m32, i32 x 4 => m32, u64 x 2 => m64, i64 x 2 => m64, f32 x 4 => m32, f64 x 2 => m64);

	impl_simd_binop!(greater_than_or_equal, cmp_ge, u8 x 16 => m8, i8 x 16 => m8, u16 x 8 => m16, i16 x 8 => m16, u32 x 4 => m32, i32 x 4 => m32, u64 x 2 => m64, i64 x 2 => m64, f32 x 4 => m32, f64 x 2 => m64);

	impl_simd_binop!(less_than, cmp_lt, u8 x 16 => m8, i8 x 16 => m8, u16 x 8 => m16, i16 x 8 => m16, u32 x 4 => m32, i32 x 4 => m32, u64 x 2 => m64, i64 x 2 => m64, f32 x 4 => m32, f64 x 2 => m64);

	impl_simd_binop!(less_than_or_equal, cmp_le, u8 x 16 => m8, i8 x 16 => m8, u16 x 8 => m16, i16 x 8 => m16, u32 x 4 => m32, i32 x 4 => m32, u64 x 2 => m64, i64 x 2 => m64, f32 x 4 => m32, f64 x 2 => m64);

	impl_scalar_binop!(conj_mul, c32, c64);

	splat!(u8 x 16, i8 x 16, u16 x 8, i16 x 8, u32 x 4, i32 x 4, u64 x 2, i64 x 2, f32 x 4, f64 x 2);

	impl_simd_binop!(max, u8 x 16, i8 x 16, u16 x 8, i16 x 8, u32 x 4, i32 x 4, f32 x 4, f64 x 2);

	impl_scalar_binop!(max, u64, i64);

	impl_simd_binop!(min, u8 x 16, i8 x 16, u16 x 8, i16 x 8, u32 x 4, i32 x 4, f32 x 4, f64 x 2);

	impl_scalar_binop!(min, u64, i64);

	impl_simd_unop!(not, m8 x 16, u8 x 16, m16 x 8, u16 x 8, m32 x 4, u32 x 4, m64 x 2, u64 x 2);

	#[inline(always)]
	fn abs2_c32s(self, a: Self::c32s) -> Self::c32s {
		let sqr = self.mul_f32s(a, a);
		let sqr_rev = self
			.sse
			._mm_shuffle_ps::<0b10_11_00_01>(cast!(sqr), cast!(sqr));
		self.add_f32s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn abs2_c64s(self, a: Self::c64s) -> Self::c64s {
		let sqr = self.mul_f64s(a, a);
		let sqr_rev = self.sse2._mm_shuffle_pd::<0b01>(cast!(sqr), cast!(sqr));
		self.add_f64s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn abs_max_c32s(self, a: Self::c32s) -> Self::c32s {
		let sqr = self.abs_f32s(a);
		let sqr_rev = self
			.sse
			._mm_shuffle_ps::<0b10_11_00_01>(cast!(sqr), cast!(sqr));
		self.max_f32s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn abs_max_c64s(self, a: Self::c64s) -> Self::c64s {
		let sqr = self.abs_f64s(a);
		let sqr_rev = self.sse2._mm_shuffle_pd::<0b01>(cast!(sqr), cast!(sqr));
		self.max_f64s(sqr, cast!(sqr_rev))
	}

	#[inline(always)]
	fn add_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		self.add_f32s(a, b)
	}

	#[inline(always)]
	fn add_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		self.add_f64s(a, b)
	}

	#[inline(always)]
	fn equal_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::m32s {
		self.equal_f32s(a, b)
	}

	#[inline(always)]
	fn equal_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::m64s {
		self.equal_f64s(a, b)
	}

	#[inline(always)]
	fn conj_c32s(self, a: Self::c32s) -> Self::c32s {
		self.xor_f32s(a, self.splat_c32s(c32 { re: 0.0, im: -0.0 }))
	}

	#[inline(always)]
	fn conj_c64s(self, a: Self::c64s) -> Self::c64s {
		self.xor_f64s(a, self.splat_c64s(c64 { re: 0.0, im: -0.0 }))
	}

	#[inline(always)]
	fn conj_mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.sse3._mm_moveldup_ps(ab);
		let bb = self.sse3._mm_movehdup_ps(ab);

		cast!(
			self.fma
				._mm_fmsubadd_ps(aa, xy, self.fma._mm_fmsubadd_ps(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn conj_mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_pd::<0b01>(xy);
		let aa = self.sse2._mm_unpacklo_pd(ab, ab);
		let bb = self.sse2._mm_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm_fmsubadd_pd(aa, xy, self.fma._mm_fmsubadd_pd(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_c32s(self, mask: MemMask<Self::m32s>, ptr: *const c32) -> Self::c32s {
		cast!(self.mask_load_ptr_u32s(mask, ptr as _))
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_c64s(self, mask: MemMask<Self::m64s>, ptr: *const c64) -> Self::c64s {
		cast!(self.mask_load_ptr_u64s(mask, ptr as _))
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *const u8) -> Self::u8s {
		Scalar128b.mask_load_ptr_u8s(mask, ptr)
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_u16s(self, mask: MemMask<Self::m16s>, ptr: *const u16) -> Self::u16s {
		Scalar128b.mask_load_ptr_u16s(mask, ptr)
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_u32s(self, mask: MemMask<Self::m32s>, ptr: *const u32) -> Self::u32s {
		#[cfg(target_arch = "x86_64")]
		if let Some(load) = mask.load {
			return cast_lossy(avx_ld_u32s(ptr, load));
		}
		cast!(self.avx2._mm_maskload_epi32(ptr as _, cast!(mask.mask)))
	}

	#[inline(always)]
	unsafe fn mask_load_ptr_u64s(self, mask: MemMask<Self::m64s>, ptr: *const u64) -> Self::u64s {
		#[cfg(target_arch = "x86_64")]
		if let Some(load) = mask.load {
			return cast_lossy(avx_ld_u32s(ptr as _, load));
		}
		cast!(self.avx2._mm_maskload_epi64(ptr as _, cast!(mask.mask)))
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_c32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut c32,
		values: Self::c32s,
	) {
		self.mask_store_ptr_u32s(mask, ptr as _, cast!(values));
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_c64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut c64,
		values: Self::c64s,
	) {
		self.mask_store_ptr_u64s(mask, ptr as _, cast!(values))
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *mut u8, values: Self::u8s) {
		Scalar128b.mask_store_ptr_u8s(mask, ptr, values);
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_u16s(
		self,
		mask: MemMask<Self::m16s>,
		ptr: *mut u16,
		values: Self::u16s,
	) {
		Scalar128b.mask_store_ptr_u16s(mask, ptr, values);
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_u32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut u32,
		values: Self::u32s,
	) {
		#[cfg(target_arch = "x86_64")]
		if let Some(store) = mask.store {
			return avx_st_u32s(ptr, cast!([values, self.splat_u32s(0)]), store);
		}
		self.avx2
			._mm_maskstore_epi32(ptr as _, cast!(mask.mask), cast!(values));
	}

	#[inline(always)]
	unsafe fn mask_store_ptr_u64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut u64,
		values: Self::u64s,
	) {
		self.mask_store_ptr_u32s(
			MemMask {
				mask: cast!(mask.mask),
				#[cfg(target_arch = "x86_64")]
				load: mask.load,
				#[cfg(target_arch = "x86_64")]
				store: mask.store,
			},
			ptr as _,
			cast!(values),
		)
	}

	#[inline(always)]
	fn mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.sse3._mm_moveldup_ps(ab);
		let bb = self.sse3._mm_movehdup_ps(ab);

		cast!(
			self.fma
				._mm_fmaddsub_ps(aa, xy, self.fma._mm_fmaddsub_ps(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_pd::<0b01>(xy);
		let aa = self.sse2._mm_unpacklo_pd(ab, ab);
		let bb = self.sse2._mm_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm_fmaddsub_pd(aa, xy, self.fma._mm_fmaddsub_pd(bb, yx, cast!(c)))
		)
	}

	#[inline(always)]
	fn mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.mul_add_f32s(a, b, c)
	}

	#[inline(always)]
	fn mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.mul_add_f64s(a, b, c)
	}

	#[inline(always)]
	fn mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.mul_add_f32x4(a, b, c)
	}

	#[inline(always)]
	fn mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.mul_add_f64x2(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.negate_mul_add_f32s(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.negate_mul_add_f64s(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s {
		self.negate_mul_add_f32x4(a, b, c)
	}

	#[inline(always)]
	fn negate_mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s {
		self.negate_mul_add_f64x2(a, b, c)
	}

	#[inline(always)]
	fn mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_ps::<0b10_11_00_01>(xy);
		let aa = self.sse3._mm_moveldup_ps(ab);
		let bb = self.sse3._mm_movehdup_ps(ab);

		cast!(
			self.fma
				._mm_fmaddsub_ps(aa, xy, self.sse._mm_mul_ps(bb, yx))
		)
	}

	#[inline(always)]
	fn mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		let ab = cast!(a);
		let xy = cast!(b);

		let yx = self.avx._mm_permute_pd::<0b01>(xy);
		let aa = self.sse2._mm_unpacklo_pd(ab, ab);
		let bb = self.sse2._mm_unpackhi_pd(ab, ab);

		cast!(
			self.fma
				._mm_fmaddsub_pd(aa, xy, self.sse2._mm_mul_pd(bb, yx))
		)
	}

	#[inline(always)]
	fn neg_c32s(self, a: Self::c32s) -> Self::c32s {
		self.xor_f32s(a, self.splat_f32s(-0.0))
	}

	#[inline(always)]
	fn neg_c64s(self, a: Self::c64s) -> Self::c64s {
		self.xor_f64s(a, self.splat_f64s(-0.0))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_load_u32s(self, slice: &[u32]) -> Self::u32s {
		cast_lossy(avx_load_u32s(self.avx2, slice))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_load_u64s(self, slice: &[u64]) -> Self::u64s {
		cast!(self.partial_load_u32s(bytemuck::cast_slice(slice)))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_store_u32s(self, slice: &mut [u32], values: Self::u32s) {
		avx_store_u32s(self.avx2, slice, cast!([values, self.splat_u32s(0)]))
	}

	#[cfg(target_arch = "x86_64")]
	#[inline(always)]
	fn partial_store_u64s(self, slice: &mut [u64], values: Self::u64s) {
		self.partial_store_u32s(bytemuck::cast_slice_mut(slice), cast!(values))
	}

	#[inline(always)]
	fn reduce_max_c32s(self, a: Self::c32s) -> c32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_max_ps(a, hi);
		cast!(self.sse2._mm_cvtsd_f64(cast!(r0)))
	}

	#[inline(always)]
	fn reduce_max_c64s(self, a: Self::c64s) -> c64 {
		cast!(a)
	}

	#[inline(always)]
	fn reduce_max_f32s(self, a: Self::f32s) -> f32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_max_ps(a, hi);
		let r0_shuffled = self.sse._mm_shuffle_ps::<0b0001>(r0, r0);
		let r = self.sse._mm_max_ss(r0, r0_shuffled);
		self.sse._mm_cvtss_f32(r)
	}

	#[inline(always)]
	fn reduce_max_f64s(self, a: Self::f64s) -> f64 {
		let a: __m128d = cast!(a);
		let hi = cast!(self.sse._mm_movehl_ps(cast!(a), cast!(a)));
		let r = self.sse2._mm_max_sd(a, hi);
		self.sse2._mm_cvtsd_f64(r)
	}

	#[inline(always)]
	fn reduce_min_c32s(self, a: Self::c32s) -> c32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_min_ps(a, hi);
		cast!(self.sse2._mm_cvtsd_f64(cast!(r0)))
	}

	#[inline(always)]
	fn reduce_min_c64s(self, a: Self::c64s) -> c64 {
		cast!(a)
	}

	#[inline(always)]
	fn reduce_min_f32s(self, a: Self::f32s) -> f32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_min_ps(a, hi);
		let r0_shuffled = self.sse._mm_shuffle_ps::<0b0001>(r0, r0);
		let r = self.sse._mm_min_ss(r0, r0_shuffled);
		self.sse._mm_cvtss_f32(r)
	}

	#[inline(always)]
	fn reduce_min_f64s(self, a: Self::f64s) -> f64 {
		let a: __m128d = cast!(a);
		let hi = cast!(self.sse._mm_movehl_ps(cast!(a), cast!(a)));
		let r = self.sse2._mm_min_sd(a, hi);
		self.sse2._mm_cvtsd_f64(r)
	}

	#[inline(always)]
	fn reduce_product_f32s(self, a: Self::f32s) -> f32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_mul_ps(a, hi);
		let r0_shuffled = self.sse._mm_shuffle_ps::<0b0001>(r0, r0);
		let r = self.sse._mm_mul_ss(r0, r0_shuffled);
		self.sse._mm_cvtss_f32(r)
	}

	#[inline(always)]
	fn reduce_product_f64s(self, a: Self::f64s) -> f64 {
		let a: __m128d = cast!(a);
		let hi = cast!(self.sse._mm_movehl_ps(cast!(a), cast!(a)));
		let r = self.sse2._mm_mul_sd(a, hi);
		self.sse2._mm_cvtsd_f64(r)
	}

	#[inline(always)]
	fn reduce_sum_c32s(self, a: Self::c32s) -> c32 {
		// a0 a1 a2 a3
		let a: __m128 = cast!(a);
		// a2 a3 a2 a3
		let hi = self.sse._mm_movehl_ps(a, a);

		// a0+a2 a1+a3 _ _
		let r0 = self.sse._mm_add_ps(a, hi);

		cast!(self.sse2._mm_cvtsd_f64(cast!(r0)))
	}

	#[inline(always)]
	fn reduce_sum_c64s(self, a: Self::c64s) -> c64 {
		cast!(a)
	}

	#[inline(always)]
	fn reduce_sum_f32s(self, a: Self::f32s) -> f32 {
		let a: __m128 = cast!(a);
		let hi = self.sse._mm_movehl_ps(a, a);
		let r0 = self.sse._mm_add_ps(a, hi);
		let r0_shuffled = self.sse._mm_shuffle_ps::<0b0001>(r0, r0);
		let r = self.sse._mm_add_ss(r0, r0_shuffled);
		self.sse._mm_cvtss_f32(r)
	}

	#[inline(always)]
	fn reduce_sum_f64s(self, a: Self::f64s) -> f64 {
		let a: __m128d = cast!(a);
		let hi = cast!(self.sse._mm_movehl_ps(cast!(a), cast!(a)));
		let r = self.sse2._mm_add_sd(a, hi);
		self.sse2._mm_cvtsd_f64(r)
	}

	#[inline(always)]
	fn rotate_right_c32s(self, a: Self::c32s, amount: usize) -> Self::c32s {
		cast!(
			self.ssse3
				._mm_shuffle_epi8(cast!(a), cast!(AVX2_128_ROTATE_IDX[8 * (amount % 2)]))
		)
	}

	#[inline(always)]
	fn rotate_right_c64s(self, a: Self::c64s, amount: usize) -> Self::c64s {
		_ = amount;
		a
	}

	#[inline(always)]
	fn rotate_right_u32s(self, a: Self::u32s, amount: usize) -> Self::u32s {
		cast!(
			self.ssse3
				._mm_shuffle_epi8(cast!(a), cast!(AVX2_128_ROTATE_IDX[4 * (amount % 4)]))
		)
	}

	#[inline(always)]
	fn rotate_right_u64s(self, a: Self::u64s, amount: usize) -> Self::u64s {
		cast!(
			self.ssse3
				._mm_shuffle_epi8(cast!(a), cast!(AVX2_128_ROTATE_IDX[8 * (amount % 2)]))
		)
	}

	#[inline(always)]
	fn select_u32s(
		self,
		mask: Self::m32s,
		if_true: Self::u32s,
		if_false: Self::u32s,
	) -> Self::u32s {
		self.select_u32x4(mask, if_true, if_false)
	}

	#[inline(always)]
	fn select_u64s(
		self,
		mask: Self::m64s,
		if_true: Self::u64s,
		if_false: Self::u64s,
	) -> Self::u64s {
		self.select_u64x2(mask, if_true, if_false)
	}

	#[inline(always)]
	fn splat_c32s(self, value: c32) -> Self::c32s {
		cast!(self.splat_f64x2(cast!(value)))
	}

	#[inline(always)]
	fn splat_c64s(self, value: c64) -> Self::c64s {
		cast!(value)
	}

	#[inline(always)]
	fn sub_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s {
		self.sub_f32x4(a, b)
	}

	#[inline(always)]
	fn sub_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s {
		self.sub_f64x2(a, b)
	}

	#[inline(always)]
	fn swap_re_im_c32s(self, a: Self::c32s) -> Self::c32s {
		cast!(self.avx._mm_permute_ps::<0b10_11_00_01>(cast!(a)))
	}

	#[inline(always)]
	fn swap_re_im_c64s(self, a: Self::c64s) -> Self::c64s {
		cast!(self.avx._mm_permute_pd::<0b01>(cast!(a)))
	}

	#[inline(always)]
	fn vectorize<Op: WithSimd>(self, op: Op) -> Op::Output {
		Simd::vectorize(self.0, op)
	}

	#[inline(always)]
	fn widening_mul_u32s(self, a: Self::u32s, b: Self::u32s) -> (Self::u32s, Self::u32s) {
		self.widening_mul_u32x4(a, b)
	}

	#[inline(always)]
	fn wrapping_dyn_shl_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s {
		self.shl_dyn_u32x4(a, self.and_u32x4(amount, self.splat_u32x4(32 - 1)))
	}

	#[inline(always)]
	fn wrapping_dyn_shr_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s {
		self.shr_dyn_u32x4(a, self.and_u32x4(amount, self.splat_u32x4(32 - 1)))
	}

	#[inline(always)]
	fn sqrt_f32s(self, a: Self::f32s) -> Self::f32s {
		self.sqrt_f32x4(a)
	}

	#[inline(always)]
	fn sqrt_f64s(self, a: Self::f64s) -> Self::f64s {
		self.sqrt_f64x2(a)
	}
}

impl Simd for V3_256b {
	type c32s = f32x8;
	type c64s = f64x4;
	type f32s = f32x8;
	type f64s = f64x4;
	type i16s = i16x16;
	type i32s = i32x8;
	type i64s = i64x4;
	type i8s = i8x32;
	type m16s = m16x16;
	type m32s = m32x8;
	type m64s = m64x4;
	type m8s = m8x32;
	type u16s = u16x16;
	type u32s = u32x8;
	type u64s = u64x4;
	type u8s = u8x32;

	const REGISTER_COUNT: usize = 16;

	impl_simd_binop!(add, f32 x 8, f64 x 4);

	impl_simd_binop!(add, wrapping_add, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	impl_simd_binop!(sub, f32 x 8, f64 x 4);

	impl_simd_binop!(sub, wrapping_sub, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	impl_simd_binop!(mul, f32 x 8, f64 x 4);

	impl_simd_binop!(mul, wrapping_mul, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	impl_simd_binop!(and, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(or, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(xor, m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(div, f32 x 8, f64 x 4);

	impl_simd_binop!(equal, cmp_eq, u8 x 32 => m8, u16 x 16 => m16, u32 x 8 => m32, u64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(greater_than, cmp_gt, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(greater_than_or_equal, cmp_ge, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(less_than, cmp_lt, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	impl_simd_binop!(less_than_or_equal, cmp_le, u8 x 32 => m8, i8 x 32 => m8, u16 x 16 => m16, i16 x 16 => m16, u32 x 8 => m32, i32 x 8 => m32, u64 x 4 => m64, i64 x 4 => m64, f32 x 8 => m32, f64 x 4 => m64);

	splat!(u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4, f32 x 8, f64 x 4);

	impl_simd_binop!(max, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, f32 x 8, f64 x 4);

	impl_simd_binop!(min, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, f32 x 8, f64 x 4);

	impl_simd_unop!(not, m8 x 32, u8 x 32, m16 x 16, u16 x 16, m32 x 8, u32 x 8, m64 x 4, u64 x 4);

	inherit!({
		fn abs2_c32s(self, a: Self::c32s) -> Self::c32s;
		fn abs2_c64s(self, a: Self::c64s) -> Self::c64s;
		fn abs_max_c32s(self, a: Self::c32s) -> Self::c32s;
		fn abs_max_c64s(self, a: Self::c64s) -> Self::c64s;
		fn add_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn add_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn conj_c32s(self, a: Self::c32s) -> Self::c32s;
		fn conj_c64s(self, a: Self::c64s) -> Self::c64s;
		fn conj_mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s;
		fn conj_mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s;
		fn conj_mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn conj_mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn equal_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::m32s;
		fn equal_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::m64s;
		fn mask_between_m32s(self, start: u32, end: u32) -> MemMask<Self::m32s>;
		fn mask_between_m64s(self, start: u64, end: u64) -> MemMask<Self::m64s>;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_c32s(
			self,
			mask: MemMask<Self::m32s>,
			ptr: *const c32,
		) -> Self::c32s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_c64s(
			self,
			mask: MemMask<Self::m64s>,
			ptr: *const c64,
		) -> Self::c64s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *const u8) -> Self::u8s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_u16s(
			self,
			mask: MemMask<Self::m16s>,
			ptr: *const u16,
		) -> Self::u16s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_u32s(
			self,
			mask: MemMask<Self::m32s>,
			ptr: *const u32,
		) -> Self::u32s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_load_ptr_u64s(
			self,
			mask: MemMask<Self::m64s>,
			ptr: *const u64,
		) -> Self::u64s;
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_c32s(
			self,
			mask: MemMask<Self::m32s>,
			ptr: *mut c32,
			values: Self::c32s,
		);
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_c64s(
			self,
			mask: MemMask<Self::m64s>,
			ptr: *mut c64,
			values: Self::c64s,
		);
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_u8s(
			self,
			mask: MemMask<Self::m8s>,
			ptr: *mut u8,
			values: Self::u8s,
		);
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_u16s(
			self,
			mask: MemMask<Self::m16s>,
			ptr: *mut u16,
			values: Self::u16s,
		);
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_u32s(
			self,
			mask: MemMask<Self::m32s>,
			ptr: *mut u32,
			values: Self::u32s,
		);
		/// # Safety
		///
		/// See the trait-level safety documentation.
		unsafe fn mask_store_ptr_u64s(
			self,
			mask: MemMask<Self::m64s>,
			ptr: *mut u64,
			values: Self::u64s,
		);
		fn mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s;
		fn mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s;
		fn mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn negate_mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn negate_mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn negate_mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn negate_mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn neg_c32s(self, a: Self::c32s) -> Self::c32s;
		fn neg_c64s(self, a: Self::c64s) -> Self::c64s;
		fn min_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn min_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s;
		fn max_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn max_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s;
		fn mul_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn mul_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s;
		fn partial_load_u32s(self, slice: &[u32]) -> Self::u32s;
		fn partial_load_u64s(self, slice: &[u64]) -> Self::u64s;
		fn partial_store_u32s(self, slice: &mut [u32], values: Self::u32s);
		fn partial_store_u64s(self, slice: &mut [u64], values: Self::u64s);
		fn reduce_max_c32s(self, a: Self::c32s) -> c32;
		fn reduce_max_c64s(self, a: Self::c64s) -> c64;
		fn reduce_max_f32s(self, a: Self::f32s) -> f32;
		fn reduce_max_f64s(self, a: Self::f64s) -> f64;
		fn reduce_min_c32s(self, a: Self::c32s) -> c32;
		fn reduce_min_c64s(self, a: Self::c64s) -> c64;
		fn reduce_min_f32s(self, a: Self::f32s) -> f32;
		fn reduce_min_f64s(self, a: Self::f64s) -> f64;
		fn reduce_product_f32s(self, a: Self::f32s) -> f32;
		fn reduce_product_f64s(self, a: Self::f64s) -> f64;
		fn reduce_sum_c32s(self, a: Self::c32s) -> c32;
		fn reduce_sum_c64s(self, a: Self::c64s) -> c64;
		fn reduce_sum_f32s(self, a: Self::f32s) -> f32;
		fn reduce_sum_f64s(self, a: Self::f64s) -> f64;
		fn rotate_right_c32s(self, a: Self::c32s, amount: usize) -> Self::c32s;
		fn rotate_right_c64s(self, a: Self::c64s, amount: usize) -> Self::c64s;
		fn rotate_right_u32s(self, a: Self::u32s, amount: usize) -> Self::u32s;
		fn rotate_right_u64s(self, a: Self::u64s, amount: usize) -> Self::u64s;
		fn select_u32s(
			self,
			mask: Self::m32s,
			if_true: Self::u32s,
			if_false: Self::u32s,
		) -> Self::u32s;
		fn select_u64s(
			self,
			mask: Self::m64s,
			if_true: Self::u64s,
			if_false: Self::u64s,
		) -> Self::u64s;
		fn splat_c32s(self, a: c32) -> Self::c32s;
		fn splat_c64s(self, a: c64) -> Self::c64s;
		fn sub_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn sub_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn swap_re_im_c32s(self, a: Self::c32s) -> Self::c32s;
		fn swap_re_im_c64s(self, a: Self::c64s) -> Self::c64s;
		fn widening_mul_u32s(self, a: Self::u32s, b: Self::u32s) -> (Self::u32s, Self::u32s);
		fn wrapping_dyn_shl_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s;
		fn wrapping_dyn_shr_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s;
	});

	#[inline(always)]
	fn vectorize<Op: WithSimd>(self, op: Op) -> Op::Output {
		Simd::vectorize(self.0, op)
	}

	#[inline(always)]
	fn sqrt_f32s(self, a: Self::f32s) -> Self::f32s {
		self.sqrt_f32x8(a)
	}

	#[inline(always)]
	fn sqrt_f64s(self, a: Self::f64s) -> Self::f64s {
		self.sqrt_f64x4(a)
	}
}

impl Simd for V3_512b {
	type c32s = f32x16;
	type c64s = f64x8;
	type f32s = f32x16;
	type f64s = f64x8;
	type i16s = i16x32;
	type i32s = i32x16;
	type i64s = i64x8;
	type i8s = i8x64;
	type m16s = m16x32;
	type m32s = m32x16;
	type m64s = m64x8;
	type m8s = m8x64;
	type u16s = u16x32;
	type u32s = u32x16;
	type u64s = u64x8;
	type u8s = u8x64;

	const REGISTER_COUNT: usize = 8;

	inherit_x2!(V3_256b(*self), {
		fn abs2_c32s(self, a: Self::c32s) -> Self::c32s;
		fn abs2_c64s(self, a: Self::c64s) -> Self::c64s;
		fn abs_max_c32s(self, a: Self::c32s) -> Self::c32s;
		fn abs_max_c64s(self, a: Self::c64s) -> Self::c64s;
		fn add_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn add_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn add_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn add_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn add_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn add_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn add_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn add_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn and_m32s(self, a: Self::m32s, b: Self::m32s) -> Self::m32s;
		fn and_m64s(self, a: Self::m64s, b: Self::m64s) -> Self::m64s;
		fn and_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn and_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn and_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn and_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn and_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn and_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn conj_c32s(self, a: Self::c32s) -> Self::c32s;
		fn conj_c64s(self, a: Self::c64s) -> Self::c64s;
		fn conj_mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s;
		fn conj_mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s;
		fn conj_mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn conj_mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn div_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn div_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn equal_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::m32s;
		fn equal_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::m64s;
		fn equal_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::m32s;
		fn equal_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::m64s;
		fn equal_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::m8s;
		fn equal_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::m8s;
		fn equal_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::m16s;
		fn equal_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::m16s;
		fn equal_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::m32s;
		fn equal_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::m32s;
		fn equal_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::m64s;
		fn equal_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::m64s;
		fn greater_than_or_equal_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::m8s;
		fn greater_than_or_equal_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::m8s;
		fn greater_than_or_equal_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::m16s;
		fn greater_than_or_equal_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::m16s;
		fn greater_than_or_equal_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::m32s;
		fn greater_than_or_equal_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::m32s;
		fn greater_than_or_equal_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::m64s;
		fn greater_than_or_equal_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::m64s;
		fn greater_than_or_equal_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::m32s;
		fn greater_than_or_equal_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::m64s;
		fn greater_than_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::m8s;
		fn greater_than_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::m8s;
		fn greater_than_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::m16s;
		fn greater_than_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::m16s;
		fn greater_than_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::m32s;
		fn greater_than_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::m32s;
		fn greater_than_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::m64s;
		fn greater_than_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::m64s;
		fn greater_than_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::m32s;
		fn greater_than_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::m64s;
		fn less_than_or_equal_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::m8s;
		fn less_than_or_equal_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::m8s;
		fn less_than_or_equal_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::m16s;
		fn less_than_or_equal_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::m16s;
		fn less_than_or_equal_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::m32s;
		fn less_than_or_equal_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::m32s;
		fn less_than_or_equal_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::m64s;
		fn less_than_or_equal_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::m64s;
		fn less_than_or_equal_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::m32s;
		fn less_than_or_equal_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::m64s;
		fn less_than_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::m8s;
		fn less_than_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::m8s;
		fn less_than_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::m16s;
		fn less_than_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::m16s;
		fn less_than_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::m32s;
		fn less_than_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::m32s;
		fn less_than_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::m64s;
		fn less_than_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::m64s;
		fn less_than_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::m32s;
		fn less_than_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::m64s;
		fn min_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn max_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn min_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::i8s;
		fn max_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::i8s;
		fn min_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn max_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn min_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::i16s;
		fn max_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::i16s;
		fn min_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn max_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn min_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::i32s;
		fn max_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::i32s;
		fn max_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn max_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn min_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn min_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn mul_add_c32s(self, a: Self::c32s, b: Self::c32s, c: Self::c32s) -> Self::c32s;
		fn mul_add_c64s(self, a: Self::c64s, b: Self::c64s, c: Self::c64s) -> Self::c64s;
		fn mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn negate_mul_add_e_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn negate_mul_add_e_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn negate_mul_add_f32s(self, a: Self::f32s, b: Self::f32s, c: Self::f32s) -> Self::f32s;
		fn negate_mul_add_f64s(self, a: Self::f64s, b: Self::f64s, c: Self::f64s) -> Self::f64s;
		fn mul_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn mul_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn mul_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn mul_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn mul_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn mul_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::i16s;
		fn mul_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn mul_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::i32s;
		fn mul_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn mul_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s;
		fn neg_c32s(self, a: Self::c32s) -> Self::c32s;
		fn neg_c64s(self, a: Self::c64s) -> Self::c64s;
		fn not_m8s(self, a: Self::m8s) -> Self::m8s;
		fn not_m16s(self, a: Self::m16s) -> Self::m16s;
		fn not_m32s(self, a: Self::m32s) -> Self::m32s;
		fn not_m64s(self, a: Self::m64s) -> Self::m64s;
		fn not_f32s(self, a: Self::f32s) -> Self::f32s;
		fn not_f64s(self, a: Self::f64s) -> Self::f64s;
		fn not_u8s(self, a: Self::u8s) -> Self::u8s;
		fn not_u16s(self, a: Self::u16s) -> Self::u16s;
		fn not_u32s(self, a: Self::u32s) -> Self::u32s;
		fn not_u64s(self, a: Self::u64s) -> Self::u64s;
		fn or_m8s(self, a: Self::m8s, b: Self::m8s) -> Self::m8s;
		fn or_m16s(self, a: Self::m16s, b: Self::m16s) -> Self::m16s;
		fn or_m32s(self, a: Self::m32s, b: Self::m32s) -> Self::m32s;
		fn or_m64s(self, a: Self::m64s, b: Self::m64s) -> Self::m64s;
		fn or_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn or_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn or_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn or_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn or_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn or_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn select_u32s(
			self,
			mask: Self::m32s,
			if_true: Self::u32s,
			if_false: Self::u32s,
		) -> Self::u32s;
		fn select_u64s(
			self,
			mask: Self::m64s,
			if_true: Self::u64s,
			if_false: Self::u64s,
		) -> Self::u64s;
		fn sub_c32s(self, a: Self::c32s, b: Self::c32s) -> Self::c32s;
		fn sub_c64s(self, a: Self::c64s, b: Self::c64s) -> Self::c64s;
		fn sub_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn sub_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn sub_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn sub_i8s(self, a: Self::i8s, b: Self::i8s) -> Self::i8s;
		fn sub_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn sub_i16s(self, a: Self::i16s, b: Self::i16s) -> Self::i16s;
		fn sub_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn sub_i32s(self, a: Self::i32s, b: Self::i32s) -> Self::i32s;
		fn sub_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
		fn sub_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s;
		fn swap_re_im_c32s(self, a: Self::c32s) -> Self::c32s;
		fn swap_re_im_c64s(self, a: Self::c64s) -> Self::c64s;
		fn wrapping_dyn_shl_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s;
		fn wrapping_dyn_shr_u32s(self, a: Self::u32s, amount: Self::u32s) -> Self::u32s;
		fn xor_m8s(self, a: Self::m8s, b: Self::m8s) -> Self::m8s;
		fn xor_m16s(self, a: Self::m16s, b: Self::m16s) -> Self::m16s;
		fn xor_m32s(self, a: Self::m32s, b: Self::m32s) -> Self::m32s;
		fn xor_m64s(self, a: Self::m64s, b: Self::m64s) -> Self::m64s;
		fn xor_f32s(self, a: Self::f32s, b: Self::f32s) -> Self::f32s;
		fn xor_f64s(self, a: Self::f64s, b: Self::f64s) -> Self::f64s;
		fn xor_u8s(self, a: Self::u8s, b: Self::u8s) -> Self::u8s;
		fn xor_u16s(self, a: Self::u16s, b: Self::u16s) -> Self::u16s;
		fn xor_u32s(self, a: Self::u32s, b: Self::u32s) -> Self::u32s;
		fn xor_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s;
	});

	inherit_x2!(V3_256b(*self), splat, {
		fn splat_c32s(self, value: c32) -> Self::c32s;
		fn splat_c64s(self, value: c64) -> Self::c64s;
		fn splat_f32s(self, value: f32) -> Self::f32s;
		fn splat_f64s(self, value: f64) -> Self::f64s;
		fn splat_u8s(self, value: u8) -> Self::u8s;
		fn splat_i8s(self, value: i8) -> Self::i8s;
		fn splat_u16s(self, value: u16) -> Self::u16s;
		fn splat_i16s(self, value: i16) -> Self::i16s;
		fn splat_u32s(self, value: u32) -> Self::u32s;
		fn splat_i32s(self, value: i32) -> Self::i32s;
		fn splat_u64s(self, value: u64) -> Self::u64s;
		fn splat_i64s(self, value: i64) -> Self::i64s;
	});

	inherit_x2!(V3_256b(*self), wide, {
		fn widening_mul_u32s(self, a: Self::u32s, b: Self::u32s) -> (Self::u32s, Self::u32s);
	});

	#[inline(always)]
	fn rotate_right_c32s(self, a: Self::c32s, amount: usize) -> Self::c32s {
		let simd = V3_256b(*self);
		let amount = amount % Self::C32_LANES;
		let [mut a0, mut a1]: [_; 2] = cast!(a);
		if amount >= Self::C32_LANES / 2 {
			core::mem::swap(&mut a0, &mut a1);
		}
		let amount = amount % (Self::C32_LANES / 2);
		let mask = simd.mask_between_m32s(0, amount as _).mask();
		let a0 = simd.rotate_right_c32s(a0, amount);
		let a1 = simd.rotate_right_c32s(a1, amount);

		cast!([
			simd.select_f32s(mask, a1, a0),
			simd.select_f32s(mask, a0, a1),
		])
	}

	#[inline(always)]
	fn rotate_right_c64s(self, a: Self::c64s, amount: usize) -> Self::c64s {
		let simd = V3_256b(*self);
		let amount = amount % Self::C64_LANES;
		let [mut a0, mut a1]: [_; 2] = cast!(a);
		if amount >= Self::C64_LANES / 2 {
			core::mem::swap(&mut a0, &mut a1);
		}
		let amount = amount % (Self::C64_LANES / 2);
		let mask = simd.mask_between_m64s(0, amount as _).mask();
		let a0 = simd.rotate_right_c64s(a0, amount);
		let a1 = simd.rotate_right_c64s(a1, amount);

		cast!([
			simd.select_f64s(mask, a1, a0),
			simd.select_f64s(mask, a0, a1),
		])
	}

	#[inline(always)]
	fn rotate_right_u32s(self, a: Self::u32s, amount: usize) -> Self::u32s {
		let simd = V3_256b(*self);
		let amount = amount % Self::U32_LANES;
		let [mut a0, mut a1]: [_; 2] = cast!(a);
		if amount >= Self::U32_LANES / 2 {
			core::mem::swap(&mut a0, &mut a1);
		}
		let amount = amount % (Self::U32_LANES / 2);
		let mask = simd.mask_between_m32s(0, amount as _).mask();
		let a0 = simd.rotate_right_u32s(a0, amount);
		let a1 = simd.rotate_right_u32s(a1, amount);

		cast!([
			simd.select_u32s(mask, a1, a0),
			simd.select_u32s(mask, a0, a1),
		])
	}

	#[inline(always)]
	fn rotate_right_u64s(self, a: Self::u64s, amount: usize) -> Self::u64s {
		let simd = V3_256b(*self);
		let amount = amount % Self::U64_LANES;
		let [mut a0, mut a1]: [_; 2] = cast!(a);
		if amount >= Self::U64_LANES / 2 {
			core::mem::swap(&mut a0, &mut a1);
		}
		let amount = amount % (Self::U64_LANES / 2);
		let mask = simd.mask_between_m64s(0, amount as _).mask();
		let a0 = simd.rotate_right_u64s(a0, amount);
		let a1 = simd.rotate_right_u64s(a1, amount);

		cast!([
			simd.select_u64s(mask, a1, a0),
			simd.select_u64s(mask, a0, a1),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_c32s(self, mask: MemMask<Self::m32s>, ptr: *const c32) -> Self::c32s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_c32s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_c32s(MemMask::new(mask[1]), ptr.wrapping_add(Self::C32_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_c64s(self, mask: MemMask<Self::m64s>, ptr: *const c64) -> Self::c64s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_c64s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_c64s(MemMask::new(mask[1]), ptr.wrapping_add(Self::C64_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *const u8) -> Self::u8s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_u8s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_u8s(MemMask::new(mask[1]), ptr.wrapping_add(V3_256b::U8_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u16s(self, mask: MemMask<Self::m16s>, ptr: *const u16) -> Self::u16s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_u16s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_u16s(MemMask::new(mask[1]), ptr.wrapping_add(V3_256b::U16_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u32s(self, mask: MemMask<Self::m32s>, ptr: *const u32) -> Self::u32s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_u32s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_u32s(MemMask::new(mask[1]), ptr.wrapping_add(Self::U32_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_load_ptr_u64s(self, mask: MemMask<Self::m64s>, ptr: *const u64) -> Self::u64s {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		cast!([
			simd.mask_load_ptr_u64s(MemMask::new(mask[0]), ptr.wrapping_add(0)),
			simd.mask_load_ptr_u64s(MemMask::new(mask[1]), ptr.wrapping_add(Self::U64_LANES)),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_c32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut c32,
		values: Self::c32s,
	) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_c32s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_c32s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(Self::C32_LANES),
				values[1]
			),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_c64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut c64,
		values: Self::c64s,
	) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_c64s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_c64s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(Self::C64_LANES),
				values[1]
			),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u8s(self, mask: MemMask<Self::m8s>, ptr: *mut u8, values: Self::u8s) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_u8s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_u8s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(V3_256b::U8_LANES),
				values[1]
			),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u16s(
		self,
		mask: MemMask<Self::m16s>,
		ptr: *mut u16,
		values: Self::u16s,
	) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_u16s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_u16s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(V3_256b::U16_LANES),
				values[1]
			),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u32s(
		self,
		mask: MemMask<Self::m32s>,
		ptr: *mut u32,
		values: Self::u32s,
	) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_u32s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_u32s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(Self::U32_LANES),
				values[1]
			),
		])
	}

	/// # Safety
	///
	/// See the trait-level safety documentation.
	#[inline(always)]
	unsafe fn mask_store_ptr_u64s(
		self,
		mask: MemMask<Self::m64s>,
		ptr: *mut u64,
		values: Self::u64s,
	) {
		let simd = V3_256b(*self);
		let mask: [_; 2] = cast!(mask.mask());
		let values: [_; 2] = cast!(values);
		cast!([
			simd.mask_store_ptr_u64s(MemMask::new(mask[0]), ptr.wrapping_add(0), values[0]),
			simd.mask_store_ptr_u64s(
				MemMask::new(mask[1]),
				ptr.wrapping_add(Self::U64_LANES),
				values[1]
			),
		])
	}

	#[inline(always)]
	fn reduce_max_c32s(self, a: Self::c32s) -> c32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_max_c32s(simd.max_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_max_c64s(self, a: Self::c64s) -> c64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_max_c64s(simd.max_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_max_f32s(self, a: Self::f32s) -> f32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_max_f32s(simd.max_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_max_f64s(self, a: Self::f64s) -> f64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_max_f64s(simd.max_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_min_c32s(self, a: Self::c32s) -> c32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_min_c32s(simd.min_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_min_c64s(self, a: Self::c64s) -> c64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_min_c64s(simd.min_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_min_f32s(self, a: Self::f32s) -> f32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_min_f32s(simd.min_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_min_f64s(self, a: Self::f64s) -> f64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_min_f64s(simd.min_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_product_f32s(self, a: Self::f32s) -> f32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_product_f32s(simd.mul_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_product_f64s(self, a: Self::f64s) -> f64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_product_f64s(simd.mul_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_sum_c32s(self, a: Self::c32s) -> c32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_sum_c32s(simd.add_c32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_sum_c64s(self, a: Self::c64s) -> c64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_sum_c64s(simd.add_c64s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_sum_f32s(self, a: Self::f32s) -> f32 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_sum_f32s(simd.add_f32s(a[0], a[1]))
	}

	#[inline(always)]
	fn reduce_sum_f64s(self, a: Self::f64s) -> f64 {
		let simd = V3_256b(*self);
		let a: [_; 2] = cast!(a);
		simd.reduce_sum_f64s(simd.add_f64s(a[0], a[1]))
	}

	#[inline(always)]
	fn vectorize<Op: WithSimd>(self, op: Op) -> Op::Output {
		Simd::vectorize(self.0, op)
	}

	#[inline(always)]
	fn max_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s {
		Scalar512b.max_u64s(a, b)
	}

	#[inline(always)]
	fn max_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s {
		Scalar512b.max_i64s(a, b)
	}

	#[inline(always)]
	fn min_u64s(self, a: Self::u64s, b: Self::u64s) -> Self::u64s {
		Scalar512b.min_u64s(a, b)
	}

	#[inline(always)]
	fn min_i64s(self, a: Self::i64s, b: Self::i64s) -> Self::i64s {
		Scalar512b.max_i64s(a, b)
	}

	#[inline(always)]
	fn sqrt_f32s(self, a: Self::f32s) -> Self::f32s {
		let a: [_; 2] = cast!(a);
		cast!([self.sqrt_f32x8(a[0]), self.sqrt_f32x8(a[1]),])
	}

	#[inline(always)]
	fn sqrt_f64s(self, a: Self::f64s) -> Self::f64s {
		let a: [_; 2] = cast!(a);
		cast!([self.sqrt_f64x4(a[0]), self.sqrt_f64x4(a[1]),])
	}
}

impl V3 {
	binop_256_nosign!(avx: add, "Adds the elements of each lane of `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: add, "Adds the elements of each lane of `a` and `b`, with wrapping on overflow.", wrapping_add, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	binop_256!(avx: and, "Returns `a & b` for each bit in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_full!(avx2: and, "Returns `a & b` for each bit in `a` and `b`.", m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4);

	binop_256!(avx: andnot, "Returns `!a & b` for each bit in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_full!(avx2: andnot, "Returns `!a & b` for each bit in `a` and `b`.", m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4);

	binop_256!(avx2: sign, r#"Applies the sign of each element of `sign` to the corresponding lane in `a`.
- If `sign` is zero, the corresponding element is zeroed.
- If `sign` is positive, the corresponding element is returned as is.
- If `sign` is negative, the corresponding element is negated."#, apply_sign, i8 x 32, i16 x 16, i32 x 8);

	binop_256!(avx2: avg, "Computes `average(a, b)` for each lane of `a` and `b`.", average, u8 x 32, u16 x 16);

	unop_256!(avx: ceil, "Returns `ceil(a)` for each lane of `a`, rounding towards positive infinity.", f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: cmpeq, "Compares the elements in each lane of `a` and `b` for equality.", cmp_eq, m8 x 32 => m8, u8 x 32 => m8, i8 x 32 => m8, m16 x 16 => m16, u16 x 16 => m16, i16 x 16 => m16, m32 x 8 => m32, u32 x 8 => m32, i32 x 8 => m32, m64 x 4 => m64, u64 x 4 => m64, i64 x 4 => m64);

	binop_256!(avx2: cmpgt, "Compares the elements in each lane of `a` and `b` for equality.", cmp_gt, i8 x 32 => m8, i16 x 16 => m16, i32 x 8 => m32, i64 x 4 => m64);

	binop_256!(avx: div, "Divides the elements of each lane of `a` and `b`.", f32 x 8, f64 x 4);

	unop_256!(avx: floor, "Rounds the elements of each lane of `a` to the nearest integer towards negative infinity.", f32 x 8, f64 x 4);

	binop_256_nosign!(avx: hadd, "[_mm_hadd_ps](core::arch::x86_64::_mm_hadd_ps)", horizontal_add_pack, f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: hadd, "[_mm_hadd_ps](core::arch::x86_64::_mm_hadd_ps)", horizontal_add_pack, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	binop_256_nosign!(avx: hsub, "[_mm_hsub_ps](core::arch::x86_64::_mm_hsub_ps)", horizontal_sub_pack, f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: hsub, "[_mm_hsub_ps](core::arch::x86_64::_mm_hsub_ps)", horizontal_sub_pack, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	binop_256!(avx: max, "Computes `max(a, b)`. for each lane in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256!(avx2: max, "Computes `max(a, b)`. for each lane in `a` and `b`.", u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	binop_256!(avx: min, "Computes `min(a, b)`. for each lane in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256!(avx2: min, "Computes `min(a, b)`. for each lane in `a` and `b`.", u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	binop_256!(avx: mul, "Computes `a * b` for each lane in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: mullo, "Computes `a * b` for each lane in `a` and `b`, with wrapping overflow.", wrapping_mul, u16 x 16, i16 x 16, u32 x 8, i32 x 8);

	binop_256!(avx: or, "Returns `a | b` for each bit in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_full!(avx2: or, "Returns `a | b` for each bit in `a` and `b`.", m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4);

	binop_256!(avx2: adds, "Adds the elements of each lane of `a` and `b`, with saturation.", saturating_add, u8 x 32, i8 x 32, u16 x 16, i16 x 16);

	binop_256!(avx2: subs, "Subtracts the elements of each lane of `a` and `b`, with saturation.", saturating_sub, u8 x 32, i8 x 32, u16 x 16, i16 x 16);

	binop_256_nosign!(avx: sub, "Subtracts the elements of each lane of `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_nosign!(avx2: sub, "Subtracts the elements of each lane of `a` and `b`, with wrapping overflow.", wrapping_sub, u8 x 32, i8 x 32, u16 x 16, i16 x 16, u32 x 8, i32 x 8, u64 x 4, i64 x 4);

	binop_256!(avx: addsub, "Alternatively subtracts and adds the elements of each lane of `a` and `b`.", subadd, f32 x 8, f64 x 4);

	unop_256!(avx2: abs, "Computes the unsigned absolute value of the elements of each lane of `a`.", unsigned_abs, i8 x 32, i16 x 16, i32 x 8);

	binop_256!(avx: xor, "Returns `a ^ b` for each bit in `a` and `b`.", f32 x 8, f64 x 4);

	binop_256_full!(avx2: xor, "Returns `a ^ b` for each bit in `a` and `b`.", m8 x 32, u8 x 32, i8 x 32, m16 x 16, u16 x 16, i16 x 16, m32 x 8, u32 x 8, i32 x 8, m64 x 4, u64 x 4, i64 x 4);

	/// Computes `abs(a)` for each lane of `a`.
	#[inline(always)]
	pub fn abs_f32x8(self, a: f32x8) -> f32x8 {
		self.and_f32x8(a, cast!(self.splat_u32x8((1 << 31) - 1)))
	}

	/// Computes `abs(a)` for each lane of `a`.
	#[inline(always)]
	pub fn abs_f64x4(self, a: f64x4) -> f64x4 {
		self.and_f64x4(a, cast!(self.splat_u64x4((1 << 63) - 1)))
	}

	/// Computes the approximate reciprocal of the elements of each lane of `a`.
	#[inline(always)]
	pub fn approx_reciprocal_f32x8(self, a: f32x8) -> f32x8 {
		cast!(self.avx._mm256_rcp_ps(cast!(a)))
	}

	/// Computes the approximate reciprocal of the square roots of the elements of each lane of `a`.
	#[inline(always)]
	pub fn approx_reciprocal_sqrt_f32x8(self, a: f32x8) -> f32x8 {
		cast!(self.avx._mm256_rsqrt_ps(cast!(a)))
	}

	/// Compares the elements in each lane of `a` and `b` for equality.
	#[inline(always)]
	pub fn cmp_eq_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_EQ_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for equality.
	#[inline(always)]
	pub fn cmp_eq_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_EQ_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_GE_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_GE_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_i16x16(self, a: i16x16, b: i16x16) -> m16x16 {
		self.not_m16x16(self.cmp_lt_i16x16(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_i32x8(self, a: i32x8, b: i32x8) -> m32x8 {
		self.not_m32x8(self.cmp_lt_i32x8(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_i64x4(self, a: i64x4, b: i64x4) -> m64x4 {
		self.not_m64x4(self.cmp_lt_i64x4(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_i8x32(self, a: i8x32, b: i8x32) -> m8x32 {
		self.not_m8x32(self.cmp_lt_i8x32(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_u16x16(self, a: u16x16, b: u16x16) -> m16x16 {
		self.not_m16x16(self.cmp_lt_u16x16(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_u32x8(self, a: u32x8, b: u32x8) -> m32x8 {
		self.not_m32x8(self.cmp_lt_u32x8(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_u64x4(self, a: u64x4, b: u64x4) -> m64x4 {
		self.not_m64x4(self.cmp_lt_u64x4(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_ge_u8x32(self, a: u8x32, b: u8x32) -> m8x32 {
		self.not_m8x32(self.cmp_lt_u8x32(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_GT_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_GT_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_u16x16(self, a: u16x16, b: u16x16) -> m16x16 {
		let k = self.splat_u16x16(0x8000);
		self.cmp_gt_i16x16(cast!(self.xor_u16x16(a, k)), cast!(self.xor_u16x16(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_u32x8(self, a: u32x8, b: u32x8) -> m32x8 {
		let k = self.splat_u32x8(0x80000000);
		self.cmp_gt_i32x8(cast!(self.xor_u32x8(a, k)), cast!(self.xor_u32x8(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_u64x4(self, a: u64x4, b: u64x4) -> m64x4 {
		let k = self.splat_u64x4(0x8000000000000000);
		self.cmp_gt_i64x4(cast!(self.xor_u64x4(a, k)), cast!(self.xor_u64x4(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for greater-than.
	#[inline(always)]
	pub fn cmp_gt_u8x32(self, a: u8x32, b: u8x32) -> m8x32 {
		let k = self.splat_u8x32(0x80);
		self.cmp_gt_i8x32(cast!(self.xor_u8x32(a, k)), cast!(self.xor_u8x32(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_LE_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_LE_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_i16x16(self, a: i16x16, b: i16x16) -> m16x16 {
		self.not_m16x16(self.cmp_gt_i16x16(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_i32x8(self, a: i32x8, b: i32x8) -> m32x8 {
		self.not_m32x8(self.cmp_gt_i32x8(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_i64x4(self, a: i64x4, b: i64x4) -> m64x4 {
		self.not_m64x4(self.cmp_gt_i64x4(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_i8x32(self, a: i8x32, b: i8x32) -> m8x32 {
		self.not_m8x32(self.cmp_gt_i8x32(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_u16x16(self, a: u16x16, b: u16x16) -> m16x16 {
		self.not_m16x16(self.cmp_gt_u16x16(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_u32x8(self, a: u32x8, b: u32x8) -> m32x8 {
		self.not_m32x8(self.cmp_gt_u32x8(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_u64x4(self, a: u64x4, b: u64x4) -> m64x4 {
		self.not_m64x4(self.cmp_gt_u64x4(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than-or-equal-to.
	#[inline(always)]
	pub fn cmp_le_u8x32(self, a: u8x32, b: u8x32) -> m8x32 {
		self.not_m8x32(self.cmp_gt_u8x32(a, b))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_LT_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_LT_OQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_i16x16(self, a: i16x16, b: i16x16) -> m16x16 {
		cast!(self.avx2._mm256_cmpgt_epi16(cast!(b), cast!(a)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_i32x8(self, a: i32x8, b: i32x8) -> m32x8 {
		cast!(self.avx2._mm256_cmpgt_epi32(cast!(b), cast!(a)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_i64x4(self, a: i64x4, b: i64x4) -> m64x4 {
		cast!(self.avx2._mm256_cmpgt_epi64(cast!(b), cast!(a)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_i8x32(self, a: i8x32, b: i8x32) -> m8x32 {
		cast!(self.avx2._mm256_cmpgt_epi8(cast!(b), cast!(a)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_u16x16(self, a: u16x16, b: u16x16) -> m16x16 {
		let k = self.splat_u16x16(0x8000);
		self.cmp_lt_i16x16(cast!(self.xor_u16x16(a, k)), cast!(self.xor_u16x16(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_u32x8(self, a: u32x8, b: u32x8) -> m32x8 {
		let k = self.splat_u32x8(0x80000000);
		self.cmp_lt_i32x8(cast!(self.xor_u32x8(a, k)), cast!(self.xor_u32x8(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_u64x4(self, a: u64x4, b: u64x4) -> m64x4 {
		let k = self.splat_u64x4(0x8000000000000000);
		self.cmp_lt_i64x4(cast!(self.xor_u64x4(a, k)), cast!(self.xor_u64x4(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for less-than.
	#[inline(always)]
	pub fn cmp_lt_u8x32(self, a: u8x32, b: u8x32) -> m8x32 {
		let k = self.splat_u8x32(0x80);
		self.cmp_lt_i8x32(cast!(self.xor_u8x32(a, k)), cast!(self.xor_u8x32(b, k)))
	}

	/// Compares the elements in each lane of `a` and `b` for inequality.
	#[inline(always)]
	pub fn cmp_not_eq_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_NEQ_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for inequality.
	#[inline(always)]
	pub fn cmp_not_eq_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_NEQ_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-greater-than-or-equal.
	#[inline(always)]
	pub fn cmp_not_ge_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_NGE_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-greater-than-or-equal.
	#[inline(always)]
	pub fn cmp_not_ge_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_NGE_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-greater-than.
	#[inline(always)]
	pub fn cmp_not_gt_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_NGT_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-greater-than.
	#[inline(always)]
	pub fn cmp_not_gt_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_NGT_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-less-than-or-equal.
	#[inline(always)]
	pub fn cmp_not_le_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_NLE_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-less-than-or-equal.
	#[inline(always)]
	pub fn cmp_not_le_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_NLE_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-less-than.
	#[inline(always)]
	pub fn cmp_not_lt_f32x8(self, a: f32x8, b: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_NLT_UQ>(cast!(a), cast!(b)))
	}

	/// Compares the elements in each lane of `a` and `b` for not-less-than.
	#[inline(always)]
	pub fn cmp_not_lt_f64x4(self, a: f64x4, b: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_NLT_UQ>(cast!(a), cast!(b)))
	}

	/// Converts a `f32x4` to `f64x4`, elementwise.
	#[inline(always)]
	pub fn convert_f32x4_to_f64x4(self, a: f32x4) -> f64x4 {
		cast!(self.avx._mm256_cvtps_pd(cast!(a)))
	}

	/// Converts a `f32x8` to `i32x8`, elementwise.
	#[inline(always)]
	pub fn convert_f32x8_to_i32x8(self, a: f32x8) -> i32x8 {
		cast!(self.avx._mm256_cvttps_epi32(cast!(a)))
	}

	/// Converts a `f64x4` to `f32x4`, elementwise.
	#[inline(always)]
	pub fn convert_f64x4_to_f32x4(self, a: f64x4) -> f32x4 {
		cast!(self.avx._mm256_cvtpd_ps(cast!(a)))
	}

	/// Converts a `f64x4` to `i32x4`, elementwise.
	#[inline(always)]
	pub fn convert_f64x4_to_i32x4(self, a: f64x4) -> i32x4 {
		cast!(self.avx._mm256_cvttpd_epi32(cast!(a)))
	}

	/// Converts a `i16x16` to `u16x16`, elementwise.
	#[inline(always)]
	pub fn convert_i16x16_to_u16x16(self, a: i16x16) -> u16x16 {
		cast!(a)
	}

	/// Converts a `i16x8` to `i32x8`, elementwise.
	#[inline(always)]
	pub fn convert_i16x8_to_i32x8(self, a: i16x8) -> i32x8 {
		cast!(self.avx2._mm256_cvtepi16_epi32(cast!(a)))
	}

	/// Converts a `i16x8` to `i64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i16x8_to_i64x4(self, a: i16x8) -> i64x4 {
		cast!(self.avx2._mm256_cvtepi16_epi64(cast!(a)))
	}

	/// Converts a `i16x8` to `u32x8`, elementwise.
	#[inline(always)]
	pub fn convert_i16x8_to_u32x8(self, a: i16x8) -> u32x8 {
		cast!(self.avx2._mm256_cvtepi16_epi32(cast!(a)))
	}

	/// Converts a `i16x8` to `u64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i16x8_to_u64x4(self, a: i16x8) -> u64x4 {
		cast!(self.avx2._mm256_cvtepi16_epi64(cast!(a)))
	}

	/// Converts a `i32x4` to `f64x4`, elementwise.
	#[inline(always)]
	pub fn convert_i32x4_to_f64x4(self, a: i32x4) -> f64x4 {
		cast!(self.avx._mm256_cvtepi32_pd(cast!(a)))
	}

	/// Converts a `i32x4` to `i64x4`, elementwise.
	#[inline(always)]
	pub fn convert_i32x4_to_i64x4(self, a: i32x4) -> i64x4 {
		cast!(self.avx2._mm256_cvtepi32_epi64(cast!(a)))
	}

	/// Converts a `i32x4` to `u64x4`, elementwise.
	#[inline(always)]
	pub fn convert_i32x4_to_u64x4(self, a: i32x4) -> u64x4 {
		cast!(self.avx2._mm256_cvtepi32_epi64(cast!(a)))
	}

	/// Converts a `i32x8` to `f32x8`, elementwise.
	#[inline(always)]
	pub fn convert_i32x8_to_f32x8(self, a: i32x8) -> f32x8 {
		cast!(self.avx._mm256_cvtepi32_ps(cast!(a)))
	}

	/// Converts a `i32x8` to `u32x8`, elementwise.
	#[inline(always)]
	pub fn convert_i32x8_to_u32x8(self, a: i32x8) -> u32x8 {
		cast!(a)
	}

	/// Converts a `i8x16` to `i16x16`, elementwise.
	#[inline(always)]
	pub fn convert_i8x16_to_i16x16(self, a: i8x16) -> i16x16 {
		cast!(self.avx2._mm256_cvtepi8_epi16(cast!(a)))
	}

	/// Converts a `i8x16` to `i32x8`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i8x16_to_i32x8(self, a: i8x16) -> i32x8 {
		cast!(self.avx2._mm256_cvtepi8_epi32(cast!(a)))
	}

	/// Converts a `i8x16` to `i64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i8x16_to_i64x4(self, a: i8x16) -> i64x4 {
		cast!(self.avx2._mm256_cvtepi8_epi64(cast!(a)))
	}

	/// Converts a `i8x16` to `u16x16`, elementwise.
	#[inline(always)]
	pub fn convert_i8x16_to_u16x16(self, a: i8x16) -> u16x16 {
		cast!(self.avx2._mm256_cvtepi8_epi16(cast!(a)))
	}

	/// Converts a `i8x16` to `u32x8`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i8x16_to_u32x8(self, a: i8x16) -> u32x8 {
		cast!(self.avx2._mm256_cvtepi8_epi32(cast!(a)))
	}

	/// Converts a `i8x16` to `u64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_i8x16_to_u64x4(self, a: i8x16) -> u64x4 {
		cast!(self.avx2._mm256_cvtepi8_epi64(cast!(a)))
	}

	/// Converts a `i8x32` to `u8x32`, elementwise.
	#[inline(always)]
	pub fn convert_i8x32_to_u8x32(self, a: i8x32) -> u8x32 {
		cast!(a)
	}

	/// Converts a `u16x16` to `i16x16`, elementwise.
	#[inline(always)]
	pub fn convert_u16x16_to_i16x16(self, a: u16x16) -> i16x16 {
		cast!(a)
	}

	/// Converts a `u16x8` to `i32x8`, elementwise.
	#[inline(always)]
	pub fn convert_u16x8_to_i32x8(self, a: u16x8) -> i32x8 {
		cast!(self.avx2._mm256_cvtepu16_epi32(cast!(a)))
	}

	/// Converts a `u16x8` to `i64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u16x8_to_i64x4(self, a: u16x8) -> i64x4 {
		cast!(self.avx2._mm256_cvtepu16_epi64(cast!(a)))
	}

	/// Converts a `u16x8` to `u32x8`, elementwise.
	#[inline(always)]
	pub fn convert_u16x8_to_u32x8(self, a: u16x8) -> u32x8 {
		cast!(self.avx2._mm256_cvtepu16_epi32(cast!(a)))
	}

	/// Converts a `u16x8` to `u64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u16x8_to_u64x4(self, a: u16x8) -> u64x4 {
		cast!(self.avx2._mm256_cvtepu16_epi64(cast!(a)))
	}

	/// Converts a `u32x4` to `i64x4`, elementwise.
	#[inline(always)]
	pub fn convert_u32x4_to_i64x4(self, a: u32x4) -> i64x4 {
		cast!(self.avx2._mm256_cvtepu32_epi64(cast!(a)))
	}

	/// Converts a `u32x4` to `u64x4`, elementwise.
	#[inline(always)]
	pub fn convert_u32x4_to_u64x4(self, a: u32x4) -> u64x4 {
		cast!(self.avx2._mm256_cvtepu32_epi64(cast!(a)))
	}

	/// Converts a `u32x8` to `i32x8`, elementwise.
	#[inline(always)]
	pub fn convert_u32x8_to_i32x8(self, a: u32x8) -> i32x8 {
		cast!(a)
	}

	/// Converts a `u8x16` to `i16x16`, elementwise.
	#[inline(always)]
	pub fn convert_u8x16_to_i16x16(self, a: u8x16) -> i16x16 {
		cast!(self.avx2._mm256_cvtepu8_epi16(cast!(a)))
	}

	/// Converts a `u8x16` to `i32x8`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u8x16_to_i32x8(self, a: u8x16) -> i32x8 {
		cast!(self.avx2._mm256_cvtepu8_epi32(cast!(a)))
	}

	/// Converts a `u8x16` to `i64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u8x16_to_i64x4(self, a: u8x16) -> i64x4 {
		cast!(self.avx2._mm256_cvtepu8_epi64(cast!(a)))
	}

	/// Converts a `u8x16` to `u16x16`, elementwise.
	#[inline(always)]
	pub fn convert_u8x16_to_u16x16(self, a: u8x16) -> u16x16 {
		cast!(self.avx2._mm256_cvtepu8_epi16(cast!(a)))
	}

	/// Converts a `u8x16` to `u32x8`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u8x16_to_u32x8(self, a: u8x16) -> u32x8 {
		cast!(self.avx2._mm256_cvtepu8_epi32(cast!(a)))
	}

	/// Converts a `u8x16` to `u64x4`, elementwise, while truncating the extra elements.
	#[inline(always)]
	pub fn convert_u8x16_to_u64x4(self, a: u8x16) -> u64x4 {
		cast!(self.avx2._mm256_cvtepu8_epi64(cast!(a)))
	}

	/// Converts a `u8x32` to `i8x32`, elementwise.
	#[inline(always)]
	pub fn convert_u8x32_to_i8x32(self, a: u8x32) -> i8x32 {
		cast!(a)
	}

	/// See [_mm_hadds_epi16].
	///
	/// [_mm_hadds_epi16]: core::arch::x86_64::_mm_hadds_epi16
	#[inline(always)]
	pub fn horizontal_saturating_add_pack_i16x16(self, a: i16x16, b: i16x16) -> i16x16 {
		cast!(self.avx2._mm256_hadds_epi16(cast!(a), cast!(b)))
	}

	/// See [_mm_hsubs_epi16].
	///
	/// [_mm_hsubs_epi16]: core::arch::x86_64::_mm_hsubs_epi16
	#[inline(always)]
	pub fn horizontal_saturating_sub_pack_i16x16(self, a: i16x16, b: i16x16) -> i16x16 {
		cast!(self.avx2._mm256_hsubs_epi16(cast!(a), cast!(b)))
	}

	/// Checks if the elements in each lane of `a` are NaN.
	#[inline(always)]
	pub fn is_nan_f32x8(self, a: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_UNORD_Q>(cast!(a), cast!(a)))
	}

	/// Checks if the elements in each lane of `a` are NaN.
	#[inline(always)]
	pub fn is_nan_f64x4(self, a: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_UNORD_Q>(cast!(a), cast!(a)))
	}

	/// Checks if the elements in each lane of `a` are not NaN.
	#[inline(always)]
	pub fn is_not_nan_f32x8(self, a: f32x8) -> m32x8 {
		cast!(self.avx._mm256_cmp_ps::<_CMP_ORD_Q>(cast!(a), cast!(a)))
	}

	/// Checks if the elements in each lane of `a` are not NaN.
	#[inline(always)]
	pub fn is_not_nan_f64x4(self, a: f64x4) -> m64x4 {
		cast!(self.avx._mm256_cmp_pd::<_CMP_ORD_Q>(cast!(a), cast!(a)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and adds the results to each lane of
	/// `c`.
	#[inline(always)]
	pub fn mul_add_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and adds the results to each lane of
	/// `c`.
	#[inline(always)]
	pub fn mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and adds the results to each lane of
	/// `c`.
	#[inline(always)]
	pub fn mul_add_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and adds the results to each lane of
	/// `c`.
	#[inline(always)]
	pub fn mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively adds/subtracts 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_addsub_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fmsubadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively adds/subtracts 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_addsub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fmsubadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively adds/subtracts 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_addsub_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fmsubadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively adds/subtracts 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_addsub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fmsubadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the results.
	#[inline(always)]
	pub fn mul_sub_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fmsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the results.
	#[inline(always)]
	pub fn mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fmsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the results.
	#[inline(always)]
	pub fn mul_sub_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fmsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the results.
	#[inline(always)]
	pub fn mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fmsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively subtracts/adds 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_subadd_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fmaddsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively subtracts/adds 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_subadd_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fmaddsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively subtracts/adds 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_subadd_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fmaddsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and alternatively subtracts/adds 'c'
	/// to/from the results.
	#[inline(always)]
	pub fn mul_subadd_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fmaddsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// See [_mm256_maddubs_epi16].
	///
	/// [_mm256_maddubs_epi16]: core::arch::x86_64::_mm256_maddubs_epi16
	#[inline(always)]
	pub fn multiply_saturating_add_adjacent_i8x32(self, a: i8x32, b: i8x32) -> i16x16 {
		cast!(self.avx2._mm256_maddubs_epi16(cast!(a), cast!(b)))
	}

	/// See [_mm256_madd_epi16].
	///
	/// [_mm256_madd_epi16]: core::arch::x86_64::_mm256_madd_epi16
	#[inline(always)]
	pub fn multiply_wrapping_add_adjacent_i16x16(self, a: i16x16, b: i16x16) -> i32x8 {
		cast!(self.avx2._mm256_madd_epi16(cast!(a), cast!(b)))
	}

	/// See [_mm256_mpsadbw_epu8].
	///
	/// [_mm256_mpsadbw_epu8]: core::arch::x86_64::_mm256_mpsadbw_epu8
	#[inline(always)]
	pub fn multisum_of_absolute_differences_u8x32<const OFFSETS: i32>(
		self,
		a: u8x32,
		b: u8x32,
	) -> u16x16 {
		cast!(self.avx2._mm256_mpsadbw_epu8::<OFFSETS>(cast!(a), cast!(b)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, negates the results, and adds them to
	/// each lane of `c`.
	#[inline(always)]
	pub fn negate_mul_add_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fnmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, negates the results, and adds them to
	/// each lane of `c`.
	#[inline(always)]
	pub fn negate_mul_add_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fnmadd_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, negates the results, and adds them to
	/// each lane of `c`.
	#[inline(always)]
	pub fn negate_mul_add_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fnmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, negates the results, and adds them to
	/// each lane of `c`.
	#[inline(always)]
	pub fn negate_mul_add_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fnmadd_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the negation of the results.
	#[inline(always)]
	pub fn negate_mul_sub_f32x4(self, a: f32x4, b: f32x4, c: f32x4) -> f32x4 {
		cast!(self.fma._mm_fnmsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the negation of the results.
	#[inline(always)]
	pub fn negate_mul_sub_f32x8(self, a: f32x8, b: f32x8, c: f32x8) -> f32x8 {
		cast!(self.fma._mm256_fnmsub_ps(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the negation of the results.
	#[inline(always)]
	pub fn negate_mul_sub_f64x2(self, a: f64x2, b: f64x2, c: f64x2) -> f64x2 {
		cast!(self.fma._mm_fnmsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Multiplies the elements in each lane of `a` and `b`, and subtracts each lane of `c` from
	/// the negation of the results.
	#[inline(always)]
	pub fn negate_mul_sub_f64x4(self, a: f64x4, b: f64x4, c: f64x4) -> f64x4 {
		cast!(self.fma._mm256_fnmsub_pd(cast!(a), cast!(b), cast!(c)))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_i16x16(self, a: i16x16) -> i16x16 {
		self.xor_i16x16(a, self.splat_i16x16(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_i32x8(self, a: i32x8) -> i32x8 {
		self.xor_i32x8(a, self.splat_i32x8(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_i64x4(self, a: i64x4) -> i64x4 {
		self.xor_i64x4(a, self.splat_i64x4(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_i8x32(self, a: i8x32) -> i8x32 {
		self.xor_i8x32(a, self.splat_i8x32(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_m16x16(self, a: m16x16) -> m16x16 {
		self.xor_m16x16(a, self.splat_m16x16(m16::new(true)))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_m32x8(self, a: m32x8) -> m32x8 {
		self.xor_m32x8(a, self.splat_m32x8(m32::new(true)))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_m64x4(self, a: m64x4) -> m64x4 {
		self.xor_m64x4(a, self.splat_m64x4(m64::new(true)))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_m8x32(self, a: m8x32) -> m8x32 {
		self.xor_m8x32(a, self.splat_m8x32(m8::new(true)))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_u16x16(self, a: u16x16) -> u16x16 {
		self.xor_u16x16(a, self.splat_u16x16(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_u32x8(self, a: u32x8) -> u32x8 {
		self.xor_u32x8(a, self.splat_u32x8(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_u64x4(self, a: u64x4) -> u64x4 {
		self.xor_u64x4(a, self.splat_u64x4(!0))
	}

	/// Returns `!a` for each bit in a.
	#[inline(always)]
	pub fn not_u8x32(self, a: u8x32) -> u8x32 {
		self.xor_u8x32(a, self.splat_u8x32(!0))
	}

	/// See [_mm256_packs_epi16].
	///
	/// [_mm256_packs_epi16]: core::arch::x86_64::_mm256_packs_epi16
	#[inline(always)]
	pub fn pack_with_signed_saturation_i16x16(self, a: i16x16, b: i16x16) -> i8x32 {
		cast!(self.avx2._mm256_packs_epi16(cast!(a), cast!(b)))
	}

	/// See [_mm256_packs_epi32].
	///
	/// [_mm256_packs_epi32]: core::arch::x86_64::_mm256_packs_epi32
	#[inline(always)]
	pub fn pack_with_signed_saturation_i32x8(self, a: i32x8, b: i32x8) -> i16x16 {
		cast!(self.avx2._mm256_packs_epi32(cast!(a), cast!(b)))
	}

	/// See [_mm256_packus_epi16].
	///
	/// [_mm256_packus_epi16]: core::arch::x86_64::_mm256_packus_epi16
	#[inline(always)]
	pub fn pack_with_unsigned_saturation_i16x16(self, a: i16x16, b: i16x16) -> u8x32 {
		cast!(self.avx2._mm256_packus_epi16(cast!(a), cast!(b)))
	}

	/// See [_mm256_packus_epi32].
	///
	/// [_mm256_packus_epi32]: core::arch::x86_64::_mm256_packus_epi32
	#[inline(always)]
	pub fn pack_with_unsigned_saturation_i32x8(self, a: i32x8, b: i32x8) -> u16x16 {
		cast!(self.avx2._mm256_packus_epi32(cast!(a), cast!(b)))
	}

	/// Rounds the elements of each lane of `a` to the nearest integer. If two values are equally
	/// close, the even value is returned.
	#[inline(always)]
	pub fn round_f32x8(self, a: f32x8) -> f32x8 {
		const ROUNDING: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
		cast!(self.avx._mm256_round_ps::<ROUNDING>(cast!(a)))
	}

	/// Rounds the elements of each lane of `a` to the nearest integer. If two values are equally
	/// close, the even value is returned.
	#[inline(always)]
	pub fn round_f64x4(self, a: f64x4) -> f64x4 {
		const ROUNDING: i32 = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
		cast!(self.avx._mm256_round_pd::<ROUNDING>(cast!(a)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_f32x8<const MASK8: i32>(self, if_true: f32x8, if_false: f32x8) -> f32x8 {
		cast!(
			self.avx
				._mm256_blend_ps::<MASK8>(cast!(if_false), cast!(if_true)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_f64x4<const MASK4: i32>(self, if_true: f64x4, if_false: f64x4) -> f64x4 {
		cast!(self.select_const_u64x4::<MASK4>(cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_i32x8<const MASK8: i32>(self, if_true: i32x8, if_false: i32x8) -> i32x8 {
		cast!(self.select_const_u32x8::<MASK8>(cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_i64x4<const MASK4: i32>(self, if_true: i64x4, if_false: i64x4) -> i64x4 {
		cast!(self.select_const_u64x4::<MASK4>(cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_u32x8<const MASK8: i32>(self, if_true: u32x8, if_false: u32x8) -> u32x8 {
		cast!(
			self.avx2
				._mm256_blend_epi32::<MASK8>(cast!(if_false), cast!(if_true)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// bit in the mask is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_const_u64x4<const MASK4: i32>(self, if_true: u64x4, if_false: u64x4) -> u64x4 {
		cast!(
			self.avx
				._mm256_blend_pd::<MASK4>(cast!(if_false), cast!(if_true)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_f32x8(self, mask: m32x8, if_true: f32x8, if_false: f32x8) -> f32x8 {
		cast!(
			self.avx
				._mm256_blendv_ps(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_f64x4(self, mask: m64x4, if_true: f64x4, if_false: f64x4) -> f64x4 {
		cast!(
			self.avx
				._mm256_blendv_pd(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_i16x16(self, mask: m16x16, if_true: i16x16, if_false: i16x16) -> i16x16 {
		cast!(self.select_u16x16(mask, cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_i32x8(self, mask: m32x8, if_true: i32x8, if_false: i32x8) -> i32x8 {
		cast!(self.select_u32x8(mask, cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_i64x4(self, mask: m64x4, if_true: i64x4, if_false: i64x4) -> i64x4 {
		cast!(self.select_u64x4(mask, cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_i8x32(self, mask: m8x32, if_true: i8x32, if_false: i8x32) -> i8x32 {
		cast!(self.select_u8x32(mask, cast!(if_true), cast!(if_false)))
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_u16x16(self, mask: m16x16, if_true: u16x16, if_false: u16x16) -> u16x16 {
		cast!(
			self.avx2
				._mm256_blendv_epi8(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_u32x8(self, mask: m32x8, if_true: u32x8, if_false: u32x8) -> u32x8 {
		cast!(
			self.avx2
				._mm256_blendv_epi8(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_u64x4(self, mask: m64x4, if_true: u64x4, if_false: u64x4) -> u64x4 {
		cast!(
			self.avx2
				._mm256_blendv_epi8(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Combines `if_true` and `if_false`, selecting elements from `if_true` if the corresponding
	/// mask in `mask` is set, otherwise selecting elements from `if_false`.
	#[inline(always)]
	pub fn select_u8x32(self, mask: m8x32, if_true: u8x32, if_false: u8x32) -> u8x32 {
		cast!(
			self.avx2
				._mm256_blendv_epi8(cast!(if_false), cast!(if_true), cast!(mask)),
		)
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_i16x16<const AMOUNT: i32>(self, a: i16x16) -> i16x16 {
		cast!(self.avx2._mm256_slli_epi16::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_i32x8<const AMOUNT: i32>(self, a: i32x8) -> i32x8 {
		cast!(self.avx2._mm256_slli_epi32::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_i64x4<const AMOUNT: i32>(self, a: i64x4) -> i64x4 {
		cast!(self.avx2._mm256_slli_epi64::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_u16x16<const AMOUNT: i32>(self, a: u16x16) -> u16x16 {
		cast!(self.avx2._mm256_slli_epi16::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_u32x8<const AMOUNT: i32>(self, a: u32x8) -> u32x8 {
		cast!(self.avx2._mm256_slli_epi32::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_const_u64x4<const AMOUNT: i32>(self, a: u64x4) -> u64x4 {
		cast!(self.avx2._mm256_slli_epi64::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_i32x4(self, a: i32x4, amount: u32x4) -> i32x4 {
		cast!(self.avx2._mm_sllv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_i32x8(self, a: i32x8, amount: u32x8) -> i32x8 {
		cast!(self.avx2._mm256_sllv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_i64x2(self, a: i64x2, amount: u64x2) -> i64x2 {
		cast!(self.avx2._mm_sllv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_i64x4(self, a: i64x4, amount: u64x4) -> i64x4 {
		cast!(self.avx2._mm256_sllv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_u32x4(self, a: u32x4, amount: u32x4) -> u32x4 {
		cast!(self.avx2._mm_sllv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_u32x8(self, a: u32x8, amount: u32x8) -> u32x8 {
		cast!(self.avx2._mm256_sllv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_u64x2(self, a: u64x2, amount: u64x2) -> u64x2 {
		cast!(self.avx2._mm_sllv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_dyn_u64x4(self, a: u64x4, amount: u64x4) -> u64x4 {
		cast!(self.avx2._mm256_sllv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_i16x16(self, a: i16x16, amount: u64x2) -> i16x16 {
		cast!(self.avx2._mm256_sll_epi16(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_i32x8(self, a: i32x8, amount: u64x2) -> i32x8 {
		cast!(self.avx2._mm256_sll_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_i64x4(self, a: i64x4, amount: u64x2) -> i64x4 {
		cast!(self.avx2._mm256_sll_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_u16x16(self, a: u16x16, amount: u64x2) -> u16x16 {
		cast!(self.avx2._mm256_sll_epi16(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_u32x8(self, a: u32x8, amount: u64x2) -> u32x8 {
		cast!(self.avx2._mm256_sll_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the left by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shl_u64x4(self, a: u64x4, amount: u64x2) -> u64x4 {
		cast!(self.avx2._mm256_sll_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by `AMOUNT`, while shifting in sign bits.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_const_i16x16<const AMOUNT: i32>(self, a: i16x16) -> i16x16 {
		cast!(self.avx2._mm256_srai_epi16::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the right by `AMOUNT`, while shifting in sign bits.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_const_i32x8<const AMOUNT: i32>(self, a: i32x8) -> i32x8 {
		cast!(self.avx2._mm256_srai_epi32::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the right by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_const_u16x16<const AMOUNT: i32>(self, a: u16x16) -> u16x16 {
		cast!(self.avx2._mm256_srli_epi16::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the right by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_const_u32x8<const AMOUNT: i32>(self, a: u32x8) -> u32x8 {
		cast!(self.avx2._mm256_srli_epi32::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the right by `AMOUNT`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_const_u64x4<const AMOUNT: i32>(self, a: u64x4) -> u64x4 {
		cast!(self.avx2._mm256_srli_epi64::<AMOUNT>(cast!(a)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in sign bits.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_dyn_i32x4(self, a: i32x4, amount: u32x4) -> i32x4 {
		cast!(self.avx2._mm_srav_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in sign bits.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_dyn_i32x8(self, a: i32x8, amount: u32x8) -> i32x8 {
		cast!(self.avx2._mm256_srav_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_dyn_u32x4(self, a: u32x4, amount: u32x4) -> u32x4 {
		cast!(self.avx2._mm_srlv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_dyn_u32x8(self, a: u32x8, amount: u32x8) -> u32x8 {
		cast!(self.avx2._mm256_srlv_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_dyn_u64x2(self, a: u64x2, amount: u64x2) -> u64x2 {
		cast!(self.avx2._mm_srlv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the element in the corresponding lane in
	/// `amount`, while shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_dyn_u64x4(self, a: u64x4, amount: u64x4) -> u64x4 {
		cast!(self.avx2._mm256_srlv_epi64(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_i16x16(self, a: i16x16, amount: u64x2) -> i16x16 {
		cast!(self.avx2._mm256_sra_epi16(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero if the
	/// sign bit is not set, and to `-1` if the sign bit is set.
	#[inline(always)]
	pub fn shr_i32x8(self, a: i32x8, amount: u64x2) -> i32x8 {
		cast!(self.avx2._mm256_sra_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_u16x16(self, a: u16x16, amount: u64x2) -> u16x16 {
		cast!(self.avx2._mm256_srl_epi16(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_u32x8(self, a: u32x8, amount: u64x2) -> u32x8 {
		cast!(self.avx2._mm256_srl_epi32(cast!(a), cast!(amount)))
	}

	/// Shift the bits of each lane of `a` to the right by the first element in `amount`, while
	/// shifting in zeros.
	/// Shifting by a value greater than the bit width of the type sets the result to zero.
	#[inline(always)]
	pub fn shr_u64x4(self, a: u64x4, amount: u64x2) -> u64x4 {
		cast!(self.avx2._mm256_srl_epi64(cast!(a), cast!(amount)))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_f32x8(self, value: f32) -> f32x8 {
		cast!(self.avx._mm256_set1_ps(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_f64x4(self, value: f64) -> f64x4 {
		cast!(self.avx._mm256_set1_pd(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_i16x16(self, value: i16) -> i16x16 {
		cast!(self.avx._mm256_set1_epi16(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_i32x8(self, value: i32) -> i32x8 {
		cast!(self.avx._mm256_set1_epi32(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_i64x4(self, value: i64) -> i64x4 {
		cast!(self.avx._mm256_set1_epi64x(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_i8x32(self, value: i8) -> i8x32 {
		cast!(self.avx._mm256_set1_epi8(value))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_m16x16(self, value: m16) -> m16x16 {
		cast!(self.avx._mm256_set1_epi16(value.0 as i16))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_m32x8(self, value: m32) -> m32x8 {
		cast!(self.avx._mm256_set1_epi32(value.0 as i32))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_m64x4(self, value: m64) -> m64x4 {
		cast!(self.avx._mm256_set1_epi64x(value.0 as i64))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_m8x32(self, value: m8) -> m8x32 {
		cast!(self.avx._mm256_set1_epi8(value.0 as i8))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_u16x16(self, value: u16) -> u16x16 {
		cast!(self.avx._mm256_set1_epi16(value as i16))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_u32x8(self, value: u32) -> u32x8 {
		cast!(self.avx._mm256_set1_epi32(value as i32))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_u64x4(self, value: u64) -> u64x4 {
		cast!(self.avx._mm256_set1_epi64x(value as i64))
	}

	/// Returns a SIMD vector with all lanes set to the given value.
	#[inline(always)]
	pub fn splat_u8x32(self, value: u8) -> u8x32 {
		cast!(self.avx._mm256_set1_epi8(value as i8))
	}

	/// Computes the square roots of the elements of each lane of `a`.
	#[inline(always)]
	pub fn sqrt_f32x8(self, a: f32x8) -> f32x8 {
		cast!(self.avx._mm256_sqrt_ps(cast!(a)))
	}

	/// Computes the square roots of the elements of each lane of `a`.
	#[inline(always)]
	pub fn sqrt_f64x4(self, a: f64x4) -> f64x4 {
		cast!(self.avx._mm256_sqrt_pd(cast!(a)))
	}

	/// See [_mm256_sad_epu8].
	///
	/// [_mm256_sad_epu8]: core::arch::x86_64::_mm256_sad_epu8
	#[inline(always)]
	pub fn sum_of_absolute_differences_u8x32(self, a: u8x32, b: u8x32) -> u64x4 {
		cast!(self.avx2._mm256_sad_epu8(cast!(a), cast!(b)))
	}

	/// Rounds the elements of each lane of `a` to the nearest integer towards zero.
	#[inline(always)]
	pub fn truncate_f32x8(self, a: f32x8) -> f32x8 {
		const ROUNDING: i32 = _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC;
		cast!(self.avx._mm256_round_ps::<ROUNDING>(cast!(a)))
	}

	/// Rounds the elements of each lane of `a` to the nearest integer towards zero.
	#[inline(always)]
	pub fn truncate_f64x4(self, a: f64x4) -> f64x4 {
		const ROUNDING: i32 = _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC;
		cast!(self.avx._mm256_round_pd::<ROUNDING>(cast!(a)))
	}

	/// Multiplies the elements of each lane of `a` and `b`, and returns separately the low and
	/// high bits of the result.
	#[inline(always)]
	pub fn widening_mul_i16x16(self, a: i16x16, b: i16x16) -> (u16x16, i16x16) {
		(
			cast!(self.avx2._mm256_mullo_epi16(cast!(a), cast!(b))),
			cast!(self.avx2._mm256_mulhi_epi16(cast!(a), cast!(b))),
		)
	}

	/// Multiplies the elements of each lane of `a` and `b`, and returns separately the low and
	/// high bits of the result.
	#[inline(always)]
	pub fn widening_mul_i32x8(self, a: i32x8, b: i32x8) -> (u32x8, i32x8) {
		let a = cast!(a);
		let b = cast!(b);
		let avx2 = self.avx2;

		// a0b0_lo a0b0_hi a2b2_lo a2b2_hi
		let ab_evens = self.avx2._mm256_mul_epi32(a, b);
		// a1b1_lo a1b1_hi a3b3_lo a3b3_hi
		let ab_odds = self.avx2._mm256_mul_epi32(
			avx2._mm256_srli_epi64::<32>(a),
			avx2._mm256_srli_epi64::<32>(b),
		);

		let ab_lo = self.avx2._mm256_blend_epi32::<0b10101010>(
			// a0b0_lo xxxxxxx a2b2_lo xxxxxxx
			ab_evens,
			// xxxxxxx a1b1_lo xxxxxxx a3b3_lo
			avx2._mm256_slli_epi64::<32>(ab_odds),
		);
		let ab_hi = self.avx2._mm256_blend_epi32::<0b10101010>(
			// a0b0_hi xxxxxxx a2b2_hi xxxxxxx
			avx2._mm256_srli_epi64::<32>(ab_evens),
			// xxxxxxx a1b1_hi xxxxxxx a3b3_hi
			ab_odds,
		);

		(cast!(ab_lo), cast!(ab_hi))
	}

	/// Multiplies the elements of each lane of `a` and `b`, and returns separately the low and
	/// high bits of the result.
	#[inline(always)]
	pub fn widening_mul_u16x16(self, a: u16x16, b: u16x16) -> (u16x16, u16x16) {
		(
			cast!(self.avx2._mm256_mullo_epi16(cast!(a), cast!(b))),
			cast!(self.avx2._mm256_mulhi_epu16(cast!(a), cast!(b))),
		)
	}

	/// Multiplies the elements of each lane of `a` and `b`, and returns separately the low and
	/// high bits of the result.
	#[inline(always)]
	pub fn widening_mul_u32x8(self, a: u32x8, b: u32x8) -> (u32x8, u32x8) {
		let a = cast!(a);
		let b = cast!(b);
		let avx2 = self.avx2;

		// a0b0_lo a0b0_hi a2b2_lo a2b2_hi
		let ab_evens = avx2._mm256_mul_epu32(a, b);
		// a1b1_lo a1b1_hi a3b3_lo a3b3_hi
		let ab_odds = avx2._mm256_mul_epu32(
			avx2._mm256_srli_epi64::<32>(a),
			avx2._mm256_srli_epi64::<32>(b),
		);

		let ab_lo = self.avx2._mm256_blend_epi32::<0b10101010>(
			// a0b0_lo xxxxxxx a2b2_lo xxxxxxx
			ab_evens,
			// xxxxxxx a1b1_lo xxxxxxx a3b3_lo
			avx2._mm256_slli_epi64::<32>(ab_odds),
		);
		let ab_hi = self.avx2._mm256_blend_epi32::<0b10101010>(
			// a0b0_hi xxxxxxx a2b2_hi xxxxxxx
			avx2._mm256_srli_epi64::<32>(ab_evens),
			// xxxxxxx a1b1_hi xxxxxxx a3b3_hi
			ab_odds,
		);

		(cast!(ab_lo), cast!(ab_hi))
	}
}