vendor/crc-fast/src/arch/x86_64/avx512_vpclmulqdq.rs at push-notes · andymccall/oxygen

◆ push-notes oxygen / vendor / crc-fast / src / arch / x86_64 / avx512_vpclmulqdq.rs

19.6 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw

// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.

//! This module provides AVX-512 and VPCLMULQDQ-specific implementations of the ArchOps trait.
//!
//! It performs folding using 4 x ZMM registers of 512-bits each.

#![cfg(target_arch = "x86_64")]

use crate::arch::x86::sse::X86SsePclmulqdqOps;
use crate::enums::Reflector;
use crate::structs::CrcState;
use crate::traits::{ArchOps, EnhancedCrcWidth};
use core::arch::x86_64::*;
use core::ops::BitXor;

/// Implements the ArchOps trait using 512-bit AVX-512 and VPCLMULQDQ instructions at 512 bits.
/// Delegates to X86SsePclmulqdqOps for standard 128-bit operations
#[derive(Debug, Copy, Clone)]
pub struct X86_64Avx512VpclmulqdqOps(X86SsePclmulqdqOps);

impl Default for X86_64Avx512VpclmulqdqOps {
    fn default() -> Self {
        Self::new()
    }
}

impl X86_64Avx512VpclmulqdqOps {
    #[inline(always)]
    pub fn new() -> Self {
        Self(X86SsePclmulqdqOps)
    }
}

// Wrapper for __m512i to make it easier to work with
#[derive(Debug, Copy, Clone)]
struct Simd512(__m512i);

impl Simd512 {
    #[inline]
    #[target_feature(enable = "avx512f")]
    #[allow(clippy::too_many_arguments)]
    unsafe fn new(x7: u64, x6: u64, x5: u64, x4: u64, x3: u64, x2: u64, x1: u64, x0: u64) -> Self {
        Self(_mm512_set_epi64(
            x7 as i64, x6 as i64, x5 as i64, x4 as i64, x3 as i64, x2 as i64, x1 as i64, x0 as i64,
        ))
    }

    #[inline]
    #[target_feature(enable = "avx512vl,vpclmulqdq")]
    unsafe fn fold_64(&self, coeff: &Self, new_data: &Self) -> Self {
        // Use 512-bit ternary logic XOR3 with carryless multiplication
        Self(_mm512_ternarylogic_epi64(
            _mm512_clmulepi64_epi128(self.0, coeff.0, 0), // Low parts
            _mm512_clmulepi64_epi128(self.0, coeff.0, 17), // High parts
            new_data.0,
            0x96, // XOR3 operation
        ))
    }

    #[cfg(feature = "std")]
    #[inline]
    #[target_feature(enable = "avx512f")]
    unsafe fn extract_u64s(&self) -> [u64; 8] {
        let mut result = [0u64; 8];
        _mm512_storeu_si512(result.as_mut_ptr().cast(), self.0);

        result
    }

    #[inline]
    #[target_feature(enable = "avx512f")]
    unsafe fn load_from_ptr(ptr: *const u8) -> Self {
        Self(_mm512_loadu_si512(ptr as *const __m512i))
    }

    #[inline]
    #[target_feature(enable = "avx512f")]
    unsafe fn to_128i_extract<const INDEX: i32>(self) -> __m128i {
        _mm512_extracti32x4_epi32(self.0, INDEX)
    }

    #[inline]
    #[target_feature(enable = "avx512f")]
    unsafe fn xor(&self, other: &Self) -> Self {
        Self(_mm512_xor_si512(self.0, other.0))
    }

    #[cfg(feature = "std")]
    #[inline]
    #[target_feature(enable = "avx512f")]
    #[allow(unused)]
    unsafe fn print_hex(&self, prefix: &str) {
        let values = self.extract_u64s();
        println!(
            "{}={:#016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}",
            prefix,
            values[7],
            values[6],
            values[5],
            values[4],
            values[3],
            values[2],
            values[1],
            values[0]
        );
    }
}

impl X86_64Avx512VpclmulqdqOps {
    /// Process aligned blocks using VPCLMULQDQ with 4 x 512-bit registers
    ///
    /// Note that #[inline(always)] loses the inlining performance boost, despite no native
    /// target_features being used directly. Odd since that's not how Rust's docs make it sound...
    #[inline]
    #[target_feature(enable = "avx512vl,avx512bw,vpclmulqdq")]
    unsafe fn process_blocks<W: EnhancedCrcWidth>(
        &self,
        state: &mut CrcState<<X86_64Avx512VpclmulqdqOps as ArchOps>::Vector>,
        first: &[__m128i; 8],
        rest: &[[__m128i; 8]],
        keys: &[u64; 23],
        reflected: bool,
    ) -> W::Value
    where
        W::Value: Copy + BitXor<Output = W::Value>,
    {
        let state_u64s = self.extract_u64s(state.value);

        let positioned_state = if reflected {
            Simd512::new(0, 0, 0, 0, 0, 0, 0, state_u64s[0])
        } else {
            Simd512::new(state_u64s[1], 0, 0, 0, 0, 0, 0, 0)
        };

        let reflector = create_reflector512(reflected);

        // Load first 256 bytes (2nd half is rest[0] since these are 128-byte blocks)
        let first_ptr = first.as_ptr() as *const u8;
        let first_rest_ptr = rest[0].as_ptr() as *const u8;

        let mut x = [
            reflect_bytes512(&reflector, Simd512::load_from_ptr(first_ptr)),
            reflect_bytes512(&reflector, Simd512::load_from_ptr(first_ptr.add(64))),
            reflect_bytes512(&reflector, Simd512::load_from_ptr(first_rest_ptr)),
            reflect_bytes512(&reflector, Simd512::load_from_ptr(first_rest_ptr.add(64))),
        ];

        x[0] = positioned_state.xor(&x[0]);

        let coeff = self.create_avx512_256byte_coefficient(keys, reflected);

        let remaining_rest = &rest[1..];
        let pair_count = remaining_rest.len() / 2;

        for i in 0..pair_count {
            let block1_ptr = remaining_rest[i * 2].as_ptr() as *const u8;
            let block2_ptr = remaining_rest[i * 2 + 1].as_ptr() as *const u8;

            x[0] = x[0].fold_64(
                &coeff,
                &reflect_bytes512(&reflector, Simd512::load_from_ptr(block1_ptr)),
            );
            x[1] = x[1].fold_64(
                &coeff,
                &reflect_bytes512(&reflector, Simd512::load_from_ptr(block1_ptr.add(64))),
            );
            x[2] = x[2].fold_64(
                &coeff,
                &reflect_bytes512(&reflector, Simd512::load_from_ptr(block2_ptr)),
            );
            x[3] = x[3].fold_64(
                &coeff,
                &reflect_bytes512(&reflector, Simd512::load_from_ptr(block2_ptr.add(64))),
            );
        }

        let processed_pairs = pair_count * 2;
        let remaining_single_count = remaining_rest.len() - processed_pairs;

        if remaining_single_count > 0 {
            // We have 1 unprocessed block (128 bytes)
            // Fold 4×512 down to 2×512 and process the remaining block with 2-register mode
            let folded_2reg = self.fold_from_4x512_to_2x256(x, keys, reflected);
            let coeff_2reg = self.create_avx512_128byte_coefficient(keys, reflected);

            let last_block_ptr = remaining_rest[processed_pairs].as_ptr() as *const u8;

            let final_x = [
                folded_2reg[0].fold_64(
                    &coeff_2reg,
                    &reflect_bytes512(&reflector, Simd512::load_from_ptr(last_block_ptr)),
                ),
                folded_2reg[1].fold_64(
                    &coeff_2reg,
                    &reflect_bytes512(&reflector, Simd512::load_from_ptr(last_block_ptr.add(64))),
                ),
            ];

            let folded = self.fold_from_2x512_to_1x128(final_x, keys, reflected);

            return W::perform_final_reduction(folded, reflected, keys, self);
        }

        // All blocks processed in pairs - fold from 4 x 512-bit to 1 x 128-bit
        let folded = self.fold_from_4x512_to_1x128(x, keys, reflected);

        W::perform_final_reduction(folded, reflected, keys, self)
    }

    /// Create a folding coefficient for AVX-512 for 128-byte folding distances
    #[inline(always)]
    unsafe fn create_avx512_128byte_coefficient(
        &self,
        keys: &[u64; 23],
        reflected: bool,
    ) -> Simd512 {
        let (k1, k2) = if reflected {
            (keys[3], keys[4])
        } else {
            (keys[4], keys[3])
        };

        // Replicate the coefficient pair
        Simd512::new(k1, k2, k1, k2, k1, k2, k1, k2)
    }

    /// Create a folding coefficient for AVX-512 for 256-byte folding distances
    #[inline(always)]
    unsafe fn create_avx512_256byte_coefficient(
        &self,
        keys: &[u64; 23],
        reflected: bool,
    ) -> Simd512 {
        let (k1, k2) = if reflected {
            (keys[21], keys[22])
        } else {
            (keys[22], keys[21])
        };

        // Replicate the coefficient pair
        Simd512::new(k1, k2, k1, k2, k1, k2, k1, k2)
    }

    /// Fold from 4 x 512-bit to 1 x 128-bit
    #[inline(always)]
    unsafe fn fold_from_4x512_to_1x128(
        &self,
        x: [Simd512; 4],
        keys: &[u64; 23],
        reflected: bool,
    ) -> __m128i {
        // Step 1: Fold 4 x 512-bit to 2 x 512-bit
        let x2 = self.fold_from_4x512_to_2x256(x, keys, reflected);

        // Step 2: Fold 2 x 512-bit to 1 x 128-bit
        self.fold_from_2x512_to_1x128(x2, keys, reflected)
    }

    /// Fold from 4 x 512-bit to 2 x 512-bit
    #[inline(always)]
    unsafe fn fold_from_4x512_to_2x256(
        &self,
        x: [Simd512; 4],
        keys: &[u64; 23],
        reflected: bool,
    ) -> [Simd512; 2] {
        // This folds registers that are 128 bytes apart (x[0] with x[2], x[1] with x[3])
        let coeff = self.create_avx512_128byte_coefficient(keys, reflected);

        // Fold pairs:
        // x[0] (bytes 0-63) + x[2] (bytes 128-191) → result[0]
        // x[1] (bytes 64-127) + x[3] (bytes 192-255) → result[1]
        [x[0].fold_64(&coeff, &x[2]), x[1].fold_64(&coeff, &x[3])]
    }

    /// Fold from 2 x 512-bit to 1 x 128-bit
    #[inline(always)]
    unsafe fn fold_from_2x512_to_1x128(
        &self,
        x: [Simd512; 2],
        keys: &[u64; 23],
        reflected: bool,
    ) -> __m128i {
        // Create the fold coefficients for different distances
        let fold_coefficients = [
            self.create_vector_from_u64_pair(keys[10], keys[9], reflected), // 112 bytes
            self.create_vector_from_u64_pair(keys[12], keys[11], reflected), // 96 bytes
            self.create_vector_from_u64_pair(keys[14], keys[13], reflected), // 80 bytes
            self.create_vector_from_u64_pair(keys[16], keys[15], reflected), // 64 bytes
            self.create_vector_from_u64_pair(keys[18], keys[17], reflected), // 48 bytes
            self.create_vector_from_u64_pair(keys[20], keys[19], reflected), // 32 bytes
            self.create_vector_from_u64_pair(keys[2], keys[1], reflected),  // 16 bytes
        ];

        // Extract the 8 x 128-bit vectors from the 2 x 512-bit vectors (this is faster than
        // using 256-bit intrinsics for 1KiB payloads)
        let v128 = if reflected {
            [
                x[0].to_128i_extract::<0>(), // 256-x0.low
                x[0].to_128i_extract::<1>(), // 256-x0.high
                x[0].to_128i_extract::<2>(), // 256-x1.low
                x[0].to_128i_extract::<3>(), // 256-x1.high
                x[1].to_128i_extract::<0>(), // 256-x2.low
                x[1].to_128i_extract::<1>(), // 256-x2.high
                x[1].to_128i_extract::<2>(), // 256-x3.low
                x[1].to_128i_extract::<3>(), // 256-x3.high
            ]
        } else {
            [
                x[0].to_128i_extract::<3>(), // 256-x1.high
                x[0].to_128i_extract::<2>(), // 256-x1.low
                x[0].to_128i_extract::<1>(), // 256-x0.high
                x[0].to_128i_extract::<0>(), // 256-x0.low
                x[1].to_128i_extract::<3>(), // 256-x3.high
                x[1].to_128i_extract::<2>(), // 256-x3.low
                x[1].to_128i_extract::<1>(), // 256-x2.high
                x[1].to_128i_extract::<0>(), // 256-x2.low
            ]
        };

        // Fold the 8 xmm registers to 1 xmm register
        let mut res = v128[7];

        for (i, &coeff) in fold_coefficients.iter().enumerate() {
            let folded_h = self.carryless_mul_00(v128[i], coeff);
            let folded_l = self.carryless_mul_11(v128[i], coeff);
            res = self.xor3_vectors(folded_h, folded_l, res);
        }

        res
    }
}

// 512-bit version of the Reflector
#[derive(Clone, Copy)]
enum Reflector512 {
    NoReflector,
    ForwardReflector { smask: Simd512 },
}

// Function to create the appropriate reflector based on CRC parameters
#[inline(always)]
unsafe fn create_reflector512(reflected: bool) -> Reflector512 {
    if reflected {
        Reflector512::NoReflector
    } else {
        // Load shuffle mask
        let smask = Simd512::new(
            0x08090a0b0c0d0e0f,
            0x0001020304050607,
            0x08090a0b0c0d0e0f,
            0x0001020304050607,
            0x08090a0b0c0d0e0f,
            0x0001020304050607,
            0x08090a0b0c0d0e0f,
            0x0001020304050607,
        );
        Reflector512::ForwardReflector { smask }
    }
}

// Function to apply reflection to a 512-bit vector
#[inline(always)]
unsafe fn reflect_bytes512(reflector: &Reflector512, data: Simd512) -> Simd512 {
    match reflector {
        Reflector512::NoReflector => data,
        Reflector512::ForwardReflector { smask } => shuffle_bytes512(data, *smask),
    }
}

// pre-compute the reverse indices for 512-bit shuffling
static REVERSE_INDICES_512: __m512i =
    unsafe { core::mem::transmute([7u64, 6u64, 5u64, 4u64, 3u64, 2u64, 1u64, 0u64]) };

// Implement a 512-bit byte shuffle function
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
unsafe fn shuffle_bytes512(data: Simd512, mask: Simd512) -> Simd512 {
    Simd512(_mm512_permutexvar_epi64(
        // Reverse the order using 512-bit permutation
        REVERSE_INDICES_512,                 // reverse indices
        _mm512_shuffle_epi8(data.0, mask.0), // shuffled data
    ))
}

// Delegate all ArchOps methods to the inner X86SsePclmulqdqOps instance
impl ArchOps for X86_64Avx512VpclmulqdqOps {
    type Vector = __m128i;

    #[inline(always)]
    unsafe fn process_enhanced_simd_blocks<W: EnhancedCrcWidth>(
        &self,
        state: &mut CrcState<Self::Vector>,
        first: &[Self::Vector; 8],
        rest: &[[Self::Vector; 8]],
        _reflector: &Reflector<Self::Vector>,
        keys: &[u64; 23],
    ) -> bool
    where
        Self::Vector: Copy,
    {
        // Update the state with the result
        *state = W::create_state(
            self.process_blocks::<W>(state, first, rest, keys, state.reflected),
            state.reflected,
            self,
        );

        // Return true to indicate we handled it
        true
    }

    // Delegate all other methods to X86SsePclmulqdqOps
    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn create_vector_from_u64_pair(
        &self,
        high: u64,
        low: u64,
        reflected: bool,
    ) -> Self::Vector {
        self.0.create_vector_from_u64_pair(high, low, reflected)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn create_vector_from_u64_pair_non_reflected(
        &self,
        high: u64,
        low: u64,
    ) -> Self::Vector {
        self.0.create_vector_from_u64_pair_non_reflected(high, low)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn create_vector_from_u64(&self, value: u64, high: bool) -> Self::Vector {
        self.0.create_vector_from_u64(value, high)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_u64s(&self, vector: Self::Vector) -> [u64; 2] {
        self.0.extract_u64s(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_poly64s(&self, vector: Self::Vector) -> [u64; 2] {
        self.0.extract_poly64s(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn xor_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.xor_vectors(a, b)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn load_bytes(&self, ptr: *const u8) -> Self::Vector {
        self.0.load_bytes(ptr)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn load_aligned(&self, ptr: *const [u64; 2]) -> Self::Vector {
        self.0.load_aligned(ptr)
    }

    #[inline]
    #[target_feature(enable = "ssse3")]
    unsafe fn shuffle_bytes(&self, data: Self::Vector, mask: Self::Vector) -> Self::Vector {
        self.0.shuffle_bytes(data, mask)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn blend_vectors(
        &self,
        a: Self::Vector,
        b: Self::Vector,
        mask: Self::Vector,
    ) -> Self::Vector {
        self.0.blend_vectors(a, b, mask)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_8(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_left_8(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn set_all_bytes(&self, value: u8) -> Self::Vector {
        self.0.set_all_bytes(value)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn create_compare_mask(&self, vector: Self::Vector) -> Self::Vector {
        self.0.create_compare_mask(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn and_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.and_vectors(a, b)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_32(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_32(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_32(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_left_32(vector)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn create_vector_from_u32(&self, value: u32, high: bool) -> Self::Vector {
        self.0.create_vector_from_u32(value, high)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_4(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_left_4(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_4(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_4(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_8(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_8(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_5(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_5(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_6(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_6(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_7(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_7(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_12(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_right_12(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_12(&self, vector: Self::Vector) -> Self::Vector {
        self.0.shift_left_12(vector)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_00(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.carryless_mul_00(a, b)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_01(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.carryless_mul_01(a, b)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_10(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.carryless_mul_10(a, b)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_11(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        self.0.carryless_mul_11(a, b)
    }

    #[inline]
    #[target_feature(enable = "avx512vl")]
    unsafe fn xor3_vectors(
        &self,
        a: Self::Vector,
        b: Self::Vector,
        c: Self::Vector,
    ) -> Self::Vector {
        // VPCLMULQDQ implementation always uses AVX512 ternary logic
        // Feature detection is handled at the dispatch level
        _mm_ternarylogic_epi64(
            a, b, c, 0x96, // XOR3 operation
        )
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.
2
3	//! This module provides AVX-512 and VPCLMULQDQ-specific implementations of the ArchOps trait.
4	//!
5	//! It performs folding using 4 x ZMM registers of 512-bits each.
6
7	#![cfg(target_arch = "x86_64")]
8
9	use crate::arch::x86::sse::X86SsePclmulqdqOps;
10	use crate::enums::Reflector;
11	use crate::structs::CrcState;
12	use crate::traits::{ArchOps, EnhancedCrcWidth};
13	use core::arch::x86_64::*;
14	use core::ops::BitXor;
15
16	/// Implements the ArchOps trait using 512-bit AVX-512 and VPCLMULQDQ instructions at 512 bits.
17	/// Delegates to X86SsePclmulqdqOps for standard 128-bit operations
18	#[derive(Debug, Copy, Clone)]
19	pub struct X86_64Avx512VpclmulqdqOps(X86SsePclmulqdqOps);
20
21	impl Default for X86_64Avx512VpclmulqdqOps {
22	fn default() -> Self {
23	Self::new()
24	}
25	}
26
27	impl X86_64Avx512VpclmulqdqOps {
28	#[inline(always)]
29	pub fn new() -> Self {
30	Self(X86SsePclmulqdqOps)
31	}
32	}
33
34	// Wrapper for __m512i to make it easier to work with
35	#[derive(Debug, Copy, Clone)]
36	struct Simd512(__m512i);
37
38	impl Simd512 {
39	#[inline]
40	#[target_feature(enable = "avx512f")]
41	#[allow(clippy::too_many_arguments)]
42	unsafe fn new(x7: u64, x6: u64, x5: u64, x4: u64, x3: u64, x2: u64, x1: u64, x0: u64) -> Self {
43	Self(_mm512_set_epi64(
44	x7 as i64, x6 as i64, x5 as i64, x4 as i64, x3 as i64, x2 as i64, x1 as i64, x0 as i64,
45	))
46	}
47
48	#[inline]
49	#[target_feature(enable = "avx512vl,vpclmulqdq")]
50	unsafe fn fold_64(&self, coeff: &Self, new_data: &Self) -> Self {
51	// Use 512-bit ternary logic XOR3 with carryless multiplication
52	Self(_mm512_ternarylogic_epi64(
53	_mm512_clmulepi64_epi128(self.0, coeff.0, 0), // Low parts
54	_mm512_clmulepi64_epi128(self.0, coeff.0, 17), // High parts
55	new_data.0,
56	0x96, // XOR3 operation
57	))
58	}
59
60	#[cfg(feature = "std")]
61	#[inline]
62	#[target_feature(enable = "avx512f")]
63	unsafe fn extract_u64s(&self) -> [u64; 8] {
64	let mut result = [0u64; 8];
65	_mm512_storeu_si512(result.as_mut_ptr().cast(), self.0);
66
67	result
68	}
69
70	#[inline]
71	#[target_feature(enable = "avx512f")]
72	unsafe fn load_from_ptr(ptr: *const u8) -> Self {
73	Self(_mm512_loadu_si512(ptr as *const __m512i))
74	}
75
76	#[inline]
77	#[target_feature(enable = "avx512f")]
78	unsafe fn to_128i_extract<const INDEX: i32>(self) -> __m128i {
79	_mm512_extracti32x4_epi32(self.0, INDEX)
80	}
81
82	#[inline]
83	#[target_feature(enable = "avx512f")]
84	unsafe fn xor(&self, other: &Self) -> Self {
85	Self(_mm512_xor_si512(self.0, other.0))
86	}
87
88	#[cfg(feature = "std")]
89	#[inline]
90	#[target_feature(enable = "avx512f")]
91	#[allow(unused)]
92	unsafe fn print_hex(&self, prefix: &str) {
93	let values = self.extract_u64s();
94	println!(
95	"{}={:#016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}_{:016x}",
96	prefix,
97	values[7],
98	values[6],
99	values[5],
100	values[4],
101	values[3],
102	values[2],
103	values[1],
104	values[0]
105	);
106	}
107	}
108
109	impl X86_64Avx512VpclmulqdqOps {
110	/// Process aligned blocks using VPCLMULQDQ with 4 x 512-bit registers
111	///
112	/// Note that #[inline(always)] loses the inlining performance boost, despite no native
113	/// target_features being used directly. Odd since that's not how Rust's docs make it sound...
114	#[inline]
115	#[target_feature(enable = "avx512vl,avx512bw,vpclmulqdq")]
116	unsafe fn process_blocks<W: EnhancedCrcWidth>(
117	&self,
118	state: &mut CrcState<<X86_64Avx512VpclmulqdqOps as ArchOps>::Vector>,
119	first: &[__m128i; 8],
120	rest: &[[__m128i; 8]],
121	keys: &[u64; 23],
122	reflected: bool,
123	) -> W::Value
124	where
125	W::Value: Copy + BitXor<Output = W::Value>,
126	{
127	let state_u64s = self.extract_u64s(state.value);
128
129	let positioned_state = if reflected {
130	Simd512::new(0, 0, 0, 0, 0, 0, 0, state_u64s[0])
131	} else {
132	Simd512::new(state_u64s[1], 0, 0, 0, 0, 0, 0, 0)
133	};
134
135	let reflector = create_reflector512(reflected);
136
137	// Load first 256 bytes (2nd half is rest[0] since these are 128-byte blocks)
138	let first_ptr = first.as_ptr() as *const u8;
139	let first_rest_ptr = rest[0].as_ptr() as *const u8;
140
141	let mut x = [
142	reflect_bytes512(&reflector, Simd512::load_from_ptr(first_ptr)),
143	reflect_bytes512(&reflector, Simd512::load_from_ptr(first_ptr.add(64))),
144	reflect_bytes512(&reflector, Simd512::load_from_ptr(first_rest_ptr)),
145	reflect_bytes512(&reflector, Simd512::load_from_ptr(first_rest_ptr.add(64))),
146	];
147
148	x[0] = positioned_state.xor(&x[0]);
149
150	let coeff = self.create_avx512_256byte_coefficient(keys, reflected);
151
152	let remaining_rest = &rest[1..];
153	let pair_count = remaining_rest.len() / 2;
154
155	for i in 0..pair_count {
156	let block1_ptr = remaining_rest[i * 2].as_ptr() as *const u8;
157	let block2_ptr = remaining_rest[i * 2 + 1].as_ptr() as *const u8;
158
159	x[0] = x[0].fold_64(
160	&coeff,
161	&reflect_bytes512(&reflector, Simd512::load_from_ptr(block1_ptr)),
162	);
163	x[1] = x[1].fold_64(
164	&coeff,
165	&reflect_bytes512(&reflector, Simd512::load_from_ptr(block1_ptr.add(64))),
166	);
167	x[2] = x[2].fold_64(
168	&coeff,
169	&reflect_bytes512(&reflector, Simd512::load_from_ptr(block2_ptr)),
170	);
171	x[3] = x[3].fold_64(
172	&coeff,
173	&reflect_bytes512(&reflector, Simd512::load_from_ptr(block2_ptr.add(64))),
174	);
175	}
176
177	let processed_pairs = pair_count * 2;
178	let remaining_single_count = remaining_rest.len() - processed_pairs;
179
180	if remaining_single_count > 0 {
181	// We have 1 unprocessed block (128 bytes)
182	// Fold 4×512 down to 2×512 and process the remaining block with 2-register mode
183	let folded_2reg = self.fold_from_4x512_to_2x256(x, keys, reflected);
184	let coeff_2reg = self.create_avx512_128byte_coefficient(keys, reflected);
185
186	let last_block_ptr = remaining_rest[processed_pairs].as_ptr() as *const u8;
187
188	let final_x = [
189	folded_2reg[0].fold_64(
190	&coeff_2reg,
191	&reflect_bytes512(&reflector, Simd512::load_from_ptr(last_block_ptr)),
192	),
193	folded_2reg[1].fold_64(
194	&coeff_2reg,
195	&reflect_bytes512(&reflector, Simd512::load_from_ptr(last_block_ptr.add(64))),
196	),
197	];
198
199	let folded = self.fold_from_2x512_to_1x128(final_x, keys, reflected);
200
201	return W::perform_final_reduction(folded, reflected, keys, self);
202	}
203
204	// All blocks processed in pairs - fold from 4 x 512-bit to 1 x 128-bit
205	let folded = self.fold_from_4x512_to_1x128(x, keys, reflected);
206
207	W::perform_final_reduction(folded, reflected, keys, self)
208	}
209
210	/// Create a folding coefficient for AVX-512 for 128-byte folding distances
211	#[inline(always)]
212	unsafe fn create_avx512_128byte_coefficient(
213	&self,
214	keys: &[u64; 23],
215	reflected: bool,
216	) -> Simd512 {
217	let (k1, k2) = if reflected {
218	(keys[3], keys[4])
219	} else {
220	(keys[4], keys[3])
221	};
222
223	// Replicate the coefficient pair
224	Simd512::new(k1, k2, k1, k2, k1, k2, k1, k2)
225	}
226
227	/// Create a folding coefficient for AVX-512 for 256-byte folding distances
228	#[inline(always)]
229	unsafe fn create_avx512_256byte_coefficient(
230	&self,
231	keys: &[u64; 23],
232	reflected: bool,
233	) -> Simd512 {
234	let (k1, k2) = if reflected {
235	(keys[21], keys[22])
236	} else {
237	(keys[22], keys[21])
238	};
239
240	// Replicate the coefficient pair
241	Simd512::new(k1, k2, k1, k2, k1, k2, k1, k2)
242	}
243
244	/// Fold from 4 x 512-bit to 1 x 128-bit
245	#[inline(always)]
246	unsafe fn fold_from_4x512_to_1x128(
247	&self,
248	x: [Simd512; 4],
249	keys: &[u64; 23],
250	reflected: bool,
251	) -> __m128i {
252	// Step 1: Fold 4 x 512-bit to 2 x 512-bit
253	let x2 = self.fold_from_4x512_to_2x256(x, keys, reflected);
254
255	// Step 2: Fold 2 x 512-bit to 1 x 128-bit
256	self.fold_from_2x512_to_1x128(x2, keys, reflected)
257	}
258
259	/// Fold from 4 x 512-bit to 2 x 512-bit
260	#[inline(always)]
261	unsafe fn fold_from_4x512_to_2x256(
262	&self,
263	x: [Simd512; 4],
264	keys: &[u64; 23],
265	reflected: bool,
266	) -> [Simd512; 2] {
267	// This folds registers that are 128 bytes apart (x[0] with x[2], x[1] with x[3])
268	let coeff = self.create_avx512_128byte_coefficient(keys, reflected);
269
270	// Fold pairs:
271	// x[0] (bytes 0-63) + x[2] (bytes 128-191) → result[0]
272	// x[1] (bytes 64-127) + x[3] (bytes 192-255) → result[1]
273	[x[0].fold_64(&coeff, &x[2]), x[1].fold_64(&coeff, &x[3])]
274	}
275
276	/// Fold from 2 x 512-bit to 1 x 128-bit
277	#[inline(always)]
278	unsafe fn fold_from_2x512_to_1x128(
279	&self,
280	x: [Simd512; 2],
281	keys: &[u64; 23],
282	reflected: bool,
283	) -> __m128i {
284	// Create the fold coefficients for different distances
285	let fold_coefficients = [
286	self.create_vector_from_u64_pair(keys[10], keys[9], reflected), // 112 bytes
287	self.create_vector_from_u64_pair(keys[12], keys[11], reflected), // 96 bytes
288	self.create_vector_from_u64_pair(keys[14], keys[13], reflected), // 80 bytes
289	self.create_vector_from_u64_pair(keys[16], keys[15], reflected), // 64 bytes
290	self.create_vector_from_u64_pair(keys[18], keys[17], reflected), // 48 bytes
291	self.create_vector_from_u64_pair(keys[20], keys[19], reflected), // 32 bytes
292	self.create_vector_from_u64_pair(keys[2], keys[1], reflected), // 16 bytes
293	];
294
295	// Extract the 8 x 128-bit vectors from the 2 x 512-bit vectors (this is faster than
296	// using 256-bit intrinsics for 1KiB payloads)
297	let v128 = if reflected {
298	[
299	x[0].to_128i_extract::<0>(), // 256-x0.low
300	x[0].to_128i_extract::<1>(), // 256-x0.high
301	x[0].to_128i_extract::<2>(), // 256-x1.low
302	x[0].to_128i_extract::<3>(), // 256-x1.high
303	x[1].to_128i_extract::<0>(), // 256-x2.low
304	x[1].to_128i_extract::<1>(), // 256-x2.high
305	x[1].to_128i_extract::<2>(), // 256-x3.low
306	x[1].to_128i_extract::<3>(), // 256-x3.high
307	]
308	} else {
309	[
310	x[0].to_128i_extract::<3>(), // 256-x1.high
311	x[0].to_128i_extract::<2>(), // 256-x1.low
312	x[0].to_128i_extract::<1>(), // 256-x0.high
313	x[0].to_128i_extract::<0>(), // 256-x0.low
314	x[1].to_128i_extract::<3>(), // 256-x3.high
315	x[1].to_128i_extract::<2>(), // 256-x3.low
316	x[1].to_128i_extract::<1>(), // 256-x2.high
317	x[1].to_128i_extract::<0>(), // 256-x2.low
318	]
319	};
320
321	// Fold the 8 xmm registers to 1 xmm register
322	let mut res = v128[7];
323
324	for (i, &coeff) in fold_coefficients.iter().enumerate() {
325	let folded_h = self.carryless_mul_00(v128[i], coeff);
326	let folded_l = self.carryless_mul_11(v128[i], coeff);
327	res = self.xor3_vectors(folded_h, folded_l, res);
328	}
329
330	res
331	}
332	}
333
334	// 512-bit version of the Reflector
335	#[derive(Clone, Copy)]
336	enum Reflector512 {
337	NoReflector,
338	ForwardReflector { smask: Simd512 },
339	}
340
341	// Function to create the appropriate reflector based on CRC parameters
342	#[inline(always)]
343	unsafe fn create_reflector512(reflected: bool) -> Reflector512 {
344	if reflected {
345	Reflector512::NoReflector
346	} else {
347	// Load shuffle mask
348	let smask = Simd512::new(
349	0x08090a0b0c0d0e0f,
350	0x0001020304050607,
351	0x08090a0b0c0d0e0f,
352	0x0001020304050607,
353	0x08090a0b0c0d0e0f,
354	0x0001020304050607,
355	0x08090a0b0c0d0e0f,
356	0x0001020304050607,
357	);
358	Reflector512::ForwardReflector { smask }
359	}
360	}
361
362	// Function to apply reflection to a 512-bit vector
363	#[inline(always)]
364	unsafe fn reflect_bytes512(reflector: &Reflector512, data: Simd512) -> Simd512 {
365	match reflector {
366	Reflector512::NoReflector => data,
367	Reflector512::ForwardReflector { smask } => shuffle_bytes512(data, *smask),
368	}
369	}
370
371	// pre-compute the reverse indices for 512-bit shuffling
372	static REVERSE_INDICES_512: __m512i =
373	unsafe { core::mem::transmute([7u64, 6u64, 5u64, 4u64, 3u64, 2u64, 1u64, 0u64]) };
374
375	// Implement a 512-bit byte shuffle function
376	#[inline]
377	#[target_feature(enable = "avx512f,avx512bw")]
378	unsafe fn shuffle_bytes512(data: Simd512, mask: Simd512) -> Simd512 {
379	Simd512(_mm512_permutexvar_epi64(
380	// Reverse the order using 512-bit permutation
381	REVERSE_INDICES_512, // reverse indices
382	_mm512_shuffle_epi8(data.0, mask.0), // shuffled data
383	))
384	}
385
386	// Delegate all ArchOps methods to the inner X86SsePclmulqdqOps instance
387	impl ArchOps for X86_64Avx512VpclmulqdqOps {
388	type Vector = __m128i;
389
390	#[inline(always)]
391	unsafe fn process_enhanced_simd_blocks<W: EnhancedCrcWidth>(
392	&self,
393	state: &mut CrcState<Self::Vector>,
394	first: &[Self::Vector; 8],
395	rest: &[[Self::Vector; 8]],
396	_reflector: &Reflector<Self::Vector>,
397	keys: &[u64; 23],
398	) -> bool
399	where
400	Self::Vector: Copy,
401	{
402	// Update the state with the result
403	*state = W::create_state(
404	self.process_blocks::<W>(state, first, rest, keys, state.reflected),
405	state.reflected,
406	self,
407	);
408
409	// Return true to indicate we handled it
410	true
411	}
412
413	// Delegate all other methods to X86SsePclmulqdqOps
414	#[inline]
415	#[target_feature(enable = "sse2")]
416	unsafe fn create_vector_from_u64_pair(
417	&self,
418	high: u64,
419	low: u64,
420	reflected: bool,
421	) -> Self::Vector {
422	self.0.create_vector_from_u64_pair(high, low, reflected)
423	}
424
425	#[inline]
426	#[target_feature(enable = "sse2")]
427	unsafe fn create_vector_from_u64_pair_non_reflected(
428	&self,
429	high: u64,
430	low: u64,
431	) -> Self::Vector {
432	self.0.create_vector_from_u64_pair_non_reflected(high, low)
433	}
434
435	#[inline]
436	#[target_feature(enable = "sse4.1")]
437	unsafe fn create_vector_from_u64(&self, value: u64, high: bool) -> Self::Vector {
438	self.0.create_vector_from_u64(value, high)
439	}
440
441	#[inline]
442	#[target_feature(enable = "sse2")]
443	unsafe fn extract_u64s(&self, vector: Self::Vector) -> [u64; 2] {
444	self.0.extract_u64s(vector)
445	}
446
447	#[inline]
448	#[target_feature(enable = "sse2")]
449	unsafe fn extract_poly64s(&self, vector: Self::Vector) -> [u64; 2] {
450	self.0.extract_poly64s(vector)
451	}
452
453	#[inline]
454	#[target_feature(enable = "sse2")]
455	unsafe fn xor_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
456	self.0.xor_vectors(a, b)
457	}
458
459	#[inline]
460	#[target_feature(enable = "sse2")]
461	unsafe fn load_bytes(&self, ptr: *const u8) -> Self::Vector {
462	self.0.load_bytes(ptr)
463	}
464
465	#[inline]
466	#[target_feature(enable = "sse2")]
467	unsafe fn load_aligned(&self, ptr: *const [u64; 2]) -> Self::Vector {
468	self.0.load_aligned(ptr)
469	}
470
471	#[inline]
472	#[target_feature(enable = "ssse3")]
473	unsafe fn shuffle_bytes(&self, data: Self::Vector, mask: Self::Vector) -> Self::Vector {
474	self.0.shuffle_bytes(data, mask)
475	}
476
477	#[inline]
478	#[target_feature(enable = "sse4.1")]
479	unsafe fn blend_vectors(
480	&self,
481	a: Self::Vector,
482	b: Self::Vector,
483	mask: Self::Vector,
484	) -> Self::Vector {
485	self.0.blend_vectors(a, b, mask)
486	}
487
488	#[inline]
489	#[target_feature(enable = "sse2")]
490	unsafe fn shift_left_8(&self, vector: Self::Vector) -> Self::Vector {
491	self.0.shift_left_8(vector)
492	}
493
494	#[inline]
495	#[target_feature(enable = "sse2")]
496	unsafe fn set_all_bytes(&self, value: u8) -> Self::Vector {
497	self.0.set_all_bytes(value)
498	}
499
500	#[inline]
501	#[target_feature(enable = "sse2")]
502	unsafe fn create_compare_mask(&self, vector: Self::Vector) -> Self::Vector {
503	self.0.create_compare_mask(vector)
504	}
505
506	#[inline]
507	#[target_feature(enable = "sse2")]
508	unsafe fn and_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
509	self.0.and_vectors(a, b)
510	}
511
512	#[inline]
513	#[target_feature(enable = "sse2")]
514	unsafe fn shift_right_32(&self, vector: Self::Vector) -> Self::Vector {
515	self.0.shift_right_32(vector)
516	}
517
518	#[inline]
519	#[target_feature(enable = "sse2")]
520	unsafe fn shift_left_32(&self, vector: Self::Vector) -> Self::Vector {
521	self.0.shift_left_32(vector)
522	}
523
524	#[inline]
525	#[target_feature(enable = "sse4.1")]
526	unsafe fn create_vector_from_u32(&self, value: u32, high: bool) -> Self::Vector {
527	self.0.create_vector_from_u32(value, high)
528	}
529
530	#[inline]
531	#[target_feature(enable = "sse2")]
532	unsafe fn shift_left_4(&self, vector: Self::Vector) -> Self::Vector {
533	self.0.shift_left_4(vector)
534	}
535
536	#[inline]
537	#[target_feature(enable = "sse2")]
538	unsafe fn shift_right_4(&self, vector: Self::Vector) -> Self::Vector {
539	self.0.shift_right_4(vector)
540	}
541
542	#[inline]
543	#[target_feature(enable = "sse2")]
544	unsafe fn shift_right_8(&self, vector: Self::Vector) -> Self::Vector {
545	self.0.shift_right_8(vector)
546	}
547
548	#[inline]
549	#[target_feature(enable = "sse2")]
550	unsafe fn shift_right_5(&self, vector: Self::Vector) -> Self::Vector {
551	self.0.shift_right_5(vector)
552	}
553
554	#[inline]
555	#[target_feature(enable = "sse2")]
556	unsafe fn shift_right_6(&self, vector: Self::Vector) -> Self::Vector {
557	self.0.shift_right_6(vector)
558	}
559
560	#[inline]
561	#[target_feature(enable = "sse2")]
562	unsafe fn shift_right_7(&self, vector: Self::Vector) -> Self::Vector {
563	self.0.shift_right_7(vector)
564	}
565
566	#[inline]
567	#[target_feature(enable = "sse2")]
568	unsafe fn shift_right_12(&self, vector: Self::Vector) -> Self::Vector {
569	self.0.shift_right_12(vector)
570	}
571
572	#[inline]
573	#[target_feature(enable = "sse2")]
574	unsafe fn shift_left_12(&self, vector: Self::Vector) -> Self::Vector {
575	self.0.shift_left_12(vector)
576	}
577
578	#[inline]
579	#[target_feature(enable = "pclmulqdq")]
580	unsafe fn carryless_mul_00(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
581	self.0.carryless_mul_00(a, b)
582	}
583
584	#[inline]
585	#[target_feature(enable = "pclmulqdq")]
586	unsafe fn carryless_mul_01(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
587	self.0.carryless_mul_01(a, b)
588	}
589
590	#[inline]
591	#[target_feature(enable = "pclmulqdq")]
592	unsafe fn carryless_mul_10(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
593	self.0.carryless_mul_10(a, b)
594	}
595
596	#[inline]
597	#[target_feature(enable = "pclmulqdq")]
598	unsafe fn carryless_mul_11(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
599	self.0.carryless_mul_11(a, b)
600	}
601
602	#[inline]
603	#[target_feature(enable = "avx512vl")]
604	unsafe fn xor3_vectors(
605	&self,
606	a: Self::Vector,
607	b: Self::Vector,
608	c: Self::Vector,
609	) -> Self::Vector {
610	// VPCLMULQDQ implementation always uses AVX512 ternary logic
611	// Feature detection is handled at the dispatch level
612	_mm_ternarylogic_epi64(
613	a, b, c, 0x96, // XOR3 operation
614	)
615	}
616	}

Keyboard