// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0. //! This module provides the CRC-64 implementation for areas where it differs from CRC-32. #![cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] use crate::algorithm; use crate::consts::{CRC_CHUNK_SIZE, CRC_HALF_CHUNK_SIZE}; use crate::crc64::consts::SIMD_CONSTANTS; use crate::enums::Reflector; use crate::structs::CrcState; use crate::traits::{ArchOps, EnhancedCrcWidth}; impl EnhancedCrcWidth for crate::structs::Width64 { #[inline(always)] fn load_constants(_reflected: bool) -> [[u64; 2]; 4] { [ [0x08090a0b0c0d0e0f, 0x0001020304050607], // smask [0x8080808080808080, 0x8080808080808080], // mask1 (mask3 reflected) [0xffffffffffffffff, 0x00000000ffffffff], // mask2 [0x0000000000000000, 0xffffffffffffffff], // mask3 (forward) ] } #[inline(always)] unsafe fn create_state( value: Self::Value, reflected: bool, ops: &T, ) -> CrcState where T::Vector: Copy, { let vector = if reflected { ops.create_vector_from_u64(value, false) // Set low 64 bits } else { ops.create_vector_from_u64(value, true) // Set high 64 bits }; CrcState { value: vector, reflected, } } #[inline(always)] unsafe fn extract_result(vector: T::Vector, reflected: bool, ops: &T) -> Self::Value where T::Vector: Copy, { let u64s = ops.extract_u64s(vector); if reflected { u64s[0] // Low 64 bits for reflected mode } else { u64s[1] // High 64 bits for non-reflected mode } } #[inline(always)] unsafe fn fold_16( state: &mut CrcState, coeff: T::Vector, data_to_xor: T::Vector, ops: &T, ) where T::Vector: Copy, { // CRC-64 specific implementation for folding 16 bytes state.value = { ops.xor3_vectors( ops.carryless_mul_00(state.value, coeff), ops.carryless_mul_11(state.value, coeff), data_to_xor, ) }; } #[inline(always)] unsafe fn fold_width(state: &mut CrcState, high: u64, low: u64, ops: &T) where T::Vector: Copy, { // CRC-64 specific implementation for folding 8 bytes let coeff = Self::create_coefficient(high, low, state.reflected, ops); if state.reflected { let h = ops.carryless_mul_01(coeff, state.value); let shifted = ops.shift_right_8(state.value); state.value = ops.xor_vectors(h, shifted); } else { let clmul = ops.carryless_mul_01(state.value, coeff); let shifted = ops.shift_left_8(state.value); state.value = ops.xor_vectors(clmul, shifted); } } #[inline(always)] unsafe fn barrett_reduction( state: &CrcState, poly: u64, mu: u64, ops: &T, ) -> Self::Value where T::Vector: Copy, { // CRC-64 Barrett reduction let x = state.value; let mu_poly = Self::create_coefficient(mu, poly, state.reflected, ops); if state.reflected { let clmul1 = ops.carryless_mul_00(x, mu_poly); let clmul2 = ops.carryless_mul_10(clmul1, mu_poly); let clmul1_shifted = ops.shift_left_8(clmul1); let final_xor = ops.xor3_vectors(clmul2, clmul1_shifted, x); ops.extract_u64s(final_xor)[1] } else { // Load mask3 for non-reflected mode let mask3 = ops.load_aligned(&[0x0000000000000000, 0xffffffffffffffff]); let x1 = ops.and_vectors(x, mask3); let clmul1 = ops.carryless_mul_11(x1, mu_poly); let clmul2 = ops.carryless_mul_01(ops.xor_vectors(clmul1, x1), mu_poly); let final_xor = ops.xor_vectors(clmul2, x); ops.extract_u64s(final_xor)[0] } } #[inline(always)] unsafe fn create_coefficient( high: u64, low: u64, reflected: bool, ops: &T, ) -> T::Vector where T::Vector: Copy, { ops.create_vector_from_u64_pair(high, low, reflected) } #[inline(always)] unsafe fn perform_final_reduction( state: T::Vector, reflected: bool, keys: &[u64; 23], ops: &T, ) -> Self::Value where T::Vector: Copy, { let mut crc_state = CrcState { value: state, reflected, }; // Fold 16 bytes into 8 bytes Self::fold_width(&mut crc_state, keys[6], keys[5], ops); // Perform Barrett reduction to finalize Self::barrett_reduction(&crc_state, keys[8], keys[7], ops) } #[inline(always)] fn get_last_bytes_table_ptr(reflected: bool, remaining_len: usize) -> (*const u8, usize) { use crate::crc64::consts::{PSBTBL_FORWARD, PSBTBL_REVERSE}; if reflected { // For reflected mode let base_ptr = &PSBTBL_REVERSE as *const _ as *const u8; let offset = if remaining_len <= CRC_CHUNK_SIZE { remaining_len } else { let real_remaining = remaining_len % CRC_CHUNK_SIZE; if real_remaining == 0 { 0 } else { real_remaining } }; (base_ptr, offset) } else { // For non-reflected mode let base_ptr = &PSBTBL_FORWARD as *const _ as *const u8; let offset = if remaining_len <= CRC_CHUNK_SIZE { CRC_CHUNK_SIZE - remaining_len } else { let real_remaining = remaining_len % CRC_CHUNK_SIZE; if real_remaining == 0 { 0 } else { CRC_CHUNK_SIZE - real_remaining } }; (base_ptr, offset) } } } /// Process inputs smaller than 16 bytes #[inline] #[cfg_attr( any(target_arch = "x86", target_arch = "x86_64"), target_feature(enable = "ssse3,sse4.1,pclmulqdq") )] #[cfg_attr(target_arch = "aarch64", target_feature(enable = "aes"))] pub(crate) unsafe fn process_0_to_15( data: &[u8], state: &mut CrcState, reflector: &Reflector, keys: &[u64; 23], ops: &T, ) -> W::Value where T::Vector: Copy, { // Create a zero-initialized aligned buffer let mut buffer = [0u8; CRC_CHUNK_SIZE]; // Copy input data into the buffer buffer[..data.len()].copy_from_slice(data); // Get the length for processing let len = data.len() as i32; // Process the buffer let xmm7 = if state.reflected { // Get base pointer and standard offset from the trait method let (base_ptr, standard_offset) = W::get_last_bytes_table_ptr(true, len as usize); // Apply special adjustment for 1-7 bytes let ptr = if len < CRC_HALF_CHUNK_SIZE as i32 { // For 1-7 bytes: Add CRC_HALF_CHUNK_SIZE (8) to the offset base_ptr.add(standard_offset + CRC_HALF_CHUNK_SIZE) } else { // For 8-15 bytes: Use the standard offset directly base_ptr.add(standard_offset) }; // Load shuffle mask let xmm0 = ops.load_bytes(ptr); // Apply shuffle to data XORed with initial state let data_with_crc = ops.xor_vectors(ops.load_bytes(buffer.as_ptr()), state.value); ops.shuffle_bytes(data_with_crc, xmm0) } else { // Calculate pointer offset for forward table // For non-reflected mode, we need to apply a special base offset let base_offset = if len < CRC_HALF_CHUNK_SIZE as i32 { 24 } else { 16 }; // Use the trait method to get the base pointer and standard offset let (base_ptr, _) = W::get_last_bytes_table_ptr(false, len as usize); // Instead of using standard_offset directly, we use our special base_offset // This is the key difference that makes the code work correctly let ptr = base_ptr.add(base_offset - len as usize); // Load and reflect data let reflected_data = algorithm::reflect_bytes(reflector, ops.load_bytes(buffer.as_ptr()), ops); let data_with_crc = ops.xor_vectors(reflected_data, state.value); // Load shuffle mask let x0 = ops.load_bytes(ptr); if len >= CRC_HALF_CHUNK_SIZE as i32 { // For lengths 8-15, XOR with mask1 let mask1 = ops.load_aligned(&SIMD_CONSTANTS[1]); ops.shuffle_bytes(data_with_crc, ops.xor_vectors(x0, mask1)) } else { // For lengths 1-7, just return shuffled data ops.shuffle_bytes(data_with_crc, x0) } }; if len >= CRC_HALF_CHUNK_SIZE as i32 { // For 8-15 bytes, perform additional folding return W::perform_final_reduction(xmm7, state.reflected, keys, ops); } let final_state = CrcState { value: xmm7, reflected: state.reflected, }; // Barrett reduction to finalize W::barrett_reduction(&final_state, keys[8], keys[7], ops) }