vendor/crc-fast/src/arch/mod.rs at switch-says · andymccall/oxygen

◆ switch-says oxygen / vendor / crc-fast / src / arch / mod.rs

16.6 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw

// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.

//! This module provides the main entry point for the SIMD CRC calculation.
//!
//! It dispatches to the appropriate architecture-specific implementation

#[cfg(all(target_arch = "aarch64", feature = "std"))]
use std::arch::is_aarch64_feature_detected;

use crate::CrcParams;

#[cfg(target_arch = "aarch64")]
use crate::arch::aarch64::aes::Aarch64AesOps;

#[cfg(target_arch = "aarch64")]
use crate::arch::aarch64::aes_sha3::Aarch64AesSha3Ops;

#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
use crate::{
    algorithm,
    structs::{Width16, Width32, Width64},
};

pub mod aarch64;
pub mod software;
pub mod x86;
pub mod x86_64;

/// Main entry point that dispatches to the appropriate architecture
///
/// # Safety
/// May use native CPU features
#[inline(always)]
#[cfg(target_arch = "aarch64")]
pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
    use crate::feature_detection::{get_arch_ops, ArchOpsInstance};

    match get_arch_ops() {
        ArchOpsInstance::Aarch64AesSha3(ops) => update_aarch64_aes_sha3(state, bytes, params, *ops),
        ArchOpsInstance::Aarch64Aes(ops) => update_aarch64_aes(state, bytes, params, *ops),
        ArchOpsInstance::SoftwareFallback => {
            #[cfg(feature = "std")]
            let has_features =
                is_aarch64_feature_detected!("aes") && is_aarch64_feature_detected!("neon");
            #[cfg(not(feature = "std"))]
            let has_features = cfg!(target_feature = "aes") && cfg!(target_feature = "neon");

            if !has_features {
                #[cfg(any(not(target_feature = "aes"), not(target_feature = "neon")))]
                {
                    // Use software implementation when no SIMD support is available
                    return crate::arch::software::update(state, bytes, params);
                }
            }

            // This should likely never happen, but just in case
            panic!("aarch64 features missing (NEON and/or AES)");
        }
    }
}

#[inline]
#[cfg(target_arch = "aarch64")]
#[target_feature(enable = "aes")]
unsafe fn update_aarch64_aes(
    state: u64,
    bytes: &[u8],
    params: &CrcParams,
    ops: Aarch64AesOps,
) -> u64 {
    match params.width {
        64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
        32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
        16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
        _ => panic!("Unsupported CRC width: {}", params.width),
    }
}

#[inline]
#[cfg(target_arch = "aarch64")]
#[target_feature(enable = "aes,sha3")]
unsafe fn update_aarch64_aes_sha3(
    state: u64,
    bytes: &[u8],
    params: &CrcParams,
    ops: Aarch64AesSha3Ops,
) -> u64 {
    match params.width {
        64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
        32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
        16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
        _ => panic!("Unsupported CRC width: {}", params.width),
    }
}

/// Main entry point for x86/x86_64
///
/// # Safety
/// May use native CPU features
#[inline(always)]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
    use crate::feature_detection::{get_arch_ops, ArchOpsInstance};

    match get_arch_ops() {
        #[cfg(target_arch = "x86_64")]
        ArchOpsInstance::X86_64Avx512Vpclmulqdq(ops) => {
            update_x86_64_avx512_vpclmulqdq(state, bytes, params, *ops)
        }
        #[cfg(target_arch = "x86_64")]
        ArchOpsInstance::X86_64Avx512Pclmulqdq(ops) => {
            update_x86_64_avx512_pclmulqdq(state, bytes, params, *ops)
        }
        ArchOpsInstance::X86SsePclmulqdq(ops) => {
            update_x86_sse_pclmulqdq(state, bytes, params, *ops)
        }
        ArchOpsInstance::SoftwareFallback => crate::arch::software::update(state, bytes, params),
    }
}

#[inline]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[target_feature(enable = "sse4.1,pclmulqdq")]
unsafe fn update_x86_sse_pclmulqdq(
    state: u64,
    bytes: &[u8],
    params: &CrcParams,
    ops: crate::arch::x86::sse::X86SsePclmulqdqOps,
) -> u64 {
    match params.width {
        64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
        32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
        16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
        _ => panic!("Unsupported CRC width: {}", params.width),
    }
}

#[inline]
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx512vl,pclmulqdq")]
unsafe fn update_x86_64_avx512_pclmulqdq(
    state: u64,
    bytes: &[u8],
    params: &CrcParams,
    ops: crate::arch::x86_64::avx512::X86_64Avx512PclmulqdqOps,
) -> u64 {
    match params.width {
        64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
        32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
        16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
        _ => panic!("Unsupported CRC width: {}", params.width),
    }
}

#[inline]
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx512vl,vpclmulqdq")]
unsafe fn update_x86_64_avx512_vpclmulqdq(
    state: u64,
    bytes: &[u8],
    params: &CrcParams,
    ops: crate::arch::x86_64::avx512_vpclmulqdq::X86_64Avx512VpclmulqdqOps,
) -> u64 {
    match params.width {
        64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
        32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
        16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
        _ => panic!("Unsupported CRC width: {}", params.width),
    }
}

#[inline(always)]
#[cfg(all(
    not(target_arch = "x86"),
    not(target_arch = "x86_64"),
    not(target_arch = "aarch64")
))]
pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
    crate::arch::software::update(state, bytes, params)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::crc32::consts::CRC32_BZIP2;
    use crate::crc64::consts::CRC64_NVME;
    use crate::test::consts::{TEST_256_BYTES_STRING, TEST_ALL_CONFIGS, TEST_CHECK_STRING};
    use crate::test::create_aligned_data;
    use crate::test::enums::AnyCrcTestConfig;
    use rand::{rng, Rng};

    #[test]
    fn test_check_value() {
        for config in TEST_ALL_CONFIGS {
            // direct update() call, which needs XOROUT applied
            let actual = unsafe {
                update(
                    config.get_init_algorithm(),
                    TEST_CHECK_STRING,
                    config.get_params(),
                ) ^ config.get_xorout()
            };

            assert_eq!(
                actual,
                config.get_check(),
                "Mismatch CRC, {}, expected {:#x}, got {:#x}",
                config.get_name(),
                config.get_check(),
                actual
            );
        }
    }

    #[test]
    fn test_256_string() {
        for config in TEST_ALL_CONFIGS {
            let actual = unsafe {
                update(
                    config.get_init_algorithm(),
                    &create_aligned_data(TEST_256_BYTES_STRING),
                    config.get_params(),
                ) ^ config.get_xorout()
            };

            assert_eq!(
                actual,
                config.checksum_with_reference(TEST_256_BYTES_STRING),
                "Mismatch CRC, {}, expected {:#x}, got {:#x}",
                config.get_name(),
                config.get_check(),
                actual
            );
        }
    }

    #[test]
    fn test_512_string() {
        let test_string = b"12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";

        for config in TEST_ALL_CONFIGS {
            let actual = unsafe {
                update(
                    config.get_init_algorithm(),
                    &create_aligned_data(test_string),
                    config.get_params(),
                ) ^ config.get_xorout()
            };

            assert_eq!(
                actual,
                config.checksum_with_reference(test_string),
                "Mismatch CRC, {}, expected {:#x}, got {:#x}",
                config.get_name(),
                config.get_check(),
                actual
            );
        }
    }

    #[test]
    fn test_1024_string() {
        let test_string = b"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345612345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";

        for config in TEST_ALL_CONFIGS {
            let actual = unsafe {
                update(
                    config.get_init_algorithm(),
                    &create_aligned_data(test_string),
                    config.get_params(),
                ) ^ config.get_xorout()
            };

            assert_eq!(
                actual,
                config.checksum_with_reference(test_string),
                "Mismatch CRC, {}, expected {:#x}, got {:#x}",
                config.get_name(),
                config.get_check(),
                actual
            );
        }
    }

    // CRC-64/NVME is a special flower in that Rust's crc library doesn't support it yet, so we have
    // tested values to check against.
    #[test]
    fn test_crc64_nvme_standard_vectors() {
        static CASES: &[(&[u8], u64)] = &[
            // from our own internal tests, since the Check value in the NVM Express® NVM Command
            // Set Specification (Revision 1.0d, December 2023) is incorrect
            // (Section 5.2.1.3.4, Figure 120, page 83).
            (b"123456789", 0xae8b14860a799888),

            // from the NVM Express® NVM Command Set Specification (Revision 1.0d, December 2023),
            // Section 5.2.1.3.5, Figure 122, page 84.
            // https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0d-2023.12.28-Ratified.pdf
            // and the Linux kernel
            // https://github.com/torvalds/linux/blob/f3813f4b287e480b1fcd62ca798d8556644b8278/crypto/testmgr.h#L3685-L3695
            (&[0; 4096], 0x6482d367eb22b64e),
            (&[255; 4096], 0xc0ddba7302eca3ac),

            // custom values
            (TEST_256_BYTES_STRING, 0xabdb9e6c30937916),
            (b"", 0),
            (b"@", 0x2808afa9582aa47),
            (b"1\x97", 0xb4af0ae0feb08e0f),
            (b"M\"\xdf", 0x85d7cd041a2a8a5d),
            (b"l\xcd\x13\xd7", 0x1860820ea79b0fa3),
            (&[0; 32], 0xcf3473434d4ecf3b),
            (&[255; 32], 0xa0a06974c34d63c4),
            (b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F", 0xb9d9d4a8492cbd7f),
            (&[0; 1024], 0x691bb2b09be5498a),
            (b"hello, world!", 0xf8046e40c403f1d0),
        ];

        for (input, expected) in CASES {
            unsafe {
                let actual = update(CRC64_NVME.init, input, &CRC64_NVME) ^ CRC64_NVME.xorout;

                assert_eq!(
                    actual, *expected,
                    "Mismatch CRC, expected {:#x}, got {:#x}, input: {:?}",
                    expected, actual, input
                );
            }
        }
    }

    /// Test the "crc32" variant used in PHP's hash() function, which is different from the
    /// crc32() function. It's really just CRC-32/BZIP2 with the output byte-reversed to little
    /// endian.
    ///
    /// https://www.php.net/manual/en/function.hash-file.php#104836
    #[test]
    fn test_crc32_php_standard_vectors() {
        static CASES: &[(&[u8], u64)] = &[
            (b"123456789", 0x181989fc),
            (&[0; 4096], 0xe3380088),
            (&[255; 4096], 0x8f2ae650),
            (b"hello, world!", 0x5eacce7),
        ];

        for (input, expected) in CASES {
            let bzip2_crc = unsafe {
                (update(CRC32_BZIP2.init, input, &CRC32_BZIP2) ^ CRC32_BZIP2.xorout) as u32
            };

            // PHP reverses the byte order of the CRC for some reason
            let actual = bzip2_crc.swap_bytes();

            assert_eq!(
                actual, *expected as u32,
                "Mismatch CRC, expected {:#x}, got {:#x}, input: {:?}",
                expected, actual, input
            );
        }
    }

    /// Test CRC-16/IBM-SDLC check value (reflected variant)
    #[test]
    fn test_crc16_ibm_sdlc_check_value() {
        use crate::crc16::consts::CRC16_IBM_SDLC;

        let actual = unsafe {
            update(CRC16_IBM_SDLC.init, TEST_CHECK_STRING, &CRC16_IBM_SDLC) ^ CRC16_IBM_SDLC.xorout
        };

        assert_eq!(
            actual, 0x906E,
            "CRC-16/IBM-SDLC check value mismatch: expected 0x906E, got {:#x}",
            actual
        );
    }

    /// Test CRC-16/T10-DIF check value (forward/non-reflected variant)
    #[test]
    fn test_crc16_t10_dif_check_value() {
        use crate::crc16::consts::CRC16_T10_DIF;

        let actual = unsafe {
            update(CRC16_T10_DIF.init, TEST_CHECK_STRING, &CRC16_T10_DIF) ^ CRC16_T10_DIF.xorout
        };

        assert_eq!(
            actual, 0xD0DB,
            "CRC-16/T10-DIF check value mismatch: expected 0xD0DB, got {:#x}",
            actual
        );
    }

    /// Skipping for Miri runs due to time constraints, underlying code already covered by other
    /// tests.
    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_small_lengths_all() {
        // Test each CRC variant
        for config in TEST_ALL_CONFIGS {
            // Test each length from 0 to 255
            for len in 0..=255 {
                test_length(len, config);
            }
        }
    }

    /// Skipping for Miri runs due to time constraints, underlying code already covered by other
    /// tests.
    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_medium_lengths() {
        // Test each CRC variant
        for config in TEST_ALL_CONFIGS {
            // Test each length from 256 to 1024, which should fold and include handling remainders
            for len in 256..=1024 {
                test_length(len, config);
            }
        }
    }

    /// Skipping for Miri runs due to time constraints, underlying code already covered by other
    /// tests.
    #[test]
    #[cfg_attr(miri, ignore)]
    fn test_large_lengths() {
        // Test each CRC variant
        for config in TEST_ALL_CONFIGS {
            // Test ~1 MiB just before, at, and just after the folding boundaries
            for len in 1048575..=1048577 {
                test_length(len, config);
            }
        }
    }

    fn test_length(length: usize, config: &AnyCrcTestConfig) {
        let mut data = vec![0u8; length];
        rng().fill(&mut data[..]);

        // Calculate expected CRC using the reference implementation
        let expected = config.checksum_with_reference(&data);

        // direct update() call, which needs XOROUT applied
        let actual = unsafe {
            update(config.get_init_algorithm(), &data, config.get_params()) ^ config.get_xorout()
        };

        assert_eq!(
            actual,
            expected,
            "\nFailed for {} with length {}\nGot: {:016x}\nExpected: {:016x}",
            config.get_name(),
            length,
            actual,
            expected
        );
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

2cd40ea1 Andy McCall 1 month ago ↶	1	// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.
	2
	3	//! This module provides the main entry point for the SIMD CRC calculation.
	4	//!
	5	//! It dispatches to the appropriate architecture-specific implementation
	6
	7	#[cfg(all(target_arch = "aarch64", feature = "std"))]
	8	use std::arch::is_aarch64_feature_detected;
	9
	10	use crate::CrcParams;
	11
	12	#[cfg(target_arch = "aarch64")]
	13	use crate::arch::aarch64::aes::Aarch64AesOps;
	14
	15	#[cfg(target_arch = "aarch64")]
	16	use crate::arch::aarch64::aes_sha3::Aarch64AesSha3Ops;
	17
	18	#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
	19	use crate::{
	20	algorithm,
	21	structs::{Width16, Width32, Width64},
	22	};
	23
	24	pub mod aarch64;
	25	pub mod software;
	26	pub mod x86;
	27	pub mod x86_64;
	28
	29	/// Main entry point that dispatches to the appropriate architecture
	30	///
	31	/// # Safety
	32	/// May use native CPU features
	33	#[inline(always)]
	34	#[cfg(target_arch = "aarch64")]
	35	pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
	36	use crate::feature_detection::{get_arch_ops, ArchOpsInstance};
	37
	38	match get_arch_ops() {
	39	ArchOpsInstance::Aarch64AesSha3(ops) => update_aarch64_aes_sha3(state, bytes, params, *ops),
	40	ArchOpsInstance::Aarch64Aes(ops) => update_aarch64_aes(state, bytes, params, *ops),
	41	ArchOpsInstance::SoftwareFallback => {
	42	#[cfg(feature = "std")]
	43	let has_features =
	44	is_aarch64_feature_detected!("aes") && is_aarch64_feature_detected!("neon");
	45	#[cfg(not(feature = "std"))]
	46	let has_features = cfg!(target_feature = "aes") && cfg!(target_feature = "neon");
	47
	48	if !has_features {
	49	#[cfg(any(not(target_feature = "aes"), not(target_feature = "neon")))]
	50	{
	51	// Use software implementation when no SIMD support is available
	52	return crate::arch::software::update(state, bytes, params);
	53	}
	54	}
	55
	56	// This should likely never happen, but just in case
	57	panic!("aarch64 features missing (NEON and/or AES)");
	58	}
	59	}
	60	}
	61
	62	#[inline]
	63	#[cfg(target_arch = "aarch64")]
	64	#[target_feature(enable = "aes")]
65	unsafe fn update_aarch64_aes(
66	state: u64,
67	bytes: &[u8],
68	params: &CrcParams,
69	ops: Aarch64AesOps,
70	) -> u64 {
71	match params.width {
72	64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
73	32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
74	16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
75	_ => panic!("Unsupported CRC width: {}", params.width),
76	}
77	}
78
79	#[inline]
80	#[cfg(target_arch = "aarch64")]
81	#[target_feature(enable = "aes,sha3")]
82	unsafe fn update_aarch64_aes_sha3(
83	state: u64,
84	bytes: &[u8],
85	params: &CrcParams,
86	ops: Aarch64AesSha3Ops,
87	) -> u64 {
88	match params.width {
89	64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
90	32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
91	16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
92	_ => panic!("Unsupported CRC width: {}", params.width),
93	}
94	}
95
96	/// Main entry point for x86/x86_64
97	///
98	/// # Safety
99	/// May use native CPU features
100	#[inline(always)]
101	#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
102	pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
103	use crate::feature_detection::{get_arch_ops, ArchOpsInstance};
104
105	match get_arch_ops() {
106	#[cfg(target_arch = "x86_64")]
107	ArchOpsInstance::X86_64Avx512Vpclmulqdq(ops) => {
108	update_x86_64_avx512_vpclmulqdq(state, bytes, params, *ops)
109	}
110	#[cfg(target_arch = "x86_64")]
111	ArchOpsInstance::X86_64Avx512Pclmulqdq(ops) => {
112	update_x86_64_avx512_pclmulqdq(state, bytes, params, *ops)
113	}
114	ArchOpsInstance::X86SsePclmulqdq(ops) => {
115	update_x86_sse_pclmulqdq(state, bytes, params, *ops)
116	}
117	ArchOpsInstance::SoftwareFallback => crate::arch::software::update(state, bytes, params),
118	}
119	}
120
121	#[inline]
122	#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
123	#[target_feature(enable = "sse4.1,pclmulqdq")]
124	unsafe fn update_x86_sse_pclmulqdq(
125	state: u64,
126	bytes: &[u8],
127	params: &CrcParams,
128	ops: crate::arch::x86::sse::X86SsePclmulqdqOps,
129	) -> u64 {
130	match params.width {
131	64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
132	32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
133	16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
134	_ => panic!("Unsupported CRC width: {}", params.width),
135	}
136	}
137
138	#[inline]
139	#[cfg(target_arch = "x86_64")]
140	#[target_feature(enable = "avx512vl,pclmulqdq")]
141	unsafe fn update_x86_64_avx512_pclmulqdq(
142	state: u64,
143	bytes: &[u8],
144	params: &CrcParams,
145	ops: crate::arch::x86_64::avx512::X86_64Avx512PclmulqdqOps,
146	) -> u64 {
147	match params.width {
148	64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
149	32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
150	16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
151	_ => panic!("Unsupported CRC width: {}", params.width),
152	}
153	}
154
155	#[inline]
156	#[cfg(target_arch = "x86_64")]
157	#[target_feature(enable = "avx512vl,vpclmulqdq")]
158	unsafe fn update_x86_64_avx512_vpclmulqdq(
159	state: u64,
160	bytes: &[u8],
161	params: &CrcParams,
162	ops: crate::arch::x86_64::avx512_vpclmulqdq::X86_64Avx512VpclmulqdqOps,
163	) -> u64 {
164	match params.width {
165	64 => algorithm::update::<_, Width64>(state, bytes, params, &ops),
166	32 => algorithm::update::<_, Width32>(state as u32, bytes, params, &ops) as u64,
167	16 => algorithm::update::<_, Width16>(state as u16, bytes, params, &ops) as u64,
168	_ => panic!("Unsupported CRC width: {}", params.width),
169	}
170	}
171
172	#[inline(always)]
173	#[cfg(all(
174	not(target_arch = "x86"),
175	not(target_arch = "x86_64"),
176	not(target_arch = "aarch64")
177	))]
178	pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: &CrcParams) -> u64 {
179	crate::arch::software::update(state, bytes, params)
180	}
181
182	#[cfg(test)]
183	mod tests {
184	use super::*;
185	use crate::crc32::consts::CRC32_BZIP2;
186	use crate::crc64::consts::CRC64_NVME;
187	use crate::test::consts::{TEST_256_BYTES_STRING, TEST_ALL_CONFIGS, TEST_CHECK_STRING};
188	use crate::test::create_aligned_data;
189	use crate::test::enums::AnyCrcTestConfig;
190	use rand::{rng, Rng};
191
192	#[test]
193	fn test_check_value() {
194	for config in TEST_ALL_CONFIGS {
195	// direct update() call, which needs XOROUT applied
196	let actual = unsafe {
197	update(
198	config.get_init_algorithm(),
199	TEST_CHECK_STRING,
200	config.get_params(),
201	) ^ config.get_xorout()
202	};
203
204	assert_eq!(
205	actual,
206	config.get_check(),
207	"Mismatch CRC, {}, expected {:#x}, got {:#x}",
208	config.get_name(),
209	config.get_check(),
210	actual
211	);
212	}
213	}
214
215	#[test]
216	fn test_256_string() {
217	for config in TEST_ALL_CONFIGS {
218	let actual = unsafe {
219	update(
220	config.get_init_algorithm(),
221	&create_aligned_data(TEST_256_BYTES_STRING),
222	config.get_params(),
223	) ^ config.get_xorout()
224	};
225
226	assert_eq!(
227	actual,
228	config.checksum_with_reference(TEST_256_BYTES_STRING),
229	"Mismatch CRC, {}, expected {:#x}, got {:#x}",
230	config.get_name(),
231	config.get_check(),
232	actual
233	);
234	}
235	}
236
237	#[test]
238	fn test_512_string() {
239	let test_string = b"12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";
240
241	for config in TEST_ALL_CONFIGS {
242	let actual = unsafe {
243	update(
244	config.get_init_algorithm(),
245	&create_aligned_data(test_string),
246	config.get_params(),
247	) ^ config.get_xorout()
248	};
249
250	assert_eq!(
251	actual,
252	config.checksum_with_reference(test_string),
253	"Mismatch CRC, {}, expected {:#x}, got {:#x}",
254	config.get_name(),
255	config.get_check(),
256	actual
257	);
258	}
259	}
260
261	#[test]
262	fn test_1024_string() {
263	let test_string = b"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345612345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456";
264
265	for config in TEST_ALL_CONFIGS {
266	let actual = unsafe {
267	update(
268	config.get_init_algorithm(),
269	&create_aligned_data(test_string),
270	config.get_params(),
271	) ^ config.get_xorout()
272	};
273
274	assert_eq!(
275	actual,
276	config.checksum_with_reference(test_string),
277	"Mismatch CRC, {}, expected {:#x}, got {:#x}",
278	config.get_name(),
279	config.get_check(),
280	actual
281	);
282	}
283	}
284
285	// CRC-64/NVME is a special flower in that Rust's crc library doesn't support it yet, so we have
286	// tested values to check against.
287	#[test]
288	fn test_crc64_nvme_standard_vectors() {
289	static CASES: &[(&[u8], u64)] = &[
290	// from our own internal tests, since the Check value in the NVM Express® NVM Command
291	// Set Specification (Revision 1.0d, December 2023) is incorrect
292	// (Section 5.2.1.3.4, Figure 120, page 83).
293	(b"123456789", 0xae8b14860a799888),
294
295	// from the NVM Express® NVM Command Set Specification (Revision 1.0d, December 2023),
296	// Section 5.2.1.3.5, Figure 122, page 84.
297	// https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0d-2023.12.28-Ratified.pdf
298	// and the Linux kernel
299	// https://github.com/torvalds/linux/blob/f3813f4b287e480b1fcd62ca798d8556644b8278/crypto/testmgr.h#L3685-L3695
300	(&[0; 4096], 0x6482d367eb22b64e),
301	(&[255; 4096], 0xc0ddba7302eca3ac),
302
303	// custom values
304	(TEST_256_BYTES_STRING, 0xabdb9e6c30937916),
305	(b"", 0),
306	(b"@", 0x2808afa9582aa47),
307	(b"1\x97", 0xb4af0ae0feb08e0f),
308	(b"M\"\xdf", 0x85d7cd041a2a8a5d),
309	(b"l\xcd\x13\xd7", 0x1860820ea79b0fa3),
310	(&[0; 32], 0xcf3473434d4ecf3b),
311	(&[255; 32], 0xa0a06974c34d63c4),
312	(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F", 0xb9d9d4a8492cbd7f),
313	(&[0; 1024], 0x691bb2b09be5498a),
314	(b"hello, world!", 0xf8046e40c403f1d0),
315	];
316
317	for (input, expected) in CASES {
318	unsafe {
319	let actual = update(CRC64_NVME.init, input, &CRC64_NVME) ^ CRC64_NVME.xorout;
320
321	assert_eq!(
322	actual, *expected,
323	"Mismatch CRC, expected {:#x}, got {:#x}, input: {:?}",
324	expected, actual, input
325	);
326	}
327	}
328	}
329
330	/// Test the "crc32" variant used in PHP's hash() function, which is different from the
331	/// crc32() function. It's really just CRC-32/BZIP2 with the output byte-reversed to little
332	/// endian.
333	///
334	/// https://www.php.net/manual/en/function.hash-file.php#104836
335	#[test]
336	fn test_crc32_php_standard_vectors() {
337	static CASES: &[(&[u8], u64)] = &[
338	(b"123456789", 0x181989fc),
339	(&[0; 4096], 0xe3380088),
340	(&[255; 4096], 0x8f2ae650),
341	(b"hello, world!", 0x5eacce7),
342	];
343
344	for (input, expected) in CASES {
345	let bzip2_crc = unsafe {
346	(update(CRC32_BZIP2.init, input, &CRC32_BZIP2) ^ CRC32_BZIP2.xorout) as u32
347	};
348
349	// PHP reverses the byte order of the CRC for some reason
350	let actual = bzip2_crc.swap_bytes();
351
352	assert_eq!(
353	actual, *expected as u32,
354	"Mismatch CRC, expected {:#x}, got {:#x}, input: {:?}",
355	expected, actual, input
356	);
357	}
358	}
359
360	/// Test CRC-16/IBM-SDLC check value (reflected variant)
361	#[test]
362	fn test_crc16_ibm_sdlc_check_value() {
363	use crate::crc16::consts::CRC16_IBM_SDLC;
364
365	let actual = unsafe {
366	update(CRC16_IBM_SDLC.init, TEST_CHECK_STRING, &CRC16_IBM_SDLC) ^ CRC16_IBM_SDLC.xorout
367	};
368
369	assert_eq!(
370	actual, 0x906E,
371	"CRC-16/IBM-SDLC check value mismatch: expected 0x906E, got {:#x}",
372	actual
373	);
374	}
375
376	/// Test CRC-16/T10-DIF check value (forward/non-reflected variant)
377	#[test]
378	fn test_crc16_t10_dif_check_value() {
379	use crate::crc16::consts::CRC16_T10_DIF;
380
381	let actual = unsafe {
382	update(CRC16_T10_DIF.init, TEST_CHECK_STRING, &CRC16_T10_DIF) ^ CRC16_T10_DIF.xorout
383	};
384
385	assert_eq!(
386	actual, 0xD0DB,
387	"CRC-16/T10-DIF check value mismatch: expected 0xD0DB, got {:#x}",
388	actual
389	);
390	}
391
392	/// Skipping for Miri runs due to time constraints, underlying code already covered by other
393	/// tests.
394	#[test]
395	#[cfg_attr(miri, ignore)]
396	fn test_small_lengths_all() {
397	// Test each CRC variant
398	for config in TEST_ALL_CONFIGS {
399	// Test each length from 0 to 255
400	for len in 0..=255 {
401	test_length(len, config);
402	}
403	}
404	}
405
406	/// Skipping for Miri runs due to time constraints, underlying code already covered by other
407	/// tests.
408	#[test]
409	#[cfg_attr(miri, ignore)]
410	fn test_medium_lengths() {
411	// Test each CRC variant
412	for config in TEST_ALL_CONFIGS {
413	// Test each length from 256 to 1024, which should fold and include handling remainders
414	for len in 256..=1024 {
415	test_length(len, config);
416	}
417	}
418	}
419
420	/// Skipping for Miri runs due to time constraints, underlying code already covered by other
421	/// tests.
422	#[test]
423	#[cfg_attr(miri, ignore)]
424	fn test_large_lengths() {
425	// Test each CRC variant
426	for config in TEST_ALL_CONFIGS {
427	// Test ~1 MiB just before, at, and just after the folding boundaries
428	for len in 1048575..=1048577 {
429	test_length(len, config);
430	}
431	}
432	}
433
434	fn test_length(length: usize, config: &AnyCrcTestConfig) {
435	let mut data = vec![0u8; length];
436	rng().fill(&mut data[..]);
437
438	// Calculate expected CRC using the reference implementation
439	let expected = config.checksum_with_reference(&data);
440
441	// direct update() call, which needs XOROUT applied
442	let actual = unsafe {
443	update(config.get_init_algorithm(), &data, config.get_params()) ^ config.get_xorout()
444	};
445
446	assert_eq!(
447	actual,
448	expected,
449	"\nFailed for {} with length {}\nGot: {:016x}\nExpected: {:016x}",
450	config.get_name(),
451	length,
452	actual,
453	expected
454	);
455	}
456	}

Keyboard