OXYGENHUB
◆ push-notes oxygen / vendor / crc-fast / src / crc32 / fusion / x86 / iscsi / sse_pclmulqdq.rs
6.2 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw
1
//! Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
2
//! with the help of Claude.ai.
3
//!
4
//! MIT licensed.
5
6
#![cfg(any(target_arch = "x86_64", target_arch = "x86"))]
7
8
#[cfg(target_arch = "x86")]
9
use core::arch::x86::*;
10
11
#[cfg(target_arch = "x86_64")]
12
use core::arch::x86_64::*;
13
14
use crate::fusion::x86::*;
15
16
/// Converted to Rust from the original C code generated by https://github.com/corsix/fast-crc32/
17
/// with the help of Claude.ai using:
18
///
19
/// ./generate -i sse -p crc32c -a v4s3x3
20
///
21
/// Modified as necessary for this Rust implementation.
22
#[inline]
23
#[target_feature(enable = "sse4.2,pclmulqdq")]
24
pub unsafe fn crc32_iscsi_sse_v4s3x3(mut crc0: u32, mut buf: *const u8, mut len: usize) -> u32 {
25
    // Align to 8-byte boundary using hardware CRC32C instructions
26
    while len > 0 && (buf as usize & 7) != 0 {
27
        crc0 = _mm_crc32_u8(crc0, *buf);
28
        buf = buf.add(1);
29
        len -= 1;
30
    }
31
32
    // Handle 8-byte alignment
33
    if (buf as usize & 8) != 0 && len >= 8 {
34
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
35
        buf = buf.add(8);
36
        len -= 8;
37
    }
38
39
    if len >= 144 {
40
        let blk = (len - 8) / 136;
41
        let klen = blk * 24;
42
        let buf2 = buf;
43
        let mut crc1 = 0u32;
44
        let mut crc2 = 0u32;
45
46
        // First vector chunk - load four 128-bit vectors (64 bytes total)
47
        let mut x0 = _mm_loadu_si128(buf2 as *const __m128i);
48
        let mut x1 = _mm_loadu_si128(buf2.add(16) as *const __m128i);
49
        let mut x2 = _mm_loadu_si128(buf2.add(32) as *const __m128i);
50
        let mut x3 = _mm_loadu_si128(buf2.add(48) as *const __m128i);
51
52
        // iSCSI-specific folding constant (same as AVX-512 version)
53
        let mut k = _mm_setr_epi32(0x740eef02u32 as i32, 0, 0x9e4addf8u32 as i32, 0);
54
55
        // XOR the CRC into the first vector's low 32 bits
56
        x0 = _mm_xor_si128(_mm_cvtsi32_si128(crc0 as i32), x0);
57
        crc0 = 0;
58
59
        let mut buf2 = buf2.add(64);
60
        len -= 136;
61
        buf = buf.add(blk * 64);
62
63
        // Main loop - process 144 bytes at a time
64
        while len >= 144 {
65
            let mut y0 = clmul_lo_sse(x0, k);
66
            x0 = clmul_hi_sse(x0, k);
67
            let mut y1 = clmul_lo_sse(x1, k);
68
            x1 = clmul_hi_sse(x1, k);
69
            let mut y2 = clmul_lo_sse(x2, k);
70
            x2 = clmul_hi_sse(x2, k);
71
            let mut y3 = clmul_lo_sse(x3, k);
72
            x3 = clmul_hi_sse(x3, k);
73
74
            // XOR operations using separate XOR instructions (no ternary logic in SSE)
75
            y0 = _mm_xor_si128(y0, _mm_loadu_si128(buf2 as *const __m128i));
76
            x0 = _mm_xor_si128(x0, y0);
77
            y1 = _mm_xor_si128(y1, _mm_loadu_si128(buf2.add(16) as *const __m128i));
78
            x1 = _mm_xor_si128(x1, y1);
79
            y2 = _mm_xor_si128(y2, _mm_loadu_si128(buf2.add(32) as *const __m128i));
80
            x2 = _mm_xor_si128(x2, y2);
81
            y3 = _mm_xor_si128(y3, _mm_loadu_si128(buf2.add(48) as *const __m128i));
82
            x3 = _mm_xor_si128(x3, y3);
83
84
            // Process scalar data in parallel using hardware CRC32C
85
            crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
86
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen) as *const u64));
87
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2) as *const u64));
88
            crc0 = mm_crc32_u64(crc0, *(buf.add(8) as *const u64));
89
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 8) as *const u64));
90
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 8) as *const u64));
91
            crc0 = mm_crc32_u64(crc0, *(buf.add(16) as *const u64));
92
            crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 16) as *const u64));
93
            crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 16) as *const u64));
94
95
            buf = buf.add(24);
96
            buf2 = buf2.add(64);
97
            len -= 136;
98
        }
99
100
        // Reduce x0 ... x3 to just x0 using iSCSI-specific constants
101
        k = _mm_setr_epi32(0xf20c0dfeu32 as i32, 0, 0x493c7d27u32 as i32, 0);
102
103
        let mut y0 = clmul_lo_sse(x0, k);
104
        x0 = clmul_hi_sse(x0, k);
105
        let mut y2 = clmul_lo_sse(x2, k);
106
        x2 = clmul_hi_sse(x2, k);
107
108
        y0 = _mm_xor_si128(y0, x1);
109
        x0 = _mm_xor_si128(x0, y0);
110
        y2 = _mm_xor_si128(y2, x3);
111
        x2 = _mm_xor_si128(x2, y2);
112
113
        k = _mm_setr_epi32(0x3da6d0cbu32 as i32, 0, 0xba4fc28eu32 as i32, 0);
114
115
        y0 = clmul_lo_sse(x0, k);
116
        x0 = clmul_hi_sse(x0, k);
117
        y0 = _mm_xor_si128(y0, x2);
118
        x0 = _mm_xor_si128(x0, y0);
119
120
        // Final scalar chunk
121
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
122
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen) as *const u64));
123
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2) as *const u64));
124
        crc0 = mm_crc32_u64(crc0, *(buf.add(8) as *const u64));
125
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 8) as *const u64));
126
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 8) as *const u64));
127
        crc0 = mm_crc32_u64(crc0, *(buf.add(16) as *const u64));
128
        crc1 = mm_crc32_u64(crc1, *(buf.add(klen + 16) as *const u64));
129
        crc2 = mm_crc32_u64(crc2, *(buf.add(klen * 2 + 16) as *const u64));
130
        buf = buf.add(24);
131
132
        let vc0 = crc_shift_iscsi_sse(crc0, klen * 2 + 8);
133
        let vc1 = crc_shift_iscsi_sse(crc1, klen + 8);
134
        let mut vc = mm_extract_epi64(_mm_xor_si128(vc0, vc1), 0);
135
136
        // Reduce 128 bits to 32 bits, and multiply by x^32
137
        // Extract the two 64-bit parts of x0 and combine them
138
        let x0_low = mm_extract_epi64(x0, 0);
139
        let x0_high = mm_extract_epi64(x0, 1);
140
        let x0_combined = mm_extract_epi64(
141
            crc_shift_iscsi_sse(mm_crc32_u64(mm_crc32_u64(0, x0_low), x0_high), klen * 3 + 8),
142
            0,
143
        );
144
        vc ^= x0_combined;
145
146
        // Final 8 bytes
147
        buf = buf.add(klen * 2);
148
        crc0 = crc2;
149
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64) ^ vc);
150
        buf = buf.add(8);
151
        len -= 8;
152
    }
153
154
    // Process remaining 8-byte chunks using hardware CRC32C
155
    while len >= 8 {
156
        crc0 = mm_crc32_u64(crc0, *(buf as *const u64));
157
        buf = buf.add(8);
158
        len -= 8;
159
    }
160
161
    // Process remaining bytes using hardware CRC32C
162
    while len > 0 {
163
        crc0 = _mm_crc32_u8(crc0, *buf);
164
        buf = buf.add(1);
165
        len -= 1;
166
    }
167
168
    crc0
169
}