vendor/crc-fast/src/arch/x86/sse.rs at v0.11.0 · andymccall/oxygen

◆ v0.11.0 oxygen / vendor / crc-fast / src / arch / x86 / sse.rs

8.5 KiB · Rust · last changed Strata: the cloud stores send through ureq, so every release target links 1 month ago source blame history raw

// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.

//! This module provides x86-specific implementations of the ArchOps trait.
//!
//! This module is designed to work with both x86 and x86_64 architectures.
//!
//! It uses the SSE4.1 instruction sets for SIMD operations.

#![cfg(any(target_arch = "x86", target_arch = "x86_64"))]

#[cfg(target_arch = "x86")]
use core::arch::x86::*;

#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;

use crate::traits::ArchOps;

// Tier-specific ArchOps implementations for x86/x86_64

/// Base x86/x86_64 SSE+PCLMULQDQ implementation - baseline performance for both architectures
/// Uses SSE4.1 and PCLMULQDQ instructions
#[derive(Debug, Copy, Clone)]
pub struct X86SsePclmulqdqOps;

// Base implementation for x86/x86_64 SSE+PCLMULQDQ tier
impl ArchOps for X86SsePclmulqdqOps {
    type Vector = __m128i;

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn create_vector_from_u64_pair(
        &self,
        high: u64,
        low: u64,
        reflected: bool,
    ) -> Self::Vector {
        // Note order is different from AArch64
        if reflected {
            self.set_epi64x(low, high)
        } else {
            self.set_epi64x(high, low)
        }
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn create_vector_from_u64_pair_non_reflected(
        &self,
        high: u64,
        low: u64,
    ) -> Self::Vector {
        // Note order is different from AArch64
        self.set_epi64x(high, low)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn create_vector_from_u64(&self, value: u64, high: bool) -> Self::Vector {
        self.create_u64_vector(value, high)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_u64s(&self, vector: Self::Vector) -> [u64; 2] {
        [self.extract_u64_low(vector), self.extract_u64_high(vector)]
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_poly64s(&self, vector: Self::Vector) -> [u64; 2] {
        // On x86, poly64s and u64s extraction is the same
        self.extract_u64s(vector)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn xor_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_xor_si128(a, b)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn load_bytes(&self, ptr: *const u8) -> Self::Vector {
        _mm_loadu_si128(ptr as *const __m128i)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn load_aligned(&self, ptr: *const [u64; 2]) -> Self::Vector {
        _mm_loadu_si128(ptr as *const __m128i)
    }

    #[inline]
    #[target_feature(enable = "ssse3")]
    unsafe fn shuffle_bytes(&self, data: Self::Vector, mask: Self::Vector) -> Self::Vector {
        _mm_shuffle_epi8(data, mask)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn blend_vectors(
        &self,
        a: Self::Vector,
        b: Self::Vector,
        mask: Self::Vector,
    ) -> Self::Vector {
        _mm_blendv_epi8(a, b, mask)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_8(&self, vector: Self::Vector) -> Self::Vector {
        _mm_slli_si128(vector, 8)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn set_all_bytes(&self, value: u8) -> Self::Vector {
        _mm_set1_epi8(value as i8)
    }

    #[inline(always)]
    unsafe fn create_compare_mask(&self, vector: Self::Vector) -> Self::Vector {
        // On x86, MSB is already used for blending, so we just return the vector
        vector
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn and_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_and_si128(a, b)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_32(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 4)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_32(&self, vector: Self::Vector) -> Self::Vector {
        _mm_slli_si128(vector, 4)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn create_vector_from_u32(&self, value: u32, high: bool) -> Self::Vector {
        if high {
            _mm_insert_epi32(_mm_set1_epi32(0), value as i32, 3)
        } else {
            _mm_set_epi32(0, 0, 0, value as i32)
        }
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_4(&self, vector: Self::Vector) -> Self::Vector {
        _mm_slli_si128(vector, 4)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_4(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 4)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_8(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 8)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_5(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 5)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_6(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 6)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_7(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 7)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_right_12(&self, vector: Self::Vector) -> Self::Vector {
        _mm_srli_si128(vector, 12)
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn shift_left_12(&self, vector: Self::Vector) -> Self::Vector {
        _mm_slli_si128(vector, 12)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_00(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_clmulepi64_si128(a, b, 0x00)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_01(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_clmulepi64_si128(a, b, 0x01)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_10(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_clmulepi64_si128(a, b, 0x10)
    }

    #[inline]
    #[target_feature(enable = "pclmulqdq")]
    unsafe fn carryless_mul_11(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
        _mm_clmulepi64_si128(a, b, 0x11)
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn xor3_vectors(
        &self,
        a: Self::Vector,
        b: Self::Vector,
        c: Self::Vector,
    ) -> Self::Vector {
        // SSE tier always uses two XOR operations
        _mm_xor_si128(_mm_xor_si128(a, b), c)
    }
}

impl X86SsePclmulqdqOps {
    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn set_epi64x(&self, e1: u64, e0: u64) -> __m128i {
        #[cfg(target_arch = "x86_64")]
        {
            _mm_set_epi64x(e1 as i64, e0 as i64)
        }

        #[cfg(target_arch = "x86")]
        {
            // _mm_set_epi32 takes (highest, higher, lower, lowest)
            // We need to ensure e0 is in the lower 64 bits and e1 in the higher 64 bits
            let lo = _mm_set_epi32(0, 0, (e0 >> 32) as i32, e0 as i32);
            let hi = _mm_set_epi32(0, 0, (e1 >> 32) as i32, e1 as i32);

            _mm_unpacklo_epi64(lo, hi)
        }
    }

    #[inline]
    #[target_feature(enable = "sse4.1")]
    unsafe fn create_u64_vector(&self, value: u64, high: bool) -> __m128i {
        if high {
            self.set_epi64x(value, 0)
        } else {
            self.set_epi64x(0, value)
        }
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_u64_low(&self, v: __m128i) -> u64 {
        #[cfg(target_arch = "x86_64")]
        {
            _mm_cvtsi128_si64(v) as u64
        }

        #[cfg(target_arch = "x86")]
        {
            let lo = _mm_cvtsi128_si32(v) as u32 as u64;
            let hi = _mm_cvtsi128_si32(_mm_srli_si128(v, 4)) as u32 as u64;
            lo | (hi << 32)
        }
    }

    #[inline]
    #[target_feature(enable = "sse2")]
    unsafe fn extract_u64_high(&self, v: __m128i) -> u64 {
        #[cfg(target_arch = "x86_64")]
        {
            _mm_cvtsi128_si64(_mm_srli_si128(v, 8)) as u64
        }

        #[cfg(target_arch = "x86")]
        {
            let lo = _mm_cvtsi128_si32(_mm_srli_si128(v, 8)) as u32 as u64;
            let hi = _mm_cvtsi128_si32(_mm_srli_si128(v, 12)) as u32 as u64;
            lo | (hi << 32)
        }
    }
}

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	// Copyright 2025 Don MacAskill. Licensed under MIT or Apache-2.0.
2
3	//! This module provides x86-specific implementations of the ArchOps trait.
4	//!
5	//! This module is designed to work with both x86 and x86_64 architectures.
6	//!
7	//! It uses the SSE4.1 instruction sets for SIMD operations.
8
9	#![cfg(any(target_arch = "x86", target_arch = "x86_64"))]
10
11	#[cfg(target_arch = "x86")]
12	use core::arch::x86::*;
13
14	#[cfg(target_arch = "x86_64")]
15	use core::arch::x86_64::*;
16
17	use crate::traits::ArchOps;
18
19	// Tier-specific ArchOps implementations for x86/x86_64
20
21	/// Base x86/x86_64 SSE+PCLMULQDQ implementation - baseline performance for both architectures
22	/// Uses SSE4.1 and PCLMULQDQ instructions
23	#[derive(Debug, Copy, Clone)]
24	pub struct X86SsePclmulqdqOps;
25
26	// Base implementation for x86/x86_64 SSE+PCLMULQDQ tier
27	impl ArchOps for X86SsePclmulqdqOps {
28	type Vector = __m128i;
29
30	#[inline]
31	#[target_feature(enable = "sse2")]
32	unsafe fn create_vector_from_u64_pair(
33	&self,
34	high: u64,
35	low: u64,
36	reflected: bool,
37	) -> Self::Vector {
38	// Note order is different from AArch64
39	if reflected {
40	self.set_epi64x(low, high)
41	} else {
42	self.set_epi64x(high, low)
43	}
44	}
45
46	#[inline]
47	#[target_feature(enable = "sse2")]
48	unsafe fn create_vector_from_u64_pair_non_reflected(
49	&self,
50	high: u64,
51	low: u64,
52	) -> Self::Vector {
53	// Note order is different from AArch64
54	self.set_epi64x(high, low)
55	}
56
57	#[inline]
58	#[target_feature(enable = "sse4.1")]
59	unsafe fn create_vector_from_u64(&self, value: u64, high: bool) -> Self::Vector {
60	self.create_u64_vector(value, high)
61	}
62
63	#[inline]
64	#[target_feature(enable = "sse2")]
65	unsafe fn extract_u64s(&self, vector: Self::Vector) -> [u64; 2] {
66	[self.extract_u64_low(vector), self.extract_u64_high(vector)]
67	}
68
69	#[inline]
70	#[target_feature(enable = "sse2")]
71	unsafe fn extract_poly64s(&self, vector: Self::Vector) -> [u64; 2] {
72	// On x86, poly64s and u64s extraction is the same
73	self.extract_u64s(vector)
74	}
75
76	#[inline]
77	#[target_feature(enable = "sse2")]
78	unsafe fn xor_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
79	_mm_xor_si128(a, b)
80	}
81
82	#[inline]
83	#[target_feature(enable = "sse2")]
84	unsafe fn load_bytes(&self, ptr: *const u8) -> Self::Vector {
85	_mm_loadu_si128(ptr as *const __m128i)
86	}
87
88	#[inline]
89	#[target_feature(enable = "sse2")]
90	unsafe fn load_aligned(&self, ptr: *const [u64; 2]) -> Self::Vector {
91	_mm_loadu_si128(ptr as *const __m128i)
92	}
93
94	#[inline]
95	#[target_feature(enable = "ssse3")]
96	unsafe fn shuffle_bytes(&self, data: Self::Vector, mask: Self::Vector) -> Self::Vector {
97	_mm_shuffle_epi8(data, mask)
98	}
99
100	#[inline]
101	#[target_feature(enable = "sse4.1")]
102	unsafe fn blend_vectors(
103	&self,
104	a: Self::Vector,
105	b: Self::Vector,
106	mask: Self::Vector,
107	) -> Self::Vector {
108	_mm_blendv_epi8(a, b, mask)
109	}
110
111	#[inline]
112	#[target_feature(enable = "sse2")]
113	unsafe fn shift_left_8(&self, vector: Self::Vector) -> Self::Vector {
114	_mm_slli_si128(vector, 8)
115	}
116
117	#[inline]
118	#[target_feature(enable = "sse2")]
119	unsafe fn set_all_bytes(&self, value: u8) -> Self::Vector {
120	_mm_set1_epi8(value as i8)
121	}
122
123	#[inline(always)]
124	unsafe fn create_compare_mask(&self, vector: Self::Vector) -> Self::Vector {
125	// On x86, MSB is already used for blending, so we just return the vector
126	vector
127	}
128
129	#[inline]
130	#[target_feature(enable = "sse2")]
131	unsafe fn and_vectors(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
132	_mm_and_si128(a, b)
133	}
134
135	#[inline]
136	#[target_feature(enable = "sse2")]
137	unsafe fn shift_right_32(&self, vector: Self::Vector) -> Self::Vector {
138	_mm_srli_si128(vector, 4)
139	}
140
141	#[inline]
142	#[target_feature(enable = "sse2")]
143	unsafe fn shift_left_32(&self, vector: Self::Vector) -> Self::Vector {
144	_mm_slli_si128(vector, 4)
145	}
146
147	#[inline]
148	#[target_feature(enable = "sse4.1")]
149	unsafe fn create_vector_from_u32(&self, value: u32, high: bool) -> Self::Vector {
150	if high {
151	_mm_insert_epi32(_mm_set1_epi32(0), value as i32, 3)
152	} else {
153	_mm_set_epi32(0, 0, 0, value as i32)
154	}
155	}
156
157	#[inline]
158	#[target_feature(enable = "sse2")]
159	unsafe fn shift_left_4(&self, vector: Self::Vector) -> Self::Vector {
160	_mm_slli_si128(vector, 4)
161	}
162
163	#[inline]
164	#[target_feature(enable = "sse2")]
165	unsafe fn shift_right_4(&self, vector: Self::Vector) -> Self::Vector {
166	_mm_srli_si128(vector, 4)
167	}
168
169	#[inline]
170	#[target_feature(enable = "sse2")]
171	unsafe fn shift_right_8(&self, vector: Self::Vector) -> Self::Vector {
172	_mm_srli_si128(vector, 8)
173	}
174
175	#[inline]
176	#[target_feature(enable = "sse2")]
177	unsafe fn shift_right_5(&self, vector: Self::Vector) -> Self::Vector {
178	_mm_srli_si128(vector, 5)
179	}
180
181	#[inline]
182	#[target_feature(enable = "sse2")]
183	unsafe fn shift_right_6(&self, vector: Self::Vector) -> Self::Vector {
184	_mm_srli_si128(vector, 6)
185	}
186
187	#[inline]
188	#[target_feature(enable = "sse2")]
189	unsafe fn shift_right_7(&self, vector: Self::Vector) -> Self::Vector {
190	_mm_srli_si128(vector, 7)
191	}
192
193	#[inline]
194	#[target_feature(enable = "sse2")]
195	unsafe fn shift_right_12(&self, vector: Self::Vector) -> Self::Vector {
196	_mm_srli_si128(vector, 12)
197	}
198
199	#[inline]
200	#[target_feature(enable = "sse2")]
201	unsafe fn shift_left_12(&self, vector: Self::Vector) -> Self::Vector {
202	_mm_slli_si128(vector, 12)
203	}
204
205	#[inline]
206	#[target_feature(enable = "pclmulqdq")]
207	unsafe fn carryless_mul_00(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
208	_mm_clmulepi64_si128(a, b, 0x00)
209	}
210
211	#[inline]
212	#[target_feature(enable = "pclmulqdq")]
213	unsafe fn carryless_mul_01(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
214	_mm_clmulepi64_si128(a, b, 0x01)
215	}
216
217	#[inline]
218	#[target_feature(enable = "pclmulqdq")]
219	unsafe fn carryless_mul_10(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
220	_mm_clmulepi64_si128(a, b, 0x10)
221	}
222
223	#[inline]
224	#[target_feature(enable = "pclmulqdq")]
225	unsafe fn carryless_mul_11(&self, a: Self::Vector, b: Self::Vector) -> Self::Vector {
226	_mm_clmulepi64_si128(a, b, 0x11)
227	}
228
229	#[inline]
230	#[target_feature(enable = "sse4.1")]
231	unsafe fn xor3_vectors(
232	&self,
233	a: Self::Vector,
234	b: Self::Vector,
235	c: Self::Vector,
236	) -> Self::Vector {
237	// SSE tier always uses two XOR operations
238	_mm_xor_si128(_mm_xor_si128(a, b), c)
239	}
240	}
241
242	impl X86SsePclmulqdqOps {
243	#[inline]
244	#[target_feature(enable = "sse2")]
245	unsafe fn set_epi64x(&self, e1: u64, e0: u64) -> __m128i {
246	#[cfg(target_arch = "x86_64")]
247	{
248	_mm_set_epi64x(e1 as i64, e0 as i64)
249	}
250
251	#[cfg(target_arch = "x86")]
252	{
253	// _mm_set_epi32 takes (highest, higher, lower, lowest)
254	// We need to ensure e0 is in the lower 64 bits and e1 in the higher 64 bits
255	let lo = _mm_set_epi32(0, 0, (e0 >> 32) as i32, e0 as i32);
256	let hi = _mm_set_epi32(0, 0, (e1 >> 32) as i32, e1 as i32);
257
258	_mm_unpacklo_epi64(lo, hi)
259	}
260	}
261
262	#[inline]
263	#[target_feature(enable = "sse4.1")]
264	unsafe fn create_u64_vector(&self, value: u64, high: bool) -> __m128i {
265	if high {
266	self.set_epi64x(value, 0)
267	} else {
268	self.set_epi64x(0, value)
269	}
270	}
271
272	#[inline]
273	#[target_feature(enable = "sse2")]
274	unsafe fn extract_u64_low(&self, v: __m128i) -> u64 {
275	#[cfg(target_arch = "x86_64")]
276	{
277	_mm_cvtsi128_si64(v) as u64
278	}
279
280	#[cfg(target_arch = "x86")]
281	{
282	let lo = _mm_cvtsi128_si32(v) as u32 as u64;
283	let hi = _mm_cvtsi128_si32(_mm_srli_si128(v, 4)) as u32 as u64;
284	lo \| (hi << 32)
285	}
286	}
287
288	#[inline]
289	#[target_feature(enable = "sse2")]
290	unsafe fn extract_u64_high(&self, v: __m128i) -> u64 {
291	#[cfg(target_arch = "x86_64")]
292	{
293	_mm_cvtsi128_si64(_mm_srli_si128(v, 8)) as u64
294	}
295
296	#[cfg(target_arch = "x86")]
297	{
298	let lo = _mm_cvtsi128_si32(_mm_srli_si128(v, 8)) as u32 as u64;
299	let hi = _mm_cvtsi128_si32(_mm_srli_si128(v, 12)) as u32 as u64;
300	lo \| (hi << 32)
301	}
302	}
303	}

Keyboard