1use std::cell::Cell;
18use std::sync::OnceLock;
19
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub(crate) enum Path {
23 Scalar,
25 Avx2,
27 Avx512,
29 Neon,
31}
32
33impl Path {
34 pub(crate) fn name(self) -> &'static str {
36 match self {
37 Path::Scalar => "scalar",
38 Path::Avx2 => "avx2",
39 Path::Avx512 => "avx512",
40 Path::Neon => "neon",
41 }
42 }
43
44 pub(crate) fn usable(self) -> bool {
46 match self {
47 Path::Scalar => true,
48 Path::Avx2 => avx2_ready(),
49 Path::Avx512 => avx512_ready(),
50 Path::Neon => neon_ready(),
51 }
52 }
53}
54
55thread_local! {
56 #[allow(clippy::missing_const_for_thread_local)]
58 static FORCE: Cell<Option<Path>> = Cell::new(None);
59}
60
61pub(crate) fn select() -> Path {
65 if let Some(path) = FORCE.with(|slot| slot.get()) {
66 return if path.usable() { path } else { Path::Scalar };
67 }
68 if let Some(path) = env_path() {
69 return path;
70 }
71 native()
72}
73
74pub(crate) fn native() -> Path {
76 if avx512_ready() {
77 Path::Avx512
78 } else if avx2_ready() {
79 Path::Avx2
80 } else if neon_ready() {
81 Path::Neon
82 } else {
83 Path::Scalar
84 }
85}
86
87fn env_path() -> Option<Path> {
88 static CACHED: OnceLock<Option<Path>> = OnceLock::new();
89 *CACHED.get_or_init(|| {
90 let raw = std::env::var("LINKCELL_SIMD").ok()?;
91 let path = parse(raw.trim())?;
92 Some(if path.usable() { path } else { Path::Scalar })
93 })
94}
95
96pub(crate) fn parse(name: &str) -> Option<Path> {
97 match name {
98 "scalar" | "0" => Some(Path::Scalar),
99 "avx2" | "avx2fma" => Some(Path::Avx2),
100 "avx512" | "avx512f" => Some(Path::Avx512),
101 "neon" => Some(Path::Neon),
102 _ => None,
103 }
104}
105
106fn avx2_ready() -> bool {
107 #[cfg(target_arch = "x86_64")]
108 {
109 std::is_x86_feature_detected!("avx2") && std::is_x86_feature_detected!("fma")
110 }
111 #[cfg(not(target_arch = "x86_64"))]
112 {
113 false
114 }
115}
116
117fn avx512_ready() -> bool {
118 #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
119 {
120 std::is_x86_feature_detected!("avx512f")
123 && std::is_x86_feature_detected!("avx512vl")
124 && std::is_x86_feature_detected!("popcnt")
125 && std::is_x86_feature_detected!("fma")
126 }
127 #[cfg(not(all(target_arch = "x86_64", linkcell_avx512)))]
128 {
129 false
130 }
131}
132
133fn neon_ready() -> bool {
134 cfg!(target_feature = "neon")
136}
137
138pub fn simd_path() -> &'static str {
141 select().name()
142}
143
144#[cfg(feature = "timing")]
145static SCAN_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
146#[cfg(feature = "timing")]
147static KNN_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
148#[cfg(feature = "timing")]
149static MESH_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
150
151#[cfg(feature = "timing")]
155pub(crate) fn scan_timing_enabled() -> bool {
156 static ON: OnceLock<bool> = OnceLock::new();
157 *ON.get_or_init(|| std::env::var_os("LINKCELL_SCAN_TIME").is_some())
158}
159
160#[cfg(feature = "timing")]
165pub(crate) fn coarse_timing_enabled() -> bool {
166 static ON: OnceLock<bool> = OnceLock::new();
167 *ON.get_or_init(|| std::env::var_os("LINKCELL_COARSE_TIME").is_some())
168}
169
170#[cfg(feature = "timing")]
171pub(crate) fn add_scan_ns(ns: u128) {
172 SCAN_NS.fetch_add(ns as u64, std::sync::atomic::Ordering::Relaxed);
173}
174
175#[cfg(feature = "timing")]
176pub(crate) fn add_knn_ns(ns: u128) {
177 KNN_NS.fetch_add(ns as u64, std::sync::atomic::Ordering::Relaxed);
178}
179
180#[cfg(feature = "timing")]
181pub(crate) fn add_mesh_ns(ns: u128) {
182 MESH_NS.fetch_add(ns as u64, std::sync::atomic::Ordering::Relaxed);
183}
184
185#[cfg(feature = "timing")]
198#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
199pub struct SimdTiming {
200 pub mesh_ns: u64,
202 pub scan_ns: u64,
204 pub knn_ns: u64,
206}
207
208#[cfg(feature = "timing")]
210pub fn simd_timing() -> SimdTiming {
211 SimdTiming {
212 mesh_ns: MESH_NS.load(std::sync::atomic::Ordering::Relaxed),
213 scan_ns: SCAN_NS.load(std::sync::atomic::Ordering::Relaxed),
214 knn_ns: KNN_NS.load(std::sync::atomic::Ordering::Relaxed),
215 }
216}
217
218#[cfg(test)]
220pub(crate) struct ForceGuard {
221 prev: Option<Path>,
222}
223
224#[cfg(test)]
229pub(crate) fn force(path: Path) -> ForceGuard {
230 let prev = FORCE.with(|slot| slot.replace(Some(path)));
231 ForceGuard { prev }
232}
233
234#[cfg(test)]
235impl Drop for ForceGuard {
236 fn drop(&mut self) {
237 let prev = self.prev;
238 FORCE.with(|slot| slot.set(prev));
239 }
240}
241
242#[cfg(test)]
244pub(crate) fn paths_to_compare() -> Vec<Path> {
245 let mut out = vec![Path::Scalar];
246 for path in [Path::Avx2, Path::Avx512, Path::Neon] {
247 if path.usable() {
248 out.push(path);
249 }
250 }
251 out
252}
253
254#[cfg(test)]
255mod tests {
256 use super::*;
257
258 #[test]
259 fn parse_names() {
260 assert_eq!(parse("scalar"), Some(Path::Scalar));
261 assert_eq!(parse("0"), Some(Path::Scalar));
262 assert_eq!(parse("avx2"), Some(Path::Avx2));
263 assert_eq!(parse("avx2fma"), Some(Path::Avx2));
264 assert_eq!(parse("avx512"), Some(Path::Avx512));
265 assert_eq!(parse("avx512f"), Some(Path::Avx512));
266 assert_eq!(parse("neon"), Some(Path::Neon));
267 assert_eq!(parse("avx"), None);
268 assert_eq!(parse(""), None);
269 }
270
271 #[test]
272 fn native_path_is_usable() {
273 let path = native();
274 assert!(path.usable());
275 assert_eq!(path.name(), simd_path());
276 }
277
278 #[cfg(feature = "timing")]
279 #[test]
280 fn simd_timing_names_mesh_scan_and_search() {
281 assert!(std::env::var_os("LINKCELL_SCAN_TIME").is_none());
282 assert!(std::env::var_os("LINKCELL_COARSE_TIME").is_none());
283 let t = simd_timing();
284 assert_eq!(
285 t,
286 SimdTiming {
287 mesh_ns: 0,
288 scan_ns: 0,
289 knn_ns: 0,
290 }
291 );
292 }
293}