Skip to main content

linkcell/
skin_simd.rs

1//! SoA candidate filter for a Verlet skin.
2//!
3//! Candidates are grouped by source atom, the same blocking vesin uses
4//! for a Verlet list. Squared distances are written once, in candidate
5//! order. AVX-512 gathers the eight target coordinates of a chunk and
6//! adds `H S` in registers. Threads then compact disjoint atom ranges
7//! into the output columns.
8
9#![allow(
10    clippy::incompatible_msrv,
11    clippy::too_many_arguments,
12    clippy::needless_range_loop
13)]
14
15use crate::cell::Cell;
16use crate::pairs::PairColumns;
17use crate::simd::{self, Path};
18use crate::Error;
19
20/// Candidates of one frame, grouped by source atom.
21pub(crate) struct Soa {
22    /// `offsets[i]..offsets[i + 1]` are the candidates whose source is `i`.
23    offsets: Vec<u32>,
24    j: Vec<u32>,
25    sx: Vec<i32>,
26    sy: Vec<i32>,
27    sz: Vec<i32>,
28    /// Packed shift in `[-3, 3]^3` when every candidate fits.
29    /// Empty when [`Self::coded`] is false and the full shift columns are used.
30    code: Vec<u8>,
31    coded: bool,
32}
33
34/// Integer code of shift `(0, 0, 0)` in [`ISHIFT`].
35#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
36const ZERO_CODE: u8 = (3 * 49 + 3 * 7 + 3) as u8;
37
38/// Eight zero-shift codes, for the aligned fast path.
39#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
40const ZERO8: u64 = {
41    let b = ZERO_CODE as u64;
42    b | (b << 8) | (b << 16) | (b << 24) | (b << 32) | (b << 40) | (b << 48) | (b << 56)
43};
44
45const fn ishift_table() -> [[i32; 3]; 343] {
46    let mut t = [[0; 3]; 343];
47    let mut s = 0;
48    while s < 343 {
49        let sx = (s / 49) as i32 - 3;
50        let sy = ((s % 49) / 7) as i32 - 3;
51        let sz = (s % 7) as i32 - 3;
52        t[s] = [sx, sy, sz];
53        s += 1;
54    }
55    t
56}
57
58/// `code -> (sx, sy, sz)` for shifts inside `[-3, 3]`.
59const ISHIFT: [[i32; 3]; 343] = ishift_table();
60
61fn encode_shift(sx: i32, sy: i32, sz: i32) -> Option<u8> {
62    if !(-3..=3).contains(&sx) || !(-3..=3).contains(&sy) || !(-3..=3).contains(&sz) {
63        return None;
64    }
65    Some(((sx + 3) as u8) * 49 + ((sy + 3) as u8) * 7 + (sz + 3) as u8)
66}
67
68impl Soa {
69    pub(crate) fn empty() -> Self {
70        Self {
71            offsets: vec![0],
72            j: Vec::new(),
73            sx: Vec::new(),
74            sy: Vec::new(),
75            sz: Vec::new(),
76            code: Vec::new(),
77            coded: true,
78        }
79    }
80
81    /// Group `cols` by source atom. Order inside an atom is the order
82    /// the columns arrived in.
83    pub(crate) fn from_columns(cols: &PairColumns, n_atoms: usize) -> Result<Self, Error> {
84        let n = cols.len();
85        if cols.j.len() != n || cols.shift.len() != n {
86            return Err(Error::BufferSize);
87        }
88        let mut counts = vec![0u32; n_atoms];
89        for &i in &cols.i {
90            let i = i as usize;
91            if i >= n_atoms {
92                return Err(Error::Overflow);
93            }
94            counts[i] = counts[i].checked_add(1).ok_or(Error::Overflow)?;
95        }
96        let mut offsets = vec![0u32; n_atoms + 1];
97        for i in 0..n_atoms {
98            offsets[i + 1] = offsets[i].checked_add(counts[i]).ok_or(Error::Overflow)?;
99        }
100        let mut cursor = offsets.clone();
101        let mut j = vec![0u32; n];
102        let mut sx = vec![0i32; n];
103        let mut sy = vec![0i32; n];
104        let mut sz = vec![0i32; n];
105        let mut code = vec![0u8; n];
106        let mut coded = true;
107        for k in 0..n {
108            let i = cols.i[k] as usize;
109            let at = cursor[i] as usize;
110            cursor[i] += 1;
111            j[at] = cols.j[k] as u32;
112            let s = cols.shift[k];
113            sx[at] = s[0];
114            sy[at] = s[1];
115            sz[at] = s[2];
116            if coded {
117                if let Some(c) = encode_shift(s[0], s[1], s[2]) {
118                    code[at] = c;
119                } else {
120                    coded = false;
121                }
122            }
123        }
124        if !coded {
125            code.clear();
126        }
127        Ok(Self {
128            offsets,
129            j,
130            sx,
131            sy,
132            sz,
133            code,
134            coded,
135        })
136    }
137
138    /// `(source, target)` of every candidate, source-major.
139    pub(crate) fn each_end(&self, mut f: impl FnMut(u32, u32)) {
140        let atoms = self.offsets.len().saturating_sub(1);
141        for i in 0..atoms {
142            let a = self.offsets[i] as usize;
143            let b = self.offsets[i + 1] as usize;
144            for k in a..b {
145                f(i as u32, self.j[k]);
146            }
147        }
148    }
149
150    /// Keep rows with squared distance strictly below `cut2`.
151    pub(crate) fn filter(
152        &self,
153        xyz: &[[f64; 3]],
154        cell: &Cell,
155        cut2: f64,
156        want_vector: bool,
157        d2: &mut Vec<f64>,
158        out: &mut PairColumns,
159        vectors: &mut Vec<[f64; 3]>,
160    ) -> Result<(), Error> {
161        let atoms = self.offsets.len().saturating_sub(1);
162        if xyz.len() != atoms {
163            return Err(Error::BufferSize);
164        }
165        let n = self.j.len();
166        let threads = filter_threads(n);
167        let ranges = atom_ranges(&self.offsets, threads);
168        let use_avx = simd::select() == Path::Avx512;
169        let ortho = cell.is_ortho();
170        let h = cell.h();
171        let widths = cell.widths();
172        if self.coded && !want_vector {
173            vectors.clear();
174            return filter_coded(
175                xyz,
176                &self.offsets,
177                &self.j,
178                &self.code,
179                &ranges,
180                cut2,
181                ortho,
182                widths,
183                h,
184                use_avx,
185                d2,
186                out,
187            );
188        }
189        if d2.len() < n {
190            d2.resize(n, 0.0);
191        }
192        let counts = count_ranges(
193            xyz,
194            &self.offsets,
195            &self.j,
196            &self.sx,
197            &self.sy,
198            &self.sz,
199            &ranges,
200            cut2,
201            ortho,
202            widths,
203            h,
204            use_avx,
205            d2,
206        );
207        let mut off = Vec::with_capacity(ranges.len() + 1);
208        off.push(0usize);
209        for c in &counts {
210            off.push(off[off.len() - 1] + c);
211        }
212        let total = *off.last().unwrap_or(&0);
213        prepare(out, vectors, total, want_vector);
214        write_ranges(
215            xyz,
216            self,
217            &ranges,
218            &off,
219            d2,
220            cut2,
221            want_vector,
222            ortho,
223            widths,
224            h,
225            use_avx,
226            out,
227            vectors,
228        );
229        Ok(())
230    }
231
232    pub(crate) fn is_coded(&self) -> bool {
233        self.coded
234    }
235
236    /// Filter every bead from this candidate list in one pass.
237    ///
238    /// `xyz` is bead-major, `beads * atoms` points. A chunk of candidates
239    /// is loaded once; each bead then gathers its own coordinates into the
240    /// same lanes and keeps the rows under `cut2`.
241    pub(crate) fn filter_beads(
242        &self,
243        xyz: &[[f64; 3]],
244        beads: usize,
245        cell: &Cell,
246        cut2: f64,
247        outs: &mut [PairColumns],
248    ) -> Result<(), Error> {
249        let atoms = self.offsets.len().saturating_sub(1);
250        if beads == 0 || outs.len() != beads || xyz.len() != beads * atoms || !self.coded {
251            return Err(Error::BufferSize);
252        }
253        let threads = filter_threads(self.j.len().saturating_mul(beads.max(1)));
254        let ranges = atom_ranges(&self.offsets, threads);
255        let use_avx = simd::select() == Path::Avx512;
256        let (tx, ty, tz) = cartesian_tables(cell.is_ortho(), cell.widths(), cell.h());
257        let counts = bead_pass::<false>(
258            xyz,
259            atoms,
260            beads,
261            &self.offsets,
262            &self.j,
263            &self.code,
264            &ranges,
265            &[],
266            cut2,
267            &tx,
268            &ty,
269            &tz,
270            use_avx,
271            &[],
272        );
273        let mut cursor = vec![0usize; beads];
274        let mut off = vec![vec![0usize; ranges.len()]; beads];
275        for (t, row) in counts.iter().enumerate() {
276            for b in 0..beads {
277                off[b][t] = cursor[b];
278                cursor[b] += row[b];
279            }
280        }
281        let mut vectors = Vec::new();
282        let mut ptrs = Vec::with_capacity(beads);
283        for b in 0..beads {
284            prepare(&mut outs[b], &mut vectors, cursor[b], false);
285            ptrs.push((
286                outs[b].i.as_mut_ptr() as usize,
287                outs[b].j.as_mut_ptr() as usize,
288                outs[b].shift.as_mut_ptr() as usize,
289                outs[b].dist2.as_mut_ptr() as usize,
290            ));
291        }
292        let written = bead_pass::<true>(
293            xyz,
294            atoms,
295            beads,
296            &self.offsets,
297            &self.j,
298            &self.code,
299            &ranges,
300            &off,
301            cut2,
302            &tx,
303            &ty,
304            &tz,
305            use_avx,
306            &ptrs,
307        );
308        debug_assert_eq!(written, counts);
309        Ok(())
310    }
311}
312
313fn cartesian_tables(
314    ortho: bool,
315    widths: [f64; 3],
316    h: [[f64; 3]; 3],
317) -> ([f64; 343], [f64; 343], [f64; 343]) {
318    let mut tx = [0.0; 343];
319    let mut ty = [0.0; 343];
320    let mut tz = [0.0; 343];
321    for (s, shift) in ISHIFT.iter().enumerate() {
322        let (x, y, z) = shift_xyz(shift[0], shift[1], shift[2], ortho, widths, h);
323        tx[s] = x;
324        ty[s] = y;
325        tz[s] = z;
326    }
327    (tx, ty, tz)
328}
329
330fn filter_coded(
331    xyz: &[[f64; 3]],
332    offsets: &[u32],
333    j: &[u32],
334    code: &[u8],
335    ranges: &[(usize, usize)],
336    cut2: f64,
337    ortho: bool,
338    widths: [f64; 3],
339    h: [[f64; 3]; 3],
340    use_avx: bool,
341    _d2: &mut Vec<f64>,
342    out: &mut PairColumns,
343) -> Result<(), Error> {
344    let (tx, ty, tz) = cartesian_tables(ortho, widths, h);
345    let counts = coded_pass::<false>(
346        xyz,
347        offsets,
348        j,
349        code,
350        ranges,
351        &[],
352        cut2,
353        &tx,
354        &ty,
355        &tz,
356        use_avx,
357        0,
358        0,
359        0,
360        0,
361        0,
362    );
363    let mut off = Vec::with_capacity(ranges.len() + 1);
364    off.push(0usize);
365    for c in &counts {
366        off.push(off[off.len() - 1] + c);
367    }
368    let total = *off.last().unwrap_or(&0);
369    let mut vectors = Vec::new();
370    prepare(out, &mut vectors, total, false);
371    let written = coded_pass::<true>(
372        xyz,
373        offsets,
374        j,
375        code,
376        ranges,
377        &off,
378        cut2,
379        &tx,
380        &ty,
381        &tz,
382        use_avx,
383        out.i.as_mut_ptr() as usize,
384        out.j.as_mut_ptr() as usize,
385        out.shift.as_mut_ptr() as usize,
386        out.dist2.as_mut_ptr() as usize,
387        0,
388    );
389    debug_assert_eq!(written, counts);
390    Ok(())
391}
392
393fn coded_pass<const WRITE: bool>(
394    xyz: &[[f64; 3]],
395    offsets: &[u32],
396    j: &[u32],
397    code: &[u8],
398    ranges: &[(usize, usize)],
399    off: &[usize],
400    cut2: f64,
401    tx: &[f64; 343],
402    ty: &[f64; 343],
403    tz: &[f64; 343],
404    use_avx: bool,
405    ip: usize,
406    jp: usize,
407    sp: usize,
408    dp: usize,
409    cand: usize,
410) -> Vec<usize> {
411    let run = |t: usize, lo: usize, hi: usize| -> usize {
412        let at = if WRITE { off[t] } else { 0 };
413        #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
414        if use_avx {
415            return unsafe {
416                coded_avx::<WRITE>(
417                    xyz, offsets, j, code, lo, hi, cut2, tx, ty, tz, at, ip, jp, sp, dp, cand,
418                )
419            };
420        }
421        let _ = use_avx;
422        coded_scalar::<WRITE>(
423            xyz, offsets, j, code, lo, hi, cut2, tx, ty, tz, at, ip, jp, sp, dp, cand,
424        )
425    };
426    #[cfg(feature = "parallel")]
427    if ranges.len() > 1 {
428        use rayon::prelude::*;
429        return ranges
430            .par_iter()
431            .enumerate()
432            .map(|(t, &(lo, hi))| run(t, lo, hi))
433            .collect();
434    }
435    ranges
436        .iter()
437        .enumerate()
438        .map(|(t, &(lo, hi))| run(t, lo, hi))
439        .collect()
440}
441
442fn coded_scalar<const WRITE: bool>(
443    xyz: &[[f64; 3]],
444    offsets: &[u32],
445    j: &[u32],
446    code: &[u8],
447    lo: usize,
448    hi: usize,
449    cut2: f64,
450    tx: &[f64; 343],
451    ty: &[f64; 343],
452    tz: &[f64; 343],
453    mut at: usize,
454    ip: usize,
455    jp: usize,
456    sp: usize,
457    dp: usize,
458    cand: usize,
459) -> usize {
460    let mut hits = 0usize;
461    for atom in lo..hi {
462        let p = xyz[atom];
463        let a = offsets[atom] as usize;
464        let b = offsets[atom + 1] as usize;
465        for k in a..b {
466            let q = xyz[j[k] as usize];
467            let c = code[k] as usize;
468            let dx = q[0] + tx[c] - p[0];
469            let dy = q[1] + ty[c] - p[1];
470            let dz = q[2] + tz[c] - p[2];
471            let dist = (dx * dx + dy * dy) + dz * dz;
472            let _ = cand;
473            if dist < cut2 {
474                if WRITE {
475                    unsafe {
476                        *(ip as *mut i32).add(at) = atom as i32;
477                        *(jp as *mut i32).add(at) = j[k] as i32;
478                        *(sp as *mut [i32; 3]).add(at) = ISHIFT[c];
479                        *(dp as *mut f64).add(at) = dist;
480                    }
481                }
482                at += 1;
483                hits += 1;
484            }
485        }
486    }
487    hits
488}
489
490#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
491#[target_feature(enable = "avx512f,avx512vl,avx2")]
492unsafe fn coded_avx<const WRITE: bool>(
493    xyz: &[[f64; 3]],
494    offsets: &[u32],
495    j: &[u32],
496    code: &[u8],
497    lo: usize,
498    hi: usize,
499    cut2: f64,
500    tx: &[f64; 343],
501    ty: &[f64; 343],
502    tz: &[f64; 343],
503    mut at: usize,
504    ip: usize,
505    jp: usize,
506    sp: usize,
507    dp: usize,
508    cand: usize,
509) -> usize {
510    use std::arch::x86_64::{
511        __m128i, __m256i, __m512i, _mm256_add_epi32, _mm256_cvtepu8_epi32, _mm256_loadu_si256,
512        _mm256_mask_compressstoreu_epi32, _mm256_maskz_compress_epi32, _mm256_mullo_epi32,
513        _mm256_set1_epi32, _mm256_setzero_si256, _mm256_storeu_si256, _mm512_add_pd,
514        _mm512_cmp_pd_mask, _mm512_i32gather_pd, _mm512_mask_compressstoreu_pd, _mm512_mul_pd,
515        _mm512_set1_pd, _mm512_setzero_si512, _mm512_storeu_pd, _mm512_storeu_si512, _mm512_sub_pd,
516        _mm_loadl_epi64, _mm_prefetch, _CMP_LT_OQ, _MM_HINT_T0,
517    };
518    let mut hits = 0usize;
519    let cut = _mm512_set1_pd(cut2);
520    let bytes = xyz.as_ptr() as *const f64;
521    let txp = tx.as_ptr();
522    let typ = ty.as_ptr();
523    let tzp = tz.as_ptr();
524    let three = _mm256_set1_epi32(3);
525    let one = _mm256_set1_epi32(1);
526    let two = _mm256_set1_epi32(2);
527    for atom in lo..hi {
528        let p = xyz[atom];
529        let px = _mm512_set1_pd(p[0]);
530        let py = _mm512_set1_pd(p[1]);
531        let pz = _mm512_set1_pd(p[2]);
532        let a = offsets[atom] as usize;
533        let b = offsets[atom + 1] as usize;
534        let mut t = a;
535        while t + 8 <= b {
536            if t + 16 <= b {
537                for s in 0..8 {
538                    let jj = unsafe { *j.as_ptr().add(t + 8 + s) } as usize;
539                    unsafe {
540                        _mm_prefetch(bytes.add(jj * 3) as *const i8, _MM_HINT_T0);
541                        _mm_prefetch(bytes.add(jj * 3 + 2) as *const i8, _MM_HINT_T0);
542                    }
543                }
544            }
545            let raw = unsafe { (code.as_ptr().add(t) as *const u64).read_unaligned() };
546            let zero = raw == ZERO8;
547            let jv = unsafe { _mm256_loadu_si256(j.as_ptr().add(t) as *const __m256i) };
548            let ix = _mm256_mullo_epi32(jv, three);
549            let qx = unsafe { _mm512_i32gather_pd(ix, bytes, 8) };
550            let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), bytes, 8) };
551            let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), bytes, 8) };
552            let (dx, dy, dz, c32) = if zero {
553                (
554                    _mm512_sub_pd(qx, px),
555                    _mm512_sub_pd(qy, py),
556                    _mm512_sub_pd(qz, pz),
557                    _mm256_set1_epi32(i32::from(ZERO_CODE)),
558                )
559            } else {
560                let c32 = _mm256_cvtepu8_epi32(unsafe {
561                    _mm_loadl_epi64(code.as_ptr().add(t) as *const __m128i)
562                });
563                let sx = unsafe { _mm512_i32gather_pd(c32, txp, 8) };
564                let sy = unsafe { _mm512_i32gather_pd(c32, typ, 8) };
565                let sz = unsafe { _mm512_i32gather_pd(c32, tzp, 8) };
566                (
567                    _mm512_sub_pd(_mm512_add_pd(qx, sx), px),
568                    _mm512_sub_pd(_mm512_add_pd(qy, sy), py),
569                    _mm512_sub_pd(_mm512_add_pd(qz, sz), pz),
570                    c32,
571                )
572            };
573            let dist = _mm512_add_pd(
574                _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
575                _mm512_mul_pd(dz, dz),
576            );
577            let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
578            let n = mask.count_ones() as usize;
579            hits += n;
580            let _ = cand;
581            if WRITE && n > 0 {
582                unsafe {
583                    let iptr = ip as *mut i32;
584                    let jptr = jp as *mut i32;
585                    let sptr = sp as *mut [i32; 3];
586                    let dptr = dp as *mut f64;
587                    let iv = _mm256_set1_epi32(atom as i32);
588                    if n == 8 && zero {
589                        _mm256_storeu_si256(iptr.add(at) as *mut __m256i, iv);
590                        _mm256_storeu_si256(jptr.add(at) as *mut __m256i, jv);
591                        _mm512_storeu_pd(dptr.add(at), dist);
592                        let sbase = (sptr as *mut i32).add(at * 3);
593                        _mm512_storeu_si512(sbase as *mut __m512i, _mm512_setzero_si512());
594                        _mm256_storeu_si256(sbase.add(16) as *mut __m256i, _mm256_setzero_si256());
595                    } else {
596                        _mm256_mask_compressstoreu_epi32(iptr.add(at), mask, iv);
597                        _mm256_mask_compressstoreu_epi32(jptr.add(at), mask, jv);
598                        _mm512_mask_compressstoreu_pd(dptr.add(at), mask, dist);
599                        if zero {
600                            for hhit in 0..n {
601                                *sptr.add(at + hhit) = [0, 0, 0];
602                            }
603                        } else {
604                            let packed = _mm256_maskz_compress_epi32(mask, c32);
605                            let mut cs = [0i32; 8];
606                            _mm256_storeu_si256(cs.as_mut_ptr() as *mut __m256i, packed);
607                            for hhit in 0..n {
608                                *sptr.add(at + hhit) = ISHIFT[cs[hhit] as usize];
609                            }
610                        }
611                    }
612                }
613                at += n;
614            }
615            t += 8;
616        }
617        let p = xyz[atom];
618        for k in t..b {
619            let q = xyz[j[k] as usize];
620            let c = code[k] as usize;
621            let dx = q[0] + tx[c] - p[0];
622            let dy = q[1] + ty[c] - p[1];
623            let dz = q[2] + tz[c] - p[2];
624            let dist = (dx * dx + dy * dy) + dz * dz;
625            let _ = cand;
626            if dist < cut2 {
627                if WRITE {
628                    unsafe {
629                        *(ip as *mut i32).add(at) = atom as i32;
630                        *(jp as *mut i32).add(at) = j[k] as i32;
631                        *(sp as *mut [i32; 3]).add(at) = ISHIFT[c];
632                        *(dp as *mut f64).add(at) = dist;
633                    }
634                }
635                at += 1;
636                hits += 1;
637            }
638        }
639    }
640    hits
641}
642
643fn bead_pass<const WRITE: bool>(
644    xyz: &[[f64; 3]],
645    atoms: usize,
646    beads: usize,
647    offsets: &[u32],
648    j: &[u32],
649    code: &[u8],
650    ranges: &[(usize, usize)],
651    off: &[Vec<usize>],
652    cut2: f64,
653    tx: &[f64; 343],
654    ty: &[f64; 343],
655    tz: &[f64; 343],
656    use_avx: bool,
657    ptrs: &[(usize, usize, usize, usize)],
658) -> Vec<Vec<usize>> {
659    let run = |t: usize, lo: usize, hi: usize| -> Vec<usize> {
660        #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
661        if use_avx {
662            return unsafe {
663                bead_avx::<WRITE>(
664                    xyz, atoms, beads, offsets, j, code, lo, hi, t, off, cut2, tx, ty, tz, ptrs,
665                )
666            };
667        }
668        let _ = use_avx;
669        bead_scalar::<WRITE>(
670            xyz, atoms, beads, offsets, j, code, lo, hi, t, off, cut2, tx, ty, tz, ptrs,
671        )
672    };
673    #[cfg(feature = "parallel")]
674    if ranges.len() > 1 {
675        use rayon::prelude::*;
676        return ranges
677            .par_iter()
678            .enumerate()
679            .map(|(t, &(lo, hi))| run(t, lo, hi))
680            .collect();
681    }
682    ranges
683        .iter()
684        .enumerate()
685        .map(|(t, &(lo, hi))| run(t, lo, hi))
686        .collect()
687}
688
689fn bead_scalar<const WRITE: bool>(
690    xyz: &[[f64; 3]],
691    atoms: usize,
692    beads: usize,
693    offsets: &[u32],
694    j: &[u32],
695    code: &[u8],
696    lo: usize,
697    hi: usize,
698    t: usize,
699    off: &[Vec<usize>],
700    cut2: f64,
701    tx: &[f64; 343],
702    ty: &[f64; 343],
703    tz: &[f64; 343],
704    ptrs: &[(usize, usize, usize, usize)],
705) -> Vec<usize> {
706    let mut hits = vec![0usize; beads];
707    let mut at: Vec<usize> = (0..beads)
708        .map(|b| if WRITE { off[b][t] } else { 0 })
709        .collect();
710    for atom in lo..hi {
711        let a = offsets[atom] as usize;
712        let b = offsets[atom + 1] as usize;
713        for k in a..b {
714            let c = code[k] as usize;
715            let target = j[k] as usize;
716            let (sx, sy, sz) = (tx[c], ty[c], tz[c]);
717            let shift = ISHIFT[c];
718            for bead in 0..beads {
719                let p = xyz[bead * atoms + atom];
720                let q = xyz[bead * atoms + target];
721                let dx = q[0] + sx - p[0];
722                let dy = q[1] + sy - p[1];
723                let dz = q[2] + sz - p[2];
724                let dist = (dx * dx + dy * dy) + dz * dz;
725                if dist < cut2 {
726                    if WRITE {
727                        let (ip, jp, sp, dp) = ptrs[bead];
728                        let atb = at[bead];
729                        unsafe {
730                            *(ip as *mut i32).add(atb) = atom as i32;
731                            *(jp as *mut i32).add(atb) = target as i32;
732                            *(sp as *mut [i32; 3]).add(atb) = shift;
733                            *(dp as *mut f64).add(atb) = dist;
734                        }
735                    }
736                    at[bead] += 1;
737                    hits[bead] += 1;
738                }
739            }
740        }
741    }
742    hits
743}
744
745#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
746#[target_feature(enable = "avx512f,avx512vl,avx2")]
747unsafe fn bead_avx<const WRITE: bool>(
748    xyz: &[[f64; 3]],
749    atoms: usize,
750    beads: usize,
751    offsets: &[u32],
752    j: &[u32],
753    code: &[u8],
754    lo: usize,
755    hi: usize,
756    t: usize,
757    off: &[Vec<usize>],
758    cut2: f64,
759    tx: &[f64; 343],
760    ty: &[f64; 343],
761    tz: &[f64; 343],
762    ptrs: &[(usize, usize, usize, usize)],
763) -> Vec<usize> {
764    use std::arch::x86_64::{
765        __m128i, __m256i, _mm256_add_epi32, _mm256_cvtepu8_epi32, _mm256_loadu_si256,
766        _mm256_mask_compressstoreu_epi32, _mm256_maskz_compress_epi32, _mm256_mullo_epi32,
767        _mm256_set1_epi32, _mm256_storeu_si256, _mm512_add_pd, _mm512_cmp_pd_mask,
768        _mm512_i32gather_pd, _mm512_mask_compressstoreu_pd, _mm512_mul_pd, _mm512_set1_pd,
769        _mm512_sub_pd, _mm_loadl_epi64, _CMP_LT_OQ,
770    };
771    let mut hits = vec![0usize; beads];
772    let mut at: Vec<usize> = (0..beads)
773        .map(|b| if WRITE { off[b][t] } else { 0 })
774        .collect();
775    let cut = _mm512_set1_pd(cut2);
776    let txp = tx.as_ptr();
777    let typ = ty.as_ptr();
778    let tzp = tz.as_ptr();
779    let three = _mm256_set1_epi32(3);
780    let one = _mm256_set1_epi32(1);
781    let two = _mm256_set1_epi32(2);
782    for atom in lo..hi {
783        let a = offsets[atom] as usize;
784        let b = offsets[atom + 1] as usize;
785        let mut k = a;
786        while k + 8 <= b {
787            let raw = unsafe { (code.as_ptr().add(k) as *const u64).read_unaligned() };
788            let zero = raw == ZERO8;
789            let jv = unsafe { _mm256_loadu_si256(j.as_ptr().add(k) as *const __m256i) };
790            let ix = _mm256_mullo_epi32(jv, three);
791            let (sxv, syv, szv, c32) = if zero {
792                (
793                    _mm512_set1_pd(0.0),
794                    _mm512_set1_pd(0.0),
795                    _mm512_set1_pd(0.0),
796                    _mm256_set1_epi32(i32::from(ZERO_CODE)),
797                )
798            } else {
799                let c32 = _mm256_cvtepu8_epi32(unsafe {
800                    _mm_loadl_epi64(code.as_ptr().add(k) as *const __m128i)
801                });
802                (
803                    unsafe { _mm512_i32gather_pd(c32, txp, 8) },
804                    unsafe { _mm512_i32gather_pd(c32, typ, 8) },
805                    unsafe { _mm512_i32gather_pd(c32, tzp, 8) },
806                    c32,
807                )
808            };
809            for bead in 0..beads {
810                let base = unsafe { xyz.as_ptr().add(bead * atoms) } as *const f64;
811                let p = xyz[bead * atoms + atom];
812                let px = _mm512_set1_pd(p[0]);
813                let py = _mm512_set1_pd(p[1]);
814                let pz = _mm512_set1_pd(p[2]);
815                let qx = unsafe { _mm512_i32gather_pd(ix, base, 8) };
816                let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), base, 8) };
817                let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), base, 8) };
818                let dx = _mm512_sub_pd(_mm512_add_pd(qx, sxv), px);
819                let dy = _mm512_sub_pd(_mm512_add_pd(qy, syv), py);
820                let dz = _mm512_sub_pd(_mm512_add_pd(qz, szv), pz);
821                let dist = _mm512_add_pd(
822                    _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
823                    _mm512_mul_pd(dz, dz),
824                );
825                let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
826                let n = mask.count_ones() as usize;
827                hits[bead] += n;
828                if WRITE && n > 0 {
829                    let (ip, jp, sp, dp) = ptrs[bead];
830                    let atb = at[bead];
831                    unsafe {
832                        let iv = _mm256_set1_epi32(atom as i32);
833                        _mm256_mask_compressstoreu_epi32((ip as *mut i32).add(atb), mask, iv);
834                        _mm256_mask_compressstoreu_epi32((jp as *mut i32).add(atb), mask, jv);
835                        _mm512_mask_compressstoreu_pd((dp as *mut f64).add(atb), mask, dist);
836                        if zero {
837                            let sptr = (sp as *mut [i32; 3]).add(atb);
838                            for hhit in 0..n {
839                                *sptr.add(hhit) = [0, 0, 0];
840                            }
841                        } else {
842                            let packed = _mm256_maskz_compress_epi32(mask, c32);
843                            let mut cs = [0i32; 8];
844                            _mm256_storeu_si256(cs.as_mut_ptr() as *mut __m256i, packed);
845                            let sptr = (sp as *mut [i32; 3]).add(atb);
846                            for hhit in 0..n {
847                                *sptr.add(hhit) = ISHIFT[cs[hhit] as usize];
848                            }
849                        }
850                    }
851                    at[bead] += n;
852                }
853            }
854            k += 8;
855        }
856        for idx in k..b {
857            let c = code[idx] as usize;
858            let target = j[idx] as usize;
859            let (sx, sy, sz) = (tx[c], ty[c], tz[c]);
860            let shift = ISHIFT[c];
861            for bead in 0..beads {
862                let p = xyz[bead * atoms + atom];
863                let q = xyz[bead * atoms + target];
864                let dx = q[0] + sx - p[0];
865                let dy = q[1] + sy - p[1];
866                let dz = q[2] + sz - p[2];
867                let dist = (dx * dx + dy * dy) + dz * dz;
868                if dist < cut2 {
869                    if WRITE {
870                        let (ip, jp, sp, dp) = ptrs[bead];
871                        let atb = at[bead];
872                        unsafe {
873                            *(ip as *mut i32).add(atb) = atom as i32;
874                            *(jp as *mut i32).add(atb) = target as i32;
875                            *(sp as *mut [i32; 3]).add(atb) = shift;
876                            *(dp as *mut f64).add(atb) = dist;
877                        }
878                    }
879                    at[bead] += 1;
880                    hits[bead] += 1;
881                }
882            }
883        }
884    }
885    hits
886}
887
888fn write_ranges(
889    xyz: &[[f64; 3]],
890    soa: &Soa,
891    ranges: &[(usize, usize)],
892    off: &[usize],
893    d2: &[f64],
894    cut2: f64,
895    want_vector: bool,
896    ortho: bool,
897    widths: [f64; 3],
898    h: [[f64; 3]; 3],
899    use_avx: bool,
900    out: &mut PairColumns,
901    vectors: &mut Vec<[f64; 3]>,
902) {
903    let ip = out.i.as_mut_ptr() as usize;
904    let jp = out.j.as_mut_ptr() as usize;
905    let sp = out.shift.as_mut_ptr() as usize;
906    let dp = out.dist2.as_mut_ptr() as usize;
907    let vp = vectors.as_mut_ptr() as usize;
908    let job = move |t: usize, lo: usize, hi: usize| {
909        let mut at = off[t];
910        for atom in lo..hi {
911            let a = soa.offsets[atom] as usize;
912            let b = soa.offsets[atom + 1] as usize;
913            let p = xyz[atom];
914            if use_avx && !want_vector && b - a >= 8 {
915                at += compact_atom(soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp);
916            } else {
917                for k in a..b {
918                    let dist = d2[k];
919                    if dist < cut2 {
920                        let s = [soa.sx[k], soa.sy[k], soa.sz[k]];
921                        unsafe {
922                            *(ip as *mut i32).add(at) = atom as i32;
923                            *(jp as *mut i32).add(at) = soa.j[k] as i32;
924                            *(sp as *mut [i32; 3]).add(at) = s;
925                            *(dp as *mut f64).add(at) = dist;
926                            if want_vector {
927                                let q = *xyz.get_unchecked(soa.j[k] as usize);
928                                let (tx, ty, tz) = shift_xyz(s[0], s[1], s[2], ortho, widths, h);
929                                *(vp as *mut [f64; 3]).add(at) =
930                                    [q[0] + tx - p[0], q[1] + ty - p[1], q[2] + tz - p[2]];
931                            }
932                        }
933                        at += 1;
934                    }
935                }
936            }
937        }
938    };
939    #[cfg(feature = "parallel")]
940    if ranges.len() > 1 {
941        use rayon::prelude::*;
942        ranges
943            .par_iter()
944            .enumerate()
945            .for_each(|(t, &(lo, hi))| job(t, lo, hi));
946        return;
947    }
948    if let Some(&(lo, hi)) = ranges.first() {
949        job(0, lo, hi);
950    }
951}
952
953fn compact_atom(
954    soa: &Soa,
955    d2: &[f64],
956    atom: usize,
957    a: usize,
958    b: usize,
959    cut2: f64,
960    at: usize,
961    ip: usize,
962    jp: usize,
963    sp: usize,
964    dp: usize,
965) -> usize {
966    #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
967    unsafe {
968        compact_avx512(soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp)
969    }
970    #[cfg(not(all(target_arch = "x86_64", linkcell_avx512)))]
971    {
972        let _ = (soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp);
973        0
974    }
975}
976
977#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
978#[target_feature(enable = "avx512f,avx512vl,avx2")]
979unsafe fn compact_avx512(
980    soa: &Soa,
981    d2: &[f64],
982    atom: usize,
983    a: usize,
984    b: usize,
985    cut2: f64,
986    mut at: usize,
987    ip: usize,
988    jp: usize,
989    sp: usize,
990    dp: usize,
991) -> usize {
992    use std::arch::x86_64::{
993        __m256i, _mm256_loadu_si256, _mm256_maskz_compress_epi32, _mm256_storeu_si256,
994        _mm512_cmp_pd_mask, _mm512_loadu_pd, _mm512_maskz_compress_pd, _mm512_set1_pd,
995        _mm512_storeu_pd, _CMP_LT_OQ,
996    };
997    let start = at;
998    let cut = _mm512_set1_pd(cut2);
999    let mut t = a;
1000    let mut js = [0i32; 8];
1001    let mut xs = [0i32; 8];
1002    let mut ys = [0i32; 8];
1003    let mut zs = [0i32; 8];
1004    let mut ds = [0f64; 8];
1005    while t + 8 <= b {
1006        let dist = unsafe { _mm512_loadu_pd(d2.as_ptr().add(t)) };
1007        let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
1008        let n = mask.count_ones() as usize;
1009        if n > 0 {
1010            unsafe {
1011                _mm256_storeu_si256(
1012                    js.as_mut_ptr() as *mut __m256i,
1013                    _mm256_maskz_compress_epi32(
1014                        mask,
1015                        _mm256_loadu_si256(soa.j.as_ptr().add(t) as *const __m256i),
1016                    ),
1017                );
1018                _mm256_storeu_si256(
1019                    xs.as_mut_ptr() as *mut __m256i,
1020                    _mm256_maskz_compress_epi32(
1021                        mask,
1022                        _mm256_loadu_si256(soa.sx.as_ptr().add(t) as *const __m256i),
1023                    ),
1024                );
1025                _mm256_storeu_si256(
1026                    ys.as_mut_ptr() as *mut __m256i,
1027                    _mm256_maskz_compress_epi32(
1028                        mask,
1029                        _mm256_loadu_si256(soa.sy.as_ptr().add(t) as *const __m256i),
1030                    ),
1031                );
1032                _mm256_storeu_si256(
1033                    zs.as_mut_ptr() as *mut __m256i,
1034                    _mm256_maskz_compress_epi32(
1035                        mask,
1036                        _mm256_loadu_si256(soa.sz.as_ptr().add(t) as *const __m256i),
1037                    ),
1038                );
1039                _mm512_storeu_pd(ds.as_mut_ptr(), _mm512_maskz_compress_pd(mask, dist));
1040            }
1041            let ii = atom as i32;
1042            for h in 0..n {
1043                unsafe {
1044                    *(ip as *mut i32).add(at) = ii;
1045                    *(jp as *mut i32).add(at) = js[h];
1046                    *(sp as *mut [i32; 3]).add(at) = [xs[h], ys[h], zs[h]];
1047                    *(dp as *mut f64).add(at) = ds[h];
1048                }
1049                at += 1;
1050            }
1051        }
1052        t += 8;
1053    }
1054    for k in t..b {
1055        let dist = d2[k];
1056        if dist < cut2 {
1057            unsafe {
1058                *(ip as *mut i32).add(at) = atom as i32;
1059                *(jp as *mut i32).add(at) = soa.j[k] as i32;
1060                *(sp as *mut [i32; 3]).add(at) = [soa.sx[k], soa.sy[k], soa.sz[k]];
1061                *(dp as *mut f64).add(at) = dist;
1062            }
1063            at += 1;
1064        }
1065    }
1066    at - start
1067}
1068
1069#[allow(clippy::uninit_vec)]
1070fn prepare(out: &mut PairColumns, vectors: &mut Vec<[f64; 3]>, total: usize, want_vector: bool) {
1071    out.i.clear();
1072    out.j.clear();
1073    out.shift.clear();
1074    out.dist2.clear();
1075    vectors.clear();
1076    out.i.reserve(total);
1077    out.j.reserve(total);
1078    out.shift.reserve(total);
1079    out.dist2.reserve(total);
1080    unsafe {
1081        out.i.set_len(total);
1082        out.j.set_len(total);
1083        out.shift.set_len(total);
1084        out.dist2.set_len(total);
1085    }
1086    if want_vector {
1087        vectors.reserve(total);
1088        unsafe { vectors.set_len(total) };
1089    }
1090}
1091
1092fn filter_threads(n: usize) -> usize {
1093    #[cfg(feature = "parallel")]
1094    {
1095        let pool = crate::pool_width().max(1);
1096        if pool > 1 && n >= 32_768 {
1097            pool
1098        } else {
1099            1
1100        }
1101    }
1102    #[cfg(not(feature = "parallel"))]
1103    {
1104        let _ = n;
1105        1
1106    }
1107}
1108
1109fn atom_ranges(offsets: &[u32], threads: usize) -> Vec<(usize, usize)> {
1110    let atoms = offsets.len().saturating_sub(1);
1111    if atoms == 0 {
1112        return Vec::new();
1113    }
1114    let threads = threads.max(1).min(atoms);
1115    let total = offsets[atoms] as usize;
1116    if threads == 1 || total == 0 {
1117        return vec![(0, atoms)];
1118    }
1119    let mut ranges = Vec::with_capacity(threads);
1120    let mut start = 0usize;
1121    for t in 1..threads {
1122        let target = total * t / threads;
1123        let mut end = start;
1124        while end < atoms && (offsets[end] as usize) < target {
1125            end += 1;
1126        }
1127        if end == start {
1128            end = (start + 1).min(atoms);
1129        }
1130        ranges.push((start, end));
1131        start = end;
1132    }
1133    if start < atoms {
1134        ranges.push((start, atoms));
1135    }
1136    ranges
1137}
1138
1139fn count_one(
1140    xyz: &[[f64; 3]],
1141    offsets: &[u32],
1142    j: &[u32],
1143    sx: &[i32],
1144    sy: &[i32],
1145    sz: &[i32],
1146    lo: usize,
1147    hi: usize,
1148    cut2: f64,
1149    ortho: bool,
1150    widths: [f64; 3],
1151    h: [[f64; 3]; 3],
1152    use_avx: bool,
1153    d2: &mut [f64],
1154) -> usize {
1155    let c0 = offsets[lo] as usize;
1156    let mut hits = 0usize;
1157    let mut local = 0usize;
1158    for atom in lo..hi {
1159        let a = offsets[atom] as usize;
1160        let b = offsets[atom + 1] as usize;
1161        let count = b - a;
1162        if count == 0 {
1163            continue;
1164        }
1165        hits += distance_block(
1166            xyz,
1167            j,
1168            sx,
1169            sy,
1170            sz,
1171            a,
1172            count,
1173            atom,
1174            cut2,
1175            ortho,
1176            widths,
1177            h,
1178            use_avx,
1179            &mut d2[local..local + count],
1180        );
1181        local += count;
1182    }
1183    let _ = c0;
1184    hits
1185}
1186
1187fn count_ranges(
1188    xyz: &[[f64; 3]],
1189    offsets: &[u32],
1190    j: &[u32],
1191    sx: &[i32],
1192    sy: &[i32],
1193    sz: &[i32],
1194    ranges: &[(usize, usize)],
1195    cut2: f64,
1196    ortho: bool,
1197    widths: [f64; 3],
1198    h: [[f64; 3]; 3],
1199    use_avx: bool,
1200    d2: &mut [f64],
1201) -> Vec<usize> {
1202    #[cfg(feature = "parallel")]
1203    if ranges.len() > 1 {
1204        use rayon::prelude::*;
1205        let ptr = d2.as_mut_ptr() as usize;
1206        return ranges
1207            .par_iter()
1208            .map(|&(lo, hi)| {
1209                let c0 = offsets[lo] as usize;
1210                let c1 = offsets[hi] as usize;
1211                let slice = unsafe {
1212                    std::slice::from_raw_parts_mut(
1213                        (ptr + c0 * std::mem::size_of::<f64>()) as *mut f64,
1214                        c1 - c0,
1215                    )
1216                };
1217                count_one(
1218                    xyz, offsets, j, sx, sy, sz, lo, hi, cut2, ortho, widths, h, use_avx, slice,
1219                )
1220            })
1221            .collect();
1222    }
1223    ranges
1224        .iter()
1225        .map(|&(lo, hi)| {
1226            let c0 = offsets[lo] as usize;
1227            let c1 = offsets[hi] as usize;
1228            count_one(
1229                xyz,
1230                offsets,
1231                j,
1232                sx,
1233                sy,
1234                sz,
1235                lo,
1236                hi,
1237                cut2,
1238                ortho,
1239                widths,
1240                h,
1241                use_avx,
1242                &mut d2[c0..c1],
1243            )
1244        })
1245        .collect()
1246}
1247
1248fn distance_block(
1249    xyz: &[[f64; 3]],
1250    j: &[u32],
1251    sx: &[i32],
1252    sy: &[i32],
1253    sz: &[i32],
1254    base: usize,
1255    count: usize,
1256    atom: usize,
1257    cut2: f64,
1258    ortho: bool,
1259    widths: [f64; 3],
1260    h: [[f64; 3]; 3],
1261    use_avx: bool,
1262    d2: &mut [f64],
1263) -> usize {
1264    #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
1265    if use_avx {
1266        return unsafe {
1267            distance_avx512(
1268                xyz, j, sx, sy, sz, base, count, atom, cut2, ortho, widths, h, d2,
1269            )
1270        };
1271    }
1272    let _ = use_avx;
1273    distance_scalar(
1274        xyz, j, sx, sy, sz, base, count, atom, cut2, ortho, widths, h, d2,
1275    )
1276}
1277
1278fn distance_scalar(
1279    xyz: &[[f64; 3]],
1280    j: &[u32],
1281    sx: &[i32],
1282    sy: &[i32],
1283    sz: &[i32],
1284    base: usize,
1285    count: usize,
1286    atom: usize,
1287    cut2: f64,
1288    ortho: bool,
1289    widths: [f64; 3],
1290    h: [[f64; 3]; 3],
1291    d2: &mut [f64],
1292) -> usize {
1293    let p = xyz[atom];
1294    let mut hits = 0usize;
1295    for t in 0..count {
1296        let q = xyz[j[base + t] as usize];
1297        let (tx, ty, tz) = shift_xyz(sx[base + t], sy[base + t], sz[base + t], ortho, widths, h);
1298        let dx = q[0] + tx - p[0];
1299        let dy = q[1] + ty - p[1];
1300        let dz = q[2] + tz - p[2];
1301        let dist = (dx * dx + dy * dy) + dz * dz;
1302        d2[t] = dist;
1303        if dist < cut2 {
1304            hits += 1;
1305        }
1306    }
1307    hits
1308}
1309
1310pub(crate) fn shift_xyz(
1311    sx: i32,
1312    sy: i32,
1313    sz: i32,
1314    ortho: bool,
1315    widths: [f64; 3],
1316    h: [[f64; 3]; 3],
1317) -> (f64, f64, f64) {
1318    if ortho {
1319        (
1320            f64::from(sx) * widths[0],
1321            f64::from(sy) * widths[1],
1322            f64::from(sz) * widths[2],
1323        )
1324    } else {
1325        let (a, b, c) = (f64::from(sx), f64::from(sy), f64::from(sz));
1326        (
1327            a * h[0][0] + b * h[1][0] + c * h[2][0],
1328            a * h[0][1] + b * h[1][1] + c * h[2][1],
1329            a * h[0][2] + b * h[1][2] + c * h[2][2],
1330        )
1331    }
1332}
1333
1334#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
1335#[target_feature(enable = "avx512f,avx512vl,avx2")]
1336unsafe fn distance_avx512(
1337    xyz: &[[f64; 3]],
1338    j: &[u32],
1339    sx: &[i32],
1340    sy: &[i32],
1341    sz: &[i32],
1342    base: usize,
1343    count: usize,
1344    atom: usize,
1345    cut2: f64,
1346    ortho: bool,
1347    widths: [f64; 3],
1348    h: [[f64; 3]; 3],
1349    d2: &mut [f64],
1350) -> usize {
1351    use std::arch::x86_64::{
1352        __m256i, _mm256_add_epi32, _mm256_loadu_si256, _mm256_mullo_epi32, _mm256_set1_epi32,
1353        _mm512_add_pd, _mm512_cmp_pd_mask, _mm512_cvtepi32_pd, _mm512_i32gather_pd, _mm512_mul_pd,
1354        _mm512_set1_pd, _mm512_storeu_pd, _mm512_sub_pd, _CMP_LT_OQ,
1355    };
1356    let p = xyz[atom];
1357    let bytes = xyz.as_ptr() as *const f64;
1358    let jptr = j.as_ptr().add(base);
1359    let sxp = sx.as_ptr().add(base);
1360    let syp = sy.as_ptr().add(base);
1361    let szp = sz.as_ptr().add(base);
1362    let outp = d2.as_mut_ptr();
1363    let cut = _mm512_set1_pd(cut2);
1364    let px = _mm512_set1_pd(p[0]);
1365    let py = _mm512_set1_pd(p[1]);
1366    let pz = _mm512_set1_pd(p[2]);
1367    let three = _mm256_set1_epi32(3);
1368    let one = _mm256_set1_epi32(1);
1369    let two = _mm256_set1_epi32(2);
1370    let mut hits = 0usize;
1371    let mut t = 0usize;
1372    while t + 8 <= count {
1373        let jv = unsafe { _mm256_loadu_si256(jptr.add(t) as *const __m256i) };
1374        let ix = _mm256_mullo_epi32(jv, three);
1375        let qx = unsafe { _mm512_i32gather_pd(ix, bytes, 8) };
1376        let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), bytes, 8) };
1377        let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), bytes, 8) };
1378        let sxf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(sxp.add(t) as *const __m256i) });
1379        let syf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(syp.add(t) as *const __m256i) });
1380        let szf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(szp.add(t) as *const __m256i) });
1381        let (tx, ty, tz) = if ortho {
1382            (
1383                _mm512_mul_pd(sxf, _mm512_set1_pd(widths[0])),
1384                _mm512_mul_pd(syf, _mm512_set1_pd(widths[1])),
1385                _mm512_mul_pd(szf, _mm512_set1_pd(widths[2])),
1386            )
1387        } else {
1388            (
1389                _mm512_add_pd(
1390                    _mm512_add_pd(
1391                        _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][0])),
1392                        _mm512_mul_pd(syf, _mm512_set1_pd(h[1][0])),
1393                    ),
1394                    _mm512_mul_pd(szf, _mm512_set1_pd(h[2][0])),
1395                ),
1396                _mm512_add_pd(
1397                    _mm512_add_pd(
1398                        _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][1])),
1399                        _mm512_mul_pd(syf, _mm512_set1_pd(h[1][1])),
1400                    ),
1401                    _mm512_mul_pd(szf, _mm512_set1_pd(h[2][1])),
1402                ),
1403                _mm512_add_pd(
1404                    _mm512_add_pd(
1405                        _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][2])),
1406                        _mm512_mul_pd(syf, _mm512_set1_pd(h[1][2])),
1407                    ),
1408                    _mm512_mul_pd(szf, _mm512_set1_pd(h[2][2])),
1409                ),
1410            )
1411        };
1412        let dx = _mm512_sub_pd(_mm512_add_pd(qx, tx), px);
1413        let dy = _mm512_sub_pd(_mm512_add_pd(qy, ty), py);
1414        let dz = _mm512_sub_pd(_mm512_add_pd(qz, tz), pz);
1415        let dist = _mm512_add_pd(
1416            _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
1417            _mm512_mul_pd(dz, dz),
1418        );
1419        unsafe { _mm512_storeu_pd(outp.add(t), dist) };
1420        hits += _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut).count_ones() as usize;
1421        t += 8;
1422    }
1423    if t < count {
1424        hits += distance_scalar(
1425            xyz,
1426            j,
1427            sx,
1428            sy,
1429            sz,
1430            base + t,
1431            count - t,
1432            atom,
1433            cut2,
1434            ortho,
1435            widths,
1436            h,
1437            &mut d2[t..],
1438        );
1439    }
1440    hits
1441}