1#![allow(
10 clippy::incompatible_msrv,
11 clippy::too_many_arguments,
12 clippy::needless_range_loop
13)]
14
15use crate::cell::Cell;
16use crate::pairs::PairColumns;
17use crate::simd::{self, Path};
18use crate::Error;
19
20pub(crate) struct Soa {
22 offsets: Vec<u32>,
24 j: Vec<u32>,
25 sx: Vec<i32>,
26 sy: Vec<i32>,
27 sz: Vec<i32>,
28 code: Vec<u8>,
31 coded: bool,
32}
33
34#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
36const ZERO_CODE: u8 = (3 * 49 + 3 * 7 + 3) as u8;
37
38#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
40const ZERO8: u64 = {
41 let b = ZERO_CODE as u64;
42 b | (b << 8) | (b << 16) | (b << 24) | (b << 32) | (b << 40) | (b << 48) | (b << 56)
43};
44
45const fn ishift_table() -> [[i32; 3]; 343] {
46 let mut t = [[0; 3]; 343];
47 let mut s = 0;
48 while s < 343 {
49 let sx = (s / 49) as i32 - 3;
50 let sy = ((s % 49) / 7) as i32 - 3;
51 let sz = (s % 7) as i32 - 3;
52 t[s] = [sx, sy, sz];
53 s += 1;
54 }
55 t
56}
57
58const ISHIFT: [[i32; 3]; 343] = ishift_table();
60
61fn encode_shift(sx: i32, sy: i32, sz: i32) -> Option<u8> {
62 if !(-3..=3).contains(&sx) || !(-3..=3).contains(&sy) || !(-3..=3).contains(&sz) {
63 return None;
64 }
65 Some(((sx + 3) as u8) * 49 + ((sy + 3) as u8) * 7 + (sz + 3) as u8)
66}
67
68impl Soa {
69 pub(crate) fn empty() -> Self {
70 Self {
71 offsets: vec![0],
72 j: Vec::new(),
73 sx: Vec::new(),
74 sy: Vec::new(),
75 sz: Vec::new(),
76 code: Vec::new(),
77 coded: true,
78 }
79 }
80
81 pub(crate) fn from_columns(cols: &PairColumns, n_atoms: usize) -> Result<Self, Error> {
84 let n = cols.len();
85 if cols.j.len() != n || cols.shift.len() != n {
86 return Err(Error::BufferSize);
87 }
88 let mut counts = vec![0u32; n_atoms];
89 for &i in &cols.i {
90 let i = i as usize;
91 if i >= n_atoms {
92 return Err(Error::Overflow);
93 }
94 counts[i] = counts[i].checked_add(1).ok_or(Error::Overflow)?;
95 }
96 let mut offsets = vec![0u32; n_atoms + 1];
97 for i in 0..n_atoms {
98 offsets[i + 1] = offsets[i].checked_add(counts[i]).ok_or(Error::Overflow)?;
99 }
100 let mut cursor = offsets.clone();
101 let mut j = vec![0u32; n];
102 let mut sx = vec![0i32; n];
103 let mut sy = vec![0i32; n];
104 let mut sz = vec![0i32; n];
105 let mut code = vec![0u8; n];
106 let mut coded = true;
107 for k in 0..n {
108 let i = cols.i[k] as usize;
109 let at = cursor[i] as usize;
110 cursor[i] += 1;
111 j[at] = cols.j[k] as u32;
112 let s = cols.shift[k];
113 sx[at] = s[0];
114 sy[at] = s[1];
115 sz[at] = s[2];
116 if coded {
117 if let Some(c) = encode_shift(s[0], s[1], s[2]) {
118 code[at] = c;
119 } else {
120 coded = false;
121 }
122 }
123 }
124 if !coded {
125 code.clear();
126 }
127 Ok(Self {
128 offsets,
129 j,
130 sx,
131 sy,
132 sz,
133 code,
134 coded,
135 })
136 }
137
138 pub(crate) fn each_end(&self, mut f: impl FnMut(u32, u32)) {
140 let atoms = self.offsets.len().saturating_sub(1);
141 for i in 0..atoms {
142 let a = self.offsets[i] as usize;
143 let b = self.offsets[i + 1] as usize;
144 for k in a..b {
145 f(i as u32, self.j[k]);
146 }
147 }
148 }
149
150 pub(crate) fn filter(
152 &self,
153 xyz: &[[f64; 3]],
154 cell: &Cell,
155 cut2: f64,
156 want_vector: bool,
157 d2: &mut Vec<f64>,
158 out: &mut PairColumns,
159 vectors: &mut Vec<[f64; 3]>,
160 ) -> Result<(), Error> {
161 let atoms = self.offsets.len().saturating_sub(1);
162 if xyz.len() != atoms {
163 return Err(Error::BufferSize);
164 }
165 let n = self.j.len();
166 let threads = filter_threads(n);
167 let ranges = atom_ranges(&self.offsets, threads);
168 let use_avx = simd::select() == Path::Avx512;
169 let ortho = cell.is_ortho();
170 let h = cell.h();
171 let widths = cell.widths();
172 if self.coded && !want_vector {
173 vectors.clear();
174 return filter_coded(
175 xyz,
176 &self.offsets,
177 &self.j,
178 &self.code,
179 &ranges,
180 cut2,
181 ortho,
182 widths,
183 h,
184 use_avx,
185 d2,
186 out,
187 );
188 }
189 if d2.len() < n {
190 d2.resize(n, 0.0);
191 }
192 let counts = count_ranges(
193 xyz,
194 &self.offsets,
195 &self.j,
196 &self.sx,
197 &self.sy,
198 &self.sz,
199 &ranges,
200 cut2,
201 ortho,
202 widths,
203 h,
204 use_avx,
205 d2,
206 );
207 let mut off = Vec::with_capacity(ranges.len() + 1);
208 off.push(0usize);
209 for c in &counts {
210 off.push(off[off.len() - 1] + c);
211 }
212 let total = *off.last().unwrap_or(&0);
213 prepare(out, vectors, total, want_vector);
214 write_ranges(
215 xyz,
216 self,
217 &ranges,
218 &off,
219 d2,
220 cut2,
221 want_vector,
222 ortho,
223 widths,
224 h,
225 use_avx,
226 out,
227 vectors,
228 );
229 Ok(())
230 }
231
232 pub(crate) fn is_coded(&self) -> bool {
233 self.coded
234 }
235
236 pub(crate) fn filter_beads(
242 &self,
243 xyz: &[[f64; 3]],
244 beads: usize,
245 cell: &Cell,
246 cut2: f64,
247 outs: &mut [PairColumns],
248 ) -> Result<(), Error> {
249 let atoms = self.offsets.len().saturating_sub(1);
250 if beads == 0 || outs.len() != beads || xyz.len() != beads * atoms || !self.coded {
251 return Err(Error::BufferSize);
252 }
253 let threads = filter_threads(self.j.len().saturating_mul(beads.max(1)));
254 let ranges = atom_ranges(&self.offsets, threads);
255 let use_avx = simd::select() == Path::Avx512;
256 let (tx, ty, tz) = cartesian_tables(cell.is_ortho(), cell.widths(), cell.h());
257 let counts = bead_pass::<false>(
258 xyz,
259 atoms,
260 beads,
261 &self.offsets,
262 &self.j,
263 &self.code,
264 &ranges,
265 &[],
266 cut2,
267 &tx,
268 &ty,
269 &tz,
270 use_avx,
271 &[],
272 );
273 let mut cursor = vec![0usize; beads];
274 let mut off = vec![vec![0usize; ranges.len()]; beads];
275 for (t, row) in counts.iter().enumerate() {
276 for b in 0..beads {
277 off[b][t] = cursor[b];
278 cursor[b] += row[b];
279 }
280 }
281 let mut vectors = Vec::new();
282 let mut ptrs = Vec::with_capacity(beads);
283 for b in 0..beads {
284 prepare(&mut outs[b], &mut vectors, cursor[b], false);
285 ptrs.push((
286 outs[b].i.as_mut_ptr() as usize,
287 outs[b].j.as_mut_ptr() as usize,
288 outs[b].shift.as_mut_ptr() as usize,
289 outs[b].dist2.as_mut_ptr() as usize,
290 ));
291 }
292 let written = bead_pass::<true>(
293 xyz,
294 atoms,
295 beads,
296 &self.offsets,
297 &self.j,
298 &self.code,
299 &ranges,
300 &off,
301 cut2,
302 &tx,
303 &ty,
304 &tz,
305 use_avx,
306 &ptrs,
307 );
308 debug_assert_eq!(written, counts);
309 Ok(())
310 }
311}
312
313fn cartesian_tables(
314 ortho: bool,
315 widths: [f64; 3],
316 h: [[f64; 3]; 3],
317) -> ([f64; 343], [f64; 343], [f64; 343]) {
318 let mut tx = [0.0; 343];
319 let mut ty = [0.0; 343];
320 let mut tz = [0.0; 343];
321 for (s, shift) in ISHIFT.iter().enumerate() {
322 let (x, y, z) = shift_xyz(shift[0], shift[1], shift[2], ortho, widths, h);
323 tx[s] = x;
324 ty[s] = y;
325 tz[s] = z;
326 }
327 (tx, ty, tz)
328}
329
330fn filter_coded(
331 xyz: &[[f64; 3]],
332 offsets: &[u32],
333 j: &[u32],
334 code: &[u8],
335 ranges: &[(usize, usize)],
336 cut2: f64,
337 ortho: bool,
338 widths: [f64; 3],
339 h: [[f64; 3]; 3],
340 use_avx: bool,
341 _d2: &mut Vec<f64>,
342 out: &mut PairColumns,
343) -> Result<(), Error> {
344 let (tx, ty, tz) = cartesian_tables(ortho, widths, h);
345 let counts = coded_pass::<false>(
346 xyz,
347 offsets,
348 j,
349 code,
350 ranges,
351 &[],
352 cut2,
353 &tx,
354 &ty,
355 &tz,
356 use_avx,
357 0,
358 0,
359 0,
360 0,
361 0,
362 );
363 let mut off = Vec::with_capacity(ranges.len() + 1);
364 off.push(0usize);
365 for c in &counts {
366 off.push(off[off.len() - 1] + c);
367 }
368 let total = *off.last().unwrap_or(&0);
369 let mut vectors = Vec::new();
370 prepare(out, &mut vectors, total, false);
371 let written = coded_pass::<true>(
372 xyz,
373 offsets,
374 j,
375 code,
376 ranges,
377 &off,
378 cut2,
379 &tx,
380 &ty,
381 &tz,
382 use_avx,
383 out.i.as_mut_ptr() as usize,
384 out.j.as_mut_ptr() as usize,
385 out.shift.as_mut_ptr() as usize,
386 out.dist2.as_mut_ptr() as usize,
387 0,
388 );
389 debug_assert_eq!(written, counts);
390 Ok(())
391}
392
393fn coded_pass<const WRITE: bool>(
394 xyz: &[[f64; 3]],
395 offsets: &[u32],
396 j: &[u32],
397 code: &[u8],
398 ranges: &[(usize, usize)],
399 off: &[usize],
400 cut2: f64,
401 tx: &[f64; 343],
402 ty: &[f64; 343],
403 tz: &[f64; 343],
404 use_avx: bool,
405 ip: usize,
406 jp: usize,
407 sp: usize,
408 dp: usize,
409 cand: usize,
410) -> Vec<usize> {
411 let run = |t: usize, lo: usize, hi: usize| -> usize {
412 let at = if WRITE { off[t] } else { 0 };
413 #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
414 if use_avx {
415 return unsafe {
416 coded_avx::<WRITE>(
417 xyz, offsets, j, code, lo, hi, cut2, tx, ty, tz, at, ip, jp, sp, dp, cand,
418 )
419 };
420 }
421 let _ = use_avx;
422 coded_scalar::<WRITE>(
423 xyz, offsets, j, code, lo, hi, cut2, tx, ty, tz, at, ip, jp, sp, dp, cand,
424 )
425 };
426 #[cfg(feature = "parallel")]
427 if ranges.len() > 1 {
428 use rayon::prelude::*;
429 return ranges
430 .par_iter()
431 .enumerate()
432 .map(|(t, &(lo, hi))| run(t, lo, hi))
433 .collect();
434 }
435 ranges
436 .iter()
437 .enumerate()
438 .map(|(t, &(lo, hi))| run(t, lo, hi))
439 .collect()
440}
441
442fn coded_scalar<const WRITE: bool>(
443 xyz: &[[f64; 3]],
444 offsets: &[u32],
445 j: &[u32],
446 code: &[u8],
447 lo: usize,
448 hi: usize,
449 cut2: f64,
450 tx: &[f64; 343],
451 ty: &[f64; 343],
452 tz: &[f64; 343],
453 mut at: usize,
454 ip: usize,
455 jp: usize,
456 sp: usize,
457 dp: usize,
458 cand: usize,
459) -> usize {
460 let mut hits = 0usize;
461 for atom in lo..hi {
462 let p = xyz[atom];
463 let a = offsets[atom] as usize;
464 let b = offsets[atom + 1] as usize;
465 for k in a..b {
466 let q = xyz[j[k] as usize];
467 let c = code[k] as usize;
468 let dx = q[0] + tx[c] - p[0];
469 let dy = q[1] + ty[c] - p[1];
470 let dz = q[2] + tz[c] - p[2];
471 let dist = (dx * dx + dy * dy) + dz * dz;
472 let _ = cand;
473 if dist < cut2 {
474 if WRITE {
475 unsafe {
476 *(ip as *mut i32).add(at) = atom as i32;
477 *(jp as *mut i32).add(at) = j[k] as i32;
478 *(sp as *mut [i32; 3]).add(at) = ISHIFT[c];
479 *(dp as *mut f64).add(at) = dist;
480 }
481 }
482 at += 1;
483 hits += 1;
484 }
485 }
486 }
487 hits
488}
489
490#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
491#[target_feature(enable = "avx512f,avx512vl,avx2")]
492unsafe fn coded_avx<const WRITE: bool>(
493 xyz: &[[f64; 3]],
494 offsets: &[u32],
495 j: &[u32],
496 code: &[u8],
497 lo: usize,
498 hi: usize,
499 cut2: f64,
500 tx: &[f64; 343],
501 ty: &[f64; 343],
502 tz: &[f64; 343],
503 mut at: usize,
504 ip: usize,
505 jp: usize,
506 sp: usize,
507 dp: usize,
508 cand: usize,
509) -> usize {
510 use std::arch::x86_64::{
511 __m128i, __m256i, __m512i, _mm256_add_epi32, _mm256_cvtepu8_epi32, _mm256_loadu_si256,
512 _mm256_mask_compressstoreu_epi32, _mm256_maskz_compress_epi32, _mm256_mullo_epi32,
513 _mm256_set1_epi32, _mm256_setzero_si256, _mm256_storeu_si256, _mm512_add_pd,
514 _mm512_cmp_pd_mask, _mm512_i32gather_pd, _mm512_mask_compressstoreu_pd, _mm512_mul_pd,
515 _mm512_set1_pd, _mm512_setzero_si512, _mm512_storeu_pd, _mm512_storeu_si512, _mm512_sub_pd,
516 _mm_loadl_epi64, _mm_prefetch, _CMP_LT_OQ, _MM_HINT_T0,
517 };
518 let mut hits = 0usize;
519 let cut = _mm512_set1_pd(cut2);
520 let bytes = xyz.as_ptr() as *const f64;
521 let txp = tx.as_ptr();
522 let typ = ty.as_ptr();
523 let tzp = tz.as_ptr();
524 let three = _mm256_set1_epi32(3);
525 let one = _mm256_set1_epi32(1);
526 let two = _mm256_set1_epi32(2);
527 for atom in lo..hi {
528 let p = xyz[atom];
529 let px = _mm512_set1_pd(p[0]);
530 let py = _mm512_set1_pd(p[1]);
531 let pz = _mm512_set1_pd(p[2]);
532 let a = offsets[atom] as usize;
533 let b = offsets[atom + 1] as usize;
534 let mut t = a;
535 while t + 8 <= b {
536 if t + 16 <= b {
537 for s in 0..8 {
538 let jj = unsafe { *j.as_ptr().add(t + 8 + s) } as usize;
539 unsafe {
540 _mm_prefetch(bytes.add(jj * 3) as *const i8, _MM_HINT_T0);
541 _mm_prefetch(bytes.add(jj * 3 + 2) as *const i8, _MM_HINT_T0);
542 }
543 }
544 }
545 let raw = unsafe { (code.as_ptr().add(t) as *const u64).read_unaligned() };
546 let zero = raw == ZERO8;
547 let jv = unsafe { _mm256_loadu_si256(j.as_ptr().add(t) as *const __m256i) };
548 let ix = _mm256_mullo_epi32(jv, three);
549 let qx = unsafe { _mm512_i32gather_pd(ix, bytes, 8) };
550 let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), bytes, 8) };
551 let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), bytes, 8) };
552 let (dx, dy, dz, c32) = if zero {
553 (
554 _mm512_sub_pd(qx, px),
555 _mm512_sub_pd(qy, py),
556 _mm512_sub_pd(qz, pz),
557 _mm256_set1_epi32(i32::from(ZERO_CODE)),
558 )
559 } else {
560 let c32 = _mm256_cvtepu8_epi32(unsafe {
561 _mm_loadl_epi64(code.as_ptr().add(t) as *const __m128i)
562 });
563 let sx = unsafe { _mm512_i32gather_pd(c32, txp, 8) };
564 let sy = unsafe { _mm512_i32gather_pd(c32, typ, 8) };
565 let sz = unsafe { _mm512_i32gather_pd(c32, tzp, 8) };
566 (
567 _mm512_sub_pd(_mm512_add_pd(qx, sx), px),
568 _mm512_sub_pd(_mm512_add_pd(qy, sy), py),
569 _mm512_sub_pd(_mm512_add_pd(qz, sz), pz),
570 c32,
571 )
572 };
573 let dist = _mm512_add_pd(
574 _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
575 _mm512_mul_pd(dz, dz),
576 );
577 let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
578 let n = mask.count_ones() as usize;
579 hits += n;
580 let _ = cand;
581 if WRITE && n > 0 {
582 unsafe {
583 let iptr = ip as *mut i32;
584 let jptr = jp as *mut i32;
585 let sptr = sp as *mut [i32; 3];
586 let dptr = dp as *mut f64;
587 let iv = _mm256_set1_epi32(atom as i32);
588 if n == 8 && zero {
589 _mm256_storeu_si256(iptr.add(at) as *mut __m256i, iv);
590 _mm256_storeu_si256(jptr.add(at) as *mut __m256i, jv);
591 _mm512_storeu_pd(dptr.add(at), dist);
592 let sbase = (sptr as *mut i32).add(at * 3);
593 _mm512_storeu_si512(sbase as *mut __m512i, _mm512_setzero_si512());
594 _mm256_storeu_si256(sbase.add(16) as *mut __m256i, _mm256_setzero_si256());
595 } else {
596 _mm256_mask_compressstoreu_epi32(iptr.add(at), mask, iv);
597 _mm256_mask_compressstoreu_epi32(jptr.add(at), mask, jv);
598 _mm512_mask_compressstoreu_pd(dptr.add(at), mask, dist);
599 if zero {
600 for hhit in 0..n {
601 *sptr.add(at + hhit) = [0, 0, 0];
602 }
603 } else {
604 let packed = _mm256_maskz_compress_epi32(mask, c32);
605 let mut cs = [0i32; 8];
606 _mm256_storeu_si256(cs.as_mut_ptr() as *mut __m256i, packed);
607 for hhit in 0..n {
608 *sptr.add(at + hhit) = ISHIFT[cs[hhit] as usize];
609 }
610 }
611 }
612 }
613 at += n;
614 }
615 t += 8;
616 }
617 let p = xyz[atom];
618 for k in t..b {
619 let q = xyz[j[k] as usize];
620 let c = code[k] as usize;
621 let dx = q[0] + tx[c] - p[0];
622 let dy = q[1] + ty[c] - p[1];
623 let dz = q[2] + tz[c] - p[2];
624 let dist = (dx * dx + dy * dy) + dz * dz;
625 let _ = cand;
626 if dist < cut2 {
627 if WRITE {
628 unsafe {
629 *(ip as *mut i32).add(at) = atom as i32;
630 *(jp as *mut i32).add(at) = j[k] as i32;
631 *(sp as *mut [i32; 3]).add(at) = ISHIFT[c];
632 *(dp as *mut f64).add(at) = dist;
633 }
634 }
635 at += 1;
636 hits += 1;
637 }
638 }
639 }
640 hits
641}
642
643fn bead_pass<const WRITE: bool>(
644 xyz: &[[f64; 3]],
645 atoms: usize,
646 beads: usize,
647 offsets: &[u32],
648 j: &[u32],
649 code: &[u8],
650 ranges: &[(usize, usize)],
651 off: &[Vec<usize>],
652 cut2: f64,
653 tx: &[f64; 343],
654 ty: &[f64; 343],
655 tz: &[f64; 343],
656 use_avx: bool,
657 ptrs: &[(usize, usize, usize, usize)],
658) -> Vec<Vec<usize>> {
659 let run = |t: usize, lo: usize, hi: usize| -> Vec<usize> {
660 #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
661 if use_avx {
662 return unsafe {
663 bead_avx::<WRITE>(
664 xyz, atoms, beads, offsets, j, code, lo, hi, t, off, cut2, tx, ty, tz, ptrs,
665 )
666 };
667 }
668 let _ = use_avx;
669 bead_scalar::<WRITE>(
670 xyz, atoms, beads, offsets, j, code, lo, hi, t, off, cut2, tx, ty, tz, ptrs,
671 )
672 };
673 #[cfg(feature = "parallel")]
674 if ranges.len() > 1 {
675 use rayon::prelude::*;
676 return ranges
677 .par_iter()
678 .enumerate()
679 .map(|(t, &(lo, hi))| run(t, lo, hi))
680 .collect();
681 }
682 ranges
683 .iter()
684 .enumerate()
685 .map(|(t, &(lo, hi))| run(t, lo, hi))
686 .collect()
687}
688
689fn bead_scalar<const WRITE: bool>(
690 xyz: &[[f64; 3]],
691 atoms: usize,
692 beads: usize,
693 offsets: &[u32],
694 j: &[u32],
695 code: &[u8],
696 lo: usize,
697 hi: usize,
698 t: usize,
699 off: &[Vec<usize>],
700 cut2: f64,
701 tx: &[f64; 343],
702 ty: &[f64; 343],
703 tz: &[f64; 343],
704 ptrs: &[(usize, usize, usize, usize)],
705) -> Vec<usize> {
706 let mut hits = vec![0usize; beads];
707 let mut at: Vec<usize> = (0..beads)
708 .map(|b| if WRITE { off[b][t] } else { 0 })
709 .collect();
710 for atom in lo..hi {
711 let a = offsets[atom] as usize;
712 let b = offsets[atom + 1] as usize;
713 for k in a..b {
714 let c = code[k] as usize;
715 let target = j[k] as usize;
716 let (sx, sy, sz) = (tx[c], ty[c], tz[c]);
717 let shift = ISHIFT[c];
718 for bead in 0..beads {
719 let p = xyz[bead * atoms + atom];
720 let q = xyz[bead * atoms + target];
721 let dx = q[0] + sx - p[0];
722 let dy = q[1] + sy - p[1];
723 let dz = q[2] + sz - p[2];
724 let dist = (dx * dx + dy * dy) + dz * dz;
725 if dist < cut2 {
726 if WRITE {
727 let (ip, jp, sp, dp) = ptrs[bead];
728 let atb = at[bead];
729 unsafe {
730 *(ip as *mut i32).add(atb) = atom as i32;
731 *(jp as *mut i32).add(atb) = target as i32;
732 *(sp as *mut [i32; 3]).add(atb) = shift;
733 *(dp as *mut f64).add(atb) = dist;
734 }
735 }
736 at[bead] += 1;
737 hits[bead] += 1;
738 }
739 }
740 }
741 }
742 hits
743}
744
745#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
746#[target_feature(enable = "avx512f,avx512vl,avx2")]
747unsafe fn bead_avx<const WRITE: bool>(
748 xyz: &[[f64; 3]],
749 atoms: usize,
750 beads: usize,
751 offsets: &[u32],
752 j: &[u32],
753 code: &[u8],
754 lo: usize,
755 hi: usize,
756 t: usize,
757 off: &[Vec<usize>],
758 cut2: f64,
759 tx: &[f64; 343],
760 ty: &[f64; 343],
761 tz: &[f64; 343],
762 ptrs: &[(usize, usize, usize, usize)],
763) -> Vec<usize> {
764 use std::arch::x86_64::{
765 __m128i, __m256i, _mm256_add_epi32, _mm256_cvtepu8_epi32, _mm256_loadu_si256,
766 _mm256_mask_compressstoreu_epi32, _mm256_maskz_compress_epi32, _mm256_mullo_epi32,
767 _mm256_set1_epi32, _mm256_storeu_si256, _mm512_add_pd, _mm512_cmp_pd_mask,
768 _mm512_i32gather_pd, _mm512_mask_compressstoreu_pd, _mm512_mul_pd, _mm512_set1_pd,
769 _mm512_sub_pd, _mm_loadl_epi64, _CMP_LT_OQ,
770 };
771 let mut hits = vec![0usize; beads];
772 let mut at: Vec<usize> = (0..beads)
773 .map(|b| if WRITE { off[b][t] } else { 0 })
774 .collect();
775 let cut = _mm512_set1_pd(cut2);
776 let txp = tx.as_ptr();
777 let typ = ty.as_ptr();
778 let tzp = tz.as_ptr();
779 let three = _mm256_set1_epi32(3);
780 let one = _mm256_set1_epi32(1);
781 let two = _mm256_set1_epi32(2);
782 for atom in lo..hi {
783 let a = offsets[atom] as usize;
784 let b = offsets[atom + 1] as usize;
785 let mut k = a;
786 while k + 8 <= b {
787 let raw = unsafe { (code.as_ptr().add(k) as *const u64).read_unaligned() };
788 let zero = raw == ZERO8;
789 let jv = unsafe { _mm256_loadu_si256(j.as_ptr().add(k) as *const __m256i) };
790 let ix = _mm256_mullo_epi32(jv, three);
791 let (sxv, syv, szv, c32) = if zero {
792 (
793 _mm512_set1_pd(0.0),
794 _mm512_set1_pd(0.0),
795 _mm512_set1_pd(0.0),
796 _mm256_set1_epi32(i32::from(ZERO_CODE)),
797 )
798 } else {
799 let c32 = _mm256_cvtepu8_epi32(unsafe {
800 _mm_loadl_epi64(code.as_ptr().add(k) as *const __m128i)
801 });
802 (
803 unsafe { _mm512_i32gather_pd(c32, txp, 8) },
804 unsafe { _mm512_i32gather_pd(c32, typ, 8) },
805 unsafe { _mm512_i32gather_pd(c32, tzp, 8) },
806 c32,
807 )
808 };
809 for bead in 0..beads {
810 let base = unsafe { xyz.as_ptr().add(bead * atoms) } as *const f64;
811 let p = xyz[bead * atoms + atom];
812 let px = _mm512_set1_pd(p[0]);
813 let py = _mm512_set1_pd(p[1]);
814 let pz = _mm512_set1_pd(p[2]);
815 let qx = unsafe { _mm512_i32gather_pd(ix, base, 8) };
816 let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), base, 8) };
817 let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), base, 8) };
818 let dx = _mm512_sub_pd(_mm512_add_pd(qx, sxv), px);
819 let dy = _mm512_sub_pd(_mm512_add_pd(qy, syv), py);
820 let dz = _mm512_sub_pd(_mm512_add_pd(qz, szv), pz);
821 let dist = _mm512_add_pd(
822 _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
823 _mm512_mul_pd(dz, dz),
824 );
825 let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
826 let n = mask.count_ones() as usize;
827 hits[bead] += n;
828 if WRITE && n > 0 {
829 let (ip, jp, sp, dp) = ptrs[bead];
830 let atb = at[bead];
831 unsafe {
832 let iv = _mm256_set1_epi32(atom as i32);
833 _mm256_mask_compressstoreu_epi32((ip as *mut i32).add(atb), mask, iv);
834 _mm256_mask_compressstoreu_epi32((jp as *mut i32).add(atb), mask, jv);
835 _mm512_mask_compressstoreu_pd((dp as *mut f64).add(atb), mask, dist);
836 if zero {
837 let sptr = (sp as *mut [i32; 3]).add(atb);
838 for hhit in 0..n {
839 *sptr.add(hhit) = [0, 0, 0];
840 }
841 } else {
842 let packed = _mm256_maskz_compress_epi32(mask, c32);
843 let mut cs = [0i32; 8];
844 _mm256_storeu_si256(cs.as_mut_ptr() as *mut __m256i, packed);
845 let sptr = (sp as *mut [i32; 3]).add(atb);
846 for hhit in 0..n {
847 *sptr.add(hhit) = ISHIFT[cs[hhit] as usize];
848 }
849 }
850 }
851 at[bead] += n;
852 }
853 }
854 k += 8;
855 }
856 for idx in k..b {
857 let c = code[idx] as usize;
858 let target = j[idx] as usize;
859 let (sx, sy, sz) = (tx[c], ty[c], tz[c]);
860 let shift = ISHIFT[c];
861 for bead in 0..beads {
862 let p = xyz[bead * atoms + atom];
863 let q = xyz[bead * atoms + target];
864 let dx = q[0] + sx - p[0];
865 let dy = q[1] + sy - p[1];
866 let dz = q[2] + sz - p[2];
867 let dist = (dx * dx + dy * dy) + dz * dz;
868 if dist < cut2 {
869 if WRITE {
870 let (ip, jp, sp, dp) = ptrs[bead];
871 let atb = at[bead];
872 unsafe {
873 *(ip as *mut i32).add(atb) = atom as i32;
874 *(jp as *mut i32).add(atb) = target as i32;
875 *(sp as *mut [i32; 3]).add(atb) = shift;
876 *(dp as *mut f64).add(atb) = dist;
877 }
878 }
879 at[bead] += 1;
880 hits[bead] += 1;
881 }
882 }
883 }
884 }
885 hits
886}
887
888fn write_ranges(
889 xyz: &[[f64; 3]],
890 soa: &Soa,
891 ranges: &[(usize, usize)],
892 off: &[usize],
893 d2: &[f64],
894 cut2: f64,
895 want_vector: bool,
896 ortho: bool,
897 widths: [f64; 3],
898 h: [[f64; 3]; 3],
899 use_avx: bool,
900 out: &mut PairColumns,
901 vectors: &mut Vec<[f64; 3]>,
902) {
903 let ip = out.i.as_mut_ptr() as usize;
904 let jp = out.j.as_mut_ptr() as usize;
905 let sp = out.shift.as_mut_ptr() as usize;
906 let dp = out.dist2.as_mut_ptr() as usize;
907 let vp = vectors.as_mut_ptr() as usize;
908 let job = move |t: usize, lo: usize, hi: usize| {
909 let mut at = off[t];
910 for atom in lo..hi {
911 let a = soa.offsets[atom] as usize;
912 let b = soa.offsets[atom + 1] as usize;
913 let p = xyz[atom];
914 if use_avx && !want_vector && b - a >= 8 {
915 at += compact_atom(soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp);
916 } else {
917 for k in a..b {
918 let dist = d2[k];
919 if dist < cut2 {
920 let s = [soa.sx[k], soa.sy[k], soa.sz[k]];
921 unsafe {
922 *(ip as *mut i32).add(at) = atom as i32;
923 *(jp as *mut i32).add(at) = soa.j[k] as i32;
924 *(sp as *mut [i32; 3]).add(at) = s;
925 *(dp as *mut f64).add(at) = dist;
926 if want_vector {
927 let q = *xyz.get_unchecked(soa.j[k] as usize);
928 let (tx, ty, tz) = shift_xyz(s[0], s[1], s[2], ortho, widths, h);
929 *(vp as *mut [f64; 3]).add(at) =
930 [q[0] + tx - p[0], q[1] + ty - p[1], q[2] + tz - p[2]];
931 }
932 }
933 at += 1;
934 }
935 }
936 }
937 }
938 };
939 #[cfg(feature = "parallel")]
940 if ranges.len() > 1 {
941 use rayon::prelude::*;
942 ranges
943 .par_iter()
944 .enumerate()
945 .for_each(|(t, &(lo, hi))| job(t, lo, hi));
946 return;
947 }
948 if let Some(&(lo, hi)) = ranges.first() {
949 job(0, lo, hi);
950 }
951}
952
953fn compact_atom(
954 soa: &Soa,
955 d2: &[f64],
956 atom: usize,
957 a: usize,
958 b: usize,
959 cut2: f64,
960 at: usize,
961 ip: usize,
962 jp: usize,
963 sp: usize,
964 dp: usize,
965) -> usize {
966 #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
967 unsafe {
968 compact_avx512(soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp)
969 }
970 #[cfg(not(all(target_arch = "x86_64", linkcell_avx512)))]
971 {
972 let _ = (soa, d2, atom, a, b, cut2, at, ip, jp, sp, dp);
973 0
974 }
975}
976
977#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
978#[target_feature(enable = "avx512f,avx512vl,avx2")]
979unsafe fn compact_avx512(
980 soa: &Soa,
981 d2: &[f64],
982 atom: usize,
983 a: usize,
984 b: usize,
985 cut2: f64,
986 mut at: usize,
987 ip: usize,
988 jp: usize,
989 sp: usize,
990 dp: usize,
991) -> usize {
992 use std::arch::x86_64::{
993 __m256i, _mm256_loadu_si256, _mm256_maskz_compress_epi32, _mm256_storeu_si256,
994 _mm512_cmp_pd_mask, _mm512_loadu_pd, _mm512_maskz_compress_pd, _mm512_set1_pd,
995 _mm512_storeu_pd, _CMP_LT_OQ,
996 };
997 let start = at;
998 let cut = _mm512_set1_pd(cut2);
999 let mut t = a;
1000 let mut js = [0i32; 8];
1001 let mut xs = [0i32; 8];
1002 let mut ys = [0i32; 8];
1003 let mut zs = [0i32; 8];
1004 let mut ds = [0f64; 8];
1005 while t + 8 <= b {
1006 let dist = unsafe { _mm512_loadu_pd(d2.as_ptr().add(t)) };
1007 let mask = _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut);
1008 let n = mask.count_ones() as usize;
1009 if n > 0 {
1010 unsafe {
1011 _mm256_storeu_si256(
1012 js.as_mut_ptr() as *mut __m256i,
1013 _mm256_maskz_compress_epi32(
1014 mask,
1015 _mm256_loadu_si256(soa.j.as_ptr().add(t) as *const __m256i),
1016 ),
1017 );
1018 _mm256_storeu_si256(
1019 xs.as_mut_ptr() as *mut __m256i,
1020 _mm256_maskz_compress_epi32(
1021 mask,
1022 _mm256_loadu_si256(soa.sx.as_ptr().add(t) as *const __m256i),
1023 ),
1024 );
1025 _mm256_storeu_si256(
1026 ys.as_mut_ptr() as *mut __m256i,
1027 _mm256_maskz_compress_epi32(
1028 mask,
1029 _mm256_loadu_si256(soa.sy.as_ptr().add(t) as *const __m256i),
1030 ),
1031 );
1032 _mm256_storeu_si256(
1033 zs.as_mut_ptr() as *mut __m256i,
1034 _mm256_maskz_compress_epi32(
1035 mask,
1036 _mm256_loadu_si256(soa.sz.as_ptr().add(t) as *const __m256i),
1037 ),
1038 );
1039 _mm512_storeu_pd(ds.as_mut_ptr(), _mm512_maskz_compress_pd(mask, dist));
1040 }
1041 let ii = atom as i32;
1042 for h in 0..n {
1043 unsafe {
1044 *(ip as *mut i32).add(at) = ii;
1045 *(jp as *mut i32).add(at) = js[h];
1046 *(sp as *mut [i32; 3]).add(at) = [xs[h], ys[h], zs[h]];
1047 *(dp as *mut f64).add(at) = ds[h];
1048 }
1049 at += 1;
1050 }
1051 }
1052 t += 8;
1053 }
1054 for k in t..b {
1055 let dist = d2[k];
1056 if dist < cut2 {
1057 unsafe {
1058 *(ip as *mut i32).add(at) = atom as i32;
1059 *(jp as *mut i32).add(at) = soa.j[k] as i32;
1060 *(sp as *mut [i32; 3]).add(at) = [soa.sx[k], soa.sy[k], soa.sz[k]];
1061 *(dp as *mut f64).add(at) = dist;
1062 }
1063 at += 1;
1064 }
1065 }
1066 at - start
1067}
1068
1069#[allow(clippy::uninit_vec)]
1070fn prepare(out: &mut PairColumns, vectors: &mut Vec<[f64; 3]>, total: usize, want_vector: bool) {
1071 out.i.clear();
1072 out.j.clear();
1073 out.shift.clear();
1074 out.dist2.clear();
1075 vectors.clear();
1076 out.i.reserve(total);
1077 out.j.reserve(total);
1078 out.shift.reserve(total);
1079 out.dist2.reserve(total);
1080 unsafe {
1081 out.i.set_len(total);
1082 out.j.set_len(total);
1083 out.shift.set_len(total);
1084 out.dist2.set_len(total);
1085 }
1086 if want_vector {
1087 vectors.reserve(total);
1088 unsafe { vectors.set_len(total) };
1089 }
1090}
1091
1092fn filter_threads(n: usize) -> usize {
1093 #[cfg(feature = "parallel")]
1094 {
1095 let pool = crate::pool_width().max(1);
1096 if pool > 1 && n >= 32_768 {
1097 pool
1098 } else {
1099 1
1100 }
1101 }
1102 #[cfg(not(feature = "parallel"))]
1103 {
1104 let _ = n;
1105 1
1106 }
1107}
1108
1109fn atom_ranges(offsets: &[u32], threads: usize) -> Vec<(usize, usize)> {
1110 let atoms = offsets.len().saturating_sub(1);
1111 if atoms == 0 {
1112 return Vec::new();
1113 }
1114 let threads = threads.max(1).min(atoms);
1115 let total = offsets[atoms] as usize;
1116 if threads == 1 || total == 0 {
1117 return vec![(0, atoms)];
1118 }
1119 let mut ranges = Vec::with_capacity(threads);
1120 let mut start = 0usize;
1121 for t in 1..threads {
1122 let target = total * t / threads;
1123 let mut end = start;
1124 while end < atoms && (offsets[end] as usize) < target {
1125 end += 1;
1126 }
1127 if end == start {
1128 end = (start + 1).min(atoms);
1129 }
1130 ranges.push((start, end));
1131 start = end;
1132 }
1133 if start < atoms {
1134 ranges.push((start, atoms));
1135 }
1136 ranges
1137}
1138
1139fn count_one(
1140 xyz: &[[f64; 3]],
1141 offsets: &[u32],
1142 j: &[u32],
1143 sx: &[i32],
1144 sy: &[i32],
1145 sz: &[i32],
1146 lo: usize,
1147 hi: usize,
1148 cut2: f64,
1149 ortho: bool,
1150 widths: [f64; 3],
1151 h: [[f64; 3]; 3],
1152 use_avx: bool,
1153 d2: &mut [f64],
1154) -> usize {
1155 let c0 = offsets[lo] as usize;
1156 let mut hits = 0usize;
1157 let mut local = 0usize;
1158 for atom in lo..hi {
1159 let a = offsets[atom] as usize;
1160 let b = offsets[atom + 1] as usize;
1161 let count = b - a;
1162 if count == 0 {
1163 continue;
1164 }
1165 hits += distance_block(
1166 xyz,
1167 j,
1168 sx,
1169 sy,
1170 sz,
1171 a,
1172 count,
1173 atom,
1174 cut2,
1175 ortho,
1176 widths,
1177 h,
1178 use_avx,
1179 &mut d2[local..local + count],
1180 );
1181 local += count;
1182 }
1183 let _ = c0;
1184 hits
1185}
1186
1187fn count_ranges(
1188 xyz: &[[f64; 3]],
1189 offsets: &[u32],
1190 j: &[u32],
1191 sx: &[i32],
1192 sy: &[i32],
1193 sz: &[i32],
1194 ranges: &[(usize, usize)],
1195 cut2: f64,
1196 ortho: bool,
1197 widths: [f64; 3],
1198 h: [[f64; 3]; 3],
1199 use_avx: bool,
1200 d2: &mut [f64],
1201) -> Vec<usize> {
1202 #[cfg(feature = "parallel")]
1203 if ranges.len() > 1 {
1204 use rayon::prelude::*;
1205 let ptr = d2.as_mut_ptr() as usize;
1206 return ranges
1207 .par_iter()
1208 .map(|&(lo, hi)| {
1209 let c0 = offsets[lo] as usize;
1210 let c1 = offsets[hi] as usize;
1211 let slice = unsafe {
1212 std::slice::from_raw_parts_mut(
1213 (ptr + c0 * std::mem::size_of::<f64>()) as *mut f64,
1214 c1 - c0,
1215 )
1216 };
1217 count_one(
1218 xyz, offsets, j, sx, sy, sz, lo, hi, cut2, ortho, widths, h, use_avx, slice,
1219 )
1220 })
1221 .collect();
1222 }
1223 ranges
1224 .iter()
1225 .map(|&(lo, hi)| {
1226 let c0 = offsets[lo] as usize;
1227 let c1 = offsets[hi] as usize;
1228 count_one(
1229 xyz,
1230 offsets,
1231 j,
1232 sx,
1233 sy,
1234 sz,
1235 lo,
1236 hi,
1237 cut2,
1238 ortho,
1239 widths,
1240 h,
1241 use_avx,
1242 &mut d2[c0..c1],
1243 )
1244 })
1245 .collect()
1246}
1247
1248fn distance_block(
1249 xyz: &[[f64; 3]],
1250 j: &[u32],
1251 sx: &[i32],
1252 sy: &[i32],
1253 sz: &[i32],
1254 base: usize,
1255 count: usize,
1256 atom: usize,
1257 cut2: f64,
1258 ortho: bool,
1259 widths: [f64; 3],
1260 h: [[f64; 3]; 3],
1261 use_avx: bool,
1262 d2: &mut [f64],
1263) -> usize {
1264 #[cfg(all(target_arch = "x86_64", linkcell_avx512))]
1265 if use_avx {
1266 return unsafe {
1267 distance_avx512(
1268 xyz, j, sx, sy, sz, base, count, atom, cut2, ortho, widths, h, d2,
1269 )
1270 };
1271 }
1272 let _ = use_avx;
1273 distance_scalar(
1274 xyz, j, sx, sy, sz, base, count, atom, cut2, ortho, widths, h, d2,
1275 )
1276}
1277
1278fn distance_scalar(
1279 xyz: &[[f64; 3]],
1280 j: &[u32],
1281 sx: &[i32],
1282 sy: &[i32],
1283 sz: &[i32],
1284 base: usize,
1285 count: usize,
1286 atom: usize,
1287 cut2: f64,
1288 ortho: bool,
1289 widths: [f64; 3],
1290 h: [[f64; 3]; 3],
1291 d2: &mut [f64],
1292) -> usize {
1293 let p = xyz[atom];
1294 let mut hits = 0usize;
1295 for t in 0..count {
1296 let q = xyz[j[base + t] as usize];
1297 let (tx, ty, tz) = shift_xyz(sx[base + t], sy[base + t], sz[base + t], ortho, widths, h);
1298 let dx = q[0] + tx - p[0];
1299 let dy = q[1] + ty - p[1];
1300 let dz = q[2] + tz - p[2];
1301 let dist = (dx * dx + dy * dy) + dz * dz;
1302 d2[t] = dist;
1303 if dist < cut2 {
1304 hits += 1;
1305 }
1306 }
1307 hits
1308}
1309
1310pub(crate) fn shift_xyz(
1311 sx: i32,
1312 sy: i32,
1313 sz: i32,
1314 ortho: bool,
1315 widths: [f64; 3],
1316 h: [[f64; 3]; 3],
1317) -> (f64, f64, f64) {
1318 if ortho {
1319 (
1320 f64::from(sx) * widths[0],
1321 f64::from(sy) * widths[1],
1322 f64::from(sz) * widths[2],
1323 )
1324 } else {
1325 let (a, b, c) = (f64::from(sx), f64::from(sy), f64::from(sz));
1326 (
1327 a * h[0][0] + b * h[1][0] + c * h[2][0],
1328 a * h[0][1] + b * h[1][1] + c * h[2][1],
1329 a * h[0][2] + b * h[1][2] + c * h[2][2],
1330 )
1331 }
1332}
1333
1334#[cfg(all(target_arch = "x86_64", linkcell_avx512))]
1335#[target_feature(enable = "avx512f,avx512vl,avx2")]
1336unsafe fn distance_avx512(
1337 xyz: &[[f64; 3]],
1338 j: &[u32],
1339 sx: &[i32],
1340 sy: &[i32],
1341 sz: &[i32],
1342 base: usize,
1343 count: usize,
1344 atom: usize,
1345 cut2: f64,
1346 ortho: bool,
1347 widths: [f64; 3],
1348 h: [[f64; 3]; 3],
1349 d2: &mut [f64],
1350) -> usize {
1351 use std::arch::x86_64::{
1352 __m256i, _mm256_add_epi32, _mm256_loadu_si256, _mm256_mullo_epi32, _mm256_set1_epi32,
1353 _mm512_add_pd, _mm512_cmp_pd_mask, _mm512_cvtepi32_pd, _mm512_i32gather_pd, _mm512_mul_pd,
1354 _mm512_set1_pd, _mm512_storeu_pd, _mm512_sub_pd, _CMP_LT_OQ,
1355 };
1356 let p = xyz[atom];
1357 let bytes = xyz.as_ptr() as *const f64;
1358 let jptr = j.as_ptr().add(base);
1359 let sxp = sx.as_ptr().add(base);
1360 let syp = sy.as_ptr().add(base);
1361 let szp = sz.as_ptr().add(base);
1362 let outp = d2.as_mut_ptr();
1363 let cut = _mm512_set1_pd(cut2);
1364 let px = _mm512_set1_pd(p[0]);
1365 let py = _mm512_set1_pd(p[1]);
1366 let pz = _mm512_set1_pd(p[2]);
1367 let three = _mm256_set1_epi32(3);
1368 let one = _mm256_set1_epi32(1);
1369 let two = _mm256_set1_epi32(2);
1370 let mut hits = 0usize;
1371 let mut t = 0usize;
1372 while t + 8 <= count {
1373 let jv = unsafe { _mm256_loadu_si256(jptr.add(t) as *const __m256i) };
1374 let ix = _mm256_mullo_epi32(jv, three);
1375 let qx = unsafe { _mm512_i32gather_pd(ix, bytes, 8) };
1376 let qy = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, one), bytes, 8) };
1377 let qz = unsafe { _mm512_i32gather_pd(_mm256_add_epi32(ix, two), bytes, 8) };
1378 let sxf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(sxp.add(t) as *const __m256i) });
1379 let syf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(syp.add(t) as *const __m256i) });
1380 let szf = _mm512_cvtepi32_pd(unsafe { _mm256_loadu_si256(szp.add(t) as *const __m256i) });
1381 let (tx, ty, tz) = if ortho {
1382 (
1383 _mm512_mul_pd(sxf, _mm512_set1_pd(widths[0])),
1384 _mm512_mul_pd(syf, _mm512_set1_pd(widths[1])),
1385 _mm512_mul_pd(szf, _mm512_set1_pd(widths[2])),
1386 )
1387 } else {
1388 (
1389 _mm512_add_pd(
1390 _mm512_add_pd(
1391 _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][0])),
1392 _mm512_mul_pd(syf, _mm512_set1_pd(h[1][0])),
1393 ),
1394 _mm512_mul_pd(szf, _mm512_set1_pd(h[2][0])),
1395 ),
1396 _mm512_add_pd(
1397 _mm512_add_pd(
1398 _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][1])),
1399 _mm512_mul_pd(syf, _mm512_set1_pd(h[1][1])),
1400 ),
1401 _mm512_mul_pd(szf, _mm512_set1_pd(h[2][1])),
1402 ),
1403 _mm512_add_pd(
1404 _mm512_add_pd(
1405 _mm512_mul_pd(sxf, _mm512_set1_pd(h[0][2])),
1406 _mm512_mul_pd(syf, _mm512_set1_pd(h[1][2])),
1407 ),
1408 _mm512_mul_pd(szf, _mm512_set1_pd(h[2][2])),
1409 ),
1410 )
1411 };
1412 let dx = _mm512_sub_pd(_mm512_add_pd(qx, tx), px);
1413 let dy = _mm512_sub_pd(_mm512_add_pd(qy, ty), py);
1414 let dz = _mm512_sub_pd(_mm512_add_pd(qz, tz), pz);
1415 let dist = _mm512_add_pd(
1416 _mm512_add_pd(_mm512_mul_pd(dx, dx), _mm512_mul_pd(dy, dy)),
1417 _mm512_mul_pd(dz, dz),
1418 );
1419 unsafe { _mm512_storeu_pd(outp.add(t), dist) };
1420 hits += _mm512_cmp_pd_mask::<_CMP_LT_OQ>(dist, cut).count_ones() as usize;
1421 t += 8;
1422 }
1423 if t < count {
1424 hits += distance_scalar(
1425 xyz,
1426 j,
1427 sx,
1428 sy,
1429 sz,
1430 base + t,
1431 count - t,
1432 atom,
1433 cut2,
1434 ortho,
1435 widths,
1436 h,
1437 &mut d2[t..],
1438 );
1439 }
1440 hits
1441}