Skip to main content

linkcell/
capi.rs

1//! C ABI. Prefix `lc_`. Caller owns every buffer.
2
3#![deny(unsafe_op_in_unsafe_fn)]
4
5use std::cell::RefCell;
6use std::ffi::{c_char, c_int, CString};
7use std::ptr;
8use std::sync::atomic::{AtomicUsize, Ordering};
9
10use crate::{Cell, Error};
11
12/// How many Rayon threads the next search should use. Zero is unset: the
13/// search runs from the calling thread with rayon's and OpenMP's own
14/// defaults. One runs on the calling thread alone.
15static WANTED_THREADS: AtomicUsize = AtomicUsize::new(0);
16
17/// One pool for the process. A different width drops the previous pool
18/// before the new one starts, so two widths never run together. The
19/// workers sleep on a condvar between searches (rayon's idle path),
20/// which is the passive wait. A one-thread search starts no pool, builds
21/// no global pool and starts no OpenMP team.
22#[cfg(not(feature = "parallel"))]
23fn with_threads<R: Send>(work: impl FnOnce() -> R + Send) -> R {
24    work()
25}
26
27#[cfg(feature = "parallel")]
28struct Slot {
29    n: usize,
30    pool: rayon::ThreadPool,
31}
32
33#[cfg(feature = "parallel")]
34static CACHE: std::sync::Mutex<Option<Slot>> = std::sync::Mutex::new(None);
35
36#[cfg(feature = "parallel")]
37fn with_threads<R: Send>(work: impl FnOnce() -> R + Send) -> R {
38    let n = WANTED_THREADS.load(Ordering::Relaxed);
39    if n == 0 {
40        return work();
41    }
42    if n == 1 {
43        return crate::run_on_caller(work);
44    }
45    if rayon::current_thread_index().is_some() && rayon::current_num_threads() == n {
46        return work();
47    }
48    let mut guard = CACHE.lock().unwrap_or_else(|err| err.into_inner());
49    if guard.as_ref().map(|slot| slot.n) != Some(n) {
50        *guard = None;
51        let pool = crate::place::build_pool(n);
52        *guard = Some(Slot { n, pool });
53    }
54    guard.as_ref().unwrap().pool.install(work)
55}
56
57/// Run a k-nearest search on the calling thread, with the workers of the
58/// pool [`lc_set_threads`] chose joining it (see `team`). The caller does
59/// not hand the search to a pool thread and sleep: it is one of the
60/// workers, and the call returns when the last stage closes.
61#[cfg(not(feature = "parallel"))]
62fn with_team<R>(work: impl FnOnce() -> R) -> R {
63    work()
64}
65
66#[cfg(feature = "parallel")]
67fn with_team<R>(work: impl FnOnce() -> R) -> R {
68    let n = WANTED_THREADS.load(Ordering::Relaxed);
69    if n == 0 || (rayon::current_thread_index().is_some() && rayon::current_num_threads() == n) {
70        return work();
71    }
72    if n == 1 {
73        return crate::run_on_caller(|| crate::team::install(1, &|job| job(), work));
74    }
75    let mut guard = CACHE.lock().unwrap_or_else(|err| err.into_inner());
76    if guard.as_ref().map(|slot| slot.n) != Some(n) {
77        *guard = None;
78        let pool = crate::place::build_pool(n);
79        *guard = Some(Slot { n, pool });
80    }
81    let pool = &guard.as_ref().unwrap().pool;
82    crate::team::install(n, &|job| pool.spawn(job), work)
83}
84
85/// Choose the Rayon width for later searches. `n == 0` leaves it unset:
86/// the search starts on the calling thread with Rayon's and OpenMP's own
87/// widths. `n == 1` runs the whole search on the calling thread and
88/// starts no thread. `n < 0` selects one thread.
89#[no_mangle]
90pub extern "C" fn lc_set_threads(n: c_int) {
91    let n = if n < 0 { 1 } else { n as usize };
92    WANTED_THREADS.store(n, Ordering::Relaxed);
93}
94
95/// Width selected by [`lc_set_threads`], or the global pool when unset.
96#[no_mangle]
97pub extern "C" fn lc_num_threads() -> c_int {
98    let n = WANTED_THREADS.load(Ordering::Relaxed);
99    if n == 0 {
100        default_width() as c_int
101    } else {
102        n as c_int
103    }
104}
105
106/// Width of the global Rayon pool, or one thread without `parallel`.
107fn default_width() -> usize {
108    #[cfg(feature = "parallel")]
109    {
110        rayon::current_num_threads()
111    }
112    #[cfg(not(feature = "parallel"))]
113    {
114        1
115    }
116}
117
118#[derive(Clone, Copy)]
119struct Continuation {
120    ctx: usize,
121    cb: usize,
122}
123
124thread_local! {
125    /// A callback the calling thread's next search runs before its pool
126    /// goes idle, so the CSR merge does not pay a second wake. Each thread
127    /// sets and clears its own hook, so concurrent searches on different
128    /// threads never run each other's callback.
129    // `const { ... }` is newer than this crate's 1.70 floor.
130    #[allow(clippy::missing_const_for_thread_local)]
131    static CONTINUATION: std::cell::Cell<Option<Continuation>> = std::cell::Cell::new(None);
132}
133
134/// The hook the calling thread set, read before the search enters the pool.
135fn take_continuation() -> Option<Continuation> {
136    CONTINUATION.with(|slot| slot.get())
137}
138
139fn run_continuation(cont: Option<Continuation>) {
140    let Some(cont) = cont else {
141        return;
142    };
143    let cb: extern "C" fn(*mut std::ffi::c_void) = unsafe { std::mem::transmute(cont.cb) };
144    cb(cont.ctx as *mut std::ffi::c_void);
145}
146
147/// Run `cb` at the end of this thread's next `lc_knearest`, still inside
148/// its pool. A null `cb` clears it. The hook belongs to the calling
149/// thread; a search started on another thread does not run it. The search
150/// clears nothing by itself: the caller drops the hook when the call
151/// returns.
152#[no_mangle]
153pub extern "C" fn lc_set_continuation(
154    ctx: *mut std::ffi::c_void,
155    cb: Option<extern "C" fn(*mut std::ffi::c_void)>,
156) {
157    CONTINUATION.with(|slot| {
158        slot.set(cb.map(|cb| Continuation {
159            ctx: ctx as usize,
160            cb: cb as usize,
161        }))
162    });
163}
164
165/// Install the device curve split used by the `device` partition. The
166/// GPU library calls this from a static initializer when it is loaded. A
167/// null `f` removes it, and the device partition then reports no device.
168#[no_mangle]
169pub extern "C" fn lc_set_gpu_curve_split(
170    f: Option<
171        unsafe extern "C" fn(
172            atom_bin: *const u32,
173            natom: usize,
174            coords: *const u32,
175            ncell: usize,
176            parts: u32,
177            assign: *mut u32,
178            out_weight: *mut u64,
179            method: u8,
180        ) -> i32,
181    >,
182) {
183    crate::partition::register_device_curve(f);
184}
185
186/// Run `cb(ctx, begin, end)` over disjoint spans of `0..n` on the pool
187/// selected by [`lc_set_threads`]. One thread runs the whole span on the
188/// caller. The spans cover `0..n` exactly once.
189#[no_mangle]
190pub extern "C" fn lc_for_slabs(
191    n: c_int,
192    ctx: *mut std::ffi::c_void,
193    cb: Option<extern "C" fn(*mut std::ffi::c_void, c_int, c_int)>,
194) {
195    let Some(cb) = cb else {
196        return;
197    };
198    if n <= 0 {
199        return;
200    }
201    let ctx_bits = ctx as usize;
202    let cb_bits = cb as usize;
203    with_threads(move || {
204        let call = |begin: c_int, end: c_int| {
205            let ctx = ctx_bits as *mut std::ffi::c_void;
206            let cb: extern "C" fn(*mut std::ffi::c_void, c_int, c_int) =
207                unsafe { std::mem::transmute(cb_bits) };
208            cb(ctx, begin, end);
209        };
210        #[cfg(not(feature = "parallel"))]
211        call(0, n);
212        #[cfg(feature = "parallel")]
213        if WANTED_THREADS.load(Ordering::Relaxed) <= 1 {
214            call(0, n);
215        } else {
216            rayon::broadcast(move |worker| {
217                let workers = worker.num_threads().max(1);
218                let t = worker.index();
219                let begin = (n as usize) * t / workers;
220                let end = (n as usize) * (t + 1) / workers;
221                call(begin as c_int, end as c_int);
222            });
223        }
224    });
225}
226
227/// Workers an [`lc_team_run`] started now may use: the `worker` index it
228/// passes is below this.
229#[no_mangle]
230pub extern "C" fn lc_team_width() -> usize {
231    with_team(|| {
232        crate::team::with_current(|width, spawn| if spawn.is_some() { width.max(1) } else { 1 })
233    })
234}
235
236/// Run `nstages` stages in one parallel region on the pool selected by
237/// [`lc_set_threads`], the caller one of the workers (see `team`). Stage
238/// `s` is `chunks[s]` calls of `fn_(ctx, s, chunk, worker)`, in any order
239/// and on any worker; every chunk of a stage returns before any chunk of
240/// the next starts. `worker` is fixed for one worker thread during the
241/// call and below [`lc_team_width`], so it can index per-worker scratch.
242/// No worker that has not started holds up a stage or the return.
243///
244/// # Safety
245///
246/// `chunks` is readable for `nstages` sizes. `fn_` does not unwind and
247/// may be called from any thread of the pool until the call returns.
248#[no_mangle]
249pub unsafe extern "C" fn lc_team_run(
250    nstages: usize,
251    chunks: *const usize,
252    fn_: Option<extern "C" fn(*mut std::ffi::c_void, usize, usize, usize)>,
253    ctx: *mut std::ffi::c_void,
254) -> c_int {
255    let Some(fn_) = fn_ else {
256        return fail_msg("null pointer");
257    };
258    if nstages == 0 {
259        clear_error();
260        return 0;
261    }
262    if chunks.is_null() {
263        return fail_msg("null pointer");
264    }
265    let sizes = unsafe { std::slice::from_raw_parts(chunks, nstages) }.to_vec();
266    let ctx_bits = ctx as usize;
267    let workers = AtomicUsize::new(0);
268    let runs: Vec<_> = (0..nstages)
269        .map(|stage| {
270            let workers = &workers;
271            move |worker: &mut Option<usize>, chunk: usize| {
272                let w = *worker.get_or_insert_with(|| workers.fetch_add(1, Ordering::Relaxed));
273                fn_(ctx_bits as *mut std::ffi::c_void, stage, chunk, w);
274            }
275        })
276        .collect();
277    let stages: Vec<crate::team::Stage<'_, Option<usize>>> = sizes
278        .iter()
279        .zip(&runs)
280        .map(|(&chunks, run)| crate::team::Stage { chunks, run })
281        .collect();
282    with_team(|| crate::team::with_current(|width, spawn| crate::team::run(width, spawn, &stages)));
283    clear_error();
284    0
285}
286
287/// Periodic parallelepiped. Lattice vectors are a, b, c (same as vesin rows).
288#[repr(C)]
289#[derive(Clone, Copy, Debug)]
290pub struct lc_cell {
291    /// Lattice vector a, x.
292    pub ax: f64,
293    /// Lattice vector a, y.
294    pub ay: f64,
295    /// Lattice vector a, z.
296    pub az: f64,
297    /// Lattice vector b, x.
298    pub bx: f64,
299    /// Lattice vector b, y.
300    pub by: f64,
301    /// Lattice vector b, z.
302    pub bz: f64,
303    /// Lattice vector c, x.
304    pub cx: f64,
305    /// Lattice vector c, y.
306    pub cy: f64,
307    /// Lattice vector c, z.
308    pub cz: f64,
309    /// Origin x.
310    pub ox: f64,
311    /// Origin y.
312    pub oy: f64,
313    /// Origin z.
314    pub oz: f64,
315}
316
317thread_local! {
318    static LAST_ERROR: RefCell<Option<CString>> = const { RefCell::new(None) };
319}
320
321fn set_error(msg: &str) {
322    LAST_ERROR.with(|slot| {
323        let cstr = CString::new(msg).unwrap_or_else(|_| {
324            CString::new("error message contained NUL").expect("fallback has no NUL")
325        });
326        *slot.borrow_mut() = Some(cstr);
327    });
328}
329
330fn clear_error() {
331    LAST_ERROR.with(|slot| {
332        *slot.borrow_mut() = None;
333    });
334}
335
336fn fail(err: Error) -> c_int {
337    set_error(&err.to_string());
338    1
339}
340
341fn fail_msg(msg: &str) -> c_int {
342    set_error(msg);
343    1
344}
345
346/// Thread-local last-error string from this thread's most recent
347/// [`lc_knearest`].
348///
349/// Returns a pointer to a NUL-terminated UTF-8 C string, or `NULL` if
350/// the last `lc_knearest` on this thread succeeded, or if none has
351/// failed yet. [`lc_version`] does not read or write the slot.
352/// Distinct threads have independent slots. The pointer is valid until
353/// the next `lc_knearest` on this thread. Do not free it.
354#[no_mangle]
355pub extern "C" fn lc_last_error() -> *const c_char {
356    LAST_ERROR.with(|slot| {
357        slot.borrow()
358            .as_ref()
359            .map(|s| s.as_ptr())
360            .unwrap_or(ptr::null())
361    })
362}
363
364/// Library version string. Process-static, NUL-terminated. Do not free.
365#[no_mangle]
366pub extern "C" fn lc_version() -> *const c_char {
367    concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char
368}
369
370/// k-nearest neighbours for `n` points.
371///
372/// `xyz` is packed row-major `n` triples `(x, y, z)` (`n * 3` doubles).
373/// `n` is the point count (`size_t`); `0` is an error.
374/// `simbox` is the periodic cell (lattice vectors a, b, c and origin).
375/// `k` is neighbours per source (`size_t`); `0` is an error.
376/// `mask` of `NULL` includes every point; otherwise `n` ints, nonzero to
377/// include that point as both source and candidate.
378/// `cell_hint` is the target cell edge. Values `<= 0` select the default
379/// (3.0 in the box units).
380/// `out_nn` is caller-owned output. Neighbours of source `i` occupy
381/// `out_nn[i*k + t]`, nearest first. Missing slots are `-1`. Length is
382/// `n * k` ints.
383///
384/// `xyz`, `simbox`, and `out_nn` must be non-null when `n > 0` and
385/// `k > 0`. `mask` may be `NULL`.
386///
387/// Returns `0` on success. Nonzero on failure. Read `lc_last_error()` on
388/// the same thread. The last-error string is thread-local: it is not
389/// shared across threads, is valid until the next `lc_*` call on this
390/// thread, and must not be freed.
391///
392/// # Safety
393///
394/// `xyz` is aligned for `double` and readable for `n * 3` doubles.
395/// `simbox` is aligned and points at one valid `lc_cell`.
396/// `out_nn` is aligned for `int` and writable for `n * k` ints.
397/// `mask`, if non-null, is aligned for `int` and readable for `n` ints.
398/// `n * 3` and `n * k` fit in `size_t`.
399#[no_mangle]
400pub unsafe extern "C" fn lc_knearest(
401    xyz: *const f64,
402    n: usize,
403    simbox: *const lc_cell,
404    k: usize,
405    mask: *const c_int,
406    cell_hint: f64,
407    out_nn: *mut c_int,
408) -> c_int {
409    // SAFETY: same contract as lc_knearest_d2 with a single frame and
410    // no distance buffer.
411    unsafe {
412        lc_knearest_many(
413            xyz,
414            n,
415            1,
416            simbox,
417            k,
418            mask,
419            cell_hint,
420            out_nn,
421            ptr::null_mut(),
422        )
423    }
424}
425
426/// Like [`lc_knearest`], and write squared distances into `out_d2`.
427///
428/// `out_d2` is caller-owned, length `n * k`. Unused slots are `NaN`.
429/// `out_d2` may be `NULL` to skip distances (same as [`lc_knearest`]).
430///
431/// # Safety
432///
433/// Same pointer contract as [`lc_knearest`]. `out_d2`, if non-null,
434/// is aligned for `double` and writable for `n * k` doubles.
435#[no_mangle]
436pub unsafe extern "C" fn lc_knearest_d2(
437    xyz: *const f64,
438    n: usize,
439    simbox: *const lc_cell,
440    k: usize,
441    mask: *const c_int,
442    cell_hint: f64,
443    out_nn: *mut c_int,
444    out_d2: *mut f64,
445) -> c_int {
446    unsafe { lc_knearest_many(xyz, n, 1, simbox, k, mask, cell_hint, out_nn, out_d2) }
447}
448
449/// Frame-major batch. `xyz` is `n_frames * n` packed triples.
450/// `out_nn` / `out_d2` are `n_frames * n * k`. One shared cell.
451/// `mask` is length `n` or `NULL`. `out_d2` may be `NULL`.
452///
453/// # Safety
454///
455/// `xyz` is readable for `n_frames * n * 3` doubles. `out_nn` is
456/// writable for `n_frames * n * k` ints. `out_d2`, if non-null, is
457/// writable for `n_frames * n * k` doubles. `simbox` and `mask`
458/// follow [`lc_knearest`].
459#[no_mangle]
460pub unsafe extern "C" fn lc_knearest_many(
461    xyz: *const f64,
462    n: usize,
463    n_frames: usize,
464    simbox: *const lc_cell,
465    k: usize,
466    mask: *const c_int,
467    cell_hint: f64,
468    out_nn: *mut c_int,
469    out_d2: *mut f64,
470) -> c_int {
471    if n == 0 || n_frames == 0 {
472        return fail(Error::Empty);
473    }
474    if k == 0 {
475        return fail(Error::ZeroK);
476    }
477    if xyz.is_null() || simbox.is_null() || out_nn.is_null() {
478        return fail_msg("null pointer");
479    }
480    let Some(n_pts) = n.checked_mul(n_frames) else {
481        return fail(Error::Overflow);
482    };
483    let Some(need) = n_pts.checked_mul(k) else {
484        return fail(Error::Overflow);
485    };
486    let max_xyz = (isize::MAX as usize) / 3;
487    let max_out = (isize::MAX as usize) / std::mem::size_of::<c_int>();
488    if n_pts > max_xyz || need > max_out {
489        return fail(Error::Overflow);
490    }
491    let box_c = unsafe { *simbox };
492    let sim = match Cell::from_vectors(
493        [box_c.ax, box_c.ay, box_c.az],
494        [box_c.bx, box_c.by, box_c.bz],
495        [box_c.cx, box_c.cy, box_c.cz],
496        [box_c.ox, box_c.oy, box_c.oz],
497    ) {
498        Ok(b) => b,
499        Err(e) => {
500            set_error(&e.to_string());
501            return 1;
502        }
503    };
504    let pts: &[[f64; 3]] = unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n_pts) };
505    let mask_vec: Option<Vec<bool>> = if mask.is_null() {
506        None
507    } else {
508        let raw = unsafe { std::slice::from_raw_parts(mask, n) };
509        Some(raw.iter().map(|&v| v != 0).collect())
510    };
511    let hint = if cell_hint > 0.0 {
512        Some(cell_hint)
513    } else {
514        None
515    };
516    let nn = unsafe { std::slice::from_raw_parts_mut(out_nn, need) };
517    let d2_store;
518    let d2 = if out_d2.is_null() {
519        None
520    } else {
521        d2_store = unsafe { std::slice::from_raw_parts_mut(out_d2, need) };
522        Some(&mut d2_store[..])
523    };
524    let cont = take_continuation();
525    let err = with_team(|| {
526        let found = if n_frames == 1 {
527            crate::knearest_into_d2(pts, &sim, k, mask_vec.as_deref(), hint, nn, d2)
528        } else {
529            crate::knearest_into_many(pts, n, n_frames, &sim, k, mask_vec.as_deref(), hint, nn, d2)
530        };
531        if found.is_ok() {
532            run_continuation(cont);
533        }
534        found
535    });
536    if let Err(e) = err {
537        set_error(&e.to_string());
538        return 1;
539    }
540    clear_error();
541    0
542}
543
544/// The k-nearest graph of `n` points as CSR.
545///
546/// Inputs are those of [`lc_knearest`]. Row `i` of the nominations is
547/// its `k` nearest, nearest first. With `mutual` nonzero, row `i` holds
548/// every `j` that row `i` lists and that lists `i`: the lower indices
549/// ascending, then the higher ones nearest first. Otherwise each edge
550/// comes from one nomination slot `s = i * k + t`: the lower index's
551/// when both rows list each other, the only one that names it when one
552/// does. A row holds its edges in slot order: edges from lower rows
553/// ascending, then its own nearest first, then edges from higher rows
554/// ascending.
555///
556/// `out_offsets` is caller-owned, `n + 1` ints. `alloc(ctx, total)` is
557/// called once, from any thread of the search, and returns `total`
558/// writable ints for the columns; null fails the call. The search, the
559/// degrees, and the fill run in one parallel region.
560///
561/// # Safety
562///
563/// Same pointer contract as [`lc_knearest`]. `out_offsets` is writable
564/// for `n + 1` ints. `alloc` returns null or `total` writable ints and
565/// does not unwind.
566#[no_mangle]
567pub unsafe extern "C" fn lc_knearest_csr(
568    xyz: *const f64,
569    n: usize,
570    simbox: *const lc_cell,
571    k: usize,
572    mask: *const c_int,
573    cell_hint: f64,
574    mutual: c_int,
575    out_offsets: *mut c_int,
576    alloc: Option<extern "C" fn(*mut std::ffi::c_void, usize) -> *mut c_int>,
577    ctx: *mut std::ffi::c_void,
578) -> c_int {
579    if n == 0 {
580        return fail(Error::Empty);
581    }
582    if k == 0 {
583        return fail(Error::ZeroK);
584    }
585    let Some(alloc) = alloc else {
586        return fail_msg("null pointer");
587    };
588    if xyz.is_null() || simbox.is_null() || out_offsets.is_null() {
589        return fail_msg("null pointer");
590    }
591    let Some(need) = n.checked_mul(k) else {
592        return fail(Error::Overflow);
593    };
594    if n > (isize::MAX as usize) / 3 || need > (isize::MAX as usize) / std::mem::size_of::<c_int>()
595    {
596        return fail(Error::Overflow);
597    }
598    let box_c = unsafe { *simbox };
599    let sim = match Cell::from_vectors(
600        [box_c.ax, box_c.ay, box_c.az],
601        [box_c.bx, box_c.by, box_c.bz],
602        [box_c.cx, box_c.cy, box_c.cz],
603        [box_c.ox, box_c.oy, box_c.oz],
604    ) {
605        Ok(b) => b,
606        Err(e) => return fail_msg(&e.to_string()),
607    };
608    let pts: &[[f64; 3]] = unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n) };
609    let mask_vec: Option<Vec<bool>> = if mask.is_null() {
610        None
611    } else {
612        let raw = unsafe { std::slice::from_raw_parts(mask, n) };
613        Some(raw.iter().map(|&v| v != 0).collect())
614    };
615    let hint = if cell_hint > 0.0 {
616        Some(cell_hint)
617    } else {
618        None
619    };
620    let offsets = unsafe { std::slice::from_raw_parts_mut(out_offsets, n + 1) };
621    let ctx_bits = ctx as usize;
622    let get = move |total: usize| alloc(ctx_bits as *mut std::ffi::c_void, total);
623    let found = with_team(|| {
624        crate::knearest::knearest_graph(
625            pts,
626            &sim,
627            k,
628            mask_vec.as_deref(),
629            hint,
630            mutual != 0,
631            offsets,
632            &get,
633        )
634    });
635    if let Err(e) = found {
636        return fail(e);
637    }
638    clear_error();
639    0
640}
641
642/// Cutoff pairs with the vesin / tonari shift `S`.
643///
644/// Each row is one atom-image: `out_i[t]`, `out_j[t]`,
645/// `out_shift[3*t + 0..3]` = `(na, nb, nc)`, and `out_d2[t]`.
646/// Displacement is `r_j - r_i + S H` in the caller's basis.
647/// `dist2` is strictly below `cutoff` squared. `half` nonzero keeps
648/// the canonical side of `(i, j, S)` versus `(j, i, -S)`.
649///
650/// Pass null `out_i`, `out_j`, `out_shift`, and `out_d2` to query.
651/// `*out_count` receives the row count and the return is 0.
652/// Otherwise all four buffers are required and `cap` is their row
653/// capacity. A short buffer sets `*out_count` to the needed row count
654/// and returns nonzero without writing.
655///
656/// A query, or a short buffer, keeps its search on the calling thread.
657/// The next call on that thread takes it when every input is the same
658/// bit for bit (`xyz`, the box, `cutoff`, `mask`, `cell_hint`, `half`)
659/// and writes the rows without searching again. Any other call drops
660/// it. A fill that writes does not keep anything.
661///
662/// # Safety
663///
664/// `xyz` is readable for `n * 3` doubles. `simbox` points at one
665/// `lc_cell`. `mask`, if non-null, is readable for `n` ints.
666/// `out_count` is writable. On a fill, `out_i` and `out_j` are
667/// writable for `cap` ints, `out_shift` for `cap * 3` ints, and
668/// `out_d2` for `cap` doubles.
669#[no_mangle]
670pub unsafe extern "C" fn lc_pairs_within(
671    xyz: *const f64,
672    n: usize,
673    simbox: *const lc_cell,
674    cutoff: f64,
675    mask: *const c_int,
676    cell_hint: f64,
677    half: c_int,
678    out_i: *mut c_int,
679    out_j: *mut c_int,
680    out_shift: *mut c_int,
681    out_d2: *mut f64,
682    cap: usize,
683    out_count: *mut usize,
684) -> c_int {
685    if out_count.is_null() {
686        return fail_msg("null pointer");
687    }
688    let (key, found) = match unsafe { pairs_call(xyz, n, simbox, cutoff, mask, cell_hint, half) } {
689        Ok(v) => v,
690        Err(code) => return code,
691    };
692    let rows = found.rows();
693    unsafe {
694        *out_count = rows;
695    }
696    let query = out_i.is_null() && out_j.is_null() && out_shift.is_null() && out_d2.is_null();
697    if query {
698        park(&key, found);
699        clear_error();
700        return 0;
701    }
702    if out_i.is_null() || out_j.is_null() || out_shift.is_null() || out_d2.is_null() {
703        return fail_msg("null pointer");
704    }
705    if cap < rows {
706        park(&key, found);
707        return fail_msg("pair buffer is shorter than the pair count");
708    }
709    match rows.checked_mul(3) {
710        Some(v) if v <= isize::MAX as usize => {}
711        _ => return fail(Error::Overflow),
712    }
713    // Safety: the caller gave `cap >= rows` slots in each buffer and
714    // three times that in `out_shift`. The fill runs on the same pool as
715    // the search, as in `lc_pairs_within_rows`.
716    let cols = ColumnOut(out_i, out_j, out_shift, out_d2);
717    with_threads(move || {
718        crate::pairs::in_pool(found.rows(), move || {
719            let (i, j, s, d) = cols.get();
720            unsafe { found.write_columns(i, j, s, d) }
721        })
722    });
723    clear_error();
724    0
725}
726
727/// One cutoff row: source `i`, target `j`, shift `S`, squared distance.
728/// The C++ `linkcell::ShiftedPair` has this layout.
729#[repr(C)]
730#[derive(Clone, Copy, Debug, Default, PartialEq)]
731pub struct lc_pair {
732    /// Source index.
733    pub i: c_int,
734    /// Target index.
735    pub j: c_int,
736    /// Integer cell shift `(na, nb, nc)` applied to the target.
737    pub shift: [c_int; 3],
738    /// Squared distance after the shift.
739    pub dist2: f64,
740}
741
742/// [`lc_pairs_within`] with one array of [`lc_pair`] rows.
743///
744/// Null `out` is a query: `*out_count` receives the row count and the
745/// return is 0. Otherwise `out` holds `cap` rows. A short buffer sets
746/// `*out_count` to the needed count and returns nonzero without
747/// writing. A query or a short buffer keeps its search on the calling
748/// thread for the next call with the same inputs, as in
749/// [`lc_pairs_within`].
750///
751/// # Safety
752///
753/// `xyz` is readable for `n * 3` doubles. `simbox` points at one
754/// `lc_cell`. `mask`, if non-null, is readable for `n` ints.
755/// `out_count` is writable. On a fill, `out` is writable for `cap`
756/// rows.
757#[no_mangle]
758pub unsafe extern "C" fn lc_pairs_within_rows(
759    xyz: *const f64,
760    n: usize,
761    simbox: *const lc_cell,
762    cutoff: f64,
763    mask: *const c_int,
764    cell_hint: f64,
765    half: c_int,
766    out: *mut lc_pair,
767    cap: usize,
768    out_count: *mut usize,
769) -> c_int {
770    if out_count.is_null() {
771        return fail_msg("null pointer");
772    }
773    let (key, found) = match unsafe { pairs_call(xyz, n, simbox, cutoff, mask, cell_hint, half) } {
774        Ok(v) => v,
775        Err(code) => return code,
776    };
777    let rows = found.rows();
778    unsafe {
779        *out_count = rows;
780    }
781    if out.is_null() {
782        park(&key, found);
783        clear_error();
784        return 0;
785    }
786    if cap < rows {
787        park(&key, found);
788        return fail_msg("pair buffer is shorter than the pair count");
789    }
790    // Safety: the caller gave `cap >= rows` rows. The fill runs on the
791    // same pool as the search. `in_pool` would otherwise open the global
792    // Rayon pool beside it.
793    let rows_out = RowOut(out);
794    with_threads(move || {
795        crate::pairs::in_pool(found.rows(), move || {
796            let out = rows_out.get();
797            unsafe { found.write_rows(out, |i, j, shift, dist2| lc_pair { i, j, shift, dist2 }) }
798        })
799    });
800    clear_error();
801    0
802}
803
804/// How many partner stencils have been built in this process.
805///
806/// A reused stencil does not increment the count. The periodic list and
807/// the open-cell list share the counter.
808#[no_mangle]
809pub extern "C" fn lc_stencil_builds() -> u64 {
810    crate::stencil_builds()
811}
812
813/// Cartesian half pair list of an open cluster.
814///
815/// Every axis is open. Each row has `i < j`, shift `(0, 0, 0)`, and
816/// `dist2` strictly below `cutoff` squared. Null `out` queries
817/// `*out_count`. A short buffer sets `*out_count` and returns nonzero
818/// without writing. The bins are the occupied Cartesian cells. Their
819/// partner stencil depends on the cutoff and `cell_hint` alone, so it is
820/// reused by frames that occupy other bins.
821///
822/// # Safety
823///
824/// `xyz` is readable for `n * 3` doubles. `out_count` is writable.
825/// On a fill, `out` is writable for `cap` rows.
826#[no_mangle]
827pub unsafe extern "C" fn lc_pairs_open(
828    xyz: *const f64,
829    n: usize,
830    cutoff: f64,
831    cell_hint: f64,
832    out: *mut lc_pair,
833    cap: usize,
834    out_count: *mut usize,
835) -> c_int {
836    let periodic = [0, 0, 0];
837    unsafe {
838        lc_pairs_within_pbc(
839            xyz,
840            n,
841            std::ptr::null(),
842            periodic.as_ptr(),
843            cutoff,
844            std::ptr::null(),
845            cell_hint,
846            1,
847            0,
848            0,
849            out,
850            cap,
851            out_count,
852        )
853    }
854}
855
856/// [`lc_pairs_within_rows`] with a periodicity mask.
857///
858/// `pbc` is three ints. Nonzero means that axis is periodic. A null
859/// `pbc` is fully periodic. An open axis is binned on the occupied
860/// Cartesian cells and is not padded into a periodic box. `closed`
861/// nonzero keeps `dist2 <= cutoff²` on an open or mixed cell.
862/// `exclude_self_images` nonzero drops `i == j`.
863///
864/// Null `out` is a count query. Otherwise `out` holds `cap` rows.
865/// A query or a short buffer keeps its rows on the calling thread for
866/// the next call with the same inputs bit for bit.
867///
868/// # Safety
869///
870/// `xyz` is readable for `n * 3` doubles. `simbox`, if any axis is
871/// periodic, points at one `lc_cell`. `pbc`, if non-null, is three
872/// ints. `mask`, if non-null, is `n` ints. `out_count` is writable.
873/// On a fill, `out` is writable for `cap` rows.
874#[no_mangle]
875pub unsafe extern "C" fn lc_pairs_within_pbc(
876    xyz: *const f64,
877    n: usize,
878    simbox: *const lc_cell,
879    pbc: *const c_int,
880    cutoff: f64,
881    mask: *const c_int,
882    cell_hint: f64,
883    half: c_int,
884    closed: c_int,
885    exclude_self_images: c_int,
886    out: *mut lc_pair,
887    cap: usize,
888    out_count: *mut usize,
889) -> c_int {
890    if out_count.is_null() {
891        return fail_msg("null pointer");
892    }
893    let periodic = if pbc.is_null() {
894        [true, true, true]
895    } else {
896        let flags = unsafe { [*pbc, *pbc.add(1), *pbc.add(2)] };
897        [flags[0] != 0, flags[1] != 0, flags[2] != 0]
898    };
899    if n == 0 {
900        unsafe { *out_count = 0 };
901        clear_error();
902        return 0;
903    }
904    if xyz.is_null() || (periodic != [false, false, false] && simbox.is_null()) {
905        return fail_msg("null pointer");
906    }
907    if n > c_int::MAX as usize || n > (isize::MAX as usize) / 3 {
908        return fail(Error::Overflow);
909    }
910    let hint = if cell_hint > 0.0 {
911        Some(cell_hint)
912    } else {
913        None
914    };
915    let pts = unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n) };
916    let mask_raw: Option<&[c_int]> = if mask.is_null() {
917        None
918    } else {
919        Some(unsafe { std::slice::from_raw_parts(mask, n) })
920    };
921    let key = OpenKey {
922        xyz: xyz_bits(pts),
923        periodic,
924        cutoff: cutoff.to_bits(),
925        mask: mask_raw.map(|m| m.iter().map(|&v| v != 0).collect()),
926        cell_hint: hint.map(f64::to_bits),
927        half: half != 0,
928        closed: closed != 0,
929        exclude_self: exclude_self_images != 0,
930        box_bits: if periodic == [false, false, false] || simbox.is_null() {
931            [0; 12]
932        } else {
933            box_bits(unsafe { &*simbox })
934        },
935    };
936    let rows = if let Some(parked) = take_open(&key) {
937        parked
938    } else {
939        match open_rows(pts, simbox, periodic, cutoff, mask_raw, hint, &key) {
940            Ok(rows) => rows,
941            Err(code) => return code,
942        }
943    };
944    unsafe { *out_count = rows.len() };
945    if out.is_null() {
946        park_open(key, rows);
947        clear_error();
948        return 0;
949    }
950    if cap < rows.len() {
951        let len = rows.len();
952        park_open(key, rows);
953        let _ = len;
954        return fail_msg("pair buffer is shorter than the pair count");
955    }
956    let dest = unsafe { std::slice::from_raw_parts_mut(out, rows.len()) };
957    dest.copy_from_slice(&rows);
958    clear_error();
959    0
960}
961
962struct OpenKey {
963    xyz: Vec<u64>,
964    periodic: [bool; 3],
965    cutoff: u64,
966    mask: Option<Vec<bool>>,
967    cell_hint: Option<u64>,
968    half: bool,
969    closed: bool,
970    exclude_self: bool,
971    box_bits: [u64; 12],
972}
973
974struct ParkedOpen {
975    key: OpenKey,
976    rows: Vec<lc_pair>,
977}
978
979thread_local! {
980    static PARKED_OPEN: RefCell<Option<ParkedOpen>> = const { RefCell::new(None) };
981}
982
983fn xyz_bits(xyz: &[[f64; 3]]) -> Vec<u64> {
984    let mut bits = Vec::with_capacity(xyz.len() * 3);
985    for point in xyz {
986        bits.push(point[0].to_bits());
987        bits.push(point[1].to_bits());
988        bits.push(point[2].to_bits());
989    }
990    bits
991}
992
993fn take_open(key: &OpenKey) -> Option<Vec<lc_pair>> {
994    let parked = PARKED_OPEN.with(|slot| slot.borrow_mut().take())?;
995    let same = parked.key.periodic == key.periodic
996        && parked.key.cutoff == key.cutoff
997        && parked.key.cell_hint == key.cell_hint
998        && parked.key.half == key.half
999        && parked.key.closed == key.closed
1000        && parked.key.exclude_self == key.exclude_self
1001        && parked.key.box_bits == key.box_bits
1002        && parked.key.mask == key.mask
1003        && parked.key.xyz == key.xyz;
1004    if same {
1005        Some(parked.rows)
1006    } else {
1007        None
1008    }
1009}
1010
1011fn park_open(key: OpenKey, rows: Vec<lc_pair>) {
1012    PARKED_OPEN.with(|slot| {
1013        *slot.borrow_mut() = Some(ParkedOpen { key, rows });
1014    });
1015}
1016
1017fn open_rows(
1018    pts: &[[f64; 3]],
1019    simbox: *const lc_cell,
1020    periodic: [bool; 3],
1021    cutoff: f64,
1022    mask: Option<&[c_int]>,
1023    hint: Option<f64>,
1024    key: &OpenKey,
1025) -> Result<Vec<lc_pair>, c_int> {
1026    let cell = if periodic == [false, false, false] {
1027        Cell::ortho(1.0, 1.0, 1.0).map_err(|err| fail(err.into()))?
1028    } else {
1029        let raw = unsafe { *simbox };
1030        crate::cell_for_pbc(
1031            [
1032                [raw.ax, raw.ay, raw.az],
1033                [raw.bx, raw.by, raw.bz],
1034                [raw.cx, raw.cy, raw.cz],
1035            ],
1036            [raw.ox, raw.oy, raw.oz],
1037            periodic,
1038        )
1039        .map_err(fail)?
1040    };
1041    let mask_vec: Option<Vec<bool>> = mask.map(|m| m.iter().map(|&v| v != 0).collect());
1042    let mut query = crate::PbcQuery::new(periodic, cutoff)
1043        .cell_hint(hint)
1044        .half(key.half)
1045        .closed(key.closed)
1046        .exclude_self_images(key.exclude_self);
1047    query.mask = mask_vec.as_deref();
1048    let found = with_threads(|| crate::pairs_within_pbc(pts, &cell, &query)).map_err(fail)?;
1049    let mut rows = Vec::with_capacity(found.len());
1050    for pair in found {
1051        if pair.i > c_int::MAX as usize || pair.j > c_int::MAX as usize {
1052            return Err(fail(Error::Overflow));
1053        }
1054        rows.push(lc_pair {
1055            i: pair.i as c_int,
1056            j: pair.j as c_int,
1057            shift: pair.shift,
1058            dist2: pair.dist2,
1059        });
1060    }
1061    Ok(rows)
1062}
1063
1064/// Check the inputs, then take the parked search or run a new one.
1065unsafe fn pairs_call<'a>(
1066    xyz: *const f64,
1067    n: usize,
1068    simbox: *const lc_cell,
1069    cutoff: f64,
1070    mask: *const c_int,
1071    cell_hint: f64,
1072    half: c_int,
1073) -> Result<(PairKey<'a>, crate::pairs::Found), c_int> {
1074    if n == 0 {
1075        return Err(fail(Error::Empty));
1076    }
1077    if xyz.is_null() || simbox.is_null() {
1078        return Err(fail_msg("null pointer"));
1079    }
1080    if n > c_int::MAX as usize || n > (isize::MAX as usize) / 3 {
1081        return Err(fail(Error::Overflow));
1082    }
1083    let box_c = unsafe { *simbox };
1084    let sim = Cell::from_vectors(
1085        [box_c.ax, box_c.ay, box_c.az],
1086        [box_c.bx, box_c.by, box_c.bz],
1087        [box_c.cx, box_c.cy, box_c.cz],
1088        [box_c.ox, box_c.oy, box_c.oz],
1089    )
1090    .map_err(|e| {
1091        set_error(&e.to_string());
1092        1
1093    })?;
1094    let pts: &'a [[f64; 3]] = unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n) };
1095    let mask_raw: Option<&'a [c_int]> = if mask.is_null() {
1096        None
1097    } else {
1098        Some(unsafe { std::slice::from_raw_parts(mask, n) })
1099    };
1100    let hint = if cell_hint > 0.0 {
1101        Some(cell_hint)
1102    } else {
1103        None
1104    };
1105    let key = PairKey {
1106        xyz: pts,
1107        simbox: box_c,
1108        cutoff,
1109        mask: mask_raw,
1110        cell_hint: hint,
1111        half: half != 0,
1112    };
1113    if let Some(found) = take_parked(&key) {
1114        return Ok((key, found));
1115    }
1116    let mask_vec: Option<Vec<bool>> = mask_raw.map(|m| m.iter().map(|&v| v != 0).collect());
1117    let found = with_threads(|| {
1118        crate::pairs::in_pool(crate::pairs::hit_estimate(n, &sim, cutoff), || {
1119            crate::pairs::search(pts, &sim, cutoff, mask_vec.as_deref(), hint, half != 0)
1120        })
1121    });
1122    match found {
1123        Ok(found) => Ok((key, found)),
1124        Err(e) => Err(fail(e)),
1125    }
1126}
1127
1128/// Caller buffers moved onto a pool thread for the fill. The caller
1129/// owns them, so the fill can run where the search ran.
1130#[derive(Clone, Copy)]
1131struct ColumnOut(*mut c_int, *mut c_int, *mut c_int, *mut f64);
1132unsafe impl Send for ColumnOut {}
1133impl ColumnOut {
1134    fn get(self) -> (*mut c_int, *mut c_int, *mut c_int, *mut f64) {
1135        (self.0, self.1, self.2, self.3)
1136    }
1137}
1138
1139#[derive(Clone, Copy)]
1140struct RowOut(*mut lc_pair);
1141unsafe impl Send for RowOut {}
1142impl RowOut {
1143    fn get(self) -> *mut lc_pair {
1144        self.0
1145    }
1146}
1147
1148/// Inputs of one [`lc_pairs_within`] call, compared bit for bit.
1149struct PairKey<'a> {
1150    xyz: &'a [[f64; 3]],
1151    simbox: lc_cell,
1152    cutoff: f64,
1153    mask: Option<&'a [c_int]>,
1154    cell_hint: Option<f64>,
1155    half: bool,
1156}
1157
1158/// A query's search, kept until the matching fill on the same thread.
1159struct ParkedPairs {
1160    xyz: Vec<[f64; 3]>,
1161    simbox: [u64; 12],
1162    cutoff: u64,
1163    mask: Option<Vec<bool>>,
1164    cell_hint: Option<u64>,
1165    half: bool,
1166    found: crate::pairs::Found,
1167}
1168
1169thread_local! {
1170    // `const { ... }` is newer than this crate's 1.70 floor.
1171    #[allow(clippy::missing_const_for_thread_local)]
1172    static PARKED_PAIRS: RefCell<Option<ParkedPairs>> = RefCell::new(None);
1173}
1174
1175fn box_bits(c: &lc_cell) -> [u64; 12] {
1176    [
1177        c.ax.to_bits(),
1178        c.ay.to_bits(),
1179        c.az.to_bits(),
1180        c.bx.to_bits(),
1181        c.by.to_bits(),
1182        c.bz.to_bits(),
1183        c.cx.to_bits(),
1184        c.cy.to_bits(),
1185        c.cz.to_bits(),
1186        c.ox.to_bits(),
1187        c.oy.to_bits(),
1188        c.oz.to_bits(),
1189    ]
1190}
1191
1192fn same_bits(a: &[[f64; 3]], b: &[[f64; 3]]) -> bool {
1193    a.len() == b.len()
1194        && a.iter().zip(b).all(|(p, q)| {
1195            p[0].to_bits() == q[0].to_bits()
1196                && p[1].to_bits() == q[1].to_bits()
1197                && p[2].to_bits() == q[2].to_bits()
1198        })
1199}
1200
1201/// The parked search when every input matches. Any other call drops it,
1202/// so a parked search never outlives the next call on this thread.
1203fn take_parked(key: &PairKey<'_>) -> Option<crate::pairs::Found> {
1204    let parked = PARKED_PAIRS.with(|slot| slot.borrow_mut().take())?;
1205    let mask_same = match (&parked.mask, key.mask) {
1206        (None, None) => true,
1207        (Some(a), Some(b)) => a.len() == b.len() && a.iter().zip(b).all(|(&x, &y)| x == (y != 0)),
1208        _ => false,
1209    };
1210    let same = parked.half == key.half
1211        && parked.cutoff == key.cutoff.to_bits()
1212        && parked.cell_hint == key.cell_hint.map(f64::to_bits)
1213        && parked.simbox == box_bits(&key.simbox)
1214        && mask_same
1215        && same_bits(&parked.xyz, key.xyz);
1216    if same {
1217        Some(parked.found)
1218    } else {
1219        None
1220    }
1221}
1222
1223fn park(key: &PairKey<'_>, found: crate::pairs::Found) {
1224    let parked = ParkedPairs {
1225        xyz: key.xyz.to_vec(),
1226        simbox: box_bits(&key.simbox),
1227        cutoff: key.cutoff.to_bits(),
1228        mask: key.mask.map(|m| m.iter().map(|&v| v != 0).collect()),
1229        cell_hint: key.cell_hint.map(f64::to_bits),
1230        half: key.half,
1231        found,
1232    };
1233    PARKED_PAIRS.with(|slot| *slot.borrow_mut() = Some(parked));
1234}
1235
1236#[allow(clippy::type_complexity)]
1237unsafe fn load_frame<'a>(
1238    xyz: *const f64,
1239    n: usize,
1240    simbox: *const lc_cell,
1241    mask: *const c_int,
1242    cell_hint: f64,
1243) -> Result<(&'a [[f64; 3]], Cell, Option<Vec<bool>>, Option<f64>), c_int> {
1244    if n == 0 {
1245        return Err(fail(Error::Empty));
1246    }
1247    if xyz.is_null() || simbox.is_null() {
1248        return Err(fail_msg("null pointer"));
1249    }
1250    if n > c_int::MAX as usize || n > (isize::MAX as usize) / 3 {
1251        return Err(fail(Error::Overflow));
1252    }
1253    let box_c = unsafe { *simbox };
1254    let sim = Cell::from_vectors(
1255        [box_c.ax, box_c.ay, box_c.az],
1256        [box_c.bx, box_c.by, box_c.bz],
1257        [box_c.cx, box_c.cy, box_c.cz],
1258        [box_c.ox, box_c.oy, box_c.oz],
1259    )
1260    .map_err(|err| fail(err.into()))?;
1261    let pts: &'a [[f64; 3]] = unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n) };
1262    let mask_vec = if mask.is_null() {
1263        None
1264    } else {
1265        let raw = unsafe { std::slice::from_raw_parts(mask, n) };
1266        Some(raw.iter().map(|&v| v != 0).collect())
1267    };
1268    let hint = if cell_hint > 0.0 {
1269        Some(cell_hint)
1270    } else {
1271        None
1272    };
1273    Ok((pts, sim, mask_vec, hint))
1274}
1275
1276fn write_grid_head(grid: &crate::CellGrid, out_nxyz: *mut c_int, out_width: *mut f64) {
1277    if !out_nxyz.is_null() {
1278        unsafe {
1279            *out_nxyz = grid.nx;
1280            *out_nxyz.add(1) = grid.ny;
1281            *out_nxyz.add(2) = grid.nz;
1282        }
1283    }
1284    if !out_width.is_null() {
1285        unsafe {
1286            *out_width = grid.bin_width[0];
1287            *out_width.add(1) = grid.bin_width[1];
1288            *out_width.add(2) = grid.bin_width[2];
1289        }
1290    }
1291}
1292
1293/// Point counts on the linked-cell mesh.
1294///
1295/// `out_nxyz` receives `nx, ny, nz`. `out_width` receives the three bin
1296/// widths. `out_weight` receives one count per cell, in the order
1297/// `((iz * ny) + iy) * nx + ix`. A null `out_weight` is a query:
1298/// `*out_ncell` is the cell count and the return is 0.
1299///
1300/// # Safety
1301///
1302/// `xyz` is readable for `n * 3` doubles. `simbox` points at one
1303/// `lc_cell`. `mask`, if non-null, is readable for `n` ints.
1304/// `out_ncell` is writable. On a fill, `out_nxyz` is writable for 3
1305/// ints, `out_width` for 3 doubles, and `out_weight` for `*out_ncell`
1306/// counts.
1307#[no_mangle]
1308pub unsafe extern "C" fn lc_cell_grid(
1309    xyz: *const f64,
1310    n: usize,
1311    simbox: *const lc_cell,
1312    cutoff: f64,
1313    mask: *const c_int,
1314    cell_hint: f64,
1315    out_nxyz: *mut c_int,
1316    out_width: *mut f64,
1317    out_weight: *mut u64,
1318    cap: usize,
1319    out_ncell: *mut usize,
1320) -> c_int {
1321    if out_ncell.is_null() {
1322        return fail_msg("null pointer");
1323    }
1324    let (pts, sim, mask_vec, hint) = match unsafe { load_frame(xyz, n, simbox, mask, cell_hint) } {
1325        Ok(v) => v,
1326        Err(rc) => return rc,
1327    };
1328    let grid = match with_threads(|| crate::cell_grid(pts, &sim, cutoff, mask_vec.as_deref(), hint))
1329    {
1330        Ok(grid) => grid,
1331        Err(err) => return fail(err),
1332    };
1333    let ncell = grid.ncell();
1334    unsafe { *out_ncell = ncell };
1335    if out_weight.is_null() {
1336        clear_error();
1337        return 0;
1338    }
1339    if cap < ncell || out_nxyz.is_null() || out_width.is_null() {
1340        return fail_msg("cell grid buffer is shorter than the cell count");
1341    }
1342    write_grid_head(&grid, out_nxyz, out_width);
1343    unsafe {
1344        std::ptr::copy_nonoverlapping(grid.weight.as_ptr(), out_weight, ncell);
1345    }
1346    clear_error();
1347    0
1348}
1349
1350/// Periodic cell adjacency. Edge weights are pair counts from
1351/// `lc_pairs_within` with `half` set.
1352///
1353/// A null `out_weight` queries `*out_ncell` and `*out_arcs` and
1354/// returns 0. Otherwise every output buffer is required. `cap_weight`
1355/// is the cell capacity and `cap_arcs` is the arc capacity.
1356/// `out_offsets` has length `ncell + 1`.
1357///
1358/// # Safety
1359///
1360/// Same point contract as [`lc_cell_grid`]. On a fill, `out_offsets`
1361/// is writable for `ncell + 1` counts, `out_neighbor` and `out_edge`
1362/// for `out_arcs` entries.
1363#[no_mangle]
1364pub unsafe extern "C" fn lc_cell_graph(
1365    xyz: *const f64,
1366    n: usize,
1367    simbox: *const lc_cell,
1368    cutoff: f64,
1369    mask: *const c_int,
1370    cell_hint: f64,
1371    out_nxyz: *mut c_int,
1372    out_width: *mut f64,
1373    out_weight: *mut u64,
1374    out_offsets: *mut u32,
1375    out_neighbor: *mut u32,
1376    out_edge: *mut u64,
1377    cap_weight: usize,
1378    cap_arcs: usize,
1379    out_ncell: *mut usize,
1380    out_arcs: *mut usize,
1381) -> c_int {
1382    if out_ncell.is_null() || out_arcs.is_null() {
1383        return fail_msg("null pointer");
1384    }
1385    let (pts, sim, mask_vec, hint) = match unsafe { load_frame(xyz, n, simbox, mask, cell_hint) } {
1386        Ok(v) => v,
1387        Err(rc) => return rc,
1388    };
1389    let graph =
1390        match with_threads(|| crate::cell_graph(pts, &sim, cutoff, mask_vec.as_deref(), hint)) {
1391            Ok(graph) => graph,
1392            Err(err) => return fail(err),
1393        };
1394    let ncell = graph.grid.ncell();
1395    let arcs = graph.neighbor.len();
1396    unsafe {
1397        *out_ncell = ncell;
1398        *out_arcs = arcs;
1399    }
1400    if out_weight.is_null() {
1401        clear_error();
1402        return 0;
1403    }
1404    if cap_weight < ncell
1405        || cap_arcs < arcs
1406        || out_nxyz.is_null()
1407        || out_width.is_null()
1408        || out_offsets.is_null()
1409        || out_neighbor.is_null()
1410        || out_edge.is_null()
1411    {
1412        return fail_msg("cell graph buffer is shorter than the graph");
1413    }
1414    write_grid_head(&graph.grid, out_nxyz, out_width);
1415    unsafe {
1416        std::ptr::copy_nonoverlapping(graph.grid.weight.as_ptr(), out_weight, ncell);
1417        std::ptr::copy_nonoverlapping(graph.offsets.as_ptr(), out_offsets, ncell + 1);
1418        if arcs > 0 {
1419            std::ptr::copy_nonoverlapping(graph.neighbor.as_ptr(), out_neighbor, arcs);
1420            std::ptr::copy_nonoverlapping(graph.edge_weight.as_ptr(), out_edge, arcs);
1421        }
1422    }
1423    clear_error();
1424    0
1425}
1426
1427/// Domain id of each cell.
1428///
1429/// `method` is 0 Morton, 1 Hilbert, 2 recursive coordinate bisection,
1430/// 3 Hilbert plus Fiduccia-Mattheyses, 4 bisection plus that pass,
1431/// 5 Scotch, 6 multilevel, 7 PT-Scotch. `balance <= 0` selects
1432/// `1 + parts * w_max / total`, where `w_max` is the heaviest cell
1433/// weight and `total` is the sum of the cell weights. `device` nonzero runs the
1434/// curve split on the CUDA device and returns nonzero when that device
1435/// is absent. A null `out_domain` queries `*out_ncell`.
1436///
1437/// # Safety
1438///
1439/// Same point contract as [`lc_cell_grid`]. On a fill, `out_domain` is
1440/// writable for `ncell` ints.
1441#[no_mangle]
1442pub unsafe extern "C" fn lc_partition(
1443    xyz: *const f64,
1444    n: usize,
1445    simbox: *const lc_cell,
1446    cutoff: f64,
1447    mask: *const c_int,
1448    cell_hint: f64,
1449    parts: usize,
1450    method: u8,
1451    balance: f64,
1452    device: c_int,
1453    out_domain: *mut c_int,
1454    cap: usize,
1455    out_ncell: *mut usize,
1456) -> c_int {
1457    if out_ncell.is_null() {
1458        return fail_msg("null pointer");
1459    }
1460    if parts == 0 || parts > u32::MAX as usize {
1461        return fail(Error::BadPartition);
1462    }
1463    let (pts, sim, mask_vec, hint) = match unsafe { load_frame(xyz, n, simbox, mask, cell_hint) } {
1464        Ok(v) => v,
1465        Err(rc) => return rc,
1466    };
1467    let method = match crate::partition::method_from_c(method) {
1468        Ok(method) => method,
1469        Err(err) => return fail(err),
1470    };
1471    let balance = if balance <= 0.0 { None } else { Some(balance) };
1472    let request = crate::PartRequest {
1473        parts: parts as u32,
1474        method,
1475        balance,
1476        device: device != 0,
1477    };
1478    let assign = match with_threads(|| {
1479        crate::partition(pts, &sim, cutoff, mask_vec.as_deref(), hint, request)
1480    }) {
1481        Ok(assign) => assign,
1482        Err(err) => return fail(err),
1483    };
1484    unsafe { *out_ncell = assign.len() };
1485    if out_domain.is_null() {
1486        clear_error();
1487        return 0;
1488    }
1489    if cap < assign.len() {
1490        return fail_msg("partition buffer is shorter than the cell count");
1491    }
1492    for (i, &domain) in assign.iter().enumerate() {
1493        if domain > c_int::MAX as u32 {
1494            return fail(Error::Overflow);
1495        }
1496        unsafe { *out_domain.add(i) = domain as c_int };
1497    }
1498    clear_error();
1499    0
1500}
1501
1502/// Bin radii for `cutoff * depth`. `out_width` receives three counts.
1503///
1504/// # Safety
1505///
1506/// `bin_width` is readable for 3 doubles. `out_width` is writable for
1507/// 3 counts.
1508#[no_mangle]
1509pub unsafe extern "C" fn lc_halo_width(
1510    cutoff: f64,
1511    depth: u32,
1512    bin_width: *const f64,
1513    out_width: *mut u32,
1514) -> c_int {
1515    if bin_width.is_null() || out_width.is_null() {
1516        return fail_msg("null pointer");
1517    }
1518    let width = unsafe { [*bin_width, *bin_width.add(1), *bin_width.add(2)] };
1519    match crate::halo_width(cutoff, depth, width) {
1520        Ok(radius) => {
1521            unsafe {
1522                *out_width = radius[0];
1523                *out_width.add(1) = radius[1];
1524                *out_width.add(2) = radius[2];
1525            }
1526            clear_error();
1527            0
1528        }
1529        Err(err) => fail(err),
1530    }
1531}
1532
1533/// Ghost cells of `domain`. A null `out_ghost` queries `*out_count`.
1534///
1535/// # Safety
1536///
1537/// `nxyz` is readable for 3 ints. `assign` is readable for `ncell`
1538/// ints. `width` is readable for 3 counts. On a fill, `out_ghost` is
1539/// writable for `*out_count` ints.
1540#[no_mangle]
1541pub unsafe extern "C" fn lc_halo(
1542    nxyz: *const c_int,
1543    assign: *const c_int,
1544    ncell: usize,
1545    domain: c_int,
1546    width: *const u32,
1547    out_ghost: *mut c_int,
1548    cap: usize,
1549    out_count: *mut usize,
1550) -> c_int {
1551    if nxyz.is_null() || assign.is_null() || width.is_null() || out_count.is_null() {
1552        return fail_msg("null pointer");
1553    }
1554    if domain < 0 {
1555        return fail(Error::BadPartition);
1556    }
1557    let nxyz = unsafe { [*nxyz, *nxyz.add(1), *nxyz.add(2)] };
1558    if nxyz[0] <= 0 || nxyz[1] <= 0 || nxyz[2] <= 0 {
1559        return fail(Error::BadGraph);
1560    }
1561    let expect = (nxyz[0] as usize)
1562        .saturating_mul(nxyz[1] as usize)
1563        .saturating_mul(nxyz[2] as usize);
1564    if expect != ncell {
1565        return fail(Error::BadGraph);
1566    }
1567    let raw = unsafe { std::slice::from_raw_parts(assign, ncell) };
1568    let mut owned = Vec::with_capacity(ncell);
1569    for &v in raw {
1570        if v < 0 {
1571            return fail(Error::BadPartition);
1572        }
1573        owned.push(v as u32);
1574    }
1575    let grid = crate::CellGrid {
1576        nx: nxyz[0],
1577        ny: nxyz[1],
1578        nz: nxyz[2],
1579        bin_width: [0.0; 3],
1580        weight: vec![0; ncell],
1581    };
1582    let radius = unsafe { [*width, *width.add(1), *width.add(2)] };
1583    let ghosts = match with_threads(|| crate::halo(&grid, &owned, domain as u32, radius)) {
1584        Ok(ghosts) => ghosts,
1585        Err(err) => return fail(err),
1586    };
1587    unsafe { *out_count = ghosts.len() };
1588    if out_ghost.is_null() {
1589        clear_error();
1590        return 0;
1591    }
1592    if cap < ghosts.len() {
1593        return fail_msg("halo buffer is shorter than the ghost count");
1594    }
1595    for (i, &cell) in ghosts.iter().enumerate() {
1596        unsafe { *out_ghost.add(i) = cell as c_int };
1597    }
1598    clear_error();
1599    0
1600}
1601
1602/// Domain of each cell and the bin of each point, from one mesh cut.
1603///
1604/// `method` and `balance` follow [`lc_partition`]. A null `out_domain`
1605/// queries `*out_ncell` and does not write. On a fill, `out_nxyz` is
1606/// three ints, `out_width` is three bin widths, `out_domain` is one
1607/// domain per cell, and `out_point_cell` is one bin per point.
1608///
1609/// # Safety
1610///
1611/// Same point contract as [`lc_cell_grid`]. On a fill, `out_domain` is
1612/// writable for `*out_ncell` ints and `out_point_cell` for `n` ints.
1613#[no_mangle]
1614pub unsafe extern "C" fn lc_mesh_cut(
1615    xyz: *const f64,
1616    n: usize,
1617    simbox: *const lc_cell,
1618    cutoff: f64,
1619    mask: *const c_int,
1620    cell_hint: f64,
1621    parts: usize,
1622    method: u8,
1623    balance: f64,
1624    device: c_int,
1625    out_nxyz: *mut c_int,
1626    out_width: *mut f64,
1627    out_domain: *mut c_int,
1628    out_point_cell: *mut c_int,
1629    cap_cell: usize,
1630    cap_point: usize,
1631    out_ncell: *mut usize,
1632) -> c_int {
1633    if out_ncell.is_null() {
1634        return fail_msg("null pointer");
1635    }
1636    if parts == 0 || parts > u32::MAX as usize {
1637        return fail(Error::BadPartition);
1638    }
1639    let (pts, sim, mask_vec, hint) = match unsafe { load_frame(xyz, n, simbox, mask, cell_hint) } {
1640        Ok(v) => v,
1641        Err(rc) => return rc,
1642    };
1643    let method = match crate::partition::method_from_c(method) {
1644        Ok(method) => method,
1645        Err(err) => return fail(err),
1646    };
1647    let request = crate::PartRequest {
1648        parts: parts as u32,
1649        method,
1650        balance: if balance <= 0.0 { None } else { Some(balance) },
1651        device: device != 0,
1652    };
1653    let cut = match with_threads(|| {
1654        crate::mesh_cut(pts, &sim, cutoff, mask_vec.as_deref(), hint, request)
1655    }) {
1656        Ok(cut) => cut,
1657        Err(err) => return fail(err),
1658    };
1659    let ncell = cut.assign.len();
1660    unsafe { *out_ncell = ncell };
1661    if out_domain.is_null() {
1662        clear_error();
1663        return 0;
1664    }
1665    if out_nxyz.is_null()
1666        || out_width.is_null()
1667        || out_point_cell.is_null()
1668        || cap_cell < ncell
1669        || cap_point < n
1670    {
1671        return fail_msg("mesh cut buffer is shorter than the cell count");
1672    }
1673    write_grid_head(&cut.grid, out_nxyz, out_width);
1674    for (i, &domain) in cut.assign.iter().enumerate() {
1675        if domain > c_int::MAX as u32 {
1676            return fail(Error::Overflow);
1677        }
1678        unsafe { *out_domain.add(i) = domain as c_int };
1679    }
1680    for (i, &cell) in cut.point_cell.iter().enumerate() {
1681        if cell > c_int::MAX as u32 {
1682            return fail(Error::Overflow);
1683        }
1684        unsafe { *out_point_cell.add(i) = cell as c_int };
1685    }
1686    clear_error();
1687    0
1688}
1689
1690/// Domain of each cell from caller-supplied weights.
1691///
1692/// `nxyz` is `nx, ny, nz`. `weight` has one count per cell, in the
1693/// order `((iz * ny) + iy) * nx + ix`. `method` and `balance` follow
1694/// [`lc_partition`]. A plain bisection reads the weights and not the
1695/// pair arcs. `out_domain` receives one domain per cell.
1696///
1697/// # Safety
1698///
1699/// `nxyz` is readable for 3 ints. `weight` is readable for `ncell`
1700/// counts. `out_domain` is writable for `ncell` ints.
1701#[no_mangle]
1702pub unsafe extern "C" fn lc_partition_weights(
1703    nxyz: *const c_int,
1704    weight: *const u64,
1705    ncell: usize,
1706    parts: usize,
1707    method: u8,
1708    balance: f64,
1709    out_domain: *mut c_int,
1710) -> c_int {
1711    if nxyz.is_null() || weight.is_null() || out_domain.is_null() {
1712        return fail_msg("null pointer");
1713    }
1714    if parts == 0 || parts > u32::MAX as usize {
1715        return fail(Error::BadPartition);
1716    }
1717    let nxyz = unsafe { [*nxyz, *nxyz.add(1), *nxyz.add(2)] };
1718    if nxyz[0] <= 0 || nxyz[1] <= 0 || nxyz[2] <= 0 {
1719        return fail(Error::BadGraph);
1720    }
1721    let expect = (nxyz[0] as usize)
1722        .saturating_mul(nxyz[1] as usize)
1723        .saturating_mul(nxyz[2] as usize);
1724    if expect != ncell {
1725        return fail(Error::BadGraph);
1726    }
1727    let method = match crate::partition::method_from_c(method) {
1728        Ok(method) => method,
1729        Err(err) => return fail(err),
1730    };
1731    let raw = unsafe { std::slice::from_raw_parts(weight, ncell) };
1732    let graph = crate::CellGraph {
1733        grid: crate::CellGrid {
1734            nx: nxyz[0],
1735            ny: nxyz[1],
1736            nz: nxyz[2],
1737            bin_width: [1.0; 3],
1738            weight: raw.to_vec(),
1739        },
1740        offsets: vec![0u32; ncell + 1],
1741        neighbor: Vec::new(),
1742        edge_weight: Vec::new(),
1743    };
1744    let request = crate::PartRequest {
1745        parts: parts as u32,
1746        method,
1747        balance: if balance <= 0.0 { None } else { Some(balance) },
1748        device: false,
1749    };
1750    let assign = match with_threads(|| crate::partition_graph(&graph, request)) {
1751        Ok(assign) => assign,
1752        Err(err) => return fail(err),
1753    };
1754    for (i, &domain) in assign.iter().enumerate() {
1755        if domain > c_int::MAX as u32 {
1756            return fail(Error::Overflow);
1757        }
1758        unsafe { *out_domain.add(i) = domain as c_int };
1759    }
1760    clear_error();
1761    0
1762}
1763
1764/// Owned points are 2. Ghosts within `reach` of an owned atom are 1.
1765///
1766/// # Safety
1767///
1768/// `xyz` is readable for `n * 3` doubles. `lengths` and `nxyz` are
1769/// readable for 3 entries. `assign` is readable for `ncell` ints.
1770/// `point_cell` is readable for `n` ints. `hwidth` is readable for 3
1771/// counts. `out_hit` is writable for `n` ints.
1772#[no_mangle]
1773pub unsafe extern "C" fn lc_within_reach(
1774    xyz: *const f64,
1775    n: usize,
1776    lengths: *const f64,
1777    nxyz: *const c_int,
1778    assign: *const c_int,
1779    ncell: usize,
1780    point_cell: *const c_int,
1781    hwidth: *const u32,
1782    domain: c_int,
1783    reach: f64,
1784    out_hit: *mut c_int,
1785) -> c_int {
1786    if xyz.is_null()
1787        || lengths.is_null()
1788        || nxyz.is_null()
1789        || assign.is_null()
1790        || point_cell.is_null()
1791        || hwidth.is_null()
1792        || out_hit.is_null()
1793        || domain < 0
1794    {
1795        return fail_msg("null pointer");
1796    }
1797    let pts: Vec<[f64; 3]> =
1798        unsafe { std::slice::from_raw_parts(xyz.cast::<[f64; 3]>(), n) }.to_vec();
1799    let lengths = unsafe { [*lengths, *lengths.add(1), *lengths.add(2)] };
1800    let nxyz = unsafe { [*nxyz, *nxyz.add(1), *nxyz.add(2)] };
1801    let hwidth = unsafe { [*hwidth, *hwidth.add(1), *hwidth.add(2)] };
1802    let assign_raw = unsafe { std::slice::from_raw_parts(assign, ncell) };
1803    let mut owned = Vec::with_capacity(ncell);
1804    for &v in assign_raw {
1805        if v < 0 {
1806            return fail(Error::BadPartition);
1807        }
1808        owned.push(v as u32);
1809    }
1810    let cells_raw = unsafe { std::slice::from_raw_parts(point_cell, n) };
1811    let mut cells = Vec::with_capacity(n);
1812    for &v in cells_raw {
1813        if v < 0 {
1814            return fail(Error::BadPartition);
1815        }
1816        cells.push(v as u32);
1817    }
1818    let hit = match with_threads(|| {
1819        crate::mark_reached(
1820            &pts,
1821            lengths,
1822            nxyz,
1823            &owned,
1824            &cells,
1825            hwidth,
1826            domain as u32,
1827            reach,
1828        )
1829    }) {
1830        Ok(hit) => hit,
1831        Err(err) => return fail(err),
1832    };
1833    for (i, &flag) in hit.iter().enumerate() {
1834        unsafe { *out_hit.add(i) = flag as c_int };
1835    }
1836    clear_error();
1837    0
1838}
1839
1840/// Owned points are 2. Points in a cell painted from the owned walk are 1.
1841///
1842/// The stencil is the one [`crate::rank_mask`] paints while it visits the
1843/// cells of `domain`. `width` is three bin radii.
1844///
1845/// # Safety
1846///
1847/// `nxyz` is readable for 3 ints. `assign` is readable for `ncell` ints.
1848/// `point_cell` is readable for `n` ints. `width` is readable for 3
1849/// counts. `out_hit` is writable for `n` ints.
1850#[no_mangle]
1851pub unsafe extern "C" fn lc_owned_mask(
1852    nxyz: *const c_int,
1853    assign: *const c_int,
1854    ncell: usize,
1855    point_cell: *const c_int,
1856    n: usize,
1857    domain: c_int,
1858    width: *const u32,
1859    out_hit: *mut c_int,
1860) -> c_int {
1861    if nxyz.is_null()
1862        || assign.is_null()
1863        || point_cell.is_null()
1864        || width.is_null()
1865        || out_hit.is_null()
1866        || domain < 0
1867    {
1868        return fail_msg("null pointer");
1869    }
1870    let nxyz = unsafe { [*nxyz, *nxyz.add(1), *nxyz.add(2)] };
1871    if nxyz[0] <= 0 || nxyz[1] <= 0 || nxyz[2] <= 0 {
1872        return fail(Error::BadGraph);
1873    }
1874    let expect = (nxyz[0] as usize)
1875        .saturating_mul(nxyz[1] as usize)
1876        .saturating_mul(nxyz[2] as usize);
1877    if expect != ncell {
1878        return fail(Error::BadGraph);
1879    }
1880    let assign_raw = unsafe { std::slice::from_raw_parts(assign, ncell) };
1881    let mut owned_assign = Vec::with_capacity(ncell);
1882    let mut owned_cells = Vec::new();
1883    for (cell, &v) in assign_raw.iter().enumerate() {
1884        if v < 0 {
1885            return fail(Error::BadPartition);
1886        }
1887        let domain_id = v as u32;
1888        if domain_id == domain as u32 {
1889            owned_cells.push(cell as u32);
1890        }
1891        owned_assign.push(domain_id);
1892    }
1893    let cells_raw = unsafe { std::slice::from_raw_parts(point_cell, n) };
1894    let mut point_bins = Vec::with_capacity(n);
1895    for &v in cells_raw {
1896        if v < 0 || v as usize >= ncell {
1897            return fail(Error::BadPartition);
1898        }
1899        point_bins.push(v as u32);
1900    }
1901    let width = unsafe { [*width, *width.add(1), *width.add(2)] };
1902    let grid = crate::CellGrid {
1903        nx: nxyz[0],
1904        ny: nxyz[1],
1905        nz: nxyz[2],
1906        bin_width: [1.0; 3],
1907        weight: vec![0; ncell],
1908    };
1909    let view = with_threads(|| {
1910        crate::rank_mask(
1911            &grid,
1912            &owned_assign,
1913            &point_bins,
1914            &owned_cells,
1915            domain as u32,
1916            width,
1917        )
1918    });
1919    for (i, &painted) in view.mask.iter().enumerate() {
1920        let flag = if !painted {
1921            0
1922        } else if owned_assign[point_bins[i] as usize] == domain as u32 {
1923            2
1924        } else {
1925            1
1926        };
1927        unsafe { *out_hit.add(i) = flag };
1928    }
1929    clear_error();
1930    0
1931}
1932
1933#[cfg(all(test, feature = "parallel"))]
1934mod tests {
1935    use super::*;
1936
1937    struct Tally {
1938        sizes: [usize; 3],
1939        hits: Vec<Vec<AtomicUsize>>,
1940        closed: [AtomicUsize; 3],
1941        early: AtomicUsize,
1942        widest: AtomicUsize,
1943    }
1944
1945    extern "C" fn tally(ctx: *mut std::ffi::c_void, stage: usize, chunk: usize, worker: usize) {
1946        let t = unsafe { &*ctx.cast::<Tally>() };
1947        if stage > 0 && t.closed[stage - 1].load(Ordering::SeqCst) != t.sizes[stage - 1] {
1948            t.early.fetch_add(1, Ordering::SeqCst);
1949        }
1950        t.widest.fetch_max(worker + 1, Ordering::SeqCst);
1951        t.hits[stage][chunk].fetch_add(1, Ordering::SeqCst);
1952        t.closed[stage].fetch_add(1, Ordering::SeqCst);
1953    }
1954
1955    #[test]
1956    fn a_team_runs_every_chunk_once_and_the_stages_in_order() {
1957        for threads in [1, 4] {
1958            lc_set_threads(threads);
1959            let sizes = [97, 0, 13];
1960            let t = Tally {
1961                sizes,
1962                hits: sizes
1963                    .iter()
1964                    .map(|&n| (0..n).map(|_| AtomicUsize::new(0)).collect())
1965                    .collect(),
1966                closed: Default::default(),
1967                early: AtomicUsize::new(0),
1968                widest: AtomicUsize::new(0),
1969            };
1970            let code = unsafe {
1971                lc_team_run(
1972                    3,
1973                    sizes.as_ptr(),
1974                    Some(tally),
1975                    (&t as *const Tally).cast_mut().cast(),
1976                )
1977            };
1978            lc_set_threads(0);
1979            assert_eq!(code, 0);
1980            assert_eq!(t.early.load(Ordering::SeqCst), 0);
1981            assert!(t
1982                .hits
1983                .iter()
1984                .flatten()
1985                .all(|h| h.load(Ordering::SeqCst) == 1));
1986            let bound = (threads as usize).max(rayon::current_num_threads());
1987            assert!(t.widest.load(Ordering::SeqCst) <= bound);
1988        }
1989        assert_ne!(
1990            unsafe { lc_team_run(1, ptr::null(), Some(tally), ptr::null_mut()) },
1991            0
1992        );
1993        assert_ne!(
1994            unsafe { lc_team_run(1, [1].as_ptr(), None, ptr::null_mut()) },
1995            0
1996        );
1997    }
1998}