mask.singeli loop renames

This commit is contained in:
dzaima 2025-02-02 22:04:58 +02:00
parent 92f40ddbe2
commit 49dd6e394a
11 changed files with 50 additions and 48 deletions

View File

@ -264,12 +264,12 @@ To test whether a mask object `M` is `mask_none` or `mask_first`, `M{0}` can be
### Loops
- `@maskedLoop{bulk}` - loop that generates its body twice, once with a `mask_none` mask, and once with a `mask_first{n}` one to handle the tail.
- `@for_masked{bulk}` - loop that generates its body twice, once with a `mask_none` mask, and once with a `mask_first{n}` one to handle the tail.
- `@muLoop{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end.
- `@for_mu{bulk, unr}` - masked & unrolled loop - generates its body three times (or two if `unr==1`) - once for unrolled main loop, once for the unrolling leftover, and once for the masked end.
Unrolling is handled by passing a tuple of indices to process as the index variable (the tail generated bodies get a tuple of the one index)
- `@muLoop{bulk, unr, fromunr}` - `fromunr` is additionally ran after exiting the unrolled loop (such that you can have separate accumulators for the unrolled bit, and convert them to single-vector acccumulators for the tail).
- `@for_mu{bulk, unr, fromunr}` - `fromunr` is additionally ran after exiting the unrolled loop (such that you can have separate accumulators for the unrolled bit, and convert them to single-vector acccumulators for the tail).
Tuples can be used in the iterated variable list for various things:
@ -287,15 +287,15 @@ Stores via those will implicitly do a masked store when required.
Loads will load past the end, so `M in 'm'` must be used to mask off elements if they're used for something other than the above stores.
For `muLoop`, `M in 'm'` still gives a single generator, but it's only ever `mask_first{n}` when there's only one index.
For `for_mu`, `M in 'm'` still gives a single generator, but it's only ever `mask_first{n}` when there's only one index.
Example usage for a loop that adds `u16` and bit boolean elements to a `u32` accumulator, early-exiting on overflow:
```
fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
fn acc_u32_u16_bit_u1(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @for_masked
def bulk = 8
def VT = [8]u32
@maskedLoop{bulk}(
@for_masked{bulk}(
r in tup{VT, r},
x in tup{VT, x},
b in tup{'b', VT, bits},
@ -309,12 +309,13 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @maskedLoop
}
1
}
export {'acc_u32_u16_bit_u1', acc_u32_u16_bit_u1}
fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x unrolled
fn acc_u32_u16_bit_u2(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @for_mu, 2x unrolled
def bulk = 8
def VT = [8]u32
@muLoop{bulk,2}(
@for_mu{bulk,2}(
r in tup{'g', VT, r},
x in tup{VT, x},
b in tup{'b', VT, bits},
@ -328,6 +329,7 @@ fn acc_u32_u16_bit(r:*u32, x:*u16, bits:*u64, len:u64) : u1 = { # @muLoop, 2x un
}
1
}
export {'acc_u32_u16_bit_u2', acc_u32_u16_bit_u2}
```
<!--

View File

@ -11,7 +11,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
e0v:= VT**e0
e1v:= VT**e1
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
@for_masked{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{blend_hom{e0v, e1v, b}}
}
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {
@ -31,7 +31,7 @@ fn blend_arr_scalar{E}(rp:*void, zero:*void, one0:u64, mask:*void, len:u64) : vo
def bulk = arch_defvw / width{E}
def VT = [bulk]E
def one = VT**cast_i{E, one0}
@maskedLoop{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
@for_masked{bulk}(r in tup{VT,*E~~rp}, zero in tup{VT,*E~~zero}, mask in tup{'b',VT,mask} over i to len) r = blend_hom{zero, one, mask}
}
}
@ -44,7 +44,7 @@ export_tab{'si_blend_arr_scalar', each{blend_arr_scalar, tup{u1, '!', '!', u8, u
def bulk = arch_defvw / 8
def V = [bulk]u8
bitalign{tup{2,8,csz}, 8, {s, align} => {
@maskedLoop{bulk}(dst in tup{V,*u8~~dst} over cam) {
@for_masked{bulk}(dst in tup{V,*u8~~dst} over cam) {
dst = align{load{*V~~src}}
ptr_add{u8, src, bulk*s/8}
}

View File

@ -6,7 +6,7 @@ include './bitops'
def copyFromBits{V=[bulk]T, loadFn, rp, l:(u64)} = {
def U = ty_u{V}
@maskedLoop{bulk}(sr in tup{'g',*T~~rp} over i to l) {
@for_masked{bulk}(sr in tup{'g',*T~~rp} over i to l) {
x:= loadFn{U, i}
sr{V~~(x & U ~~ V**1)}
}
@ -31,11 +31,11 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
# show{'R==u64', X, R}
assert{X==u8 or X==u16 or X==u32}
# TODO could maybe read 256 bits and use unpack to write >256
@maskedLoop{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)}
@for_masked{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x | RV**(cbqn_c32Tag{}<<48)}
} else if (X==u1 and R==u1) {
# show{'u1u1', X, R}
def V64 = [vw/64]u64
@maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
@for_masked{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over cdiv{l,64}) sr{x}
} else if (X==u1) {
# show{'X==u1', X, R}
copyFromBits{[bulk]R, load_expand_bits{., xp, .}, r, l}
@ -54,10 +54,10 @@ fn copy{X, R}(r: *void, x: *void, l:u64, xRaw: *void) : void = {
}
} else if (width{X}<=width{R}) {
# show{'w{X}<=w{R}', X, R}
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x}
@for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over l) sr{x}
} else {
# show{'w{X}>w{R}', X, R}
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over l) sr{x}
@for_mu{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over l) sr{x}
}
}

View File

@ -130,13 +130,13 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
if (R==u1) {
def bulk = vw/64
def VT = [bulk]u64
@maskedLoop{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over cdiv{len, 64}) r{F{cw,cx}}
@for_masked{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over cdiv{len, 64}) r{F{cw,cx}}
} else if (same{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6
def bulk = vw / width{W}
def TU = ty_u{R}
def TV = [bulk]TU
@muLoop{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over len) {
@for_mu{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over len) {
sr{each{&, cw, cx}}
}
} else {
@ -147,7 +147,7 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
def run = runner{same{overflow, 0}, R, F}
def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
@for_mu{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
sr{arithProcess{F, run, overflow, M, is, cw, cx, TY}}
}
}
@ -182,7 +182,7 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
cw:= ty_sc{W, TY}**getW{w}
def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
@for_mu{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
def cws = length{is}**cw
sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}}
}
@ -199,7 +199,7 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
def T8 = [bulk]u8
def T64 = [bulk/8]u64
maskFull:= T8~~T64**maskU64
@maskedLoop{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
@for_masked{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over len) sr{cx & maskFull}
}
export{'simd_andBytes', andBytes{arch_defvw}}

View File

@ -20,13 +20,13 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
if (W==u1) {
if (X==u1) { # bitarr ≡ bitarr
def BT = [vw/8]u8
@maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyne_bit{w,x,M}) return{0}
@for_masked{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyne_bit{w,x,M}) return{0}
} else if (X==f64) { # bitarr ≡ f64arr
def TF = [vw/64]f64
def TU = [vw/64]u64
f0:= TF**0.0
f1:= TF**1.0
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
@for_masked_pos{bulk}(M in 'm' over i to l) {
wu:= (if (hasarch{'AVX2'}) {
cw:= load_bits_lo{bulk, *u64~~w, i}
blend_top{f0, f1, TU**cw << make{TU,63-iota{vcount{TU}}}}
@ -46,7 +46,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
# TODO compare with doing the comparison in vector registers
badBits:= T ** ~(X~~1)
@maskedLoop{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
@for_masked{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
if (~and_bit_none{M{x}, badBits}) return{0}
if (anyne{promote{u64,mask{sh{x}}}, promote{u64,cw}, M}) return{0}
}
@ -57,7 +57,7 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
def R = [bulk]X
@maskedLoopPositive{bulk}(M in 'm' over i to l) {
@for_masked_pos{bulk}(M in 'm' over i to l) {
cw:= load_widen{*W~~w, i, R}
cx:= load_widen{*X~~x, i, R}
if (anyne_positive{cw,cx,M}) return{0}

View File

@ -89,7 +89,7 @@ local def ml_exec{i, iter, vars0, bulk, M} = {
}
# i0 - initial batch index; not used as begin because it's in a different scale compared to end
def maskedLoop{bulk, i0}{vars,begin==0,end,iter} = {
def for_masked{bulk, i0}{vars,begin==0,end,iter} = {
l:u64 = end
m:u64 = l / bulk
@ -98,10 +98,10 @@ def maskedLoop{bulk, i0}{vars,begin==0,end,iter} = {
left:= l & (bulk-1)
if (left!=0) ml_exec{m, iter, vars, bulk, mask_first{left}}
}
def maskedLoop{bulk} = maskedLoop{bulk,0}
def for_masked{bulk} = for_masked{bulk,0}
def maskedLoopPositive{bulk}{vars,begin==0,end:L,iter} = {
def for_masked_pos{bulk}{vars,begin==0,end:L,iter} = {
assert{end > 0}
i:L = 0
while(i < (end-1)/bulk) {
@ -121,7 +121,7 @@ def maskedLoopPositive{bulk}{vars,begin==0,end:L,iter} = {
# begin must be 0
# end is scalar element count
# index given is a tuple of batch indexes to process
def muLoop{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
def for_mu{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
l:u64 = end
m:u64 = l / bulk
@ -152,4 +152,4 @@ def muLoop{bulk, unr, fromunr}{vars,begin==0,end,iter} = {
if (left!=0) ml_exec{tup{m}, iter, vars, bulk, mask_first{left}}
}
}
def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0}
def for_mu{bulk, unr} = for_mu{bulk, unr, {}=>0}

View File

@ -8,7 +8,7 @@ include './mask'
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T}
def VT = [bulk]T
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
@for_mu{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
if (T!=f64 and any_hom{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{select{is,0}*bulk}
sr{each{abs, cx}}
}

View File

@ -329,7 +329,7 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
d4:usz = width{V} % l
m:= make{V, scan{{a,_} => a>>d | a<<(l-d), tup{ms, ...iota{3}}}}
c:= V**0
@maskedLoop{4} (x in tup{V, xp},
@for_masked{4} (x in tup{V, xp},
r in tup{V, rp} over promote{u64,nw}) {
s := scan_words{x, m}
pc:= c; c = shuf{-(s>>63), 3,0,1,2}

View File

@ -25,7 +25,7 @@ def search{E, x, n:(u64), OP} = {
tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + promote{u64, ctz_ext{hom_to_int_ext{c}}}}
}
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
@for_mu{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
eq:= each{OP, x}
if (any_hom{M{tree_fold{|, eq}}}) {
findFirst{
@ -53,7 +53,7 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
def E = f64
def bulk = arch_defvw/width{E}
def VT = [bulk]E
@maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
@for_masked{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
if (any_hom{M{x!=x}}) return{1}
sr{x + VT**0}
}
@ -324,7 +324,7 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
def unr = tern{E==f64 and hasarch{'X86_64'}, 1, 2}
def minA = acc{2, VT**min1}
def maxA = acc{2, VT**min1}
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
@for_mu{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
if (E==f64 and any_hom{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend

View File

@ -57,7 +57,7 @@ fn wrap_inds{TI if issigned{TI}}(src:*void, dst:*void, n:u64, cyc0:u64) : void =
if (has_simd) {
def bulk = arch_defvw / width{TI}
def VT = [bulk]TI
@maskedLoop{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
@for_masked{bulk}(src in vptr{VT, src}, dst in vptr{VT, dst} over n) {
dst = blend_hom{src, src + VT**cyc, src < VT**0}
}
} else {
@ -198,7 +198,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
# show{TD, nt, ni, G}
# lprintf{'LUT of ', VI, ' ⊏ ', [nt]TD, ' with ', wl, ' ≡ ≠𝕨, ', xl, ' ≡ ≠𝕩'}
def lut = G{x}
@maskedLoop{ni}(w0 in tup{VI,w}, M in 'm' over wl) {
@for_masked{ni}(w0 in tup{VI,w}, M in 'm' over wl) {
def w = wrapChk{w0, xlf, M}
def rs = lut{ty_u{w}}
r = masked_multistore{r, rs, M, {} => return{1}}
@ -220,7 +220,7 @@ fn select_fn{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def VD = [bulk]TDE
def xlf = VI**cast_i{TIE, xl}
@maskedLoop{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over wl) {
@for_masked{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
got:= gather{VD**0, x, cw, M}
if (TDE!=TD) got&= VD**((1<<wd)-1)
@ -276,7 +276,7 @@ export_tab{'si_select_tab', join{table{select_fn,
if (hasarch{'AARCH64'}) {
def xrev = rbit{load{*VU ~~ x0}}
@maskedLoop{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
@for_masked{16}(cw0 in w, r in *u16~~r0, M in 'm' over i to wl) {
def cw = ty_u{wrapChk{cw0, xlf, M}}
def byte = shuf{[16]u8, xrev, cw>>3}
r = hom_to_int{ty_s{byte << (cw & VU**7)} < VI**0}
@ -284,7 +284,7 @@ export_tab{'si_select_tab', join{table{select_fn,
} else {
if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, expand_bits{[32]u8, load{*u32~~x0}}, 0,1,0,1}
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
@for_masked{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
sr = hom_to_int{shuf{[16]i8, xb, cw}}
}
@ -292,7 +292,7 @@ export_tab{'si_select_tab', join{table{select_fn,
x:= shuf{[4]u64, load{*VI ~~ x0}, 0,1,0,1}
low:= VI**7
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
@maskedLoop{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
@for_masked{32}(cw0 in w, sr in *u32~~r0, M in 'm' over wl) {
cw:= wrapChk{cw0, xlf, M}
byte:= shuf{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
mask:= shuf{[16]i8, b, cw & low}
@ -308,7 +308,7 @@ export_tab{'si_select_tab', join{table{select_fn,
def bulk = arch_defvw / 8
def V = [bulk]u8
def lut = shuf_u8bits{inds, indn}
@maskedLoop{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
@for_masked{bulk}(src in tup{V,*u8~~src}, dst in tup{V,*u8~~dst} over rows) {
dst = lut{src}
}
}

View File

@ -81,7 +81,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
if (CHR) { # c16/c32/B → char
mt:= XV**0 # unused for c16
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
def orx = M{tree_fold{|, xs}}
if (B) {
if (any_nonC32{M, ...xs}) return{3}
@ -100,21 +100,21 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
def EH = w_h{X}
def acc = int_acc{XV}
if (acc{'minmax'}) {
@muLoop{bulk, 4}(xs in xb, M in 'm' over len) {
@for_mu{bulk, 4}(xs in xb, M in 'm' over len) {
minc:= tree_fold{min, xs}
maxc:= tree_fold{max, xs}
if (M{0}==0 and any_hom{M{(minc < XV**minvalue{EH}) | (maxc > XV**maxvalue{EH})}}) return{0xffff_ffff}
acc{M, minc, maxc}
}
} else {
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
def mixed = acc{M, xs}
if (M{0}==0 and any_hom{mixed > XV**maxvalue{EH}}) return{0xffff_ffff}
}
}
acc{}
} else if (X==i8) {
@muLoop{bulk, 2}(xs in xb, M in 'm' over len) {
@for_mu{bulk, 2}(xs in xb, M in 'm' over len) {
if (~and_bit_none{M{tree_fold{|, xs}}, XV ** -2}) return{2}
}
0
@ -122,14 +122,14 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:ux) : u32 = {
def case_B = make_branch{tup{ux}, {bulkCont} => {
def i0 = bulkCont*bulk
x:= i0 + *u64~~x0
if (B) @muLoop{bulk, 2}(xs in tup{[bulk]u64, x}, M in 'm' over len-i0) {
if (B) @for_mu{bulk, 2}(xs in tup{[bulk]u64, x}, M in 'm' over len-i0) {
if (any_sNaN{M, ...xs}) return{0xffff_fffe} # not even a number
}
return{0xffff_ffff} # float
}}
def acc = int_acc{re_el{i32, XV}}
@muLoop{bulk, 2}(xs in xb, M in 'm' over is to len) {
@for_mu{bulk, 2}(xs in xb, M in 'm' over is to len) {
if (hasarch{'X86_64'}) {
def ns = each{narrow{i32,.}, xs}
if (any_hom{M{tree_fold{|, each{{ns,x} => widen{XV,ns}!=x, ns, xs}}}}) case_B{select{is, 0}}