@-forms of maskedLoop and muLoop

This commit is contained in:
dzaima 2023-06-11 20:46:05 +03:00
parent fe8892ce6c
commit 2a52dff482
10 changed files with 140 additions and 132 deletions

View File

@ -187,6 +187,8 @@ def widen{...x} = assert{'widen not supported', show{...x}}
def narrow{...x} = assert{'narrow not supported', show{...x}}
def unord{...x} = assert{'unord not supported', show{...x}}
def floor{...x} = assert{'floor not supported', show{...x}}
def loadBatchBit{...x} = assert{'loadBatchBit not supported', show{...x}}
def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}}
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
oper &~ andnot infix none 35
@ -230,6 +232,12 @@ def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}
def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'}
def load{p,i & kgen{p}} = p{i}
def store{p,i,x & kgen{p}} = p{i,x}
def tptr{l,s} = { # create "pointer" generator with given load & store definitions
def r{i} = l{i}
def r{i,x} = s{i,x}
}
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
def f{i,l & i==l} = 0

View File

@ -19,11 +19,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
e0v:= VT**e0
e1v:= VT**e1
maskedLoop{bulk, len, {i, M} => {
cb:= loadBatchBit{VT, bits, i}
storeBatch{r, i, homBlend{e0v, e1v, cb}, M}
}}
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}}
}
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {

View File

@ -16,11 +16,10 @@ def copyFromBits{T, loadFn, rp, l:u64} = {
def bulk = vcount{T}
def TU = ty_u{T}
maskedLoop{bulk, l, {i, M} => {
@maskedLoop{bulk}(sr in tup{'g',*eltype{T}~~rp} over i to l) {
x:= loadFn{TU, i}
r:= x & TU ~~ T**1
storeBatch{*eltype{T}~~rp, i, T~~r, M}
}}
sr{T~~(x & TU ~~ T**1)}
}
}
@ -41,18 +40,12 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
} else if (R==u64) {
# show{'R==u64', X, R}
assert{((X==u8) | (X==u16)) | (X==u32)}
maskedLoop{bulk, l, {i, M} => { # TODO could maybe read 256 bits and use unpack to write >256
v:= loadBatch{xp, i, RV}
v|= RV ** (cbqn_c32Tag{} << 48)
storeBatch{rp, i, v, M}
}}
# TODO could maybe read 256 bits and use unpack to write >256
@maskedLoop{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x | RV**(cbqn_c32Tag{}<<48)}
} else if (X==u1 and R==u1) {
# show{'u1u1', X, R}
def V64 = [vw/64]u64
maskedLoop{vcount{V64}, cdiv{l, 64}, {i, M} => {
v:= loadBatch{xp, i, V64}
storeBatch{rp, i, v, M}
}}
@maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over _ to cdiv{l,64}) sr{x}
} else if (X==u1) {
# show{'X==u1', X, R}
copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l}
@ -66,16 +59,10 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
}
} else if (width{X}<=width{R}) {
# show{'w{X}<=w{R}', X, R}
muLoop{bulk, ur, l, {is, M} => {
def v = loadBatch{xp, is, RV}
storeBatch{rp, is, v, M}
}}
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x}
} else {
# show{'w{X}>w{R}', X, R}
muLoop{bulk, ur, l, {is, M} => {
def v = loadBatch{xp, is, XV}
storeBatch{rp, is, v, M}
}}
@muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over _ to l) sr{x}
}
}

View File

@ -143,22 +143,16 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
# show{F, mode, W, X, R}
if (R==u1) {
def bulk = vw/64
def TY = [bulk]u64
maskedLoop{bulk, cdiv{len, 64}, {i, M} => {
cw:= loadBatch{*u64~~w, i, TY}
cx:= loadBatch{*u64~~x, i, TY}
storeBatch{*u64~~r, i, F{cw, cx}, M}
}}
def VT = [bulk]u64
@maskedLoop{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over _ to cdiv{len, 64}) r{F{cw,cx}}
} else if (match{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6
def bulk = vw / width{W}
def TU = ty_u{R}
def TV = [bulk]TU
muLoop{bulk, 2, len, {is, M} => {
def cw = loadBatch{*TU~~w, is, TV}
def cx = loadBatchBit{TV, *u64~~x, is}
storeBatch{*TU~~r, is, each{&, cw, cx}, M}
}}
@muLoop{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over _ to len) {
sr{each{&, cw, cx}}
}
} else {
def bulk = vw / max{max{width{W}, width{X}}, width{R}}
def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}}
@ -167,11 +161,9 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
def run = runner{match{overflow, 0}, R, F}
def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty
muLoop{bulk, unr, len, {is, M} => {
def cw = loadBatch{*W~~w, is, ty_sc{W, TY}}
def cx = loadBatch{*X~~x, is, ty_sc{X, TY}}
storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, cw, cx, TY}, M}
}}
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
sr{arithProcess{F, run, overflow, M, is, cw, cx, TY}}
}
}
}
@ -204,11 +196,10 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
cw:= ty_sc{W, TY}**getW{w}
def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl
muLoop{bulk, unr, len, {is, M} => {
def cx = loadBatch{*X~~x, is, ty_sc{X, TY}}
@muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
def cws = tuplen{is}**cw
storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}, M}
}}
sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}}
}
if (mode==1 or mode==3) 0
else len
@ -222,9 +213,7 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
def T8 = [bulk]u8
def T64 = [bulk/8]u64
maskFull:= T8~~T64**maskU64
maskedLoop{bulk, len, {i, M} => {
storeBatch{r, i, loadBatch{x, i, T8} & maskFull, M}
}}
@maskedLoop{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over _ to len) sr{cx & maskFull}
}
export{'simd_andBytes', andBytes{arch_defvw}}

View File

@ -27,11 +27,8 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
if (W==u1) {
if (X==u1) { # bitarr ≡ bitarr
maskedLoop{vw, l, {i, M} => {
cw:= load{*[vw/8]u8 ~~ w, i}
cx:= load{*[vw/8]u8 ~~ x, i}
if (anyneBit{cw,cx,M}) return{0}
}}
def BT = [vw/8]u8
@maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyneBit{w,x,M}) return{0}
} else if (X==f64) { # bitarr ≡ f64arr
def TF = [vw/64]f64
def TU = [vw/64]u64
@ -57,12 +54,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
# TODO compare with doing the comparison in vector registers
badBits:= T ** ~(X~~1)
maskedLoop{bulk, l, {i, M} => {
cw:= b_getBatch{bulk, *u64~~w, i}
cx:= load{*T ~~ x, i}
if (~andAllZero{M{cx}, badBits}) return{0}
if (anyne{promote{u64,mask{sh{cx}}}, promote{u64,cw}, M}) return{0}
}}
@maskedLoop{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
if (~andAllZero{M{x}, badBits}) return{0}
if (anyne{promote{u64,mask{sh{x}}}, promote{u64,cw}, M}) return{0}
}
1
}
} else { # everything not involving bitarrs (i.e. floats/ints, and chars)

View File

@ -68,16 +68,6 @@ def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n,
def maskedLoop{bulk, l, step} = maskedLoop{bulk, 0, l, step}
def maskedLoop{bulk, i0, l, step} = {
m:u64 = l/bulk
@for (i from i0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}
def maskedLoopPositive{bulk, l:L, step} = {
i:L = 0
while(i < (l-1)/bulk) {
@ -87,37 +77,88 @@ def maskedLoopPositive{bulk, l:L, step} = {
step{i, maskAfter{l - i*bulk}}
}
# "harmless" pointer cast that'll only cast void*
def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got ',p}}
def hCast{T,p:P & match{T,eltype{P}}} = p
def hCast{T,p:P & match{P,*void}} = *T~~p
# masked loop arguments
# p:*T - regular pointer, loaded by the batch index(es)
# tup{VT,p:*T} - loadBatch/storeBatch vector data
# tup{'b',p:P} - b_getBatch
# tup{'b',VT,p:P} - loadBatchBit
# tup{'g',VT,p:*T} - gives a generator supporting g{} for loadBatch and g{newValue} for storeBatch
# tup{'g',p:*T} - the above, but without load support
def mlExec{i, block, vars0, bulk, M} = {
def vproc2{T,p:P} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}}
def vproc2{S=='b', p:P} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'}
def vproc2{S=='b',T,p:P} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'}
def vproc2{S=='g', p:P} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'}
def vproc2{S=='g',T,p:P} = tptr{{i} => {
def dv{} = loadBatch{p, i, T}
def dv{x} = storeBatch{p, i, x, M}
}, '!'}
def vproc{p:P & isptr{P}} = p
def vproc{S=='m'} = tptr{{_}=>M, '!'}
def vproc{t & ktup{t}} = vproc2{...t}
exec{i, each{{c} => vproc{c}, vars0}, block}
}
# i0 - initial batch index; not used as begin because it's in a different scale compared to end
def maskedLoop{bulk, i0}{vars,begin==0,end,block} = {
l:u64 = end
m:u64 = l / bulk
@for (i from i0 to m) mlExec{i, block, vars, bulk, maskNone}
left:= l & (bulk-1)
if (left!=0) mlExec{m, block, vars, bulk, maskAfter{left}}
}
def maskedLoop{bulk} = maskedLoop{bulk,0}
# masked unrolled loop
# bulk: vector count
# unr: unroll amount
# l: length
# step: {is, M}=>operation
# 'is' is a tuple of bulk indexes to process, usable as an arg to loadBatch/storeBatch to get a tuple of vectors
# fromunr (optional): {}=>{transition from unrolled to non-unrolled}
def muLoop{bulk, unr, l, step, fromunr} = {
# loop args:
# begin must be 0
# end is scalar element count
# index given is a tuple of batch indexes to process
def muLoop{bulk, unr, fromunr}{vars,begin==0,end,block} = {
l:u64 = end
def step = 123123123
m:u64 = l / bulk
if (unr==1) {
maskedLoop{bulk, l, {i, M} => step{tup{i}, M}}
@for (i from 0 to m) mlExec{tup{i}, block, vars, bulk, maskNone}
left:= l & (bulk-1)
if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}}
} else {
m:u64 = l/bulk
if (m > 0) {
i:u64 = 0
if (unr <= m) {
while ((i+unr) <= m) {
def is = each{{j}=>i+j, iota{unr}}
step{each{{j}=>i+j, iota{unr}}, maskNone}
mlExec{each{{j}=>i+j, iota{unr}}, block, vars, bulk, maskNone}
i+= unr
}
fromunr{}
}
if (unr==2) {
if (i!=m) step{tup{i}, maskNone}
if (i!=m) mlExec{tup{i}, block, vars, bulk, maskNone}
} else {
@for(j from i to m) step{tup{j}, maskNone}
@for(j from i to m) mlExec{tup{j}, block, vars, bulk, maskNone}
}
}
left:= l & (bulk-1)
if (left!=0) step{tup{m}, maskAfter{left}}
if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}}
}
}
def muLoop{bulk, unr, l, step} = muLoop{bulk, unr, l, step, {}=>0}
def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0}

View File

@ -17,11 +17,10 @@ include './mask'
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T}
def VT = [bulk]T
muLoop{bulk, tern{T==f64, 2, 1}, len, {is, M} => {
def cx = loadBatch{*T~~x, is, VT}
@muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk}
storeBatch{*T~~r, is, each{abs, cx}, M}
}}
sr{each{abs, cx}}
}
len
}

View File

@ -37,9 +37,8 @@ def search{E, x, n:u64, OP} = {
tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}}
}
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => {
eq:= each{OP, loadBatch{*E~~x, is, VT}}
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
eq:= each{OP, x}
if (homAny{M{tree_fold{|, eq}}}) {
findFirst{
{i,c} => homAny{c},
@ -48,7 +47,7 @@ def search{E, x, n:u64, OP} = {
is, eq
}
}
}}
}
n
}
@ -66,11 +65,10 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
def E = f64
def bulk = arch_defvw/width{E}
def VT = [bulk]E
maskedLoop{bulk, len, {i, M} => {
c:= loadBatch{x, i, VT}
if (homAny{M{c!=c}}) return{1}
storeBatch{r, i, c + VT**0, M}
}}
@maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
if (homAny{M{x!=x}}) return{1}
sr{x + VT**0}
}
0
}
@ -324,12 +322,11 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2}
def minA = acc{2, VT**min1}
def maxA = acc{2, VT**min1}
muLoop{bulk, unr, n, {is, M} => {
def cx = loadBatch{x, is, VT}
@muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend
}, {} => { minA{'tr',min}; maxA{'tr',max} }}
}
min1 = vfold{min, minA{'get'}}
max1 = vfold{max, maxA{'get'}}
} else {

View File

@ -36,8 +36,8 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI
def ext = ri/rd
xlf:= VI**cast_i{TI, xl}
maskedLoop{ri, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
@maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) {
cw:= wrapChk{cw0, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i)
def se{e, c, o} = {
c2:= shuf{[4]u64, c+c, 4b3120}
@ -49,20 +49,20 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
}
def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl}
se{1, cw, 0}
}}
}
}
def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI
def ext = ri/rd
xlf:= VI**cast_i{TI, xl}
maskedLoop{ri, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
@maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) {
cw:= wrapChk{cw0, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i)
def part{o} = widen{[8]i32, to_el{i8, shuf{[4]u64, cw, 4b3210+o}}}
def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl}
each{se, iota{ext}}
}}
}
}
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
@ -121,12 +121,12 @@ fn select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def VD = [bulk]TDE
def xlf = VI**cast_i{TIE, xl}
maskedLoop{bulk, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
@maskedLoop{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over _ to wl) {
cw:= wrapChk{cw0, VI,xlf, M}
got:= gather{VD**0, x, cw, M}
if (TDE!=TD) got&= VD**((1<<wd)-1)
storeBatch{r, i, got, M}
}}
sr{got}
}
}
1
}
@ -146,20 +146,20 @@ fn avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
store{r, i, homMask{sel{[16]i8, xb, cw}}}
}}
@maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) {
cw:= wrapChk{cw0, VI,xlf, M}
sr = homMask{sel{[16]i8, xb, cw}}
}
} else {
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
low:= VI**7
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
@maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) {
cw:= wrapChk{cw0, VI,xlf, M}
byte:= sel{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
mask:= sel{[16]i8, b, cw & low}
store{r, i, homMask{(mask & byte) == mask}}
}}
sr = homMask{(mask & byte) == mask}
}
}
1
}

View File

@ -59,33 +59,32 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
r1:= EV**0
if (CHR) { # c8, c16, c32
def hw = width{E}/2
maskedLoop{bulk, len, {i, M} => {
c:= EV~~loadBatch{xp, i, XV}
@maskedLoop{bulk}(xv in tup{XV,xp}, M in 'm' over _ to len) {
c:= EV~~xv
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
if (B) {
if (anyNonChar{M, c}) return{3}
} else {
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
}
}}
}
r2:= foldTotal{u32, r1}
if (X>u32 and r2>=65536) return{2}
if (X>u16 and r2>=256) return{1}
0
} else { # i8, i16, i32, f64
if (X==i8) { # i8
maskedLoop{bulk, len, {i, M} => {
v0:= loadBatch{xp, i, XV}
@maskedLoop{bulk}(v0 in tup{XV,xp}, M in 'm' over _ to len) {
if (anynePositive{EV**0xfe & EV~~v0, EV**0, M}) return{2}
}}
}
0
} else { # i16, i32, f64
def case_B = makeOptBranch{B, tup{Size}, {iCont} => {
maskedLoop{bulk, iCont, len, {i, M} => {
def XU = [bulk]u64
v:= XU ~~ loadBatch{xp, i, XV}
def XU = [bulk]u64
@maskedLoop{bulk, iCont}(xv in tup{XV,xp}, M in 'm' over _ to len) {
v:= XU ~~ xv
if (anySNaN{M, v}) return{0xffff_fffe} # not even a number
}}
}
return{0xffff_ffff} # float
}}
@ -94,15 +93,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
}
if (isint{X}) { # i16, i32
muLoop{bulk, 1, len, {is, M} => {
def v0 = loadBatch{xp, is, XV}
@muLoop{bulk, 1}(v0 in tup{XV,xp}, M in 'm' over _ to len) {
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
}}
}
} else { # f64
r2:= EV2**0
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => {
def v0 = loadBatch{xp, is, XV}
@muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) {
def int = {
def {int, wdn} = {
if (hasarch{'AARCH64'} and tuplen{is}==2) {
@ -140,7 +136,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
def acc = { if (tuplen{is}==2) r2; else r1 }
acc|= M{getAcc{type{acc}, int}}
}, {} => { r1 = half{r2,0}|half{r2,1} }}
}
}