diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index f136a511..40064119 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -187,6 +187,8 @@ def widen{...x} = assert{'widen not supported', show{...x}} def narrow{...x} = assert{'narrow not supported', show{...x}} def unord{...x} = assert{'unord not supported', show{...x}} def floor{...x} = assert{'floor not supported', show{...x}} +def loadBatchBit{...x} = assert{'loadBatchBit not supported', show{...x}} +def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}} def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b oper &~ andnot infix none 35 @@ -230,6 +232,12 @@ def arch_defvw = tern{hasarch{'AVX2'}, 256, 128} def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} +def load{p,i & kgen{p}} = p{i} +def store{p,i,x & kgen{p}} = p{i,x} +def tptr{l,s} = { # create "pointer" generator with given load & store definitions + def r{i} = l{i} + def r{i,x} = s{i,x} +} def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def f{i,l & i==l} = 0 diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 9709927d..0101d795 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -19,11 +19,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = { e0v:= VT**e0 e1v:= VT**e1 - - maskedLoop{bulk, len, {i, M} => { - cb:= loadBatchBit{VT, bits, i} - storeBatch{r, i, homBlend{e0v, e1v, cb}, M} - }} + @maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}} } fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 7fbd5a22..7a13920a 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -16,11 +16,10 @@ def copyFromBits{T, loadFn, rp, l:u64} = { def bulk = vcount{T} def TU = ty_u{T} - maskedLoop{bulk, l, {i, M} => { + @maskedLoop{bulk}(sr in tup{'g',*eltype{T}~~rp} over i to l) { x:= loadFn{TU, i} - r:= x & TU ~~ T**1 - storeBatch{*eltype{T}~~rp, i, T~~r, M} - }} + sr{T~~(x & TU ~~ T**1)} + } } @@ -41,18 +40,12 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { } else if (R==u64) { # show{'R==u64', X, R} assert{((X==u8) | (X==u16)) | (X==u32)} - maskedLoop{bulk, l, {i, M} => { # TODO could maybe read 256 bits and use unpack to write >256 - v:= loadBatch{xp, i, RV} - v|= RV ** (cbqn_c32Tag{} << 48) - storeBatch{rp, i, v, M} - }} + # TODO could maybe read 256 bits and use unpack to write >256 + @maskedLoop{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x | RV**(cbqn_c32Tag{}<<48)} } else if (X==u1 and R==u1) { # show{'u1u1', X, R} def V64 = [vw/64]u64 - maskedLoop{vcount{V64}, cdiv{l, 64}, {i, M} => { - v:= loadBatch{xp, i, V64} - storeBatch{rp, i, v, M} - }} + @maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over _ to cdiv{l,64}) sr{x} } else if (X==u1) { # show{'X==u1', X, R} copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l} @@ -66,16 +59,10 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { } } else if (width{X}<=width{R}) { # show{'w{X}<=w{R}', X, R} - muLoop{bulk, ur, l, {is, M} => { - def v = loadBatch{xp, is, RV} - storeBatch{rp, is, v, M} - }} + @muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x} } else { # show{'w{X}>w{R}', X, R} - muLoop{bulk, ur, l, {is, M} => { - def v = loadBatch{xp, is, XV} - storeBatch{rp, is, v, M} - }} + @muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over _ to l) sr{x} } } diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 35ae63e9..33f0183d 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -143,22 +143,16 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = { # show{F, mode, W, X, R} if (R==u1) { def bulk = vw/64 - def TY = [bulk]u64 - maskedLoop{bulk, cdiv{len, 64}, {i, M} => { - cw:= loadBatch{*u64~~w, i, TY} - cx:= loadBatch{*u64~~x, i, TY} - storeBatch{*u64~~r, i, F{cw, cx}, M} - }} + def VT = [bulk]u64 + @maskedLoop{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over _ to cdiv{len, 64}) r{F{cw,cx}} } else if (match{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6 def bulk = vw / width{W} def TU = ty_u{R} def TV = [bulk]TU - muLoop{bulk, 2, len, {is, M} => { - def cw = loadBatch{*TU~~w, is, TV} - def cx = loadBatchBit{TV, *u64~~x, is} - storeBatch{*TU~~r, is, each{&, cw, cx}, M} - }} + @muLoop{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over _ to len) { + sr{each{&, cw, cx}} + } } else { def bulk = vw / max{max{width{W}, width{X}}, width{R}} def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}} @@ -167,11 +161,9 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = { def run = runner{match{overflow, 0}, R, F} def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty - muLoop{bulk, unr, len, {is, M} => { - def cw = loadBatch{*W~~w, is, ty_sc{W, TY}} - def cx = loadBatch{*X~~x, is, ty_sc{X, TY}} - storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, cw, cx, TY}, M} - }} + @muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) { + sr{arithProcess{F, run, overflow, M, is, cw, cx, TY}} + } } } @@ -204,11 +196,10 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64 cw:= ty_sc{W, TY}**getW{w} def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl - muLoop{bulk, unr, len, {is, M} => { - def cx = loadBatch{*X~~x, is, ty_sc{X, TY}} + @muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) { def cws = tuplen{is}**cw - storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}, M} - }} + sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}} + } if (mode==1 or mode==3) 0 else len @@ -222,9 +213,7 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = { def T8 = [bulk]u8 def T64 = [bulk/8]u64 maskFull:= T8~~T64**maskU64 - maskedLoop{bulk, len, {i, M} => { - storeBatch{r, i, loadBatch{x, i, T8} & maskFull, M} - }} + @maskedLoop{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over _ to len) sr{cx & maskFull} } export{'simd_andBytes', andBytes{arch_defvw}} diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 420483ee..f82182d2 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -27,11 +27,8 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { if (W==u1) { if (X==u1) { # bitarr ≡ bitarr - maskedLoop{vw, l, {i, M} => { - cw:= load{*[vw/8]u8 ~~ w, i} - cx:= load{*[vw/8]u8 ~~ x, i} - if (anyneBit{cw,cx,M}) return{0} - }} + def BT = [vw/8]u8 + @maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyneBit{w,x,M}) return{0} } else if (X==f64) { # bitarr ≡ f64arr def TF = [vw/64]f64 def TU = [vw/64]u64 @@ -57,12 +54,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { # TODO compare with doing the comparison in vector registers badBits:= T ** ~(X~~1) - maskedLoop{bulk, l, {i, M} => { - cw:= b_getBatch{bulk, *u64~~w, i} - cx:= load{*T ~~ x, i} - if (~andAllZero{M{cx}, badBits}) return{0} - if (anyne{promote{u64,mask{sh{cx}}}, promote{u64,cw}, M}) return{0} - }} + @maskedLoop{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) { + if (~andAllZero{M{x}, badBits}) return{0} + if (anyne{promote{u64,mask{sh{x}}}, promote{u64,cw}, M}) return{0} + } 1 } } else { # everything not involving bitarrs (i.e. floats/ints, and chars) diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index d8be8521..0c46a163 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -68,16 +68,6 @@ def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n, -def maskedLoop{bulk, l, step} = maskedLoop{bulk, 0, l, step} - -def maskedLoop{bulk, i0, l, step} = { - m:u64 = l/bulk - @for (i from i0 to m) step{i, maskNone} - - left:= l & (bulk-1) - if (left!=0) step{m, maskAfter{left}} -} - def maskedLoopPositive{bulk, l:L, step} = { i:L = 0 while(i < (l-1)/bulk) { @@ -87,37 +77,88 @@ def maskedLoopPositive{bulk, l:L, step} = { step{i, maskAfter{l - i*bulk}} } + +# "harmless" pointer cast that'll only cast void* +def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got ',p}} +def hCast{T,p:P & match{T,eltype{P}}} = p +def hCast{T,p:P & match{P,*void}} = *T~~p + +# masked loop arguments +# p:*T - regular pointer, loaded by the batch index(es) +# tup{VT,p:*T} - loadBatch/storeBatch vector data +# tup{'b',p:P} - b_getBatch +# tup{'b',VT,p:P} - loadBatchBit +# tup{'g',VT,p:*T} - gives a generator supporting g{} for loadBatch and g{newValue} for storeBatch +# tup{'g',p:*T} - the above, but without load support +def mlExec{i, block, vars0, bulk, M} = { + def vproc2{T,p:P} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}} + def vproc2{S=='b', p:P} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'} + def vproc2{S=='b',T,p:P} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'} + def vproc2{S=='g', p:P} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'} + def vproc2{S=='g',T,p:P} = tptr{{i} => { + def dv{} = loadBatch{p, i, T} + def dv{x} = storeBatch{p, i, x, M} + }, '!'} + + def vproc{p:P & isptr{P}} = p + def vproc{S=='m'} = tptr{{_}=>M, '!'} + def vproc{t & ktup{t}} = vproc2{...t} + + exec{i, each{{c} => vproc{c}, vars0}, block} +} + +# i0 - initial batch index; not used as begin because it's in a different scale compared to end +def maskedLoop{bulk, i0}{vars,begin==0,end,block} = { + l:u64 = end + + m:u64 = l / bulk + @for (i from i0 to m) mlExec{i, block, vars, bulk, maskNone} + + left:= l & (bulk-1) + if (left!=0) mlExec{m, block, vars, bulk, maskAfter{left}} +} +def maskedLoop{bulk} = maskedLoop{bulk,0} + + + # masked unrolled loop # bulk: vector count # unr: unroll amount -# l: length -# step: {is, M}=>operation -# 'is' is a tuple of bulk indexes to process, usable as an arg to loadBatch/storeBatch to get a tuple of vectors # fromunr (optional): {}=>{transition from unrolled to non-unrolled} -def muLoop{bulk, unr, l, step, fromunr} = { +# loop args: +# begin must be 0 +# end is scalar element count +# index given is a tuple of batch indexes to process +def muLoop{bulk, unr, fromunr}{vars,begin==0,end,block} = { + l:u64 = end + def step = 123123123 + + m:u64 = l / bulk if (unr==1) { - maskedLoop{bulk, l, {i, M} => step{tup{i}, M}} + @for (i from 0 to m) mlExec{tup{i}, block, vars, bulk, maskNone} + + left:= l & (bulk-1) + if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}} } else { - m:u64 = l/bulk if (m > 0) { i:u64 = 0 if (unr <= m) { while ((i+unr) <= m) { def is = each{{j}=>i+j, iota{unr}} - step{each{{j}=>i+j, iota{unr}}, maskNone} + mlExec{each{{j}=>i+j, iota{unr}}, block, vars, bulk, maskNone} i+= unr } fromunr{} } if (unr==2) { - if (i!=m) step{tup{i}, maskNone} + if (i!=m) mlExec{tup{i}, block, vars, bulk, maskNone} } else { - @for(j from i to m) step{tup{j}, maskNone} + @for(j from i to m) mlExec{tup{j}, block, vars, bulk, maskNone} } } left:= l & (bulk-1) - if (left!=0) step{tup{m}, maskAfter{left}} + if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}} } } -def muLoop{bulk, unr, l, step} = muLoop{bulk, unr, l, step, {}=>0} +def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0} diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index eb458af9..9e1f036b 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -17,11 +17,10 @@ include './mask' fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { def bulk = arch_defvw/width{T} def VT = [bulk]T - muLoop{bulk, tern{T==f64, 2, 1}, len, {is, M} => { - def cx = loadBatch{*T~~x, is, VT} + @muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) { if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk} - storeBatch{*T~~r, is, each{abs, cx}, M} - }} + sr{each{abs, cx}} + } len } diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 2b063827..67db0711 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -37,9 +37,8 @@ def search{E, x, n:u64, OP} = { tup{u64, ty_u{VT}}, {i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}} } - - muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { - eq:= each{OP, loadBatch{*E~~x, is, VT}} + @muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) { + eq:= each{OP, x} if (homAny{M{tree_fold{|, eq}}}) { findFirst{ {i,c} => homAny{c}, @@ -48,7 +47,7 @@ def search{E, x, n:u64, OP} = { is, eq } } - }} + } n } @@ -66,11 +65,10 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = { def E = f64 def bulk = arch_defvw/width{E} def VT = [bulk]E - maskedLoop{bulk, len, {i, M} => { - c:= loadBatch{x, i, VT} - if (homAny{M{c!=c}}) return{1} - storeBatch{r, i, c + VT**0, M} - }} + @maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) { + if (homAny{M{x!=x}}) return{1} + sr{x + VT**0} + } 0 } @@ -324,12 +322,11 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = { def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2} def minA = acc{2, VT**min1} def maxA = acc{2, VT**min1} - muLoop{bulk, unr, n, {is, M} => { - def cx = loadBatch{x, is, VT} + @muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) { if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend - }, {} => { minA{'tr',min}; maxA{'tr',max} }} + } min1 = vfold{min, minA{'get'}} max1 = vfold{max, maxA{'get'}} } else { diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index c38fba1e..8c3cd9e7 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -36,8 +36,8 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { def VI = [ri]TI def ext = ri/rd xlf:= VI**cast_i{TI, xl} - maskedLoop{ri, wl, {i, M} => { - cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + @maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) { + cw:= wrapChk{cw0, VI,xlf, M} is:= (if (ext>1) i< { - cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + @maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) { + cw:= wrapChk{cw0, VI,xlf, M} is:= (if (ext>1) i< { - cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + @maskedLoop{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over _ to wl) { + cw:= wrapChk{cw0, VI,xlf, M} got:= gather{VD**0, x, cw, M} if (TDE!=TD) got&= VD**((1<32 and xl<=16) { xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010} - maskedLoop{32, wl, {i, M} => { - cw:= wrapChk{load{w, i}, VI,xlf, M} - store{r, i, homMask{sel{[16]i8, xb, cw}}} - }} + @maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) { + cw:= wrapChk{cw0, VI,xlf, M} + sr = homMask{sel{[16]i8, xb, cw}} + } } else { x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} low:= VI**7 b := VI~~make{[32]u8, 1 << (iota{32} & 7)} - maskedLoop{32, wl, {i, M} => { - cw:= wrapChk{load{w, i}, VI,xlf, M} + @maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) { + cw:= wrapChk{cw0, VI,xlf, M} byte:= sel{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)} mask:= sel{[16]i8, b, cw & low} - store{r, i, homMask{(mask & byte) == mask}} - }} + sr = homMask{(mask & byte) == mask} + } } 1 } diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index a2dd60dd..661bfc7f 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -59,33 +59,32 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { r1:= EV**0 if (CHR) { # c8, c16, c32 def hw = width{E}/2 - maskedLoop{bulk, len, {i, M} => { - c:= EV~~loadBatch{xp, i, XV} + @maskedLoop{bulk}(xv in tup{XV,xp}, M in 'm' over _ to len) { + c:= EV~~xv if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end if (B) { if (anyNonChar{M, c}) return{3} } else { if (anynePositive{EV**((1<u32 and r2>=65536) return{2} if (X>u16 and r2>=256) return{1} 0 } else { # i8, i16, i32, f64 if (X==i8) { # i8 - maskedLoop{bulk, len, {i, M} => { - v0:= loadBatch{xp, i, XV} + @maskedLoop{bulk}(v0 in tup{XV,xp}, M in 'm' over _ to len) { if (anynePositive{EV**0xfe & EV~~v0, EV**0, M}) return{2} - }} + } 0 } else { # i16, i32, f64 def case_B = makeOptBranch{B, tup{Size}, {iCont} => { - maskedLoop{bulk, iCont, len, {i, M} => { - def XU = [bulk]u64 - v:= XU ~~ loadBatch{xp, i, XV} + def XU = [bulk]u64 + @maskedLoop{bulk, iCont}(xv in tup{XV,xp}, M in 'm' over _ to len) { + v:= XU ~~ xv if (anySNaN{M, v}) return{0xffff_fffe} # not even a number - }} + } return{0xffff_ffff} # float }} @@ -94,15 +93,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { } if (isint{X}) { # i16, i32 - muLoop{bulk, 1, len, {is, M} => { - def v0 = loadBatch{xp, is, XV} + @muLoop{bulk, 1}(v0 in tup{XV,xp}, M in 'm' over _ to len) { r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}} - }} + } } else { # f64 r2:= EV2**0 - muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => { - def v0 = loadBatch{xp, is, XV} - + @muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) { def int = { def {int, wdn} = { if (hasarch{'AARCH64'} and tuplen{is}==2) { @@ -140,7 +136,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { def acc = { if (tuplen{is}==2) r2; else r1 } acc|= M{getAcc{type{acc}, int}} - }, {} => { r1 = half{r2,0}|half{r2,1} }} + } }