@-forms of maskedLoop and muLoop

This commit is contained in:
dzaima 2023-06-11 20:46:05 +03:00
parent fe8892ce6c
commit 2a52dff482
10 changed files with 140 additions and 132 deletions

View File

@ -187,6 +187,8 @@ def widen{...x} = assert{'widen not supported', show{...x}}
def narrow{...x} = assert{'narrow not supported', show{...x}} def narrow{...x} = assert{'narrow not supported', show{...x}}
def unord{...x} = assert{'unord not supported', show{...x}} def unord{...x} = assert{'unord not supported', show{...x}}
def floor{...x} = assert{'floor not supported', show{...x}} def floor{...x} = assert{'floor not supported', show{...x}}
def loadBatchBit{...x} = assert{'loadBatchBit not supported', show{...x}}
def b_getBatch{...x} = assert{'b_getBatch not supported', show{...x}}
def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b
oper &~ andnot infix none 35 oper &~ andnot infix none 35
@ -230,6 +232,12 @@ def arch_defvw = tern{hasarch{'AVX2'}, 256, 128}
def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'}
def load{p,i & kgen{p}} = p{i}
def store{p,i,x & kgen{p}} = p{i,x}
def tptr{l,s} = { # create "pointer" generator with given load & store definitions
def r{i} = l{i}
def r{i,x} = s{i,x}
}
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {
def f{i,l & i==l} = 0 def f{i,l & i==l} = 0

View File

@ -19,11 +19,7 @@ def bitsel{VL, T, r, bits, e0, e1, len} = {
e0v:= VT**e0 e0v:= VT**e0
e1v:= VT**e1 e1v:= VT**e1
@maskedLoop{bulk}(r in tup{'g',r}, b in tup{'b',VT,bits} over i to len) r{homBlend{e0v, e1v, b}}
maskedLoop{bulk, len, {i, M} => {
cb:= loadBatchBit{VT, bits, i}
storeBatch{r, i, homBlend{e0v, e1v, cb}, M}
}}
} }
fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = { fn bitsel_i{VL,T}(r:*void, bits:*u64, e0:u64, e1:u64, len:u64) : void = {

View File

@ -16,11 +16,10 @@ def copyFromBits{T, loadFn, rp, l:u64} = {
def bulk = vcount{T} def bulk = vcount{T}
def TU = ty_u{T} def TU = ty_u{T}
maskedLoop{bulk, l, {i, M} => { @maskedLoop{bulk}(sr in tup{'g',*eltype{T}~~rp} over i to l) {
x:= loadFn{TU, i} x:= loadFn{TU, i}
r:= x & TU ~~ T**1 sr{T~~(x & TU ~~ T**1)}
storeBatch{*eltype{T}~~rp, i, T~~r, M} }
}}
} }
@ -41,18 +40,12 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
} else if (R==u64) { } else if (R==u64) {
# show{'R==u64', X, R} # show{'R==u64', X, R}
assert{((X==u8) | (X==u16)) | (X==u32)} assert{((X==u8) | (X==u16)) | (X==u32)}
maskedLoop{bulk, l, {i, M} => { # TODO could maybe read 256 bits and use unpack to write >256 # TODO could maybe read 256 bits and use unpack to write >256
v:= loadBatch{xp, i, RV} @maskedLoop{bulk}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x | RV**(cbqn_c32Tag{}<<48)}
v|= RV ** (cbqn_c32Tag{} << 48)
storeBatch{rp, i, v, M}
}}
} else if (X==u1 and R==u1) { } else if (X==u1 and R==u1) {
# show{'u1u1', X, R} # show{'u1u1', X, R}
def V64 = [vw/64]u64 def V64 = [vw/64]u64
maskedLoop{vcount{V64}, cdiv{l, 64}, {i, M} => { @maskedLoop{vcount{V64}}(sr in tup{'g',rp}, x in tup{V64,xp} over _ to cdiv{l,64}) sr{x}
v:= loadBatch{xp, i, V64}
storeBatch{rp, i, v, M}
}}
} else if (X==u1) { } else if (X==u1) {
# show{'X==u1', X, R} # show{'X==u1', X, R}
copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l} copyFromBits{[bulk]R, {T, i} => loadBatchBit{T, xp, i}, r, l}
@ -66,16 +59,10 @@ fn copy{X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
} }
} else if (width{X}<=width{R}) { } else if (width{X}<=width{R}) {
# show{'w{X}<=w{R}', X, R} # show{'w{X}<=w{R}', X, R}
muLoop{bulk, ur, l, {is, M} => { @muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{RV,xp} over _ to l) sr{x}
def v = loadBatch{xp, is, RV}
storeBatch{rp, is, v, M}
}}
} else { } else {
# show{'w{X}>w{R}', X, R} # show{'w{X}>w{R}', X, R}
muLoop{bulk, ur, l, {is, M} => { @muLoop{bulk,ur}(sr in tup{'g',rp}, x in tup{XV,xp} over _ to l) sr{x}
def v = loadBatch{xp, is, XV}
storeBatch{rp, is, v, M}
}}
} }
} }

View File

@ -143,22 +143,16 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
# show{F, mode, W, X, R} # show{F, mode, W, X, R}
if (R==u1) { if (R==u1) {
def bulk = vw/64 def bulk = vw/64
def TY = [bulk]u64 def VT = [bulk]u64
maskedLoop{bulk, cdiv{len, 64}, {i, M} => { @maskedLoop{bulk}(r in tup{'g',*u64~~r}, cw in tup{VT,*u64~~w}, cx in tup{VT,*u64~~x} over _ to cdiv{len, 64}) r{F{cw,cx}}
cw:= loadBatch{*u64~~w, i, TY}
cx:= loadBatch{*u64~~x, i, TY}
storeBatch{*u64~~r, i, F{cw, cx}, M}
}}
} else if (match{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6 } else if (match{F,__mul} and W!=u1 and X==u1 and W==R) { # 0‿1‿1‿1‿1‿0‿1‿1×3‿1‿4‿1‿5‿9‿2‿6
def bulk = vw / width{W} def bulk = vw / width{W}
def TU = ty_u{R} def TU = ty_u{R}
def TV = [bulk]TU def TV = [bulk]TU
muLoop{bulk, 2, len, {is, M} => { @muLoop{bulk, 2}(sr in tup{'g',*TU~~r}, cw in tup{TV,*TU~~w}, cx in tup{'b',TV,x} over _ to len) {
def cw = loadBatch{*TU~~w, is, TV} sr{each{&, cw, cx}}
def cx = loadBatchBit{TV, *u64~~x, is} }
storeBatch{*TU~~r, is, each{&, cw, cx}, M}
}}
} else { } else {
def bulk = vw / max{max{width{W}, width{X}}, width{R}} def bulk = vw / max{max{width{W}, width{X}}, width{R}}
def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}} def overflow = tern{mode==1, {i}=>return{i}, tern{mode==2, {i}=>return{1}, 0}}
@ -167,11 +161,9 @@ def arithAAimpl{vw, mode, F, W, X, R, w, x, r, len} = {
def run = runner{match{overflow, 0}, R, F} def run = runner{match{overflow, 0}, R, F}
def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty def unr = tern{mode==0, 2, 1} # 2x unroll non-overflowing cases; suppresses clang's default unrolling, which unrolls a lot more; 2x appears to be plenty
muLoop{bulk, unr, len, {is, M} => { @muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cw in tup{ty_sc{W,TY},*W~~w}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
def cw = loadBatch{*W~~w, is, ty_sc{W, TY}} sr{arithProcess{F, run, overflow, M, is, cw, cx, TY}}
def cx = loadBatch{*X~~x, is, ty_sc{X, TY}} }
storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, cw, cx, TY}, M}
}}
} }
} }
@ -204,11 +196,10 @@ fn arithSAf{vw, mode, F, swap, W, X, R}(r:*void, w:u64, x:*void, len:u64) : u64
cw:= ty_sc{W, TY}**getW{w} cw:= ty_sc{W, TY}**getW{w}
def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl def unr = tern{mode>=2, 2, 1} # same as in arithAAimpl
muLoop{bulk, unr, len, {is, M} => { @muLoop{bulk, unr}(sr in tup{'g',*R~~r}, cx in tup{ty_sc{X,TY},*X~~x}, M in 'm' over is to len) {
def cx = loadBatch{*X~~x, is, ty_sc{X, TY}}
def cws = tuplen{is}**cw def cws = tuplen{is}**cw
storeBatch{*R~~r, is, arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}, M} sr{arithProcess{F, run, overflow, M, is, tern{swap,cx,cws}, tern{swap,cws,cx}, TY}}
}} }
if (mode==1 or mode==3) 0 if (mode==1 or mode==3) 0
else len else len
@ -222,9 +213,7 @@ fn andBytes{vw}(r: *u8, x: *u8, maskU64:u64, len:u64) : void = {
def T8 = [bulk]u8 def T8 = [bulk]u8
def T64 = [bulk/8]u64 def T64 = [bulk/8]u64
maskFull:= T8~~T64**maskU64 maskFull:= T8~~T64**maskU64
maskedLoop{bulk, len, {i, M} => { @maskedLoop{bulk}(sr in tup{'g',r}, cx in tup{T8,x} over _ to len) sr{cx & maskFull}
storeBatch{r, i, loadBatch{x, i, T8} & maskFull, M}
}}
} }
export{'simd_andBytes', andBytes{arch_defvw}} export{'simd_andBytes', andBytes{arch_defvw}}

View File

@ -27,11 +27,8 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
if (W==u1) { if (W==u1) {
if (X==u1) { # bitarr ≡ bitarr if (X==u1) { # bitarr ≡ bitarr
maskedLoop{vw, l, {i, M} => { def BT = [vw/8]u8
cw:= load{*[vw/8]u8 ~~ w, i} @maskedLoop{vw}(w in *BT~~w, x in *BT~~x, M in 'm' over l) if (anyneBit{w,x,M}) return{0}
cx:= load{*[vw/8]u8 ~~ x, i}
if (anyneBit{cw,cx,M}) return{0}
}}
} else if (X==f64) { # bitarr ≡ f64arr } else if (X==f64) { # bitarr ≡ f64arr
def TF = [vw/64]f64 def TF = [vw/64]f64
def TU = [vw/64]u64 def TU = [vw/64]u64
@ -57,12 +54,10 @@ fn equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
# TODO compare with doing the comparison in vector registers # TODO compare with doing the comparison in vector registers
badBits:= T ** ~(X~~1) badBits:= T ** ~(X~~1)
maskedLoop{bulk, l, {i, M} => { @maskedLoop{bulk}(cw in tup{'b',w}, x in *T~~x, M in 'm' over i to l) {
cw:= b_getBatch{bulk, *u64~~w, i} if (~andAllZero{M{x}, badBits}) return{0}
cx:= load{*T ~~ x, i} if (anyne{promote{u64,mask{sh{x}}}, promote{u64,cw}, M}) return{0}
if (~andAllZero{M{cx}, badBits}) return{0} }
if (anyne{promote{u64,mask{sh{cx}}}, promote{u64,cw}, M}) return{0}
}}
1 1
} }
} else { # everything not involving bitarrs (i.e. floats/ints, and chars) } else { # everything not involving bitarrs (i.e. floats/ints, and chars)

View File

@ -68,16 +68,6 @@ def storeBatch{ptr:P, ns, xs, M & istup{ns}} = each{{n,x} => storeBatch{ptr, n,
def maskedLoop{bulk, l, step} = maskedLoop{bulk, 0, l, step}
def maskedLoop{bulk, i0, l, step} = {
m:u64 = l/bulk
@for (i from i0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}
def maskedLoopPositive{bulk, l:L, step} = { def maskedLoopPositive{bulk, l:L, step} = {
i:L = 0 i:L = 0
while(i < (l-1)/bulk) { while(i < (l-1)/bulk) {
@ -87,37 +77,88 @@ def maskedLoopPositive{bulk, l:L, step} = {
step{i, maskAfter{l - i*bulk}} step{i, maskAfter{l - i*bulk}}
} }
# "harmless" pointer cast that'll only cast void*
def hCast{T,p} = assert{show{'expected pointer with element',T,'or void but got ',p}}
def hCast{T,p:P & match{T,eltype{P}}} = p
def hCast{T,p:P & match{P,*void}} = *T~~p
# masked loop arguments
# p:*T - regular pointer, loaded by the batch index(es)
# tup{VT,p:*T} - loadBatch/storeBatch vector data
# tup{'b',p:P} - b_getBatch
# tup{'b',VT,p:P} - loadBatchBit
# tup{'g',VT,p:*T} - gives a generator supporting g{} for loadBatch and g{newValue} for storeBatch
# tup{'g',p:*T} - the above, but without load support
def mlExec{i, block, vars0, bulk, M} = {
def vproc2{T,p:P} = tptr{{i} => loadBatch{p, i, T}, {i,x} => storeBatch{p, i, x, M}}
def vproc2{S=='b', p:P} = tptr{{i} => b_getBatch{bulk, hCast{u64,p}, i}, '!'}
def vproc2{S=='b',T,p:P} = tptr{{i} => loadBatchBit{T, hCast{u64,p}, i}, '!'}
def vproc2{S=='g', p:P} = tptr{{i} => ({x} => storeBatch{p, i, x, M}), '!'}
def vproc2{S=='g',T,p:P} = tptr{{i} => {
def dv{} = loadBatch{p, i, T}
def dv{x} = storeBatch{p, i, x, M}
}, '!'}
def vproc{p:P & isptr{P}} = p
def vproc{S=='m'} = tptr{{_}=>M, '!'}
def vproc{t & ktup{t}} = vproc2{...t}
exec{i, each{{c} => vproc{c}, vars0}, block}
}
# i0 - initial batch index; not used as begin because it's in a different scale compared to end
def maskedLoop{bulk, i0}{vars,begin==0,end,block} = {
l:u64 = end
m:u64 = l / bulk
@for (i from i0 to m) mlExec{i, block, vars, bulk, maskNone}
left:= l & (bulk-1)
if (left!=0) mlExec{m, block, vars, bulk, maskAfter{left}}
}
def maskedLoop{bulk} = maskedLoop{bulk,0}
# masked unrolled loop # masked unrolled loop
# bulk: vector count # bulk: vector count
# unr: unroll amount # unr: unroll amount
# l: length
# step: {is, M}=>operation
# 'is' is a tuple of bulk indexes to process, usable as an arg to loadBatch/storeBatch to get a tuple of vectors
# fromunr (optional): {}=>{transition from unrolled to non-unrolled} # fromunr (optional): {}=>{transition from unrolled to non-unrolled}
def muLoop{bulk, unr, l, step, fromunr} = { # loop args:
# begin must be 0
# end is scalar element count
# index given is a tuple of batch indexes to process
def muLoop{bulk, unr, fromunr}{vars,begin==0,end,block} = {
l:u64 = end
def step = 123123123
m:u64 = l / bulk
if (unr==1) { if (unr==1) {
maskedLoop{bulk, l, {i, M} => step{tup{i}, M}} @for (i from 0 to m) mlExec{tup{i}, block, vars, bulk, maskNone}
left:= l & (bulk-1)
if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}}
} else { } else {
m:u64 = l/bulk
if (m > 0) { if (m > 0) {
i:u64 = 0 i:u64 = 0
if (unr <= m) { if (unr <= m) {
while ((i+unr) <= m) { while ((i+unr) <= m) {
def is = each{{j}=>i+j, iota{unr}} def is = each{{j}=>i+j, iota{unr}}
step{each{{j}=>i+j, iota{unr}}, maskNone} mlExec{each{{j}=>i+j, iota{unr}}, block, vars, bulk, maskNone}
i+= unr i+= unr
} }
fromunr{} fromunr{}
} }
if (unr==2) { if (unr==2) {
if (i!=m) step{tup{i}, maskNone} if (i!=m) mlExec{tup{i}, block, vars, bulk, maskNone}
} else { } else {
@for(j from i to m) step{tup{j}, maskNone} @for(j from i to m) mlExec{tup{j}, block, vars, bulk, maskNone}
} }
} }
left:= l & (bulk-1) left:= l & (bulk-1)
if (left!=0) step{tup{m}, maskAfter{left}} if (left!=0) mlExec{tup{m}, block, vars, bulk, maskAfter{left}}
} }
} }
def muLoop{bulk, unr, l, step} = muLoop{bulk, unr, l, step, {}=>0} def muLoop{bulk, unr} = muLoop{bulk, unr, {}=>0}

View File

@ -17,11 +17,10 @@ include './mask'
fn absFn{T}(r:*void, x:*void, len:u64) : u64 = { fn absFn{T}(r:*void, x:*void, len:u64) : u64 = {
def bulk = arch_defvw/width{T} def bulk = arch_defvw/width{T}
def VT = [bulk]T def VT = [bulk]T
muLoop{bulk, tern{T==f64, 2, 1}, len, {is, M} => { @muLoop{bulk, tern{T==f64, 2, 1}}(cx in tup{VT,*T~~x}, sr in tup{'g',*T~~r}, M in 'm' over is to len) {
def cx = loadBatch{*T~~x, is, VT}
if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk} if (T!=f64 and homAny{M{tree_fold{|, eachx{==, cx, VT**minvalue{T}}}}}) return{tupsel{0,is}*bulk}
storeBatch{*T~~r, is, each{abs, cx}, M} sr{each{abs, cx}}
}} }
len len
} }

View File

@ -37,9 +37,8 @@ def search{E, x, n:u64, OP} = {
tup{u64, ty_u{VT}}, tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}} {i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}}
} }
@muLoop{bulk, tern{arch_defvw>=256, 1, 2}}(x in tup{VT,*E~~x}, M in 'm' over is to n) {
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => { eq:= each{OP, x}
eq:= each{OP, loadBatch{*E~~x, is, VT}}
if (homAny{M{tree_fold{|, eq}}}) { if (homAny{M{tree_fold{|, eq}}}) {
findFirst{ findFirst{
{i,c} => homAny{c}, {i,c} => homAny{c},
@ -48,7 +47,7 @@ def search{E, x, n:u64, OP} = {
is, eq is, eq
} }
} }
}} }
n n
} }
@ -66,11 +65,10 @@ fn copyOrdered{}(r:*f64, x:*f64, len:u64) : u1 = {
def E = f64 def E = f64
def bulk = arch_defvw/width{E} def bulk = arch_defvw/width{E}
def VT = [bulk]E def VT = [bulk]E
maskedLoop{bulk, len, {i, M} => { @maskedLoop{bulk}(x in tup{VT,x}, sr in tup{'g',r}, M in 'm' over i to len) {
c:= loadBatch{x, i, VT} if (homAny{M{x!=x}}) return{1}
if (homAny{M{c!=c}}) return{1} sr{x + VT**0}
storeBatch{r, i, c + VT**0, M} }
}}
0 0
} }
@ -324,12 +322,11 @@ fn getRange{E}(x0:*void, res:*i64, n:u64) : u1 = {
def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2} def unr = tern{(E==f64) & hasarch{'X86_64'}, 1, 2}
def minA = acc{2, VT**min1} def minA = acc{2, VT**min1}
def maxA = acc{2, VT**min1} def maxA = acc{2, VT**min1}
muLoop{bulk, unr, n, {is, M} => { @muLoop{bulk, unr, {} => { minA{'tr',min}; maxA{'tr',max} }}(cx in tup{VT,x}, M in 'm' over is to n) {
def cx = loadBatch{x, is, VT}
if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0} if (E==f64 and homAny{M{tree_fold{|, each{{c} => ~isI64{c}, cx}}}}) return{0}
minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend minA{'upd', is, {a} => eachx{maskBlend, a, each{min, a, cx}, M}} # blend
maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend maxA{'upd', is, {a} => eachx{maskBlend, a, each{max, a, cx}, M}} # blend
}, {} => { minA{'tr',min}; maxA{'tr',max} }} }
min1 = vfold{min, minA{'get'}} min1 = vfold{min, minA{'get'}}
max1 = vfold{max, maxA{'get'}} max1 = vfold{max, maxA{'get'}}
} else { } else {

View File

@ -36,8 +36,8 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI def VI = [ri]TI
def ext = ri/rd def ext = ri/rd
xlf:= VI**cast_i{TI, xl} xlf:= VI**cast_i{TI, xl}
maskedLoop{ri, wl, {i, M} => { @maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} cw:= wrapChk{cw0, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i) is:= (if (ext>1) i<<lb{ext}; else i)
def se{e, c, o} = { def se{e, c, o} = {
c2:= shuf{[4]u64, c+c, 4b3120} c2:= shuf{[4]u64, c+c, 4b3120}
@ -49,20 +49,20 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
} }
def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl} def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl}
se{1, cw, 0} se{1, cw, 0}
}} }
} }
def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI def VI = [ri]TI
def ext = ri/rd def ext = ri/rd
xlf:= VI**cast_i{TI, xl} xlf:= VI**cast_i{TI, xl}
maskedLoop{ri, wl, {i, M} => { @maskedLoop{ri}(cw0 in tup{VI,w}, M in 'm' over i to wl) {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} cw:= wrapChk{cw0, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i) is:= (if (ext>1) i<<lb{ext}; else i)
def part{o} = widen{[8]i32, to_el{i8, shuf{[4]u64, cw, 4b3210+o}}} def part{o} = widen{[8]i32, to_el{i8, shuf{[4]u64, cw, 4b3210+o}}}
def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl} def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl}
each{se, iota{ext}} each{se, iota{ext}}
}} }
} }
def makeselx{VI, VD, nsel, xd, logv, cshuf} = { def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
@ -121,12 +121,12 @@ fn select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def VD = [bulk]TDE def VD = [bulk]TDE
def xlf = VI**cast_i{TIE, xl} def xlf = VI**cast_i{TIE, xl}
maskedLoop{bulk, wl, {i, M} => { @maskedLoop{bulk}(cw0 in tup{VI,w}, sr in tup{'g',r}, M in 'm' over _ to wl) {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} cw:= wrapChk{cw0, VI,xlf, M}
got:= gather{VD**0, x, cw, M} got:= gather{VD**0, x, cw, M}
if (TDE!=TD) got&= VD**((1<<wd)-1) if (TDE!=TD) got&= VD**((1<<wd)-1)
storeBatch{r, i, got, M} sr{got}
}} }
} }
1 1
} }
@ -146,20 +146,20 @@ fn avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
if (wl>32 and xl<=16) { if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010} xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
maskedLoop{32, wl, {i, M} => { @maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) {
cw:= wrapChk{load{w, i}, VI,xlf, M} cw:= wrapChk{cw0, VI,xlf, M}
store{r, i, homMask{sel{[16]i8, xb, cw}}} sr = homMask{sel{[16]i8, xb, cw}}
}} }
} else { } else {
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
low:= VI**7 low:= VI**7
b := VI~~make{[32]u8, 1 << (iota{32} & 7)} b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
maskedLoop{32, wl, {i, M} => { @maskedLoop{32}(cw0 in w, sr in r, M in 'm' over _ to wl) {
cw:= wrapChk{load{w, i}, VI,xlf, M} cw:= wrapChk{cw0, VI,xlf, M}
byte:= sel{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)} byte:= sel{[16]i8, x, VI~~(([8]u32~~(cw&~low))>>3)}
mask:= sel{[16]i8, b, cw & low} mask:= sel{[16]i8, b, cw & low}
store{r, i, homMask{(mask & byte) == mask}} sr = homMask{(mask & byte) == mask}
}} }
} }
1 1
} }

View File

@ -59,33 +59,32 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
r1:= EV**0 r1:= EV**0
if (CHR) { # c8, c16, c32 if (CHR) { # c8, c16, c32
def hw = width{E}/2 def hw = width{E}/2
maskedLoop{bulk, len, {i, M} => { @maskedLoop{bulk}(xv in tup{XV,xp}, M in 'm' over _ to len) {
c:= EV~~loadBatch{xp, i, XV} c:= EV~~xv
if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end if (X!=u16) r1|= M{c} # for u64, just accept the garbage top 32 bits and deal with them at the end
if (B) { if (B) {
if (anyNonChar{M, c}) return{3} if (anyNonChar{M, c}) return{3}
} else { } else {
if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2} if (anynePositive{EV**((1<<hw-1)<<hw) & c, EV**0, M}) return{lb{hw}-2}
} }
}} }
r2:= foldTotal{u32, r1} r2:= foldTotal{u32, r1}
if (X>u32 and r2>=65536) return{2} if (X>u32 and r2>=65536) return{2}
if (X>u16 and r2>=256) return{1} if (X>u16 and r2>=256) return{1}
0 0
} else { # i8, i16, i32, f64 } else { # i8, i16, i32, f64
if (X==i8) { # i8 if (X==i8) { # i8
maskedLoop{bulk, len, {i, M} => { @maskedLoop{bulk}(v0 in tup{XV,xp}, M in 'm' over _ to len) {
v0:= loadBatch{xp, i, XV}
if (anynePositive{EV**0xfe & EV~~v0, EV**0, M}) return{2} if (anynePositive{EV**0xfe & EV~~v0, EV**0, M}) return{2}
}} }
0 0
} else { # i16, i32, f64 } else { # i16, i32, f64
def case_B = makeOptBranch{B, tup{Size}, {iCont} => { def case_B = makeOptBranch{B, tup{Size}, {iCont} => {
maskedLoop{bulk, iCont, len, {i, M} => {
def XU = [bulk]u64 def XU = [bulk]u64
v:= XU ~~ loadBatch{xp, i, XV} @maskedLoop{bulk, iCont}(xv in tup{XV,xp}, M in 'm' over _ to len) {
v:= XU ~~ xv
if (anySNaN{M, v}) return{0xffff_fffe} # not even a number if (anySNaN{M, v}) return{0xffff_fffe} # not even a number
}} }
return{0xffff_ffff} # float return{0xffff_ffff} # float
}} }}
@ -94,15 +93,12 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
} }
if (isint{X}) { # i16, i32 if (isint{X}) { # i16, i32
muLoop{bulk, 1, len, {is, M} => { @muLoop{bulk, 1}(v0 in tup{XV,xp}, M in 'm' over _ to len) {
def v0 = loadBatch{xp, is, XV}
r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}} r1|= M{tree_fold{|, each{{v} => getAcc{EV, v}, v0}}}
}} }
} else { # f64 } else { # f64
r2:= EV2**0 r2:= EV2**0
muLoop{bulk, hasarch{'AARCH64'}+1, len, {is, M} => { @muLoop{bulk, hasarch{'AARCH64'}+1, {} => { r1 = half{r2,0}|half{r2,1} }}(v0 in tup{XV,xp}, M in 'm' over is to len) {
def v0 = loadBatch{xp, is, XV}
def int = { def int = {
def {int, wdn} = { def {int, wdn} = {
if (hasarch{'AARCH64'} and tuplen{is}==2) { if (hasarch{'AARCH64'} and tuplen{is}==2) {
@ -140,7 +136,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
def acc = { if (tuplen{is}==2) r2; else r1 } def acc = { if (tuplen{is}==2) r2; else r1 }
acc|= M{getAcc{type{acc}, int}} acc|= M{getAcc{type{acc}, int}}
}, {} => { r1 = half{r2,0}|half{r2,1} }} }
} }