homMaskX, merging multi-arg homMask
This commit is contained in:
parent
fdea9cbc09
commit
8542ba071d
@ -24,6 +24,11 @@ def ctz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_ctz', x}
|
||||
def clz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_clzll', x}
|
||||
def clz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_clz', x}
|
||||
|
||||
def truncBits{n, v & n<=8} = cast_i{u8, v}
|
||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||
|
||||
# base-2 log of a constant power of two
|
||||
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
||||
def lb{n==1} = 0
|
||||
@ -44,6 +49,13 @@ def unreachable{} = emit{void, 'si_unreachable'}
|
||||
def assert{x:u1} = { if (not x) emit{void, 'si_unreachable'} }
|
||||
|
||||
# various checks
|
||||
def oneType{vs} = {
|
||||
def T = type{tupsel{0,vs}}
|
||||
each{{x:C} => assert{C==T}, vs}
|
||||
T
|
||||
}
|
||||
def oneType{vs & tuplen{vs}==0} = 1
|
||||
|
||||
def isreg = kreg
|
||||
def isconst = kcon
|
||||
def istype = ktyp
|
||||
@ -187,6 +199,15 @@ def cvt{T, x:X & T==eltype{X}} = x
|
||||
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
|
||||
|
||||
def homMask{...vs & tuplen{vs}>1} = {
|
||||
def n = tuplen{vs}
|
||||
def T = oneType{vs}
|
||||
def RT = ty_u{max{8,vcount{T}*n}}
|
||||
tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}}
|
||||
}
|
||||
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
|
||||
def ctzX{t} = { def {n,v}=t; ctz{v}/n } # ctz for a result of homMaskX
|
||||
|
||||
def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count)
|
||||
def TU = ty_u{TS}
|
||||
(TU~~(x-TS**start)) < TU**count
|
||||
|
||||
@ -82,10 +82,6 @@ def loadBatchBit{T, x:*u64, n:(Size)} = { # vector with type T with each element
|
||||
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
|
||||
}
|
||||
|
||||
def truncBits{n, v & n<=8} = cast_i{u8, v}
|
||||
def truncBits{n, v & n==16} = cast_i{u16, v}
|
||||
def truncBits{n, v & n==32} = cast_i{u32, v}
|
||||
def truncBits{n, v & n==64} = cast_i{u64, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
|
||||
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it
|
||||
|
||||
@ -78,9 +78,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
|
||||
while (ri < am) {
|
||||
r:u64 = 0
|
||||
if (hasarch{'AARCH64'}) {
|
||||
def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}}
|
||||
def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}}
|
||||
r = promote{u64, homMask{step{0, unr}}}
|
||||
r = promote{u64, homMask{...each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
|
||||
} else {
|
||||
@unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})
|
||||
}
|
||||
@ -130,7 +128,7 @@ def table{aa, F, w, G_bit, G_vec} = {
|
||||
...each{{E}=>{
|
||||
def E2 = (if (aa and isunsigned{E} and ((match{F,__eq} | match{F,__ne}) | (E==u32))) {ty_s{E}} else E)
|
||||
def bulk = w/width{E2}
|
||||
G_vec{[bulk]E2, max{1, 8/bulk}}
|
||||
G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, 8/bulk}}
|
||||
}, tup{i8, i16, i32, f64, u8, u16, u32}}
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,8 +21,11 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
|
||||
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
|
||||
|
||||
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~homMask{ x==y }} << (32-M{'count'} )) != 0
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~homMask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
|
||||
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = {
|
||||
def {n,m} = homMaskX{x==y}
|
||||
def E = tern{type{m}==u64, u64, u32}
|
||||
(promote{E,~m} << (width{E}-M{'count'}*n)) != 0
|
||||
}
|
||||
|
||||
def maskNone{x} = x
|
||||
def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x}
|
||||
|
||||
@ -45,9 +45,13 @@ def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
|
||||
def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b}
|
||||
def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b}
|
||||
def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
|
||||
def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b}
|
||||
|
||||
def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshl', '_n', T}, a, s}; def __shl{a:T,s==0 & nveci{T}} = a
|
||||
def __shr{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshr', '_n', T}, a, s}; def __shr{a:T,s==0 & nveci{T}} = a
|
||||
def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=v_half{ty_half{T}}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
|
||||
def shrm{a:T, d:T, s & w128u{T}} = emit{T, ntyp{'vsri', '_n', T}, a, d, s} # (a>>s) | (d & newZeroesIn(a>>s))
|
||||
|
||||
def __shl{a:T,b & nveci{T} & knum{b} & b>0 & b<elwidth{T}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a
|
||||
def __shr{a:T,b & nveci{T} & knum{b} & b>0 & b<elwidth{T}} = emit{T, ntyp{'vshr', '_n', T}, a, b}; def __shr{a:T,b==0 & nveci{T}} = a
|
||||
def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M,elwidth{T}} & width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
|
||||
def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m}
|
||||
|
||||
@ -160,11 +164,34 @@ def iota{T & nvec{T}} = make{T, ...iota{vcount{T}}}
|
||||
|
||||
|
||||
def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
|
||||
cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
|
||||
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
|
||||
}
|
||||
def homMask{x:T & nvecu{T} & T==[16]u8} = {
|
||||
fold_add{addpw{x & make{[16]u8, 1<<(iota{16}&7)}} << make{[8]u16, merge{4**0, 4**8}}}
|
||||
}
|
||||
def homMask{a:T,b:T & T==[16]u8} = {
|
||||
m:= make{[16]u8, 1<<(iota{16}&7)}
|
||||
fold_add{addpw{addpw{addp{a&m, b&m}}}<<make{[4]u32,iota{4}*8}}
|
||||
}
|
||||
def homMask{a:T,b:T,c:T,d:T & T==[16]u8} = {
|
||||
m:= make{[16]u8, 1<<(iota{16}&7)}
|
||||
t1:= addp{a&m, b&m}
|
||||
t2:= addp{c&m, d&m}
|
||||
t3:= addp{t1, t2}
|
||||
extract{[2]u64~~addp{t3,t3},0}
|
||||
}
|
||||
def homMask{...as & tuplen{as}>1 & elwidth{type{tupsel{0,as}}}>=32} = homMask{...each{{i}=>narrowPair{tupsel{i*2,as},tupsel{i*2+1,as}}, iota{tuplen{as}/2}}}
|
||||
def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = {
|
||||
def n = vcount{T}
|
||||
truncBits{n*2, fold_add{shrm{b,a,elwidth{T}-n} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
|
||||
}
|
||||
|
||||
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
|
||||
|
||||
def homMaskX{a:T & eltype{T}!=u64} = {
|
||||
def h = elwidth{T}/2
|
||||
tup{h, truncBits{vcount{T}*h, extract{[1]u64~~shrn{ty_dbl{T}~~a, h}, 0}}}
|
||||
}
|
||||
|
||||
|
||||
def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}
|
||||
|
||||
@ -230,7 +230,7 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = {
|
||||
def cr = eachx{+, widenFull{R, s1}, cv}
|
||||
cv = toLast{tupsel{-1, cr}}
|
||||
|
||||
each{{c:T} => assert{T==type{cv}}, cr}
|
||||
assert{type{cv} == oneType{cr}}
|
||||
assert{vcount{type{cv}} * tuplen{cr} == bulk}
|
||||
|
||||
each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}}
|
||||
|
||||
@ -35,7 +35,7 @@ def search{E, x, n:u64, OP} = {
|
||||
def VT = [bulk]E
|
||||
def end = makeBranch{
|
||||
tup{u64, ty_u{VT}},
|
||||
{i,c} => return{i*bulk + ctzi{homMask{c}}}
|
||||
{i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}}
|
||||
}
|
||||
|
||||
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => {
|
||||
|
||||
@ -170,6 +170,7 @@ def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
|
||||
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
|
||||
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
|
||||
def homMask{x:T & w128{T}} = topMask{x}
|
||||
def homMaskX{a:T & elwidth{T}==16} = tup{2, homMask{to_el{u8,a}}}
|
||||
|
||||
def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0
|
||||
def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
|
||||
|
||||
Loading…
Reference in New Issue
Block a user