homMaskX, merging multi-arg homMask

This commit is contained in:
dzaima 2023-06-07 17:39:38 +03:00
parent fdea9cbc09
commit 8542ba071d
8 changed files with 61 additions and 15 deletions

View File

@ -24,6 +24,11 @@ def ctz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_ctz', x}
def clz{x:T & isint{T} & width{T}==64} = emit{u8, '__builtin_clzll', x}
def clz{x:T & isint{T} & width{T}<=32} = emit{u8, '__builtin_clz', x}
def truncBits{n, v & n<=8} = cast_i{u8, v}
def truncBits{n, v & n==16} = cast_i{u16, v}
def truncBits{n, v & n==32} = cast_i{u32, v}
def truncBits{n, v & n==64} = cast_i{u64, v}
# base-2 log of a constant power of two
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
def lb{n==1} = 0
@ -44,6 +49,13 @@ def unreachable{} = emit{void, 'si_unreachable'}
def assert{x:u1} = { if (not x) emit{void, 'si_unreachable'} }
# various checks
def oneType{vs} = {
def T = type{tupsel{0,vs}}
each{{x:C} => assert{C==T}, vs}
T
}
def oneType{vs & tuplen{vs}==0} = 1
def isreg = kreg
def isconst = kcon
def istype = ktyp
@ -187,6 +199,15 @@ def cvt{T, x:X & T==eltype{X}} = x
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
def homMask{...vs & tuplen{vs}>1} = {
def n = tuplen{vs}
def T = oneType{vs}
def RT = ty_u{max{8,vcount{T}*n}}
tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}}
}
def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times
def ctzX{t} = { def {n,v}=t; ctz{v}/n } # ctz for a result of homMaskX
def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count)
def TU = ty_u{TS}
(TU~~(x-TS**start)) < TU**count

View File

@ -82,10 +82,6 @@ def loadBatchBit{T, x:*u64, n:(Size)} = { # vector with type T with each element
spreadBits{T, b_getBatchLo{vcount{T}, x, n}}
}
def truncBits{n, v & n<=8} = cast_i{u8, v}
def truncBits{n, v & n==16} = cast_i{u16, v}
def truncBits{n, v & n==32} = cast_i{u32, v}
def truncBits{n, v & n==64} = cast_i{u64, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
# load bits starting at bit i, leaving garbage at the top. Only the bottom 57 bits are guaranteed correct; 58 and 60 will be correct if `i` is a multiple of it

View File

@ -78,9 +78,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
while (ri < am) {
r:u64 = 0
if (hasarch{'AARCH64'}) {
def step{j, n & n==1} = op{wV{xi+j}, xV{xi+j}}
def step{j, n & n>=2} = packLo{step{j, n/2}, step{j+n/2, n/2}}
r = promote{u64, homMask{step{0, unr}}}
r = promote{u64, homMask{...each{{j}=>op{wV{xi+j}, xV{xi+j}}, iota{unr}}}}
} else {
@unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})
}
@ -130,7 +128,7 @@ def table{aa, F, w, G_bit, G_vec} = {
...each{{E}=>{
def E2 = (if (aa and isunsigned{E} and ((match{F,__eq} | match{F,__ne}) | (E==u32))) {ty_s{E}} else E)
def bulk = w/width{E2}
G_vec{[bulk]E2, max{1, 8/bulk}}
G_vec{[bulk]E2, max{tern{hasarch{'AARCH64'},2,1}, 8/bulk}}
}, tup{i8, i16, i32, f64, u8, u16, u32}}
}
}

View File

@ -21,8 +21,11 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~homMask{ x==y }} << (32-M{'count'} )) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~homMask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T}} = {
def {n,m} = homMaskX{x==y}
def E = tern{type{m}==u64, u64, u32}
(promote{E,~m} << (width{E}-M{'count'}*n)) != 0
}
def maskNone{x} = x
def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x}

View File

@ -45,9 +45,13 @@ def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b}
def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b}
def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b}
def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshl', '_n', T}, a, s}; def __shl{a:T,s==0 & nveci{T}} = a
def __shr{a:T, s & nveci{T} & knum{s} & s>0 & s<elwidth{T}} = emit{T, ntyp{'vshr', '_n', T}, a, s}; def __shr{a:T,s==0 & nveci{T}} = a
def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=v_half{ty_half{T}}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed
def shrm{a:T, d:T, s & w128u{T}} = emit{T, ntyp{'vsri', '_n', T}, a, d, s} # (a>>s) | (d & newZeroesIn(a>>s))
def __shl{a:T,b & nveci{T} & knum{b} & b>0 & b<elwidth{T}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a
def __shr{a:T,b & nveci{T} & knum{b} & b>0 & b<elwidth{T}} = emit{T, ntyp{'vshr', '_n', T}, a, b}; def __shr{a:T,b==0 & nveci{T}} = a
def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M,elwidth{T}} & width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m}
@ -160,11 +164,34 @@ def iota{T & nvec{T}} = make{T, ...iota{vcount{T}}}
def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
truncBits{vcount{T}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
}
def homMask{x:T & nvecu{T} & T==[16]u8} = {
fold_add{addpw{x & make{[16]u8, 1<<(iota{16}&7)}} << make{[8]u16, merge{4**0, 4**8}}}
}
def homMask{a:T,b:T & T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
fold_add{addpw{addpw{addp{a&m, b&m}}}<<make{[4]u32,iota{4}*8}}
}
def homMask{a:T,b:T,c:T,d:T & T==[16]u8} = {
m:= make{[16]u8, 1<<(iota{16}&7)}
t1:= addp{a&m, b&m}
t2:= addp{c&m, d&m}
t3:= addp{t1, t2}
extract{[2]u64~~addp{t3,t3},0}
}
def homMask{...as & tuplen{as}>1 & elwidth{type{tupsel{0,as}}}>=32} = homMask{...each{{i}=>narrowPair{tupsel{i*2,as},tupsel{i*2+1,as}}, iota{tuplen{as}/2}}}
def homMask{a:T,b:T & vcount{T}*2<=elwidth{T}} = {
def n = vcount{T}
truncBits{n*2, fold_add{shrm{b,a,elwidth{T}-n} & make{T, (1<<iota{n}) | (1<<(iota{n}+n))}}}
}
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def homMaskX{a:T & eltype{T}!=u64} = {
def h = elwidth{T}/2
tup{h, truncBits{vcount{T}*h, extract{[1]u64~~shrn{ty_dbl{T}~~a, h}, 0}}}
}
def homMaskStoreF{p:P, m:M, v:T & nveci{M} & nvec{T,elwidth{M}} & eltype{P}==T} = store{p, 0, homBlend{load{p}, v, m}}

View File

@ -230,7 +230,7 @@ def simd_plus_scan{X, b, R}{x:*X, c:(R), r:*R, len:u64} = {
def cr = eachx{+, widenFull{R, s1}, cv}
cv = toLast{tupsel{-1, cr}}
each{{c:T} => assert{T==type{cv}}, cr}
assert{type{cv} == oneType{cr}}
assert{vcount{type{cv}} * tuplen{cr} == bulk}
each{{c:T, j} => store{*T~~(r+i), j, c}, cr, iota{tuplen{cr}}}

View File

@ -35,7 +35,7 @@ def search{E, x, n:u64, OP} = {
def VT = [bulk]E
def end = makeBranch{
tup{u64, ty_u{VT}},
{i,c} => return{i*bulk + ctzi{homMask{c}}}
{i,c} => return{i*bulk + promote{u64, ctzX{homMaskX{c}}}}
}
muLoop{bulk, tern{arch_defvw>=256, 1, 2}, n, {is, M} => {

View File

@ -170,6 +170,7 @@ def topMask{x:T & w128{T, 16}} = topMask{packs{[8]i16~~x, [8]i16**0}}
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T & w128{T}} = topMask{x}
def homMaskX{a:T & elwidth{T}==16} = tup{2, homMask{to_el{u8,a}}}
def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff