getmask → homMask/topMask

This commit is contained in:
dzaima 2022-12-18 22:27:42 +02:00
parent a07b94b889
commit f9c919d5ce
9 changed files with 34 additions and 32 deletions

View File

@ -128,11 +128,12 @@ def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m}
# mask stuff
def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
def topMask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def topMask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}
def homMask{x:T & w256{T}} = topMask{x}
def homAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} != 0
def homAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff
def homAny{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0
def homAll{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff
def topAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0
def topAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<<vcount{T})-1
def topAny{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} != 0
def topAll{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} == (1<<vcount{T})-1

View File

@ -130,17 +130,17 @@ def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_ep
# mask stuff
def getmask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
def getmask{x:T & w256{T, 16}} = {
msk:u32 = getmask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
def topMask{x:T & w256{T, 8}} = emit{u32, '_mm256_movemask_epi8', x}
def topMask{x:T & w256{T, 16}} = {
msk:u32 = topMask{emit{[32]u8, '_mm256_packs_epi16', x, [16]u16**0}}
(msk&255) | (msk>>8)
}
def homAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0
def homAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff
def homAny{x:T & w256i{T}} = homMask{[32]u8 ~~ x} != 0
def homAll{x:T & w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff
def topAny{x:T & w256i{T}} = getmask{x} != 0
def topAll{x:T & w256i{T}} = getmask{x} == (1<<vcount{T})-1
def topAny{x:T & w256i{T}} = topMask{x} != 0
def topAll{x:T & w256i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T & w256i{T,16}} = homAny{[16]i16~~x < [16]i16**0}
def topAll{x:T & w256i{T,16}} = homAll{[16]i16~~x < [16]i16**0}

View File

@ -71,7 +71,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
assert{am>0}
while (ri < am) {
r:u64 = 0
@unroll (j to unr) r|= promote{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})
@unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})
b_setBatch{bulk, dst, ri, r}
xi+= unr
ri+= 1

View File

@ -50,7 +50,7 @@ copy{vw, X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = {
def XU = ty_u{XV}
@forNZ (i to cdiv{l,vcount{XV}}) {
v:= loadBatch{xp, i, XV}
r:= getmask{(XU~~v) == XU~~XV**1}
r:= homMask{(XU~~v) == XU~~XV**1}
b_setBatch{vcount{XV}, rp, i, r} # TODO something more special for f64
}
} else if (width{X}<=width{R}) {

View File

@ -46,8 +46,8 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
def T = [bulk]X
def sh{c} = c << (width{X}-1)
def sh{c & X==u8} = T ~~ (to_el{u16,c}<<7)
def topMask{x:X & hasarch{'X86_64'}} = getmask{x}
def topMask{x:X & hasarch{'AARCH64'}} = getmask{andnz{x, ~T**0}}
def mask{x:X & hasarch{'X86_64'}} = topMask{x}
def mask{x:X & hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}}
# TODO compare with doing the comparison in vector registers
badBits:= T ** ~(X~~1)
@ -55,7 +55,7 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
cw:= b_getBatch{bulk, *u64~~w, i}
cx:= load{*T ~~ x, i}
if (~andAllZero{M{cx}, badBits}) return{0}
if (anyne{promote{u64,topMask{sh{cx}}}, promote{u64,cw}, M}) return{0}
if (anyne{promote{u64,mask{sh{cx}}}, promote{u64,cw}, M}) return{0}
}}
1
}

View File

@ -20,8 +20,8 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0
def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'}
def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M}
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~homMask{ x==y }} << (32-M{'count'} )) != 0
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~homMask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def maskNone{x} = x
def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x}

View File

@ -128,10 +128,10 @@ def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = {
def make{T, x & nvec{T} & istup{x}} = make{T, ...x}
def getmask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1<<iota{vcount{T}}}}}
}
def getmask{x:T & nvecu{T} & T==[16]u8} = {
def homMask{x:T & nvecu{T} & T==[16]u8} = {
fold_add{addpw{x & make{[16]u8, 1<<(iota{16}&7)}} << make{[8]u16, merge{4**0, 4**8}}}
}
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}

View File

@ -151,7 +151,7 @@ avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
store{r, i, getmask{sel{[16]i8, xb, cw}}}
store{r, i, homMask{sel{[16]i8, xb, cw}}}
}}
} else {
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
@ -161,7 +161,7 @@ avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
cw:= wrapChk{load{w, i}, VI,xlf, M}
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
mask:= sel{[16]i8, b, cw & low}
store{r, i, getmask{(mask & byte) == mask}}
store{r, i, homMask{(mask & byte) == mask}}
}}
}
1

View File

@ -121,16 +121,17 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
# mask stuff
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}
def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def topMask{x:T & w128{T, 16}} = topMask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}
def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}}
def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}}
def homMask{x:T & w128{T}} = topMask{x}
def homAny{x:T & w128i{T}} = getmask{[16]u8 ~~ x} != 0
def homAll{x:T & w128i{T}} = getmask{[16]u8 ~~ x} == 0xffff
def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0
def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff
def topAny{x:T & w128i{T}} = getmask{x} != 0
def topAll{x:T & w128i{T}} = getmask{x} == (1<<vcount{T})-1
def topAny{x:T & w128i{T}} = topMask{x} != 0
def topAll{x:T & w128i{T}} = topMask{x} == (1<<vcount{T})-1
def topAny{x:T & w128i{T, 16}} = homAny{[8]i16~~x < [8]i16**0}
def topAll{x:T & w128i{T, 16}} = homAll{[8]i16~~x < [8]i16**0}