From f9c919d5ce729aa40fa91845c07ca78f6c697022 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sun, 18 Dec 2022 22:27:42 +0200 Subject: [PATCH] =?UTF-8?q?getmask=20=E2=86=92=20homMask/topMask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/avx.singeli | 13 +++++++------ src/singeli/src/avx2.singeli | 14 +++++++------- src/singeli/src/cmp.singeli | 2 +- src/singeli/src/copy.singeli | 2 +- src/singeli/src/equal.singeli | 6 +++--- src/singeli/src/mask.singeli | 4 ++-- src/singeli/src/neon.singeli | 4 ++-- src/singeli/src/select.singeli | 4 ++-- src/singeli/src/sse3.singeli | 17 +++++++++-------- 9 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 315d35ec..816ae970 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -128,11 +128,12 @@ def homBlend{f:T, t:T, m:M & w256{T}} = topBlend{f, t, m} # mask stuff def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} -def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} -def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} +def topMask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} +def topMask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} +def homMask{x:T & w256{T}} = topMask{x} -def homAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} != 0 -def homAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{[8]u32 ~~ x} == 0xff +def homAny{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} != 0 +def homAll{x:T & w256i{T} & elwidth{T}>=32} = homMask{[8]u32 ~~ x} == 0xff -def topAny{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} != 0 -def topAll{x:T & w256i{T} & elwidth{T}>=32} = getmask{x} == (1<=32} = topMask{x} != 0 +def topAll{x:T & w256i{T} & elwidth{T}>=32} = topMask{x} == (1<>8) } -def homAny{x:T & w256i{T}} = getmask{[32]u8 ~~ x} != 0 -def homAll{x:T & w256i{T}} = getmask{[32]u8 ~~ x} == 0xffff_ffff +def homAny{x:T & w256i{T}} = homMask{[32]u8 ~~ x} != 0 +def homAll{x:T & w256i{T}} = homMask{[32]u8 ~~ x} == 0xffff_ffff -def topAny{x:T & w256i{T}} = getmask{x} != 0 -def topAll{x:T & w256i{T}} = getmask{x} == (1<0} while (ri < am) { r:u64 = 0 - @unroll (j to unr) r|= promote{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) + @unroll (j to unr) r|= promote{u64, homMask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}) b_setBatch{bulk, dst, ri, r} xi+= unr ri+= 1 diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 3bdc7279..1201d0ff 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -50,7 +50,7 @@ copy{vw, X, R}(x: *void, r: *void, l:u64, xRaw: *void) : void = { def XU = ty_u{XV} @forNZ (i to cdiv{l,vcount{XV}}) { v:= loadBatch{xp, i, XV} - r:= getmask{(XU~~v) == XU~~XV**1} + r:= homMask{(XU~~v) == XU~~XV**1} b_setBatch{vcount{XV}, rp, i, r} # TODO something more special for f64 } } else if (width{X}<=width{R}) { diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index dd6c48e8..31fc89f2 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -46,8 +46,8 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { def T = [bulk]X def sh{c} = c << (width{X}-1) def sh{c & X==u8} = T ~~ (to_el{u16,c}<<7) - def topMask{x:X & hasarch{'X86_64'}} = getmask{x} - def topMask{x:X & hasarch{'AARCH64'}} = getmask{andnz{x, ~T**0}} + def mask{x:X & hasarch{'X86_64'}} = topMask{x} + def mask{x:X & hasarch{'AARCH64'}} = homMask{andnz{x, ~T**0}} # TODO compare with doing the comparison in vector registers badBits:= T ** ~(X~~1) @@ -55,7 +55,7 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = { cw:= b_getBatch{bulk, *u64~~w, i} cx:= load{*T ~~ x, i} if (~andAllZero{M{cx}, badBits}) return{0} - if (anyne{promote{u64,topMask{sh{cx}}}, promote{u64,cw}, M}) return{0} + if (anyne{promote{u64,mask{sh{cx}}}, promote{u64,cw}, M}) return{0} }} 1 } diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index cd1b68b9..189effd8 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -20,8 +20,8 @@ def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} def anynePositive{x:T, y:T, M & M{0}==0} = anyne{x, y, M} -def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~getmask{ x==y }} << (32-M{'count'} )) != 0 -def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 +def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (promote{u32,~homMask{ x==y }} << (32-M{'count'} )) != 0 +def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~homMask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 def maskNone{x} = x def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 9de18b23..e19d47ad 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -128,10 +128,10 @@ def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = { def make{T, x & nvec{T} & istup{x}} = make{T, ...x} -def getmask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = { +def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = { cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1< { cw:= wrapChk{load{w, i}, VI,xlf, M} - store{r, i, getmask{sel{[16]i8, xb, cw}}} + store{r, i, homMask{sel{[16]i8, xb, cw}}} }} } else { x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} @@ -161,7 +161,7 @@ avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { cw:= wrapChk{load{w, i}, VI,xlf, M} byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)} mask:= sel{[16]i8, b, cw & low} - store{r, i, getmask{(mask & byte) == mask}} + store{r, i, homMask{(mask & byte) == mask}} }} } 1 diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 68f832b3..58b3145a 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -121,16 +121,17 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} # mask stuff def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} -def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} -def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}} -def getmask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} -def getmask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} +def topMask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} +def topMask{x:T & w128{T, 16}} = topMask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}} +def topMask{x:T & w128{T, 32}} = emit{u8, '_mm_movemask_ps', v2f{x}} +def topMask{x:T & w128{T, 64}} = emit{u8, '_mm_movemask_pd', v2d{x}} +def homMask{x:T & w128{T}} = topMask{x} -def homAny{x:T & w128i{T}} = getmask{[16]u8 ~~ x} != 0 -def homAll{x:T & w128i{T}} = getmask{[16]u8 ~~ x} == 0xffff +def homAny{x:T & w128i{T}} = homMask{[16]u8 ~~ x} != 0 +def homAll{x:T & w128i{T}} = homMask{[16]u8 ~~ x} == 0xffff -def topAny{x:T & w128i{T}} = getmask{x} != 0 -def topAll{x:T & w128i{T}} = getmask{x} == (1<