From 6e3cb9a42509a9a471c3f8ea3a651a7e553d1bde Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 16 Dec 2022 00:44:32 +0200 Subject: [PATCH] =?UTF-8?q?andIsZero=E2=86=92andAllZero;=20NEON=20andnz,?= =?UTF-8?q?=20blendF,=20sel,=20bitops.singeli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/avx.singeli | 2 +- src/singeli/src/avx2.singeli | 8 ++++---- src/singeli/src/base.singeli | 6 ++++-- src/singeli/src/bitops.singeli | 9 ++++++++- src/singeli/src/equal.singeli | 2 +- src/singeli/src/mask.singeli | 4 ++-- src/singeli/src/neon.singeli | 8 +++++++- src/singeli/src/sse3.singeli | 6 +++--- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 719dd3ad..2b6f9945 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -125,7 +125,7 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_ble def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}} # mask stuff -def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} +def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 63817db6..ec85a7aa 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -121,12 +121,12 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_ble def blendF{f:T, t:T, m:M & w256{T} & w256{M} & elwidth{M}!=16} = blend{f, t, m} def blendF{f:T, t:T, m:M & w256{T} & w256{M,16}} = blend{f, t, [32]i8 ~~ m} -def shuf{L==[4]u32, x:T, n & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n} -def shuf{L==[4]u64, x:T, n & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n} +def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n} +def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n} def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_permute2x128_si256', v2i{x}, v2i{y}, n} -def sel{L, x:T, i:I & w256{T} & w256{L,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i} -def sel{L, x:T, i:I & w256{T} & w128{L, 8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i} +def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i} +def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i} def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i} def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 787926fb..d1129191 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -86,13 +86,15 @@ def ty_half{T & isvec{T}} = to_el{ty_half{eltype{T}}, T} # double/halve vector count def v_dbl {T & isvec{T}} = [vcount{T}*2](eltype{T}) def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T}) - +# test if vector has a specific width & element type +def lvec{T, n, w} = 0 +def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 # base vector utility definitions def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x def undefPromote{T, x:X & T==X} = T~~x - +def andnz{x, y} = assert{'andnz not supported', show{x, y}} def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 18507fe6..c7685e1d 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -4,6 +4,7 @@ def b_get{x:*u64, n:(Size)} = { ((load{x,n>>6}>>(n&63)) & 1) != 0 } +def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) & 3 def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15 def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n} @@ -56,7 +57,13 @@ def spreadBits{T==[32]u8, a:u32} = { e:= make{[32]u8, 1<=8}} + andnz{b, make{[16]u8, 1<<(iota{16}&7)}} +} + +def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = { b:= make{T, 1< { cw:= load{*ty_u{bulk} ~~ w, i} cx:= load{*T ~~ x, i} - if (~andIsZero{M{cx}, badBits}) return{0} + if (~andAllZero{M{cx}, badBits}) return{0} if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0} }} 1 diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index 64c43f27..cd1b68b9 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -24,11 +24,11 @@ def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (pro def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0 def maskNone{x} = x -def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x} +def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x} def maskAfter{n} = { def mask{x:X & isvec{X}} = x & (X~~maskOf{X,n}) def mask{x:X & anyInt{x}} = x & ((1<=vcount{T}} = { def getmask{x:T & nvecu{T} & T==[16]u8} = { fold_add{addpw{x & make{[16]u8, 1<<(iota{16}&7)}} << make{[8]u16, merge{4**0, 4**8}}} } -def andIsZero{x:T, y:T & nveci{T}} = ~bitAny{x&y} +def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y} def maskstore{p:P, m:M, x:T & eltype{P}==T & nvec{T}} = store{p, 0, bblend{load{p}, x, m}} def maskstoreF{p:P, m:M, x:T & nvec{T}} = maskstore{p, m, x} \ No newline at end of file diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 8d38044d..e3ecbecb 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -103,8 +103,8 @@ def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T # shuffle stuff -def shuf{L, x:T, n & w128{T} & w128i{L, 32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} -def sel{L, x:T, i:[16]i8 & w128{T} & w128{L, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} +def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n} +def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i} # arith def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b} @@ -119,7 +119,7 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} # mask stuff -def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} +def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y} def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x} def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}