andIsZero→andAllZero; NEON andnz, blendF, sel, bitops.singeli

This commit is contained in:
dzaima 2022-12-16 00:44:32 +02:00
parent cbd891b768
commit 6e3cb9a425
8 changed files with 30 additions and 15 deletions

View File

@ -125,7 +125,7 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = T ~~ emit{[8]f32, '_mm256_ble
def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = T ~~ emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}
# mask stuff
def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
def andAllZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y}
def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}}
def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}}

View File

@ -121,12 +121,12 @@ def blend{f:T, t:T, m:M & w256{T} & w256i{M, 8}} = T ~~ emit{[32]i8, '_mm256_ble
def blendF{f:T, t:T, m:M & w256{T} & w256{M} & elwidth{M}!=16} = blend{f, t, m}
def blendF{f:T, t:T, m:M & w256{T} & w256{M,16}} = blend{f, t, [32]i8 ~~ m}
def shuf{L==[4]u32, x:T, n & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L==[4]u64, x:T, n & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
def shuf{L, x:T, n & lvec{L,4,32} & w256{T} & knum{n}} = T ~~ emit{[8]i32, '_mm256_shuffle_epi32', v2i{x}, n}
def shuf{L, x:T, n & lvec{L,4,64} & w256{T} & knum{n}} = T ~~ emit{[4]f64, '_mm256_permute4x64_pd', v2d{x}, n}
def shufHalves{x:T, y:T, n & w256{T} & knum{n}} = T ~~ emit{[4]i64, '_mm256_permute2x128_si256', v2i{x}, v2i{y}, n}
def sel{L, x:T, i:I & w256{T} & w256{L,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
def sel{L, x:T, i:I & w256{T} & w128{L, 8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
def sel{L, x:T, i:I & w256{T} & lvec{L,8,32} & w256{I,32}} = T ~~ emit{[32]u8, '_mm256_permutevar8x32_epi32', v2i{x}, i}
def sel{L, x:T, i:I & w256{T} & lvec{L,16,8} & w256{I, 8}} = T ~~ emit{[32]u8, '_mm256_shuffle_epi8', v2i{x}, i}
def extract{x:T, i & w256i{T,8 } & knum{i}} = emit{eltype{T}, '_mm256_extract_epi8', x, i}
def extract{x:T, i & w256i{T,16} & knum{i}} = emit{eltype{T}, '_mm256_extract_epi16', x, i}

View File

@ -86,13 +86,15 @@ def ty_half{T & isvec{T}} = to_el{ty_half{eltype{T}}, T}
# double/halve vector count
def v_dbl {T & isvec{T}} = [vcount{T}*2](eltype{T})
def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T})
# test if vector has a specific width & element type
def lvec{T, n, w} = 0
def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1
# base vector utility definitions
def widen{T, x:X & T==X} = x
def narrow{T, x:X & T==eltype{X}} = x
def undefPromote{T, x:X & T==X} = T~~x
def andnz{x, y} = assert{'andnz not supported', show{x, y}}
def unroll{vars,begin,end,block & knum{begin} & knum{end}} = {

View File

@ -4,6 +4,7 @@ def b_get{x:*u64, n:(Size)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatch{sz, x:*u64, n:(Size) & sz==2} = (load{*u8~~x, n>>2} >> cast_i{u8, (n&3)*2}) & 3
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15
def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
@ -56,7 +57,13 @@ def spreadBits{T==[32]u8, a:u32} = {
e:= make{[32]u8, 1<<tail{3, idxs}}
e == (d&e)
}
def spreadBits{T, a & vcount{T} <= elwidth{T} & width{T}==256 & quality{eltype{T}}=='u'} = {
def spreadBits{T==[16]u8, a:u16} = {
b:= sel{[16]u8, [16]u8~~[8]u16**a, make{[16]i8, iota{16}>=8}}
andnz{b, make{[16]u8, 1<<(iota{16}&7)}}
}
def spreadBits{T, a & vcount{T} <= elwidth{T} & quality{eltype{T}}=='u'} = {
b:= make{T, 1<<iota{vcount{T}}}
b == (b & T ~~ to_el{type{a}, T}**a) # not just T**a so that if a is read from RAM, it can use the single instruction for broadcasting from RAM; the extra bits don't matter
}

View File

@ -44,7 +44,7 @@ equal{W, X}(w:*void, x:*void, l:u64, d:u64) : u1 = {
maskedLoop{bulk, l, {i, M} => {
cw:= load{*ty_u{bulk} ~~ w, i}
cx:= load{*T ~~ x, i}
if (~andIsZero{M{cx}, badBits}) return{0}
if (~andAllZero{M{cx}, badBits}) return{0}
if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0}
}}
1

View File

@ -24,11 +24,11 @@ def anynePositive{x:T, y:T, M & M{0}==1 & isvec{T} } = (pro
def anynePositive{x:T, y:T, M & M{0}==1 & width{T}==256 & elwidth{T}==16} = (promote{u32,~getmask{[32]u8~~(x==y)}} << (32-M{'count'}*2)) != 0
def maskNone{x} = x
def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x}
def maskNone{x, mode=='all bits zeroes'} = andAllZero{x, x}
def maskAfter{n} = {
def mask{x:X & isvec{X}} = x & (X~~maskOf{X,n})
def mask{x:X & anyInt{x}} = x & ((1<<n) - 1)
def mask{x:X, mode=='all bits zeroes'} = andIsZero{x, X~~maskOfBit{X,n}}
def mask{x:X, mode=='all bits zeroes'} = andAllZero{x, X~~maskOfBit{X,n}}
def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{mode=='count'} = n
def mask{x==0} = 1

View File

@ -38,6 +38,7 @@ def __or{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorr', T}, a, b}
def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b}
def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b}
def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b}
def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b}
def min{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmin', T}, a, b} # TODO float - there are multiple options
def max{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmax', T}, a, b}
def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}}
@ -45,6 +46,7 @@ def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, nty
def __shl{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a
def __shr{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshr', '_n', T}, a, b}; def __shr{a:T,b==0 & nveci{T}} = a
def bblend{f:T, t:T, m:M & nvec{T} & nvecu{M} & width{T}==width{M} & elwidth{T}==elwidth{M}} = emit{T, ntyp{'vbsl', T}, m, t, f}
def blendF{f:T, t:T, m:M & nvec{M}} = bblend{f, t, m}
def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a}
def addpw{a:T & nveci{T} & elwidth{T}<=32} = emit{ty_dbl{T}, ntyp{'vpaddl', T}, a}
@ -88,7 +90,11 @@ def undefPromote{T, x:X & n64{X} & n128{T} & eltype{T}==eltype{X}} = emit{T, nty
def half{x:T, n==0 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}}, x}
def half{x:T, n==1 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x}
def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n}
def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} }
def sel{L, x:T, i:I & lvec{L,16,8} & n128{T} & nvec{I, 8}} = to_el{eltype{T}, emit{I, ntyp{'vqtbl1',I}, to_el{eltype{I},x}, i}}
local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers
@ -125,7 +131,7 @@ def getmask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = {
def getmask{x:T & nvecu{T} & T==[16]u8} = {
fold_add{addpw{x & make{[16]u8, 1<<(iota{16}&7)}} << make{[8]u16, merge{4**0, 4**8}}}
}
def andIsZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def andAllZero{x:T, y:T & nveci{T}} = ~bitAny{x&y}
def maskstore{p:P, m:M, x:T & eltype{P}==T & nvec{T}} = store{p, 0, bblend{load{p}, x, m}}
def maskstoreF{p:P, m:M, x:T & nvec{T}} = maskstore{p, m, x}

View File

@ -103,8 +103,8 @@ def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T
# shuffle stuff
def shuf{L, x:T, n & w128{T} & w128i{L, 32} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def sel{L, x:T, i:[16]i8 & w128{T} & w128{L, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
def shuf{L, x:T, n & lvec{L,4,32} & w128{T} & knum{n}} = T ~~ emit{[4]i32, '_mm_shuffle_epi32', v2i{x}, n}
def sel{L, x:T, i:I & lvec{L,16,8} & w128{T} & w128i{I, 8}} = T ~~ emit{[16]u8, '_mm_shuffle_epi8', v2i{x}, i}
# arith
def __add{a:T,b:T & w128i{T, 8}} = emit{T, '_mm_add_epi8', a, b}
@ -119,7 +119,7 @@ def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b}
# mask stuff
def andIsZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def andAllZero{x:T, y:T & w128i{T}} = emit{u1, '_mm_testz_si128', x, y}
def getmask{x:T & w128{T, 8}} = emit{u16, '_mm_movemask_epi8', x}
def getmask{x:T & w128{T, 16}} = getmask{emit{[16]u8, '_mm_packs_epi16', x, [8]u16**0}}