diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 5be6a3d9..aa45ed54 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -87,10 +87,10 @@ def __add{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_add_epi32', a, b}; def __sub def __add{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_add_epi64', a, b}; def __sub{a:T,b:T & w256i{T, 64}} = emit{T, '_mm256_sub_epi64', a, b} # saturating add/sub -def __adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def __adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b} -def __subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def __subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} -def __adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def __adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} -def __subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def __subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b} +def adds{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_adds_epi8', a, b}; def adds{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_adds_epu8', a, b} +def subs{a:T,b:T & T==[32]i8 } = emit{T, '_mm256_subs_epi8', a, b}; def subs{a:T,b:T & T==[32]u8 } = emit{T, '_mm256_subs_epu8', a, b} +def adds{a:T,b:T & T==[16]i16} = emit{T, '_mm256_adds_epi16', a, b}; def adds{a:T,b:T & T==[16]u16} = emit{T, '_mm256_adds_epu16', a, b} +def subs{a:T,b:T & T==[16]i16} = emit{T, '_mm256_subs_epi16', a, b}; def subs{a:T,b:T & T==[16]u16} = emit{T, '_mm256_subs_epu16', a, b} # structural operations diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 40064119..0a8c5b93 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -140,6 +140,10 @@ def ty_half{T & isvec{T}} = to_el{ty_half{eltype{T}}, T} # double/halve vector count def v_dbl {T & isvec{T}} = [vcount{T}*2](eltype{T}) def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T}) +# double element width without changing count +def el_dbl {T & isvec{T}} = [vcount{T}](ty_dbl{eltype{T}}) +def el_half{T & isvec{T}} = [vcount{T}](ty_dbl{eltype{T}}) + # test if vector has a specific width & element type def lvec{T, n, w} = 0 def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index 33f0183d..4c5c2363 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -31,8 +31,8 @@ def bqn_or{a, b} = (a+b)-(a*b) # + & - def arithChk1{F==__add, M, w:T, x:T, r:T} = tup{'topAny', M{(w^r) & (x^r)}} def arithChk1{F==__sub, M, w:T, x:T, r:T} = tup{'topAny', M{(w^x) & (w^r)}} -def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', __adds{w,x}, r} -def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', __subs{w,x}, r} +def arithChk1{F==__add, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', adds{w,x}, r} +def arithChk1{F==__sub, M, w:T, x:T, r:T & isvec{T} & tern{hasarch{'X86_64'}, elwidth{T}<=16, 1}} = tup{'anyne', subs{w,x}, r} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index e953f736..1e1e2d0b 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -23,12 +23,6 @@ def ntyp0{S, T} = merge{S, '_', nty{T}} def load{a:T, n & nvec{eltype{T}}} = emit{eltype{T}, ntyp{'vld1', eltype{T}}, *eltype{eltype{T}} ~~ (a+n)} def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a+n), v} -def __adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} -def __subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} -def mul1{a:T,b:T & w64i{T}} = emit{ty_dbl{v_dbl{T}}, ntyp{'vmull', T}, a, b} -def mul2{a:T,b:T & w128i{T}} = emit{ty_dbl{T}, ntyp0{'vmull_high', T}, a, b} -def mul12{a:T,b:T & w128{T}} = tup{mul1{half{a,0}, half{b,0}}, mul2{a,b}} - def __or{a:T,b:T & nvecf{T}} = T~~ __or{ty_u{a}, ty_u{b}} def __and{a:T,b:T & nvecf{T}} = T~~__and{ty_u{a}, ty_u{b}} def __xor{a:T,b:T & nvecf{T}} = T~~__xor{ty_u{a}, ty_u{b}} @@ -45,11 +39,19 @@ def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b} def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} +def adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} +def subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} def addp{a:T,b:T & nvec{T}} = emit{T, ntyp{'vpadd', T}, a, b} +def addw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vaddl', T}, a, b} +def subw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vsubl', T}, a, b} +def mulw{a:T,b:T & w64i{T}} = emit{el_dbl{T}, ntyp{'vmull', T}, a, b} + +def mulw2{a:T,b:T & w128i{T}} = emit{ty_dbl{T}, ntyp0{'vmull_high', T}, a, b} +def mul12{a:T,b:T & w128{T}} = tup{mulw{half{a,0}, half{b,0}}, mulw2{a,b}} def __shl{a:T, s & nveci{T} & knum{s} & s>0 & s0 & s8} = { def H=v_half{ty_half{T}}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed +def shrn{a:T, s & w128i{T} & elwidth{T}>8} = { def H=el_half{T}; emit{H, ntyp0{'vshrn_n', T}, a, s} } # a>>s, narrowed def shrm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsri', '_n', T}, d, a, s} # (a>>s) | (d & (mask of new zeroes)) def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a< def widen{T, x:X & w128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}} -def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{ty_half{v_half{X}}, ntyp0{'vmovn', X}, x} +def narrow{T, x:X & w128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{el_half{X}, ntyp0{'vmovn', X}, x} def narrow{T, x:X & w128{X} & isfloat{T}!=isfloat{eltype{X}} & width{T}b} def max{a:T,b:T & w128i{T}} = homBlend{a, b, a>31; (t^a) - t } def absu{a:T & w128i{T}} = ty_u{abs{a}} # saturating -def __adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def __adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b} -def __subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def __subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b} -def __adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def __adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b} -def __subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def __subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b} +def adds{a:T,b:T & T==[16]i8 } = emit{T, '_mm_adds_epi8', a, b}; def adds{a:T,b:T & T==[16]u8 } = emit{T, '_mm_adds_epu8', a, b} +def subs{a:T,b:T & T==[16]i8 } = emit{T, '_mm_subs_epi8', a, b}; def subs{a:T,b:T & T==[16]u8 } = emit{T, '_mm_subs_epu8', a, b} +def adds{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_adds_epi16', a, b}; def adds{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_adds_epu16', a, b} +def subs{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_subs_epi16', a, b}; def subs{a:T,b:T & T==[ 8]u16} = emit{T, '_mm_subs_epu16', a, b} # float arith def __add{a:T,b:T & T==[4]f32} = emit{T, '_mm_add_ps', a, b}; def __add{a:T,b:T & T==[2]f64} = emit{T, '_mm_add_pd', a, b}