diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index a9068635..08967f80 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -1,20 +1,11 @@ def w256{T} = 0 def w256{T & isvec{T}} = width{T}==256 - -local def wgen256{F} = { - def r{T} = 0 - def r{T & w256{T}} = F{eltype{T}} - def r{T,w} = 0 - def r{T,w & w256{T}} = F{eltype{T}} & (elwidth{T}==w) - def r{T & ~isvec{T}} = 0 - r -} def w256{T,w} = 0 def w256{T,w & w256{T}} = elwidth{T}==w -def w256i = wgen256{{T} => isint{T}} -def w256s = wgen256{{T} => isint{T} & issigned{T}} -def w256u = wgen256{{T} => isint{T} & isunsigned{T}} -def w256f = wgen256{{T} => isfloat{T}} +def w256i = genchk{w256, {T} => isint{T}} +def w256s = genchk{w256, {T} => issigned{T}} +def w256u = genchk{w256, {T} => isunsigned{T}} +def w256f = genchk{w256, {T} => isfloat{T}} def v2i{x:T & w256{T}} = [32]u8 ~~ x # for compact casting for the annoying intrinsic type system def v2f{x:T & w256{T}} = [8]f32 ~~ x diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 5887b480..f731f18e 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -9,6 +9,15 @@ def load{x} = *x def elwidth{T} = width{eltype{T}} +def genchk{B, F} = { + def r{T} = 0 + def r{T & B{T}} = F{eltype{T}} + def r{T,w} = 0 + def r{T,w & B{T}} = F{eltype{T}} & (elwidth{T}==w) + def r{T & ~isvec{T}} = 0 + r +} + # ceiling divide def cdiv{a,b} = (a+b-1)/b @@ -67,6 +76,12 @@ def ty_s{x:T & isvec{T}} = ty_s{T}~~x def to_el{E, T & isvec{T}} = [width{T}/width{E}]E def to_el{E, x:T} = to_el{E,T} ~~ x +# change width of primitive type +def to_w{T, w} = primtype{quality{T}, w} +# double (element) type width +def ty_dbl{T & isprim{T}} = to_w{T, width{T}*2} +def ty_dbl{T & isvec{T}} = to_el{ty_dbl{eltype{T}}, T} + def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def f{i,l & i==l} = 0 def f{i,l & i!=l} = { diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 2ba32b21..c5fe650f 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -1,20 +1,16 @@ -def n128{T} = 0 -def n128{T & isvec{T}} = width{T}==128 +def nvec{T} = 0 +def nvec{T & isvec{T}} = (width{T}==64) | (width{T}==128) +def nvec{T,w} = 0 +def nvec{T,w & nvec{T}} = elwidth{T}==w +def nveci = genchk{nvec, {T} => isint{T}} +def nvecs = genchk{nvec, {T} => issigned{T}} +def nvecu = genchk{nvec, {T} => isunsigned{T}} +def nvecf = genchk{nvec, {T} => isfloat{T}} -local def ngen128{F} = { - def r{T} = 0 - def r{T & n128{T}} = F{eltype{T}} - def r{T,w} = 0 - def r{T,w & n128{T}} = F{eltype{T}} & (elwidth{T}==w) - def r{T & ~isvec{T}} = 0 - r -} -def n128{T,w} = 0 -def n128{T,w & n128{T}} = elwidth{T}==w -def n128i = ngen128{{T} => isint{T}} -def n128s = ngen128{{T} => isint{T} & issigned{T}} -def n128u = ngen128{{T} => isint{T} & isunsigned{T}} -def n128f = ngen128{{T} => isfloat{T}} +def n128{T} = 0 +def n128{T&isvec{T}} = width{T}==128 +def n64{T} = 0 +def n64{T&isvec{T}} = width{T}==64 def reinterpret{T, v & match{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v } @@ -24,50 +20,65 @@ def nty{T==u32} = 'u32'; def nty{T==i32} = 's32' def nty{T==u64} = 'u64'; def nty{T==i64} = 's64' def nty{T==f32} = 'f32'; def nty{T==f64} = 'f64' def nty{T & isvec{T}} = nty{eltype{T}} -def nty{T & isptr{T}} = nty{eltype{T}} -def ntyp{S, T} = merge{S, '_', nty{T}} +def ntyp{S, ...S2, T & n128{T}} = merge{S, 'q', ...S2, '_', nty{T}} +def ntyp{S, ...S2, T & n64{T}} = merge{S, ...S2, '_', nty{T}} -def load{a:T, n & n128{eltype{T}}} = emit{eltype{T}, ntyp{'vld1q', T}, *eltype{eltype{T}} ~~ (a+n)} -def store{a:*V, n, v:V & n128{V}} = emit{void, ntyp{'vst1q', V}, *eltype{V} ~~ (a+n), v} +def load{a:T, n & nvec{eltype{T}}} = emit{eltype{T}, ntyp{'vld1', eltype{T}}, *eltype{eltype{T}} ~~ (a+n)} +def store{a:*V, n, v:V & nvec{V}} = emit{void, ntyp{'vst1', V}, *eltype{V} ~~ (a+n), v} -def __adds{a:T,b:T & n128i{T}} = emit{T, ntyp{'vqaddq', T}, a, b} -def __subs{a:T,b:T & n128i{T}} = emit{T, ntyp{'vqsubq', T}, a, b} +def __adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} +def __subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} -def __add{a:T,b:T & n128 {T}} = emit{T, ntyp{'vaddq', T}, a, b} -def __sub{a:T,b:T & n128 {T}} = emit{T, ntyp{'vsubq', T}, a, b} -def __mul{a:T,b:T & n128 {T}} = emit{T, ntyp{'vmulq', T}, a, b} -def __div{a:T,b:T & n128f{T}} = emit{T, ntyp{'vdivq', T}, a, b} -def __and{a:T,b:T & n128i{T}} = emit{T, ntyp{'vandq', T}, a, b} -def __or{a:T,b:T & n128i{T}} = emit{T, ntyp{'vorrq', T}, a, b} -def __xor{a:T,b:T & n128i{T}} = emit{T, ntyp{'veorq', T}, a, b} -def andnot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vbicq', T}, a, b} -def ornot{a:T,b:T & n128i{T}} = emit{T, ntyp{'vornq', T}, a, b} -def min{a:T,b:T & n128i{T}} = emit{T, ntyp{'vminq', T}, a, b} # TODO float - there are multiple options -def max{a:T,b:T & n128i{T}} = emit{T, ntyp{'vmaxq', T}, a, b} -def __shl{a:T,b:S & n128i{T} & n128i{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshlq', T}, a, ty_s{b}} +def __add{a:T,b:T & nvec {T}} = emit{T, ntyp{'vadd', T}, a, b} +def __sub{a:T,b:T & nvec {T}} = emit{T, ntyp{'vsub', T}, a, b} +def __mul{a:T,b:T & nvec {T}} = emit{T, ntyp{'vmul', T}, a, b} +def __div{a:T,b:T & nvecf{T}} = emit{T, ntyp{'vdiv', T}, a, b} +def __and{a:T,b:T & nveci{T}} = emit{T, ntyp{'vand', T}, a, b} +def __or{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorr', T}, a, b} +def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b} +def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b} +def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b} +def min{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmin', T}, a, b} # TODO float - there are multiple options +def max{a:T,b:T & nveci{T}} = emit{T, ntyp{'vmax', T}, a, b} +def __shl{a:T,b:S & nveci{T} & nveci{S} & elwidth{T}==elwidth{S}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} -def __shl{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshlq_n', T}, a, b}; def __shl{a:T,b==0 & n128i{T}} = a -def __shr{a:T,b & n128i{T} & knum{b}} = emit{T, ntyp{'vshrq_n', T}, a, b}; def __shr{a:T,b==0 & n128i{T}} = a -def bblend{f:T, t:T, m:M & n128{T} & n128u{M} & elwidth{T}==elwidth{M}} = emit{T, ntyp{'vbslq', T}, m, t, f} +def __shl{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshl', '_n', T}, a, b}; def __shl{a:T,b==0 & nveci{T}} = a +def __shr{a:T,b & nveci{T} & knum{b}} = emit{T, ntyp{'vshr', '_n', T}, a, b}; def __shr{a:T,b==0 & nveci{T}} = a +def bblend{f:T, t:T, m:M & nvec{T} & nvecu{M} & width{T}==width{M} & elwidth{T}==elwidth{M}} = emit{T, ntyp{'vbsl', T}, m, t, f} -def __neg{a:T & (n128s{T}|n128f{T})} = emit{T, ntyp{'vnegq', T}, a} -def __not{a:T & n128u{T}} = emit{T, ntyp{'vmvnq', T}, a} -def sqrt{a:T & n128f{T}} = emit{T, ntyp{'vsqrtq', T}, a} -def floor{a:T & n128f{T}} = emit{T, ntyp{'vrndmq', T}, a} -def ceil{a:T & n128f{T}} = emit{T, ntyp{'vrndpq', T}, a} -def abs{a:T & (n128s{T}|n128f{T})} = emit{T, ntyp{'vabsq', T}, a} -def absu{a:T & n128i{T}} = ty_u{abs{a}} +def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a} +def addpw{a:T & nveci{T} & elwidth{T}<=32} = emit{ty_dbl{T}, ntyp{'vpaddl', T}, a} +def __not{a:T & nvecu{T}} = emit{T, ntyp{'vmvn', T}, a} +def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a} +def floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a} +def ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a} +def abs{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vabs', T}, a} +def absu{a:T & nveci{T}} = ty_u{abs{a}} -def __eq{a:T,b:T & n128{T}} = emit{ty_u{T}, ntyp{'vceqq', T}, a, b} -def __ge{a:T,b:T & n128{T}} = emit{ty_u{T}, ntyp{'vcgeq', T}, a, b} -def __gt{a:T,b:T & n128{T}} = emit{ty_u{T}, ntyp{'vcgtq', T}, a, b} -def __lt{a:T,b:T & n128{T}} = emit{ty_u{T}, ntyp{'vcltq', T}, a, b} -def __le{a:T,b:T & n128{T}} = emit{ty_u{T}, ntyp{'vcleq', T}, a, b} -def __ne{a:T,b:T & n128{T}} = ~(a==b) +def __eq{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vceq', T}, a, b} +def __ge{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcge', T}, a, b} +def __gt{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcgt', T}, a, b} +def __lt{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vclt', T}, a, b} +def __le{a:T,b:T & nvec{T}} = emit{ty_u{T}, ntyp{'vcle', T}, a, b} +def __ne{a:T,b:T & nvec{T}} = ~(a==b) -def broadcast{T, x & n128{T}} = emit{T, ntyp{'vdupq_n', T}, x} -def make{T, ...xs & tuplen{xs}==vcount{T}} = { +def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a} +def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} +def fold_min {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vminv', T}, a} +def fold_max {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vmaxv', T}, a} + +def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x} + +def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = { def TE = eltype{T} load{*T ~~ *TE ~~ each{{c}=>promote{eltype{T},c}, xs}, 0} } -def make{T, x & n128{T} & istup{x}} = make{T, ...x} +def make{T, x & nvec{T} & istup{x}} = make{T, ...x} + + +def getmask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = { + cast_i{ty_u{max{8,vcount{T}}}, fold_add{x & make{T, 1< isint{T}} -def w128s = wgen128{{T} => isint{T} & issigned{T}} -def w128u = wgen128{{T} => isint{T} & isunsigned{T}} -def w128f = wgen128{{T} => isfloat{T}} +def w128i = genchk{w128, {T} => isint{T}} +def w128s = genchk{w128, {T} => issigned{T}} +def w128u = genchk{w128, {T} => isunsigned{T}} +def w128f = genchk{w128, {T} => isfloat{T}} def v2i{x:T & w128{T}} = [16]u8 ~~ x # for compact casting for the annoying intrinsic type system def v2f{x:T & w128{T}} = [4]f32 ~~ x