diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index daa1d1a5..e80db12c 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -176,7 +176,7 @@ def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1 # base cases def { - andAllZero,andnz,b_getBatch,clmul,cvt,fold_addw,half, + absu,andAllZero,andnz,b_getBatch,clmul,cvt,extract,fold_addw,half, homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,mulw,mulHi,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow, @@ -196,11 +196,6 @@ def packQ{{a, b}} = packQ{a, b} def pair{{a, b}} = pair{a, b} def cvt{T, x:X & T==eltype{X}} = x -def min{...x} = __min{...x} -def max{...x} = __max{...x} -def __min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} - def homMask{...vs & tuplen{vs}>1} = { def n = tuplen{vs} def T = oneType{vs} @@ -234,6 +229,34 @@ def broadcast{n, v & knum{n}} = each{{_}=>v, range{n}} def arch_defvw = tern{hasarch{'AVX2'}, 256, 128} def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} +if (hasarch{'X86_64'}) { + include 'arch/iintrinsic/basic' + def adds = __adds + def subs = __subs + def sqrt = __sqrt + def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} + def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} + include './sse2' + include './sse' + include './avx' + include './avx2' +} else if (hasarch{'AARCH64'}) { + include './neon' +} else { + def {adds,subs,sqrt} +} + +def floor = __floor +def ceil = __ceil +def abs = __abs +def min = __min +def max = __max +def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} + +def absu{a:T & isvec{T}} = ty_u{abs{a}} +def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} + def load{p,i & kgen{p}} = p{i} def store{p,i,x & kgen{p}} = p{i,x} diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index a7d140b3..c0e5ed4c 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -1,8 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} include './mask' include 'util/tup' diff --git a/src/singeli/src/bits.singeli b/src/singeli/src/bits.singeli index 8c3d5dd1..e3198fc6 100644 --- a/src/singeli/src/bits.singeli +++ b/src/singeli/src/bits.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './bitops' include './mask' diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index 8c6b6f86..21cf4150 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './bitops' diff --git a/src/singeli/src/constrep.singeli b/src/singeli/src/constrep.singeli index 2b634fcd..0b612860 100644 --- a/src/singeli/src/constrep.singeli +++ b/src/singeli/src/constrep.singeli @@ -1,9 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include 'util/tup' def incl{a,b} = slice{iota{b+1},a} diff --git a/src/singeli/src/copy.singeli b/src/singeli/src/copy.singeli index 77f0dfe8..a029b44f 100644 --- a/src/singeli/src/copy.singeli +++ b/src/singeli/src/copy.singeli @@ -1,9 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './cbqnDefs' include './bitops' diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli index 2a4acb32..90c068a0 100644 --- a/src/singeli/src/count.singeli +++ b/src/singeli/src/count.singeli @@ -1,5 +1,4 @@ include './base' -include './x86_vec' include 'util/tup' include './vecfold' diff --git a/src/singeli/src/dyarith.singeli b/src/singeli/src/dyarith.singeli index e4adfee7..eea7d28e 100644 --- a/src/singeli/src/dyarith.singeli +++ b/src/singeli/src/dyarith.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './bitops' include './mask' diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index 4a847a4c..ade621f3 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './mask' include './bitops' diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index e8f79cd6..f513360d 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,9 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' def opsh64{op}{v:[4]f64, perm} = op{v, shuf{[4]u64, v, perm}} diff --git a/src/singeli/src/monarith.singeli b/src/singeli/src/monarith.singeli index 9afcfd75..ecdee8a3 100644 --- a/src/singeli/src/monarith.singeli +++ b/src/singeli/src/monarith.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './bitops' include './mask' diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index f8f7ff3a..cfbc2c07 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -36,8 +36,8 @@ def __xor{a:T,b:T & nveci{T}} = emit{T, ntyp{'veor', T}, a, b} def andnot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vbic', T}, a, b} def ornot{a:T,b:T & nveci{T}} = emit{T, ntyp{'vorn', T}, a, b} def andnz{a:T,b:T & nveci{T}} = emit{T, ntyp{'vtst', T}, a, b} -def min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} -def max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} +def __min{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmin', T}, a, b} +def __max{a:T,b:T & nvec{T}} = emit{T, ntyp{'vmax', T}, a, b} def __shl{a:T,b:S & nveci{T} & nveci{S,elwidth{T}}} = emit{T, ntyp{'vshl', T}, a, ty_s{b}} def adds{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqadd', T}, a, b} def subs{a:T,b:T & nveci{T}} = emit{T, ntyp{'vqsub', T}, a, b} @@ -57,13 +57,12 @@ def shlm{a:T, s, d:T & nvecu{T}} = emit{T, ntyp{'vsli', '_n', T}, d, a, s} # (a def bitBlend{f:T, t:T, m:M & nvec{T} & nvecu{M,elwidth{T}} & width{T}==width{M}} = emit{T, ntyp{'vbsl', T}, m, t, f} def homBlend{f:T, t:T, m:M & nvec{M}} = bitBlend{f, t, m} -def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a} -def __not{a:T & nvecu{T}} = T~~emit{re_el{u8,T}, ntyp{'vmvn', re_el{u8,T}}, a} -def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a} -def floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a} -def ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a} -def abs{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vabs', T}, a} -def absu{a:T & nveci{T}} = ty_u{abs{a}} +def __neg{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vneg', T}, a} +def __not{a:T & nvecu{T}} = T~~emit{re_el{u8,T}, ntyp{'vmvn', re_el{u8,T}}, a} +def sqrt{a:T & nvecf{T}} = emit{T, ntyp{'vsqrt', T}, a} +def __floor{a:T & nvecf{T}} = emit{T, ntyp{'vrndm', T}, a} +def __ceil{a:T & nvecf{T}} = emit{T, ntyp{'vrndp', T}, a} +def __abs{a:T & (nvecs{T}|nvecf{T})} = emit{T, ntyp{'vabs', T}, a} def addpw { x:T & nveci{T} & elwidth{T}<=32 } = emit{el_m{T}, ntyp{'vpaddl', T}, x} # add pairwise widening def addpwa{a:D, x:T & nveci{T} & elwidth{T}<=32 & D==el_m{T}} = emit{D, ntyp{'vpadal', T}, a, x} # add pairwise widening + accumulate def mla{a:T, x:T, y:T & nvec{T}} = emit{T, ntyp{'vmla', T}, a, x, y} # a + x*y @@ -168,7 +167,6 @@ def make{T, ...xs & nvec{T} & tuplen{xs}==vcount{T}} = { load{*T ~~ *TE ~~ each{promote{eltype{T},.}, xs}, 0} } def make{T, x & nvec{T} & istup{x}} = make{T, ...x} -def iota{T & nvec{T}} = make{T, ...iota{vcount{T}}} def homMask{x:T & nvecu{T} & elwidth{T}>=vcount{T}} = { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 12ea2969..92e16230 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -1,7 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} include './clmul' include './mask' include './f64' diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 0900000d..21ef5e2d 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -1,9 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './vecfold' diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 50cfc105..db8544a8 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -1,6 +1,5 @@ include './base' include './cbqnDefs' -include './x86_vec' include './mask' include './bitops' include 'util/tup' diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 5e85383b..879229ec 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -1,10 +1,7 @@ include './base' if (hasarch{'X86_64'}) { - include './x86_vec' if (hasarch{'PCLMUL'}) include './clmul' if (hasarch{'BMI2'}) include './bmi2' -} else if (hasarch{'AARCH64'}) { - include './neon' } include './mask' include 'util/tup' diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index 1ed62054..1a9a3818 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -1,9 +1,4 @@ include './base' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './mask' include './cbqnDefs' include 'util/tup' diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index a03519a3..08dbc977 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -1,10 +1,5 @@ include './base' include './cbqnDefs' -if (hasarch{'X86_64'}) { - include './x86_vec' -} else if (hasarch{'AARCH64'}) { - include './neon' -} include './f64' include './mask' include './bitops' diff --git a/src/singeli/src/x86_vec.singeli b/src/singeli/src/x86_vec.singeli deleted file mode 100644 index ecdfe658..00000000 --- a/src/singeli/src/x86_vec.singeli +++ /dev/null @@ -1,15 +0,0 @@ -include 'arch/iintrinsic/basic' -def adds = __adds -def subs = __subs -def sqrt = __sqrt -def floor = __floor -def ceil = __ceil -def abs = __abs -def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} -def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} -def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} -def absu{a:T & isvec{T}} = ty_u{abs{a}} -include './sse2' -include './sse' -include './avx' -include './avx2'