diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 06367618..17eaf98e 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -22,6 +22,9 @@ def isprim{T} = 0; def isprim{T & istype{T}} = same{typekind{T},'primitive'} def isptr {T} = 0; def isptr {T & istype{T}} = same{typekind{T},'pointer'} def elwidth{T} = width{eltype{T}} +oper &~ andnot infix none 35 +def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b + def load{p:P, n & isvec{eltype{P}}} = assert{0} @@ -41,37 +44,6 @@ def exportN{f, ...ns} = each{export{.,f}, ns} def exportT{name, fs} = { v:*type{tupsel{0,fs}} = fs; export{name, v} } -# more arith -def cdiv{a,b} = (a+b-1)/b # ceiling divide -def cdiv{a,b & knum{a} & knum{b}} = ((a+b-1)/b)>>0 -def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x} -def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x} -def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x} -def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x} -def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x} -def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x} -# count-leading-zeros complement, less type-dependent -def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} -def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} - -def ceil_log2{n} = clzc{n-1} - -def truncBits{n, v & n<=8} = cast_i{u8, v} -def truncBits{n, v & n==16} = cast_i{u16, v} -def truncBits{n, v & n==32} = cast_i{u32, v} -def truncBits{n, v & n==64} = cast_i{u64, v} - -# base-2 log of a constant power of two -def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 -def lb{n==1} = 0 - -def zlow{n,x} = (x >> n) << n # zero out n least significant bits -def tail{n,x} = x & ((1<v, range{n}} +def collect{vars,begin,end,iter & knum{begin} & knum{end}} = { + each{iter{., vars}, range{end-begin}+begin} +} + +# convert tuple to number in little-endian base b +def base{b,l} = if (0==tuplen{l}) 0 else tupsel{0,l}+b*base{b,slice{l,1}} + + + +# vector definitions +def arch_defvw = if (hasarch{'AVX2'}) 256 else 128 +def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} + +# test if vector has a specific width & element type +def lvec{T, n, w} = 0 +def lvec{T, n, w & isvec{T} & vcount{T}==n & elwidth{T}==w} = 1 + # base cases def { absu,andAllZero,andnz,b_getBatch,clmul,cvt,extract,fold_addw,half, @@ -185,51 +172,9 @@ def { zip,zipHi,zipLo } -def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}} -def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b} -def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} -def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b -oper &~ andnot infix none 35 -def widen{T, x:X & T==X} = x -def narrow{T, x:X & T==eltype{X}} = x -def undefPromote{T, x:X & T==X} = T~~x -def packQ{{a, b}} = packQ{a, b} -def pair{{a, b}} = pair{a, b} -def cvt{T, x:X & T==eltype{X}} = x - -def homMask{...vs & tuplen{vs}>1} = { - def n = tuplen{vs} - def T = oneType{vs} - def RT = ty_u{max{8,vcount{T}*n}} - tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}} -} def homMaskX{a:T} = tup{1, homMask{a}} # tup{n,mask}; mask with each bit repeated n times def ctzX{{n,v}} = ctz{v}/n # ctz for a result of homMaskX -def inRangeLen{x:TS, start, count & issigned{eltype{TS}}} = { # ∊ [start;start+count) - def TU = ty_u{TS} - (TU~~(x-TS**start)) < TU**count -} -def inRangeLen{x:TU, start, count & isunsigned{eltype{TU}}} = { # ∊ [start;start+count) - def TS = ty_s{TU} - def h = 1 << (elwidth{TU}-1) - (TS~~(x-TU**(start-h))) < TS**(count-h) -} -def inRangeIncl{x:T, start, end} = inRangeLen{x, start, end-start+1} # ∊ [start;end] -def inRangeExcl{x:T, start, end} = inRangeLen{x, start, end-start} # ∊ [start;end) - - -# tuple operations -def broadcast{T, v & isprim{T}} = v -def iota{n & knum{n}} = range{n} -def collect{vars,begin,end,iter & knum{begin} & knum{end}} = { - each{iter{., vars}, range{end-begin}+begin} -} -def broadcast{n, v & knum{n}} = each{{_}=>v, range{n}} - -def arch_defvw = tern{hasarch{'AVX2'}, 256, 128} -def has_simd = hasarch{'X86_64'} | hasarch{'AARCH64'} - if (hasarch{'X86_64'}) { include 'arch/iintrinsic/basic' include './sse2' @@ -243,6 +188,23 @@ if (hasarch{'X86_64'}) { def {__adds,__subs,__sqrt,vec_broadcast,vec_make} } +def homMask{...vs & tuplen{vs}>1} = { + def n = tuplen{vs} + def T = oneType{vs} + def RT = ty_u{max{8,vcount{T}*n}} + tree_fold{|, each{{v,i}=>promote{RT,homMask{v}}<<(i*vcount{T}), vs, iota{n}}} +} +def mzip {a:T, b:T} = each{{v}=>el_m{T}~~v, zip{a, b}} +def mzipLo{a:T, b:T} = el_m{T} ~~ zipLo{a, b} +def mzipHi{a:T, b:T} = el_m{T} ~~ zipHi{a, b} +def packQ{{a, b}} = packQ{a, b} +def pair{{a, b}} = pair{a, b} + +def widen{T, x:X & T==X} = x +def narrow{T, x:X & T==eltype{X}} = x +def undefPromote{T, x:X & T==X} = T~~x +def cvt{T, x:X & T==eltype{X}} = x + def broadcast{T, v & isvec{T}} = vec_broadcast{T, promote{eltype{T},v}} def make{T, ...xs & isvec{T}} = vec_make{T, ...xs} def iota{T & isvec{T}} = make{T, ...iota{vcount{T}}} @@ -256,8 +218,52 @@ def max = __max def adds = __adds def subs = __subs def sqrt = __sqrt + + + +# more arith def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} +def cdiv{a,b} = (a+b-1)/b # ceiling divide +def cdiv{a,b & knum{a} & knum{b}} = ceil{a/b} +def popc{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_popcountll', x} +def popc{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_popcount', x} +def ctz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_ctzll', x} +def ctz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_ctz', x} +def clz{x:T & isint{T} & width{T}==64} = emit{ux, '__builtin_clzll', x} +def clz{x:T & isint{T} & width{T}<=32} = emit{ux, '__builtin_clz', x} +# count-leading-zeros complement, less type-dependent +def clzc{x:T & isint{T} & width{T}==64} = 64-clz{x} +def clzc{x:T & isint{T} & width{T}<=32} = 32-clz{x} + +def ceil_log2{n} = clzc{n-1} + +def truncBits{n, v & n<=8} = cast_i{u8, v} +def truncBits{n, v & n==16} = cast_i{u16, v} +def truncBits{n, v & n==32} = cast_i{u32, v} +def truncBits{n, v & n==64} = cast_i{u64, v} + +# base-2 log of a constant power of two +def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 +def lb{n==1} = 0 + +def zlow{n,x} = (x >> n) << n # zero out n least significant bits +def tail{n,x} = x & ((1< {if(ktup{x}) tuplen{x}; else 0}, args}} - each{F, ...each{{x} => {if (istup{x}) x; else l**x}, args}} + def l = tree_fold{max, each{{x} => if(ktup{x}) tuplen{x} else 0, args}} + each{F, ...each{{x} => if (istup{x}) x else l**x, args}} } def undef{T, n & istype{T}} = @collect(n) undef{T} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 9fe3e8e6..a6a812c2 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -10,11 +10,10 @@ def nvecf = genchk{nvec, isfloat} def reinterpret{T, v & same{'pointer',typekind{T}} & ktup{v}} = { tmp:T=v } -def nty{T== u8} = 'u8'; def nty{T== i8} = 's8' -def nty{T==u16} = 'u16'; def nty{T==i16} = 's16' -def nty{T==u32} = 'u32'; def nty{T==i32} = 's32' -def nty{T==u64} = 'u64'; def nty{T==i64} = 's64' -def nty{T==f32} = 'f32'; def nty{T==f64} = 'f64' +def nty{T} = { + def q = quality{T} + merge{if (q=='i') 's' else q, fmtnat{width{T}}} +} def nty{T & isvec{T}} = nty{eltype{T}} def ntyp{S, ...S2, T & w128{T}} = merge{S, 'q', ...S2, '_', nty{T}} def ntyp{S, ...S2, T & w64{T}} = merge{S, ...S2, '_', nty{T}}