diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 4abb12a0..719dd3ad 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -11,8 +11,7 @@ def v2i{x:T & w256{T}} = [32]u8 ~~ x # for compact casting for the annoying intr def v2f{x:T & w256{T}} = [8]f32 ~~ x def v2d{x:T & w256{T}} = [4]f64 ~~ x -def undefPromote{T, x:X & w128{X} & w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} -def undefPromote{T, x:X & width{T}==width{X}} = T~~x +def undefPromote{T, x:X & w128{X} & w256{T} & eltype{T}==eltype{X}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} # load & store def load {a:T, n & w256i{eltype{T} }} = emit{eltype{T}, '_mm256_loadu_si256', a+n} @@ -28,7 +27,7 @@ def storea{a:T, n, v & w256f{eltype{T},64}} = emit{void, '_mm256_store_pd', *f6 def store {a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_storeu_ps', *f32 ~~ (a+n), v} def storea{a:T, n, v & w256f{eltype{T},32}} = emit{void, '_mm256_store_ps', *f32 ~~ (a+n), v} -def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*[16]u8 ~~ ptr, w}} +def loadLow{ptr:P, w & w256{eltype{P}} & w<=128} = undefPromote{eltype{P}, loadLow{*v_half{eltype{P}} ~~ ptr, w}} def loadLow{ptr:P, w & w256{eltype{P}} & w==256} = load{ptr} def storeLow{ptr:P, w, x:T & w256{T} & w<=128} = storeLow{ptr, w, half{x, 0}} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index e71e1fb4..787926fb 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -91,6 +91,9 @@ def v_half{T & isvec{T}} = [vcount{T}/2](eltype{T}) # base vector utility definitions def widen{T, x:X & T==X} = x def narrow{T, x:X & T==eltype{X}} = x +def undefPromote{T, x:X & T==X} = T~~x + + def unroll{vars,begin,end,block & knum{begin} & knum{end}} = { def f{i,l & i==l} = 0 diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 0479fcc0..a41a8957 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -67,10 +67,6 @@ def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} def fold_min {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vminv', T}, a} def fold_max {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vmaxv', T}, a} -def half{x:T, n==0 & nvec{T} & width{T}==128} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}}, x} -def half{x:T, n==1 & nvec{T} & width{T}==128} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x} -def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n} - # TODO don't rely on regular stores being unaligned local def storeu{ptr:P, e:T} = store{ptr, 0, e} local def loadu{ptr:P} = load{ptr} @@ -87,6 +83,24 @@ def loadLow{ptr:P, w & w==elwidth{P}} = load{ptr} + +def undefPromote{T, x:X & n64{X} & n128{T} & eltype{T}==eltype{X}} = emit{T, ntyp{'vcombine', X}, x, x} # arm_neon.h doesn't actually provide a way to do this in a 0-instruction way. ¯\_(ツ)_/¯ +def half{x:T, n==0 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_low', v_half{T}}, x} +def half{x:T, n==1 & n128{T}} = emit{v_half{T}, ntyp{'vget', '_high', v_half{T}}, x} +def extract{x:T,n & nvec{T} & knum{n}} = emit{eltype{T}, ntyp{'vget', '_lane', T}, x, n} +def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} } + +local def eqqi{A, B} = isint{A} & (quality{A}==quality{B}) # equal quality integers + +def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}==elwidth{X}*2} = emit{T, ntyp{'vmovl', X}, x} +def widen{T, x:X & n64{X} & eqqi{eltype{T},eltype{X}} & elwidth{T}> elwidth{X}*2} = widen{T, widen{ty_half{T}, x}} +def widen{T, x:X & n128{X} & vcount{X}>vcount{T}} = widen{T, half{x,0}} + +def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2< elwidth{X}} = narrow{T, undefPromote{ty_half{X}, narrow{ty_half{eltype{X}}, x}}} +def narrow{T, x:X & n128{X} & eqqi{T,eltype{X}} & width{T}*2==elwidth{X}} = emit{ty_half{v_half{X}}, ntyp{'vmovn', v_half{X}}, x} + + + def bitAny{x:T} = fold_max{to_el{u32, x}}!=0 def bitAll{x:T} = fold_min{to_el{u32, x}}==0xffff_ffff @@ -95,7 +109,6 @@ def topAll{x:T & nvec{T}} = fold_max{ty_s{x}}<0 def homAny{x:T & nvec{T}} = bitAny{x} def homAll{x:T & nvec{T}} = bitAll{x} -def pack{x:T, y:T & nvec{T}} = { def H=ty_half{T}; emit{H, ntyp{'vuzp1', H}, H~~x, H~~y} } def broadcast{T, x & nvec{T}} = emit{T, ntyp{'vdup', '_n', T}, x}