diff --git a/src/singeli/README.md b/src/singeli/README.md index 78b7db29..93b66413 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -182,10 +182,10 @@ For unaligned scalar loads & stores, `loadu` & `storeu` should be used. - `loadu{p:*E} : E` - load scalar from unaligned memory - `storeu{p:*E, a:E} : void` - store scalar to unaligned memory -- `load{p:*V} : V` - load full vector -- `store{p:*V, a:V} : void` - store full vector -- `load_low{p:*V, w} : V` - load to low `w` bits -- `store_low{p:*E, w, a:[n]E}` - store low `w` bits +- `load{[k]E, p:*E} : [k]E` - load full vector +- `store{p:*E, a:[k]E} : void` - store full vector +- `load{[k]E, p:*E, vl} : [k]E` - load first `vl` elements of vector (memory of upper ones won't be touched, and their values are unspecified) +- `store{p:*E, a:[k]E, vl} : void` - store first `vl` elements (upper ones won't be touched) - `store_masked_hom{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements - `store_masked_top{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements - `store_blended_hom` - `store_masked_hom` but may touch masked-off elements and thus be supported on more types diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 5f975a0b..441566fa 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -6,13 +6,8 @@ def v2d{x:T if w256{T}} = [4]f64 ~~ x def undef_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}} def zero_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}} -# load & store -def load_low{ptr:*V, w if w256{V} and w<=128} = undef_promote{V, load_low{*n_h{V} ~~ ptr, w}} -def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr} - -def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}} -def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x} - +def load{V=[_]E, ptr:*E, vl if w256{V} and vl*width{E}<=128} = undef_promote{V, load{n_h{V}, ptr, vl}} +def store{ptr:*E, x:V=[k]E, vl if w256{V} and vl*width{E}<=128} = store{ptr, half{x, 0}, vl} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index c3f1c2d7..da828c85 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -54,8 +54,8 @@ def eachx{F, ...args} = { } -def load {p:*[_]_, n } = assert{0,'bad load',p,n} -def store{p:*[_]_, n, v} = assert{0,'bad store',p,n,v} +def load {...args if match (...args) { {ptr:*E, idx if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad load', ...args} +def store{...args if match (...args) { {ptr:*E, idx, val if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad store', ...args} def load{p:*_} = load{p, 0} # def store{p:*_, v} = store{p, 0, v} def loadu {p:*T if isunsigned{T}} = emit{T, merge{'loadu_u', fmtnat{width{T}}}, p} @@ -198,11 +198,17 @@ def hom_to_int{...vs if length{vs}>1} = { } def hom_to_int{x if ktup{x}} = hom_to_int{...x} -def load{V=[_]_, ptr:*_} = assert{0, 'bad load', V, ptr} -def store{ptr:*_, val:[_]_} = assert{0, 'bad store', ptr, val} + + +def load{V=[k]E, ptr:*E, k} = load{V, ptr} +def store{ptr:*E, val:V=[k]E, k} = store{ptr, val} + def load{V=[k]E, ptr:*E} = load{*V~~ptr} def store{ptr:*E, val:V=[k]E} = store{*V~~ptr, 0, val} +def load_low{ptr:*V=[k]E, w} = load{V, *E~~ptr, w/width{E}} +def store_low{ptr:*E, w, val:V=[k]E} = store{ptr, val, w/width{E}} + if_inline (hasarch{'X86_64'}) { include 'arch/iintrinsic/basic' include 'arch/iintrinsic/select' diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 812fa568..0ca386bf 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -66,14 +66,16 @@ def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x} def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x} def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x} -def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } -def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} - -def load_low{ptr:*V, w if w<=64} = { # implemented via a broadcast load - def L=re_el{ty_u{w}, V} - V ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} +def store{ptr:*E, x:V=[_]E, vl if nvec{V} and vl*width{E}<=64} = { + def E=ty_u{vl*width{E}} + storeu{*E~~ptr, extract{re_el{E,V}~~x, 0}} +} + +def load{V=[_]E, ptr:*E, vl if nvec{V} and vl*width{E}<=64} = { # implemented via a broadcast load + def U = ty_u{vl*width{E}} + def L = re_el{U, V} + V ~~ emit{L, ntyp{'vld1', '_dup', L}, *U~~ptr} } -def load_low{ptr:*V, (width{V})} = load{ptr} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 926afb0f..5adcc9bb 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -5,15 +5,13 @@ def v2d{x:T if w128{T}} = [2]f64 ~~ x # load & store -def load_low{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr} -def load_low{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr} -def load_low{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def load_low{ptr:*V, w if w128{V} and w==128} = load{ptr} +def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr} +def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr} +def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} -def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} -def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} -def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x} +def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} +def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} +def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}