define load_low/store_low in terms of vl-based load/store

This commit is contained in:
dzaima 2025-02-03 03:37:38 +02:00
parent 848813331a
commit 61565bfa17
5 changed files with 31 additions and 30 deletions

View File

@ -182,10 +182,10 @@ For unaligned scalar loads & stores, `loadu` & `storeu` should be used.
- `loadu{p:*E} : E` - load scalar from unaligned memory
- `storeu{p:*E, a:E} : void` - store scalar to unaligned memory
- `load{p:*V} : V` - load full vector
- `store{p:*V, a:V} : void` - store full vector
- `load_low{p:*V, w} : V` - load to low `w` bits
- `store_low{p:*E, w, a:[n]E}` - store low `w` bits
- `load{[k]E, p:*E} : [k]E` - load full vector
- `store{p:*E, a:[k]E} : void` - store full vector
- `load{[k]E, p:*E, vl} : [k]E` - load first `vl` elements of vector (memory of upper ones won't be touched, and their values are unspecified)
- `store{p:*E, a:[k]E, vl} : void` - store first `vl` elements (upper ones won't be touched)
- `store_masked_hom{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements
- `store_masked_top{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements
- `store_blended_hom` - `store_masked_hom` but may touch masked-off elements and thus be supported on more types

View File

@ -6,13 +6,8 @@ def v2d{x:T if w256{T}} = [4]f64 ~~ x
def undef_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_castsi128_si256', v2i{x}}
def zero_promote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}}
# load & store
def load_low{ptr:*V, w if w256{V} and w<=128} = undef_promote{V, load_low{*n_h{V} ~~ ptr, w}}
def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr}
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}}
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x}
def load{V=[_]E, ptr:*E, vl if w256{V} and vl*width{E}<=128} = undef_promote{V, load{n_h{V}, ptr, vl}}
def store{ptr:*E, x:V=[k]E, vl if w256{V} and vl*width{E}<=128} = store{ptr, half{x, 0}, vl}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}

View File

@ -54,8 +54,8 @@ def eachx{F, ...args} = {
}
def load {p:*[_]_, n } = assert{0,'bad load',p,n}
def store{p:*[_]_, n, v} = assert{0,'bad store',p,n,v}
def load {...args if match (...args) { {ptr:*E, idx if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad load', ...args}
def store{...args if match (...args) { {ptr:*E, idx, val if not isvec{E}}=>0; {..._}=>1 } } = assert{0, 'bad store', ...args}
def load{p:*_} = load{p, 0}
# def store{p:*_, v} = store{p, 0, v}
def loadu {p:*T if isunsigned{T}} = emit{T, merge{'loadu_u', fmtnat{width{T}}}, p}
@ -198,11 +198,17 @@ def hom_to_int{...vs if length{vs}>1} = {
}
def hom_to_int{x if ktup{x}} = hom_to_int{...x}
def load{V=[_]_, ptr:*_} = assert{0, 'bad load', V, ptr}
def store{ptr:*_, val:[_]_} = assert{0, 'bad store', ptr, val}
def load{V=[k]E, ptr:*E, k} = load{V, ptr}
def store{ptr:*E, val:V=[k]E, k} = store{ptr, val}
def load{V=[k]E, ptr:*E} = load{*V~~ptr}
def store{ptr:*E, val:V=[k]E} = store{*V~~ptr, 0, val}
def load_low{ptr:*V=[k]E, w} = load{V, *E~~ptr, w/width{E}}
def store_low{ptr:*E, w, val:V=[k]E} = store{ptr, val, w/width{E}}
if_inline (hasarch{'X86_64'}) {
include 'arch/iintrinsic/basic'
include 'arch/iintrinsic/select'

View File

@ -66,14 +66,16 @@ def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x}
def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x}
def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def load_low{ptr:*V, w if w<=64} = { # implemented via a broadcast load
def L=re_el{ty_u{w}, V}
V ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
def store{ptr:*E, x:V=[_]E, vl if nvec{V} and vl*width{E}<=64} = {
def E=ty_u{vl*width{E}}
storeu{*E~~ptr, extract{re_el{E,V}~~x, 0}}
}
def load{V=[_]E, ptr:*E, vl if nvec{V} and vl*width{E}<=64} = { # implemented via a broadcast load
def U = ty_u{vl*width{E}}
def L = re_el{U, V}
V ~~ emit{L, ntyp{'vld1', '_dup', L}, *U~~ptr}
}
def load_low{ptr:*V, (width{V})} = load{ptr}

View File

@ -5,15 +5,13 @@ def v2d{x:T if w128{T}} = [2]f64 ~~ x
# load & store
def load_low{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def load_low{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def load_low{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def load_low{ptr:*V, w if w128{V} and w==128} = load{ptr}
def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def load{V=[_]E, ptr:*E, vl if w128{V} and vl*width{E}==64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x}
def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def store{ptr:*E, x:V=[_]E, vl if w128{V} and vl*width{E}==64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}