(load|store)Low → (load|store)_low

This commit is contained in:
dzaima 2025-02-01 22:16:07 +02:00
parent ad79ef71cf
commit 10ec04a76e
9 changed files with 27 additions and 29 deletions

View File

@ -186,8 +186,8 @@ For unaligned scalar loads & stores, `loadu` & `storeu` should be used.
- `storeu{p:*E, a:E} : void` - store scalar to unaligned memory
- `load{p:*V} : V` - load full vector
- `store{p:*V, a:V} : void` - store full vector
- `loadLow{p:*V, w} : V` - load to low `w` bits
- `storeLow{p:*E, w, a:[n]E}` - store low `w` bits
- `load_low{p:*V, w} : V` - load to low `w` bits
- `store_low{p:*E, w, a:[n]E}` - store low `w` bits
- `store_masked_hom{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements
- `store_masked_top{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements
- `store_blended_hom` - `store_masked_hom` but may touch masked-off elements and thus be supported on more types

View File

@ -7,11 +7,11 @@ def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm
def zeroPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}}
# load & store
def loadLow{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, loadLow{*n_h{V} ~~ ptr, w}}
def loadLow{ptr:*V, w if w256{V} and w==256} = load{ptr}
def load_low{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, load_low{*n_h{V} ~~ ptr, w}}
def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr}
def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}}
def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x}
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}}
def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x}
# float comparison

View File

@ -175,8 +175,8 @@ def {
all_bit,any_bit,blend_bit,
all_hom,any_hom,blend_hom,hom_to_int,store_masked_hom,store_blended_hom,
all_top,any_top,blend_top,top_to_int,store_masked_top,store_blended_top,
loadBatchBit,loadLow,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair,
packQ,pair,pdep,pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow,
loadBatchBit,load_low,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair,
packQ,pair,pdep,pext,popcRand,rbit,sel,shl,shr,shufInd,store_low,
unord,unzip,vfold,vec_select,vec_shuffle,widen,widen_upper,multishift,
}

View File

@ -42,8 +42,6 @@ def maskAfter{n} = {
def loadLowBatch{V=[_]T, ptr:*T, w, n} = loadLow{*V ~~ (ptr + n*(w/width{T})), w}
# store the i-th batch of k elements to ptr, narrowing elements if needed; masked by M
def storeBatch{ptr:*E0, i, x:[k]E1, M} = {
def rpos = ptr + i*k
@ -51,14 +49,14 @@ def storeBatch{ptr:*E0, i, x:[k]E1, M} = {
xu:= narrow{E0, x}
if (M{0}) store_blended_hom{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}}
else storeLow{rpos, k*width{E0}, xu}
else store_low{rpos, k*width{E0}, xu}
}
# (sign/zero)-extend the i-th batch of k elements at ptr to [k]E1
def loadBatch{ptr:*E0, i, [k]E1} = {
def rpos = ptr + i*k
def TF = re_el{E0, [k]E1}
widen{[k]E1, loadLow{*TF~~rpos, k*width{E0}}}
widen{[k]E1, load_low{*TF~~rpos, k*width{E0}}}
}
def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns}

View File

@ -59,14 +59,14 @@ def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x}
def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x}
def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x}
def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} }
def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x}
def loadLow{ptr:*V, w if w<=64} = { # implemented via a broadcast load
def load_low{ptr:*V, w if w<=64} = { # implemented via a broadcast load
def L=re_el{ty_u{w}, V}
V ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr}
}
def loadLow{ptr:*V, (width{V})} = load{ptr}
def load_low{ptr:*V, (width{V})} = load{ptr}

View File

@ -111,7 +111,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64
rv = zip{...(@collect (j to 2) xor64{xv, j, c}), 0}
}
if ((words & 1) == 1) {
storeLow{*u64~~(rv+e), 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c}
store_low{*u64~~(rv+e), 64, clmul{load_low{xv+e, 64}, m, 0} ^ c}
}
}
fn scan_neq{if hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = {

View File

@ -168,7 +168,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max}
if (max >= (inds_buf_max>>sh)) return{0}
@for_backwards (i to inds_buf_max/(width{WV}/8)) {
def s = loadLow{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}}
def s = load_low{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}}
def v = raw_widen_inds{bulk, sh, s}
store{*WV~~dst, i, v}
}

View File

@ -5,15 +5,15 @@ def v2d{x:T if w128{T}} = [2]f64 ~~ x
# load & store
def loadLow{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def loadLow{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def loadLow{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def loadLow{ptr:*V, w if w128{V} and w==128} = load{ptr}
def load_low{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr}
def load_low{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr}
def load_low{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr}
def load_low{ptr:*V, w if w128{V} and w==128} = load{ptr}
def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}}
def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x}

View File

@ -38,17 +38,17 @@ def transpose_square{VT, l, x if avx2} = unpack_to{1, l/2, x}
def load2{a:*T, b:*T} = match (width{T}) {
{64} => {
def v = each{{p}=>loadLow{*[2]u64~~p, 64}, tup{a,b}}
def v = each{{p}=>load_low{*[2]u64~~p, 64}, tup{a,b}}
n_d{T}~~zip{...v, 0}
}
{128} => pair{load{a}, load{b}}
}
def store2{a:*T, b:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) {
{ 64} => each{{p, v} => storeLow{*u64~~p, 64, [2]u64~~v}, tup{a,b}, tup{v, shuf{u64, v, 1,0}}}
{ 64} => each{{p, v} => store_low{*u64~~p, 64, [2]u64~~v}, tup{a,b}, tup{v, shuf{u64, v, 1,0}}}
{128} => each{{p, i} => store{p, 0, T~~half{v,i}}, tup{a,b}, iota{2}}
}
def store1of2{a:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) {
{ 64} => storeLow{*u64~~a, 64, [2]u64~~v}
{ 64} => store_low{*u64~~a, 64, [2]u64~~v}
{128} => store{a, 0, T~~half{v,0}}
}
def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}