From 10ec04a76e3404e406510a8cbd55c12f01d7caa4 Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 1 Feb 2025 22:16:07 +0200 Subject: [PATCH] =?UTF-8?q?(load|store)Low=20=E2=86=92=20(load|store)=5Flo?= =?UTF-8?q?w?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/README.md | 4 ++-- src/singeli/src/avx.singeli | 8 ++++---- src/singeli/src/base.singeli | 4 ++-- src/singeli/src/mask.singeli | 6 ++---- src/singeli/src/neon.singeli | 8 ++++---- src/singeli/src/scan.singeli | 2 +- src/singeli/src/select.singeli | 2 +- src/singeli/src/sse2.singeli | 16 ++++++++-------- src/singeli/src/transpose.singeli | 6 +++--- 9 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/singeli/README.md b/src/singeli/README.md index e6d1f869..794d5824 100644 --- a/src/singeli/README.md +++ b/src/singeli/README.md @@ -186,8 +186,8 @@ For unaligned scalar loads & stores, `loadu` & `storeu` should be used. - `storeu{p:*E, a:E} : void` - store scalar to unaligned memory - `load{p:*V} : V` - load full vector - `store{p:*V, a:V} : void` - store full vector -- `loadLow{p:*V, w} : V` - load to low `w` bits -- `storeLow{p:*E, w, a:[n]E}` - store low `w` bits +- `load_low{p:*V, w} : V` - load to low `w` bits +- `store_low{p:*E, w, a:[n]E}` - store low `w` bits - `store_masked_hom{p:*V, m:mt{V}, a:V}` - conditionally store elements based on mask; won't touch masked-off elements - `store_masked_top{p:*V, m:V, a:V}` - conditionally store elements based on top bit of `m`; won't touch masked-off elements - `store_blended_hom` - `store_masked_hom` but may touch masked-off elements and thus be supported on more types diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 944269b5..f432bdf6 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -7,11 +7,11 @@ def undefPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm def zeroPromote{T=[_]E, x:X=[_]E if w128{X} and w256{T}} = T~~emit{[32]u8, '_mm256_zextsi128_si256', v2i{x}} # load & store -def loadLow{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, loadLow{*n_h{V} ~~ ptr, w}} -def loadLow{ptr:*V, w if w256{V} and w==256} = load{ptr} +def load_low{ptr:*V, w if w256{V} and w<=128} = undefPromote{V, load_low{*n_h{V} ~~ ptr, w}} +def load_low{ptr:*V, w if w256{V} and w==256} = load{ptr} -def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = storeLow{ptr, w, half{x, 0}} -def storeLow{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x} +def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w<=128} = store_low{ptr, w, half{x, 0}} +def store_low{ptr:*E, w, x:X=[_]E if w256{X} and w==256} = store{*X~~ptr, 0, x} # float comparison diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index e1b367c9..5258f2e8 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -175,8 +175,8 @@ def { all_bit,any_bit,blend_bit, all_hom,any_hom,blend_hom,hom_to_int,store_masked_hom,store_blended_hom, all_top,any_top,blend_top,top_to_int,store_masked_top,store_blended_top, - loadBatchBit,loadLow,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair, - packQ,pair,pdep,pext,popcRand,rbit,sel,shl,shr,shufInd,storeLow, + loadBatchBit,load_low,make,mask_to_hom,mulw_split,mulh,narrow,narrow_trunc,narrow_pair, + packQ,pair,pdep,pext,popcRand,rbit,sel,shl,shr,shufInd,store_low, unord,unzip,vfold,vec_select,vec_shuffle,widen,widen_upper,multishift, } diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index bc1b4bee..9cbcfb36 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -42,8 +42,6 @@ def maskAfter{n} = { -def loadLowBatch{V=[_]T, ptr:*T, w, n} = loadLow{*V ~~ (ptr + n*(w/width{T})), w} - # store the i-th batch of k elements to ptr, narrowing elements if needed; masked by M def storeBatch{ptr:*E0, i, x:[k]E1, M} = { def rpos = ptr + i*k @@ -51,14 +49,14 @@ def storeBatch{ptr:*E0, i, x:[k]E1, M} = { xu:= narrow{E0, x} if (M{0}) store_blended_hom{*TF~~rpos, M{TF, 'to homogeneous bits'}, undefPromote{TF, xu}} - else storeLow{rpos, k*width{E0}, xu} + else store_low{rpos, k*width{E0}, xu} } # (sign/zero)-extend the i-th batch of k elements at ptr to [k]E1 def loadBatch{ptr:*E0, i, [k]E1} = { def rpos = ptr + i*k def TF = re_el{E0, [k]E1} - widen{[k]E1, loadLow{*TF~~rpos, k*width{E0}}} + widen{[k]E1, load_low{*TF~~rpos, k*width{E0}}} } def loadBatch {ptr:*E, {...ns}, T } = each{loadBatch {ptr, ., T }, ns} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 2a97d35a..852b4472 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -59,14 +59,14 @@ def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, min}} = fold_min{x} def vfold{F, x:T if nvec{T} and ~nveci{T,64} and same{F, max}} = fold_max{x} def vfold{F, x:T if nvec{T} and same{F, +}} = fold_add{x} -def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } -def storeLow{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} +def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w<=64} = { def E=ty_u{w}; storeu{*E~~ptr, extract{re_el{E,T}~~x, 0}} } +def store_low{ptr:*E, w, x:T=[_]E if nvec{T} and w==width{T}} = store{*T~~ptr, 0, x} -def loadLow{ptr:*V, w if w<=64} = { # implemented via a broadcast load +def load_low{ptr:*V, w if w<=64} = { # implemented via a broadcast load def L=re_el{ty_u{w}, V} V ~~ emit{L, ntyp{'vld1', '_dup', L}, *ty_u{w}~~ptr} } -def loadLow{ptr:*V, (width{V})} = load{ptr} +def load_low{ptr:*V, (width{V})} = load{ptr} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 58bcd0e9..cc70a6f6 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -111,7 +111,7 @@ fn clmul_scan_ne_any{if hasarch{'PCLMUL'}}(x:*void, r:*void, init:u64, words:u64 rv = zip{...(@collect (j to 2) xor64{xv, j, c}), 0} } if ((words & 1) == 1) { - storeLow{*u64~~(rv+e), 64, clmul{loadLow{xv+e, 64}, m, 0} ^ c} + store_low{*u64~~(rv+e), 64, clmul{load_low{xv+e, 64}, m, 0} ^ c} } } fn scan_neq{if hasarch{'PCLMUL'}}(init:u64, x:*u64, r:*u64, nw:u64) : void = { diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 3b424763..3cffff6a 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -168,7 +168,7 @@ export{'INDS_BUF_MAX_COPY', ux~~inds_buf_max} if (max >= (inds_buf_max>>sh)) return{0} @for_backwards (i to inds_buf_max/(width{WV}/8)) { - def s = loadLow{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}} + def s = load_low{*arch_minv{V}~~(*u8~~src + i*bulk), width{V}} def v = raw_widen_inds{bulk, sh, s} store{*WV~~dst, i, v} } diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index d385aa61..cd333e60 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -5,15 +5,15 @@ def v2d{x:T if w128{T}} = [2]f64 ~~ x # load & store -def loadLow{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr} -def loadLow{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr} -def loadLow{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr} -def loadLow{ptr:*V, w if w128{V} and w==128} = load{ptr} +def load_low{ptr:*V, w if w128{V} and w== 16} = V ~~ emit{[16]u8, '_mm_loadu_si16', ptr} +def load_low{ptr:*V, w if w128{V} and w== 32} = V ~~ emit{[16]u8, '_mm_loadu_si32', ptr} +def load_low{ptr:*V, w if w128{V} and w== 64} = V ~~ emit{[16]u8, '_mm_loadu_si64', ptr} +def load_low{ptr:*V, w if w128{V} and w==128} = load{ptr} -def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} -def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} -def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} -def storeLow{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x} +def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 16} = emit{void, '_mm_storeu_si16', ptr, v2i{x}} +def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 32} = emit{void, '_mm_storeu_si32', ptr, v2i{x}} +def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w== 64} = emit{void, '_mm_storeu_si64', ptr, v2i{x}} +def store_low{ptr:*E, w, x:T=[_]E if w128{T} and w==128} = store{*T~~ptr, 0, x} diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index 2de87f8b..7907ee7e 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -38,17 +38,17 @@ def transpose_square{VT, l, x if avx2} = unpack_to{1, l/2, x} def load2{a:*T, b:*T} = match (width{T}) { {64} => { - def v = each{{p}=>loadLow{*[2]u64~~p, 64}, tup{a,b}} + def v = each{{p}=>load_low{*[2]u64~~p, 64}, tup{a,b}} n_d{T}~~zip{...v, 0} } {128} => pair{load{a}, load{b}} } def store2{a:*T, b:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) { - { 64} => each{{p, v} => storeLow{*u64~~p, 64, [2]u64~~v}, tup{a,b}, tup{v, shuf{u64, v, 1,0}}} + { 64} => each{{p, v} => store_low{*u64~~p, 64, [2]u64~~v}, tup{a,b}, tup{v, shuf{u64, v, 1,0}}} {128} => each{{p, i} => store{p, 0, T~~half{v,i}}, tup{a,b}, iota{2}} } def store1of2{a:*T, v:T2 if 2*width{T} == width{T2}} = match (width{T}) { - { 64} => storeLow{*u64~~a, 64, [2]u64~~v} + { 64} => store_low{*u64~~a, 64, [2]u64~~v} {128} => store{a, 0, T~~half{v,0}} } def load_k {VT, src, l, w} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}}