diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index eca00fcc..bf2bf7de 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -23,8 +23,10 @@ def elwidth{T} = width{eltype{T}} +def load{p:P, n & isvec{eltype{P}}} = assert{0} +def store{p:P, n, v & isvec{eltype{P}}} = assert{0} def load{p:P & isptr{P}} = load{p, 0} -def store{p:P, v:V & isptr{P} & eltype{P}==v} = store{p, 0, v} +# def store{p:P, v & isptr{P}} = store{p, 0, v} def loadu{p:T & isunsigned{eltype{T}}} = emit{eltype{T}, merge{'loadu_u',fmtnat{elwidth{T}}}, p} def storeu{p:T, v:eltype{T} & isunsigned{eltype{T}}} = emit{void, merge{'storeu_u',fmtnat{elwidth{T}}}, p, v} def loadu{p:T & elwidth{T}==8} = load{p} diff --git a/src/singeli/src/bins.singeli b/src/singeli/src/bins.singeli index 9e27e388..29bbd6b1 100644 --- a/src/singeli/src/bins.singeli +++ b/src/singeli/src/bins.singeli @@ -196,7 +196,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { def ms = if (up) 256-(1<<(1+iota{8})) else (1<127), 8**0}}} # Exact values for multiples of 8 - store{*U~~t0, 0, vpopc{vb}} + store{*V~~t0, 0, vpopc{vb}} st:i8=0; @for_dir{up} (t0 over 256/8) { st += t0; t0 = st } def sel_c = getsel{swap{load{*V~~t0, 0} - V**dup}} # Top 5 bits select bytes from tables; bottom 3 select from mask @@ -216,7 +216,7 @@ def bins_vectab_i8{up, w, wn, x, xn, rp, t0, t, done & hasarch{'AVX2'}} = { if (nu > 32) ind = homBlend{topBlend{...each{isel,ui2}, b}, ind, i0 < V**vl} } } - store{*U~~(*T~~rp+j), 0, ind} + store{*V~~(*T~~rp+j), 0, ind} } goto{done} setlabel{no_bittab} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index e73ccbc2..b55b5e35 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -225,10 +225,9 @@ fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: i:u64 = 0 {j} => { v:=load{*V~~x, i}; ++i; sel{V, v, j} } } else if (T==i32) { - def VT = [8]T - i := VT**x - ii := VT**8 - {j} => { v:=i+VT~~j; i+=ii; v } + i := V ** ty_u{x} + ii := V ** 8 + {j} => { v:=i+V~~j; i+=ii; v } } def tab = if (tw==32) itab else i64tab def step{r, w} = { diff --git a/src/singeli/src/transpose.singeli b/src/singeli/src/transpose.singeli index c0d8faf0..d1ea188b 100644 --- a/src/singeli/src/transpose.singeli +++ b/src/singeli/src/transpose.singeli @@ -36,10 +36,10 @@ def transpose_square{VT, l, x & hasarch{'AVX2'}} = unpack_to{1, l/2, x} def load2{a:T, b:T} = pair{load{a}, load{b}} def store2{a:T, b:T, v:T2 & w128i{eltype{T}} & w256{T2}} = { - each{{p, i} => store{p, 0, half{v, i}}, tup{a,b}, iota{2}} + each{{p:P, i} => store{p, 0, eltype{P}~~half{v,i}}, tup{a,b}, iota{2}} } -def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} -def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, v}, iota{l}, x} +def load_k {VT, src, l, w & w256{VT}} = each{{i} =>load {*VT~~(src+i*w), 0 }, iota{l}} +def store_k{VT, dst, x, l, h & w256{VT}} = each{{i,v}=>store{*VT~~(dst+i*h), 0, VT~~v}, iota{l}, x} def load_k {VT, src, l, w & w128{VT}} = each{{i} =>{p:=src+ i*w; load2 {*VT~~p, *VT~~(p+l*w) }}, iota{l}} def store_k{VT, dst, x, l, h & w128{VT}} = each{{i,v}=>{p:=dst+2*i*h; store2{*VT~~p, *VT~~(p+ h), v}}, iota{l}, x} @@ -112,7 +112,7 @@ def transpose_with_kernel{T, k, kh, call_base, rp:*T, xp:*T, w, h, ws, hs} = { assert{k == kh} def VT = [k]T def line_vecs = line_bytes / (width{VT}/8) - def store_line{p, vs} = each{store{p, ...}, iota{line_vecs}, vs} + def store_line{p:P, vs} = each{{i,v} => store{p, i, eltype{P}~~v}, iota{line_vecs}, vs} def get_lines{loadx} = { def vt{i} = transpose_square{VT, k, each{loadx, k*i + iota{k}}} each{tup, ...each{vt, iota{line_vecs}}}