Move 4-shuffle data to a table
This commit is contained in:
parent
9947de0146
commit
d28a74b572
@ -57,6 +57,17 @@ include './mask'
|
||||
include 'util/tup'
|
||||
def incl{a,b} = slice{iota{b+1},a}
|
||||
|
||||
# 1+˝∨`⌾⌽0=div|⌜range
|
||||
def makefact{divisor, range} = {
|
||||
def t = table{{a,b}=>0==b%a, divisor, range}
|
||||
fold{+, 1, reverse{scan{|, reverse{t}}}}
|
||||
}
|
||||
def basic_rep = incl{2,7}
|
||||
def fact_size = 128
|
||||
def fact_inds = slice{iota{fact_size},8}
|
||||
def fact_tab = makefact{basic_rep, fact_inds}
|
||||
factors:*u8 = fact_tab
|
||||
|
||||
def get_shufs{step, wv, len} = {
|
||||
def i = iota{len*step}
|
||||
split{step, (i - i%wv)/wv}
|
||||
@ -67,10 +78,6 @@ def get_shuf_data{wv, elbytes, len} = {
|
||||
each{expand{elbytes}, get_shufs{32 / elbytes, wv, len}}
|
||||
}
|
||||
def get_shuf_data{wv, b} = get_shuf_data{wv, b, wv}
|
||||
def get_shuf_Vs{V, wv, len} = {
|
||||
def eb = width{eltype{V}}/8
|
||||
each{bind{make,[32]i8}, get_shuf_data{wv, eb, len}}
|
||||
}
|
||||
def rep_iter_from_sh{sh} = {
|
||||
def l = tuplen{sh}
|
||||
def h = l >> 1
|
||||
@ -81,7 +88,6 @@ def rep_iter_from_sh{sh} = {
|
||||
b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}}
|
||||
}
|
||||
}
|
||||
def get_rep_iter{V, wv} = rep_iter_from_sh{get_shuf_Vs{V, wv, wv}} # Unused
|
||||
def get_rep_iter{V, wv==2}{x, gen} = {
|
||||
def s = shuf{[4]u64, x, 4b3120}
|
||||
each{{q}=>gen{V~~q}, unpackQ{s, s}}
|
||||
@ -115,10 +121,8 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = {
|
||||
}
|
||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n}
|
||||
|
||||
def rcsh_min = 3
|
||||
def rcsh_max = 7
|
||||
def rcsh_vals = incl{rcsh_min, rcsh_max}
|
||||
def rcsh_offs = shiftright{0, scan{+,rcsh_vals}}
|
||||
def rcsh_vals = incl{3, 7}
|
||||
rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}}
|
||||
def rcsh_table = table{get_shuf_data, rcsh_vals, tup{1,2,4}}
|
||||
rcsh_data:*i8 = join{join{join{rcsh_table}}}
|
||||
rcsh_sub{wv}(x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
@ -126,18 +130,21 @@ rcsh_sub{wv}(x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
def st = each{bind{load,sh}, iota{wv}}
|
||||
rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n}
|
||||
}
|
||||
def rcsh_subs = each{rcsh_sub, rcsh_vals}
|
||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & width{eltype{V}}<=32 & rcsh_min<=wv & wv<=rcsh_max} = {
|
||||
def ri = wv - tupsel{0,rcsh_vals}
|
||||
def eb = width{eltype{V}}/8
|
||||
def ti = eb-1 - (eb==4)
|
||||
tupsel{ri, rcsh_subs}(
|
||||
*i8~~xv, *i8~~rv, n*(width{eltype{V}}/8),
|
||||
*[32]i8~~rcsh_data + (tupsel{ri,rcsh_offs}*3 + ti*wv)
|
||||
)
|
||||
rep_const_shuffle_full(wv:i32, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = {
|
||||
def try{k} = { if (wv==k) rcsh_sub{k}(x, r, n, sh) }
|
||||
each{try, rcsh_vals}
|
||||
}
|
||||
|
||||
rep_const_part4_sub(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i8, sh3:[32]i8) : void = {
|
||||
def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}}
|
||||
rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 1, 4}, rcsh4_dom}}}
|
||||
rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}}
|
||||
rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = {
|
||||
shp := *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}
|
||||
def double{x} = { s:=shuf{[4]u64, x, 4b3120}; s+=s; each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} }
|
||||
sh0 := load{shp, 0}; sh1:=sh0; sh2:=sh0; sh3:=sh0
|
||||
if (elbytes<=2) sh1=load{shp,1}; else tup{sh0,sh1}=double{sh0};
|
||||
if (elbytes<=1) tup{sh2,sh3}=each{bind{load,shp},tup{2,3}};
|
||||
else { t:=sh1; tup{sh0,sh1}=double{sh0}; tup{sh2,sh3}=double{t} }
|
||||
def h = 4; def sh = tup{sh0,sh1,sh2,sh3}
|
||||
def V = [32]i8
|
||||
def step = vcount{V} # Bytes written
|
||||
@ -165,11 +172,6 @@ rep_const_part4_sub(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64, sh0:[32]i8, sh1:[3
|
||||
q := (re+step) - r
|
||||
if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s}
|
||||
}
|
||||
def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & wv >= 11} = {
|
||||
def T = eltype{V}
|
||||
def sh = get_shuf_Vs{V, wv, 4}
|
||||
apply{bind{call, rep_const_part4_sub, wv, width{T}/8, *i8~~xv, *i8~~rv, n}, sh}
|
||||
}
|
||||
|
||||
rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = {
|
||||
@for (x over n) {
|
||||
@ -195,16 +197,6 @@ rep_const_broadcast{T, V}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = {
|
||||
}
|
||||
}
|
||||
|
||||
# 1+˝∨`⌾⌽0=div|⌜range
|
||||
def makefact{divisor, range} = {
|
||||
def t = table{{a,b}=>0==b%a, divisor, range}
|
||||
fold{+, 1, reverse{scan{|, reverse{t}}}}
|
||||
}
|
||||
def basic_rep = incl{2,7}
|
||||
def fact_size = 128
|
||||
def fact_inds = slice{iota{fact_size},8}
|
||||
def fact_tab = makefact{basic_rep, fact_inds}
|
||||
factors:*u8 = fact_tab
|
||||
rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = {
|
||||
assert{wv>=2}
|
||||
if (wv>=8 and wv<=fact_size) {
|
||||
@ -218,13 +210,29 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = {
|
||||
return{}
|
||||
}
|
||||
}
|
||||
def vn = 256/width{T}
|
||||
def wT = width{T}
|
||||
def vn = 256/wT
|
||||
def V = [vn]T
|
||||
def max_shuffle = min{2*vn, 64}
|
||||
def max_shuffle = 2*vn
|
||||
if (wv <= max_shuffle) {
|
||||
def pr = merge{basic_rep, replicate{(fact_inds<=max_shuffle)&(fact_tab==1), fact_inds}}
|
||||
def try{k} = { if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} }
|
||||
each{try, pr}
|
||||
def specialize{k} = {
|
||||
if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}}
|
||||
}
|
||||
if (wT<=32) {
|
||||
def elbytes = wT/8
|
||||
specialize{2}
|
||||
if (wv <= tupsel{-1,rcsh_vals}) {
|
||||
ri := wv - tupsel{0,rcsh_vals}
|
||||
def ti = elbytes-1 - (elbytes==4)
|
||||
shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri}*3 + ti*wv
|
||||
rep_const_shuffle_full(wv, x, r, n*elbytes, shp)
|
||||
} else {
|
||||
rep_const_shuffle_partial4(wv, elbytes, x, r, n)
|
||||
}
|
||||
} else {
|
||||
assert{max_shuffle <= tupsel{0, fact_inds}}
|
||||
each{specialize, basic_rep}
|
||||
}
|
||||
} else {
|
||||
kv := wv / vn
|
||||
@unroll (k from (max_shuffle/vn) to 4) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user