From 29886f355a2825d982df0c2ba6d6cbb811999aff Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 29 Sep 2022 19:58:14 -0400 Subject: [PATCH] Combine small-replication shuffle vector tables across types --- src/singeli/src/slash.singeli | 53 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index a83958db..a519c2bc 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -72,12 +72,8 @@ def get_shufs{step, wv, len} = { def i = iota{len*step} split{step, (i - i%wv)/wv} } -def get_shuf_data{wv, elbytes, len} = { - def expand{ex}{s} = join{table{+, ex*s, iota{ex}}} - def expand{ex==1}{s} = s - each{expand{elbytes}, get_shufs{32 / elbytes, wv, len}} -} -def get_shuf_data{wv, b} = get_shuf_data{wv, b, wv} +def get_shuf_data{wv, len} = get_shufs{32, wv, len} +def get_shuf_data{wv} = get_shuf_data{wv, wv} def rep_iter_from_sh{sh} = { def l = tuplen{sh} def h = l >> 1 @@ -98,6 +94,24 @@ def get_rep_iter{V==[4]u64, wv} = { def sh = each{base4, get_shufs{step, wv, wv}} {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } +def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { + def double{x} = { + s:=shuf{[4]u64, x, 4b3120}; s+=s + each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} + } + def doubles{n,tup} = slice{join{each{double,tup}}, 0, n} + def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}} + def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up + def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} } + def ext{e} = { + def m = tlen{2*e}; def n = tlen{e} # m=,64}, replicate{fact_tab==1, fact_inds}} -rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 1, 4}, rcsh4_dom}}} +rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { - shp := *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv} - def double{x} = { s:=shuf{[4]u64, x, 4b3120}; s+=s; each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} } - sh0 := load{shp, 0}; sh1:=sh0; sh2:=sh0; sh3:=sh0 - if (elbytes<=2) sh1=load{shp,1}; else tup{sh0,sh1}=double{sh0}; - if (elbytes<=1) tup{sh2,sh3}=each{bind{load,shp},tup{2,3}}; - else { t:=sh1; tup{sh0,sh1}=double{sh0}; tup{sh2,sh3}=double{t} } - def h = 4; def sh = tup{sh0,sh1,sh2,sh3} + def h = 4 + def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}} def V = [32]i8 def step = vcount{V} # Bytes written def wvb = wv * elbytes @@ -212,9 +220,8 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { specialize{2} if (wv <= tupsel{-1,rcsh_vals}) { ri := wv - tupsel{0,rcsh_vals} - def ti = elbytes-1 - (elbytes==4) - shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri}*3 + ti*wv - rep_const_shuffle_full(wv, x, r, n*elbytes, shp) + shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri} + rep_const_shuffle_full(wv, elbytes, x, r, n*elbytes, shp) } else { rep_const_shuffle_partial4(wv, elbytes, x, r, n) }