From 851f50635eb01d2dd742d05470df4fe7773a9a7b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 26 Sep 2022 10:30:10 -0400 Subject: [PATCH 01/22] Some simplifications in base.singeli --- src/singeli/src/base.singeli | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 1319486b..fb002cd8 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -1,5 +1,6 @@ include 'skin/c' include 'arch/c' +include 'util/kind' oper ~~ reinterpret infix right 55 def Size = u64 @@ -12,8 +13,8 @@ def cdiv{a,b} = (a+b-1)/b def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 def lb{n==1} = 0 -def tail{n,x} = x - (x>>n << n) # get the n least significant bits -def bit {k,x} = tail{1,x>>k} << k # get the k-th bit +def tail{n,x} = x & ((1< Date: Mon, 26 Sep 2022 10:33:54 -0400 Subject: [PATCH 02/22] Singeli shuffle-based code for constant replicates <=7 --- src/builtins/slash.c | 7 ++++ src/singeli/src/slash.singeli | 75 ++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7cb5a879..8a662231 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -720,6 +720,13 @@ B slash_c2(B t, B w, B x) { u8 xk = xl-3; void* rv = m_tyarrv(&r, 1<> 1 + {x, gen} => { + def fs{v, s} = gen{sel{[16]i8, v, s}} + a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}} + fs{x, tupsel{h, sh}} + b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} + } +} +def get_rep_iter{V, wv & wv%2 == 0} = { + def r2 = get_rep_iter{V, 2} + def rk = get_rep_iter{V, wv/2} + {x, gen} => rk{x, {a}=>r2{a,gen}} +} +def get_rep_iter{V, wv==2}{x, gen} = { + def s = shuf{[4]u64, x, 4b3120} + each{{q}=>gen{V~~q}, unpackQ{s, s}} +} +def get_rep_iter{V==[4]u64, wv} = { + def step = 4 + def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} } + def sh = each{base4, get_shufs{step, wv}} + {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} +} + +rep_const{T, wv}(x:*void, r:*void, n:u64) : void = { + def step = 256/width{T} + def V = [step]T + xv := *V~~x + rv := *V~~r + def onreps = get_rep_iter{V, wv} + + nv := n / step + j:u64 = 0 + def write{v} = { store{rv, j, v}; ++j } + @for (xv over nv) onreps{xv, write} + if (nv*step < n) { + nr := n * wv + e := nr / step + s := broadcast{V, 0} + def end = makelabel{} + onreps{load{xv,nv}, {v} => { + s = v + if (j == e) goto{end} + write{s} + }} + setlabel{end} + q := nr & (step-1) + if (q) maskstoreF{rv, maskOf{V, q}, e, s} + } +} +rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { + assert{wv>=2} + @unroll (k from 2 to 8) { + if (wv==k) return{rep_const{T, k}(x, r, n)} + } + def rc = rep_const{T} +} + +'rep_u8' = rep_const{i8 }; 'rep_u16' = rep_const{i16} +'rep_u32' = rep_const{i32}; 'rep_u64' = rep_const{u64} From 15c405e7bf8cb7433af6402704075761b64aa66e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 26 Sep 2022 11:15:37 -0400 Subject: [PATCH 03/22] Factorize constant replicate lengths under 32 --- src/builtins/slash.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 8a662231..a36f2be5 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -717,6 +717,13 @@ B slash_c2(B t, B w, B x) { else { BOOL_REP_OVER(wv, xlen) } goto decX_ret; } else { + #if SINGELI + static const u8 factors[] = {4, 3, 5, 1, 6, 1, 7, 5, 4, 1, 6, 1, 5, 7, 2, 1, 6, 5, 2, 3, 7, 1, 6, 1, 4}; + u8 fa; + if (xlen>=12 && wv>=8 && wv<32 && (fa=factors[wv-8])>1) { + return slash_c2(m_f64(0), m_f64(fa), slash_c2(m_f64(0), m_f64(wv/fa), x)); + } + #endif u8 xk = xl-3; void* rv = m_tyarrv(&r, 1< Date: Mon, 26 Sep 2022 17:19:56 -0400 Subject: [PATCH 04/22] Handle all constant Replicate in Singeli --- src/builtins/slash.c | 16 ++------- src/singeli/src/slash.singeli | 65 +++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index a36f2be5..31d3a0be 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -717,24 +717,14 @@ B slash_c2(B t, B w, B x) { else { BOOL_REP_OVER(wv, xlen) } goto decX_ret; } else { - #if SINGELI - static const u8 factors[] = {4, 3, 5, 1, 6, 1, 7, 5, 4, 1, 6, 1, 5, 7, 2, 1, 6, 5, 2, 3, 7, 1, 6, 1, 4}; - u8 fa; - if (xlen>=12 && wv>=8 && wv<32 && (fa=factors[wv-8])>1) { - return slash_c2(m_f64(0), m_f64(fa), slash_c2(m_f64(0), m_f64(wv/fa), x)); - } - #endif u8 xk = xl-3; void* rv = m_tyarrv(&r, 1< each{{s}=>gen{shuf{V, x, s}}, sh} } - -rep_const{T, wv}(x:*void, r:*void, n:u64) : void = { - def step = 256/width{T} - def V = [step]T - xv := *V~~x - rv := *V~~r +def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64} = { def onreps = get_rep_iter{V, wv} - nv := n / step j:u64 = 0 def write{v} = { store{rv, j, v}; ++j } @@ -112,12 +106,63 @@ rep_const{T, wv}(x:*void, r:*void, n:u64) : void = { if (q) maskstoreF{rv, maskOf{V, q}, e, s} } } + +rep_const_broadcast{T, V}(wv:u64, x:*T, r:*T, n:u64) : void = { + kv := cdiv{wv, vcount{V}} + @for (x over n) { + v := broadcast{V, x} + @for (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} + } +} +rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = { + @for (x over n) { + v := broadcast{V, x} + @unroll (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} + } +} +rep_const_broadcast{T, V, kv==0}(wv:u64, x:*T, r:*T, n:u64) : void = { + @for (x over n-1) { + store{*V~~r, 0, broadcast{V, x}} + r += wv + } + maskstoreF{*V~~r, maskOf{V, wv}, 0, broadcast{V, load{x,n-1}}} +} + +rep_const{T, wv}(x:*void, r:*void, n:u64) : void = { + def step = 256/width{T} + def V = [step]T + if ((step>=16 and wv==17) or wv <= max{step/2, 6}) { + rep_const_shuffle{V, step, wv, *V~~x, *V~~r, n} + } else { + def kv = (wv - wv%step) / step + rep_const_broadcast{T, V, kv}(wv, *T~~x, *T~~r, n) + } +} + +# 1+˝∨`⌾⌽(2+↕6)(0=|⌜)8+↕25 +def makefact{divisor, range} = { + def t = table{{a,b}=>0==b%a, divisor, range} + fold{+, 1, reverse{scan{|, reverse{t}}}} +} +factors:*u8 = makefact{2+iota{6}, 8+iota{25+32}} rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { assert{wv>=2} - @unroll (k from 2 to 8) { - if (wv==k) return{rep_const{T, k}(x, r, n)} + if (wv*width{T} > 8*256 or wv > 64) return{rep_const_broadcast{T,[256/width{T}]T}(wv,x,r,n)} + def pr = tup{2,3,4,5,6,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61} + def try{k} = { if (wv==k) return{rep_const{T, k}(x, r, n)} } + each{try, pr} + k := u32~~wv + fa := promote{u32, load{factors,k-8}} + if (fa > 1) { + fi := promote{u64, k / fa} + def t = *T~~r + (promote{u64,k}-fi)*n + rep_const{T}(fi,x,t,n) + rep_const{T}(fa,t,r,fi*n) } - def rc = rep_const{T} } 'rep_u8' = rep_const{i8 }; 'rep_u16' = rep_const{i16} From 68aeb0f7cb4cac8385717b92e71af6c4fdbb3718 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 27 Sep 2022 15:07:07 -0400 Subject: [PATCH 05/22] Add large-constant replicate method always using 4 shuffles --- src/singeli/src/slash.singeli | 99 ++++++++++++++++++++++++++--------- 1 file changed, 73 insertions(+), 26 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index c2e62333..ba636a31 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -55,13 +55,13 @@ include './avx' include './avx2' include './mask' include 'util/tup' +def idiv{a,b} = (a-a%b)/b def get_shufs{step, wv} = split{step, replicate{wv, iota{step}}} +def expandsel{ex}{s} = make{[32]i8, join{table{+, ex*s, iota{ex}}}} def get_rep_iter{V, wv} = { def step = vcount{V} - def ie = 32 / step # index expansion - def tosel{s} = make{[32]i8, join{table{+, ie*s, iota{ie}}}} - def sh = each{tosel, get_shufs{step, wv}} + def sh = each{expandsel{32 / step}, get_shufs{step, wv}} def h = wv >> 1 {x, gen} => { def fs{v, s} = gen{sel{[16]i8, v, s}} @@ -106,16 +106,39 @@ def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64} = { if (q) maskstoreF{rv, maskOf{V, q}, e, s} } } - -rep_const_broadcast{T, V}(wv:u64, x:*T, r:*T, n:u64) : void = { - kv := cdiv{wv, vcount{V}} - @for (x over n) { - v := broadcast{V, x} - @for (j to kv) store{*V~~r, j, v} - r += wv - store{*V~~r, -1, v} +rep_const_sub{T, V}(wv:u64, x:*T, r:*T, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i8, sh3:[32]i8) : void = { + def sh = tup{sh0,sh1,sh2,sh3} + def h = 4 + def step = vcount{V} + def hs = idiv{h*step, wv} # Actual step size + re := r + n*wv - h*step + i:u64 = 0 + while (r <= re) { + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + @unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}} + i += hs + r += hs*wv } + re += (h-1)*step + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + s := broadcast{V, 0} + def end = makelabel{} + @unroll (j to h) { + s = sel{[16]i8, a, tupsel{j,sh}} + if (r > re) goto{end} + store{*V~~r, 0, s} + r += step + } + setlabel{end} + q := (re+step) - r + if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } +def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64 & wv >= 11} = { + def T = eltype{V} + def sh = each{expandsel{32/step}, slice{get_shufs{step, wv}, 0, 4}} + apply{bind{call, rep_const_sub{T, V}, wv, *T~~xv, *T~~rv, n}, sh} +} + rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = { @for (x over n) { v := broadcast{V, x} @@ -131,14 +154,29 @@ rep_const_broadcast{T, V, kv==0}(wv:u64, x:*T, r:*T, n:u64) : void = { } maskstoreF{*V~~r, maskOf{V, wv}, 0, broadcast{V, load{x,n-1}}} } +rep_const_broadcast{T, V}(wv:u64, x:*T, r:*T, n:u64) : void = { + kv := wv / vcount{V} + if (kv < 4) { + @unroll (k from 0 to 4) { + if (kv == k) rep_const_broadcast{T, V, k}(wv, x, r, n) + } + } else { + @for (x over n) { + v := broadcast{V, x} + @for (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} + } + } +} -rep_const{T, wv}(x:*void, r:*void, n:u64) : void = { +def rep_const{T, wv, x, r, n} = { def step = 256/width{T} def V = [step]T - if ((step>=16 and wv==17) or wv <= max{step/2, 6}) { + if (wv <= max{2*step, 6}) { rep_const_shuffle{V, step, wv, *V~~x, *V~~r, n} } else { - def kv = (wv - wv%step) / step + def kv = idiv{wv, step} rep_const_broadcast{T, V, kv}(wv, *T~~x, *T~~r, n) } } @@ -148,21 +186,30 @@ def makefact{divisor, range} = { def t = table{{a,b}=>0==b%a, divisor, range} fold{+, 1, reverse{scan{|, reverse{t}}}} } -factors:*u8 = makefact{2+iota{6}, 8+iota{25+32}} +def basic_rep = 2+iota{6} +def fact_size = 128 +def fact_inds = slice{iota{fact_size},8} +def fact_tab = makefact{basic_rep, fact_inds} +factors:*u8 = fact_tab rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { assert{wv>=2} - if (wv*width{T} > 8*256 or wv > 64) return{rep_const_broadcast{T,[256/width{T}]T}(wv,x,r,n)} - def pr = tup{2,3,4,5,6,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61} - def try{k} = { if (wv==k) return{rep_const{T, k}(x, r, n)} } - each{try, pr} - k := u32~~wv - fa := promote{u32, load{factors,k-8}} - if (fa > 1) { - fi := promote{u64, k / fa} - def t = *T~~r + (promote{u64,k}-fi)*n - rep_const{T}(fi,x,t,n) - rep_const{T}(fa,t,r,fi*n) + if (wv>=8 and wv<=fact_size) { + k := u32~~wv + fa := promote{u32, load{factors,k-8}} + if (fa > 1) { + fi := promote{u64, k / fa} + def t = *T~~r + (promote{u64,k}-fi)*n + rep_const{T}(fi,x,t,n) + rep_const{T}(fa,t,r,fi*n) + return{} + } } + def vn = 256/width{T} + def max_special = min{4*vn, 64} + if (wv > max_special) return{rep_const_broadcast{T,[vn]T}(wv,x,r,n)} + def pr = merge{basic_rep, replicate{(fact_inds<=max_special)&(fact_tab==1), fact_inds}} + def try{k} = { if (wv==k) return{rep_const{T, k, x, r, n}} } + each{try, pr} } 'rep_u8' = rep_const{i8 }; 'rep_u16' = rep_const{i16} From c430922a7e51cbb0a612b60413c844116a54e8ac Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 27 Sep 2022 17:38:19 -0400 Subject: [PATCH 06/22] Use shared functions and shuffle array for sizes 3 to 7 in constant Replicate --- src/singeli/src/slash.singeli | 85 +++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index ba636a31..123f7320 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -55,26 +55,28 @@ include './avx' include './avx2' include './mask' include 'util/tup' -def idiv{a,b} = (a-a%b)/b +def incl{a,b} = slice{iota{b+1},a} def get_shufs{step, wv} = split{step, replicate{wv, iota{step}}} -def expandsel{ex}{s} = make{[32]i8, join{table{+, ex*s, iota{ex}}}} -def get_rep_iter{V, wv} = { - def step = vcount{V} - def sh = each{expandsel{32 / step}, get_shufs{step, wv}} - def h = wv >> 1 +def get_shuf_data{wv, elbytes} = { + def expand{ex}{s} = join{table{+, ex*s, iota{ex}}} + each{expand{elbytes}, get_shufs{32 / elbytes, wv}} +} +def get_shuf_Vs{V, wv, len} = { + def s = get_shuf_data{wv, width{eltype{V}}/8} + each{bind{make,[32]i8}, slice{s,0,len}} +} +def rep_iter_from_sh{sh} = { + def l = tuplen{sh} + def h = l >> 1 {x, gen} => { def fs{v, s} = gen{sel{[16]i8, v, s}} a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}} - fs{x, tupsel{h, sh}} + if (l%2) fs{x, tupsel{h, sh}} b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} } } -def get_rep_iter{V, wv & wv%2 == 0} = { - def r2 = get_rep_iter{V, 2} - def rk = get_rep_iter{V, wv/2} - {x, gen} => rk{x, {a}=>r2{a,gen}} -} +def get_rep_iter{V, wv} = rep_iter_from_sh{get_shuf_Vs{V, wv, wv}} # Unused def get_rep_iter{V, wv==2}{x, gen} = { def s = shuf{[4]u64, x, 4b3120} each{{q}=>gen{V~~q}, unpackQ{s, s}} @@ -85,8 +87,8 @@ def get_rep_iter{V==[4]u64, wv} = { def sh = each{base4, get_shufs{step, wv}} {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } -def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64} = { - def onreps = get_rep_iter{V, wv} +def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { + def step = vcount{V} nv := n / step j:u64 = 0 def write{v} = { store{rv, j, v}; ++j } @@ -106,18 +108,43 @@ def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64} = { if (q) maskstoreF{rv, maskOf{V, q}, e, s} } } -rep_const_sub{T, V}(wv:u64, x:*T, r:*T, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i8, sh3:[32]i8) : void = { - def sh = tup{sh0,sh1,sh2,sh3} - def h = 4 - def step = vcount{V} - def hs = idiv{h*step, wv} # Actual step size - re := r + n*wv - h*step +def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} + +def rcsh_min = 3 +def rcsh_max = 7 +def rcsh_vals = incl{rcsh_min, rcsh_max} +def rcsh_offs = shiftright{0, scan{+,rcsh_vals}} +def rcsh_table = table{get_shuf_data, rcsh_vals, tup{1,2,4}} +rcsh_data:*i8 = join{join{join{rcsh_table}}} +rcsh_sub{wv}(x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { + def V = [32]i8 + def st = each{bind{load,sh}, iota{wv}} + rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} +} +def rcsh_subs = each{rcsh_sub, rcsh_vals} +def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & width{eltype{V}}<=32 & rcsh_min<=wv & wv<=rcsh_max} = { + def ri = wv - tupsel{0,rcsh_vals} + def eb = width{eltype{V}}/8 + def ti = eb-1 - (eb==4) + tupsel{ri, rcsh_subs}( + *i8~~xv, *i8~~rv, n*(width{eltype{V}}/8), + *[32]i8~~rcsh_data + (tupsel{ri,rcsh_offs}*3 + ti*wv) + ) +} + +rep_const_part4_sub(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i8, sh3:[32]i8) : void = { + def h = 4; def sh = tup{sh0,sh1,sh2,sh3} + def V = [32]i8 + def step = vcount{V} # Bytes written + def wvb = wv * elbytes + def hs = (h*step) / wvb # Actual step size in argument elements + re := r + n*wvb - h*step i:u64 = 0 while (r <= re) { a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} @unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}} - i += hs - r += hs*wv + i += hs*elbytes + r += hs*wvb } re += (h-1)*step a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} @@ -133,10 +160,10 @@ rep_const_sub{T, V}(wv:u64, x:*T, r:*T, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i q := (re+step) - r if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } -def rep_const_shuffle{V, step, wv, xv:*V, rv:*V, n:u64 & wv >= 11} = { +def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & wv >= 11} = { def T = eltype{V} - def sh = each{expandsel{32/step}, slice{get_shufs{step, wv}, 0, 4}} - apply{bind{call, rep_const_sub{T, V}, wv, *T~~xv, *T~~rv, n}, sh} + def sh = get_shuf_Vs{V, wv, 4} + apply{bind{call, rep_const_part4_sub, wv, width{T}/8, *i8~~xv, *i8~~rv, n}, sh} } rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = { @@ -174,19 +201,19 @@ def rep_const{T, wv, x, r, n} = { def step = 256/width{T} def V = [step]T if (wv <= max{2*step, 6}) { - rep_const_shuffle{V, step, wv, *V~~x, *V~~r, n} + rep_const_shuffle{V, wv, *V~~x, *V~~r, n} } else { - def kv = idiv{wv, step} + def kv = (wv-wv%step) / step rep_const_broadcast{T, V, kv}(wv, *T~~x, *T~~r, n) } } -# 1+˝∨`⌾⌽(2+↕6)(0=|⌜)8+↕25 +# 1+˝∨`⌾⌽0=div|⌜range def makefact{divisor, range} = { def t = table{{a,b}=>0==b%a, divisor, range} fold{+, 1, reverse{scan{|, reverse{t}}}} } -def basic_rep = 2+iota{6} +def basic_rep = incl{2,7} def fact_size = 128 def fact_inds = slice{iota{fact_size},8} def fact_tab = makefact{basic_rep, fact_inds} From db3e0a632437baa9a17af8df9be738808e3dd70d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 27 Sep 2022 20:47:36 -0400 Subject: [PATCH 07/22] Move all the algorithm decision logic into main rep_const and simplify --- src/singeli/src/slash.singeli | 48 ++++++++++++++--------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 123f7320..7bdf4f66 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -181,30 +181,12 @@ rep_const_broadcast{T, V, kv==0}(wv:u64, x:*T, r:*T, n:u64) : void = { } maskstoreF{*V~~r, maskOf{V, wv}, 0, broadcast{V, load{x,n-1}}} } -rep_const_broadcast{T, V}(wv:u64, x:*T, r:*T, n:u64) : void = { - kv := wv / vcount{V} - if (kv < 4) { - @unroll (k from 0 to 4) { - if (kv == k) rep_const_broadcast{T, V, k}(wv, x, r, n) - } - } else { - @for (x over n) { - v := broadcast{V, x} - @for (j to kv) store{*V~~r, j, v} - r += wv - store{*V~~r, -1, v} - } - } -} - -def rep_const{T, wv, x, r, n} = { - def step = 256/width{T} - def V = [step]T - if (wv <= max{2*step, 6}) { - rep_const_shuffle{V, wv, *V~~x, *V~~r, n} - } else { - def kv = (wv-wv%step) / step - rep_const_broadcast{T, V, kv}(wv, *T~~x, *T~~r, n) +rep_const_broadcast{T, V}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = { + @for (x over n) { + v := broadcast{V, x} + @for (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} } } @@ -232,11 +214,19 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { } } def vn = 256/width{T} - def max_special = min{4*vn, 64} - if (wv > max_special) return{rep_const_broadcast{T,[vn]T}(wv,x,r,n)} - def pr = merge{basic_rep, replicate{(fact_inds<=max_special)&(fact_tab==1), fact_inds}} - def try{k} = { if (wv==k) return{rep_const{T, k, x, r, n}} } - each{try, pr} + def V = [vn]T + def max_shuffle = min{2*vn, 64} + if (wv <= max_shuffle) { + def pr = merge{basic_rep, replicate{(fact_inds<=max_shuffle)&(fact_tab==1), fact_inds}} + def try{k} = { if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} } + each{try, pr} + } else { + kv := wv / vn + @unroll (k from (max_shuffle/vn) to 4) { + if (kv == k) return{rep_const_broadcast{T, V, k}(wv, x, r, n)} + } + rep_const_broadcast{T, V}(kv, wv, x, r, n) + } } 'rep_u8' = rep_const{i8 }; 'rep_u16' = rep_const{i16} From 9947de0146060c6866052c442cea8ecb3bc20376 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 27 Sep 2022 21:29:24 -0400 Subject: [PATCH 08/22] Performance improvements when compiling slash.singeli --- src/singeli/src/slash.singeli | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 7bdf4f66..8a2eaab4 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -57,14 +57,19 @@ include './mask' include 'util/tup' def incl{a,b} = slice{iota{b+1},a} -def get_shufs{step, wv} = split{step, replicate{wv, iota{step}}} -def get_shuf_data{wv, elbytes} = { - def expand{ex}{s} = join{table{+, ex*s, iota{ex}}} - each{expand{elbytes}, get_shufs{32 / elbytes, wv}} +def get_shufs{step, wv, len} = { + def i = iota{len*step} + split{step, (i - i%wv)/wv} } +def get_shuf_data{wv, elbytes, len} = { + def expand{ex}{s} = join{table{+, ex*s, iota{ex}}} + def expand{ex==1}{s} = s + each{expand{elbytes}, get_shufs{32 / elbytes, wv, len}} +} +def get_shuf_data{wv, b} = get_shuf_data{wv, b, wv} def get_shuf_Vs{V, wv, len} = { - def s = get_shuf_data{wv, width{eltype{V}}/8} - each{bind{make,[32]i8}, slice{s,0,len}} + def eb = width{eltype{V}}/8 + each{bind{make,[32]i8}, get_shuf_data{wv, eb, len}} } def rep_iter_from_sh{sh} = { def l = tuplen{sh} @@ -84,7 +89,7 @@ def get_rep_iter{V, wv==2}{x, gen} = { def get_rep_iter{V==[4]u64, wv} = { def step = 4 def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} } - def sh = each{base4, get_shufs{step, wv}} + def sh = each{base4, get_shufs{step, wv, wv}} {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { From d28a74b57233559e60b5eaeb530ffbc4af19fa21 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 28 Sep 2022 13:22:55 -0400 Subject: [PATCH 09/22] Move 4-shuffle data to a table --- src/singeli/src/slash.singeli | 86 +++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 8a2eaab4..1d71e724 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -57,6 +57,17 @@ include './mask' include 'util/tup' def incl{a,b} = slice{iota{b+1},a} +# 1+˝∨`⌾⌽0=div|⌜range +def makefact{divisor, range} = { + def t = table{{a,b}=>0==b%a, divisor, range} + fold{+, 1, reverse{scan{|, reverse{t}}}} +} +def basic_rep = incl{2,7} +def fact_size = 128 +def fact_inds = slice{iota{fact_size},8} +def fact_tab = makefact{basic_rep, fact_inds} +factors:*u8 = fact_tab + def get_shufs{step, wv, len} = { def i = iota{len*step} split{step, (i - i%wv)/wv} @@ -67,10 +78,6 @@ def get_shuf_data{wv, elbytes, len} = { each{expand{elbytes}, get_shufs{32 / elbytes, wv, len}} } def get_shuf_data{wv, b} = get_shuf_data{wv, b, wv} -def get_shuf_Vs{V, wv, len} = { - def eb = width{eltype{V}}/8 - each{bind{make,[32]i8}, get_shuf_data{wv, eb, len}} -} def rep_iter_from_sh{sh} = { def l = tuplen{sh} def h = l >> 1 @@ -81,7 +88,6 @@ def rep_iter_from_sh{sh} = { b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} } } -def get_rep_iter{V, wv} = rep_iter_from_sh{get_shuf_Vs{V, wv, wv}} # Unused def get_rep_iter{V, wv==2}{x, gen} = { def s = shuf{[4]u64, x, 4b3120} each{{q}=>gen{V~~q}, unpackQ{s, s}} @@ -115,10 +121,8 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { } def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} -def rcsh_min = 3 -def rcsh_max = 7 -def rcsh_vals = incl{rcsh_min, rcsh_max} -def rcsh_offs = shiftright{0, scan{+,rcsh_vals}} +def rcsh_vals = incl{3, 7} +rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} def rcsh_table = table{get_shuf_data, rcsh_vals, tup{1,2,4}} rcsh_data:*i8 = join{join{join{rcsh_table}}} rcsh_sub{wv}(x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { @@ -126,18 +130,21 @@ rcsh_sub{wv}(x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { def st = each{bind{load,sh}, iota{wv}} rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} } -def rcsh_subs = each{rcsh_sub, rcsh_vals} -def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & width{eltype{V}}<=32 & rcsh_min<=wv & wv<=rcsh_max} = { - def ri = wv - tupsel{0,rcsh_vals} - def eb = width{eltype{V}}/8 - def ti = eb-1 - (eb==4) - tupsel{ri, rcsh_subs}( - *i8~~xv, *i8~~rv, n*(width{eltype{V}}/8), - *[32]i8~~rcsh_data + (tupsel{ri,rcsh_offs}*3 + ti*wv) - ) +rep_const_shuffle_full(wv:i32, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { + def try{k} = { if (wv==k) rcsh_sub{k}(x, r, n, sh) } + each{try, rcsh_vals} } -rep_const_part4_sub(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64, sh0:[32]i8, sh1:[32]i8, sh2:[32]i8, sh3:[32]i8) : void = { +def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} +rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 1, 4}, rcsh4_dom}}} +rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} +rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + shp := *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv} + def double{x} = { s:=shuf{[4]u64, x, 4b3120}; s+=s; each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} } + sh0 := load{shp, 0}; sh1:=sh0; sh2:=sh0; sh3:=sh0 + if (elbytes<=2) sh1=load{shp,1}; else tup{sh0,sh1}=double{sh0}; + if (elbytes<=1) tup{sh2,sh3}=each{bind{load,shp},tup{2,3}}; + else { t:=sh1; tup{sh0,sh1}=double{sh0}; tup{sh2,sh3}=double{t} } def h = 4; def sh = tup{sh0,sh1,sh2,sh3} def V = [32]i8 def step = vcount{V} # Bytes written @@ -165,11 +172,6 @@ rep_const_part4_sub(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64, sh0:[32]i8, sh1:[3 q := (re+step) - r if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } -def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64 & wv >= 11} = { - def T = eltype{V} - def sh = get_shuf_Vs{V, wv, 4} - apply{bind{call, rep_const_part4_sub, wv, width{T}/8, *i8~~xv, *i8~~rv, n}, sh} -} rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = { @for (x over n) { @@ -195,16 +197,6 @@ rep_const_broadcast{T, V}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = { } } -# 1+˝∨`⌾⌽0=div|⌜range -def makefact{divisor, range} = { - def t = table{{a,b}=>0==b%a, divisor, range} - fold{+, 1, reverse{scan{|, reverse{t}}}} -} -def basic_rep = incl{2,7} -def fact_size = 128 -def fact_inds = slice{iota{fact_size},8} -def fact_tab = makefact{basic_rep, fact_inds} -factors:*u8 = fact_tab rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { assert{wv>=2} if (wv>=8 and wv<=fact_size) { @@ -218,13 +210,29 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { return{} } } - def vn = 256/width{T} + def wT = width{T} + def vn = 256/wT def V = [vn]T - def max_shuffle = min{2*vn, 64} + def max_shuffle = 2*vn if (wv <= max_shuffle) { - def pr = merge{basic_rep, replicate{(fact_inds<=max_shuffle)&(fact_tab==1), fact_inds}} - def try{k} = { if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} } - each{try, pr} + def specialize{k} = { + if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} + } + if (wT<=32) { + def elbytes = wT/8 + specialize{2} + if (wv <= tupsel{-1,rcsh_vals}) { + ri := wv - tupsel{0,rcsh_vals} + def ti = elbytes-1 - (elbytes==4) + shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri}*3 + ti*wv + rep_const_shuffle_full(wv, x, r, n*elbytes, shp) + } else { + rep_const_shuffle_partial4(wv, elbytes, x, r, n) + } + } else { + assert{max_shuffle <= tupsel{0, fact_inds}} + each{specialize, basic_rep} + } } else { kv := wv / vn @unroll (k from (max_shuffle/vn) to 4) { From 4836f70e0d8ea3349a4af513e2d00243acf1b9a6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 28 Sep 2022 14:14:39 -0400 Subject: [PATCH 10/22] Simplify rep_const_broadcast, removing unused kv==0 case --- src/singeli/src/slash.singeli | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 1d71e724..a83958db 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -173,29 +173,18 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } -rep_const_broadcast{T, V, kv}(wv:u64, x:*T, r:*T, n:u64) : void = { +def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { + assert{kv > 0} + def V = [256/width{T}]T @for (x over n) { v := broadcast{V, x} - @unroll (j to kv) store{*V~~r, j, v} - r += wv - store{*V~~r, -1, v} - } -} -rep_const_broadcast{T, V, kv==0}(wv:u64, x:*T, r:*T, n:u64) : void = { - @for (x over n-1) { - store{*V~~r, 0, broadcast{V, x}} - r += wv - } - maskstoreF{*V~~r, maskOf{V, wv}, 0, broadcast{V, load{x,n-1}}} -} -rep_const_broadcast{T, V}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = { - @for (x over n) { - v := broadcast{V, x} - @for (j to kv) store{*V~~r, j, v} + @loop (j to kv) store{*V~~r, j, v} r += wv store{*V~~r, -1, v} } } +rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n} +rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n} rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { assert{wv>=2} @@ -236,9 +225,9 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { } else { kv := wv / vn @unroll (k from (max_shuffle/vn) to 4) { - if (kv == k) return{rep_const_broadcast{T, V, k}(wv, x, r, n)} + if (kv == k) return{rep_const_broadcast{T, k}(wv, x, r, n)} } - rep_const_broadcast{T, V}(kv, wv, x, r, n) + rep_const_broadcast{T}(kv, wv, x, r, n) } } From 09aba9bc371f9f4587eaf4fe23f668ccdd182889 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 28 Sep 2022 17:18:42 -0400 Subject: [PATCH 11/22] slash.singeli is only used with __BMI2__ for now --- src/builtins/slash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 31d3a0be..9374d42d 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -720,7 +720,7 @@ B slash_c2(B t, B w, B x) { u8 xk = xl-3; void* rv = m_tyarrv(&r, 1< Date: Wed, 28 Sep 2022 17:18:55 -0400 Subject: [PATCH 12/22] Adjust Table thresholds for fast constant Replicate --- src/builtins/md1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 01c7481d..be50fdc9 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -61,7 +61,7 @@ B tbl_c2(Md1D* d, B w, B x) { B f = d->f; BBB2B fc2 = c2fn(f); if (!EACH_FILLS && isFun(f) && isPervasiveDy(f) && TI(w,arrD1)) { - if (TI(x,arrD1) && xia<80 && wia>130) { + if (TI(x,arrD1) && wia>130 && xia<2560>>arrTypeBitsLog(TY(x))) { Arr* wd = arr_shVec(TI(w,slice)(incG(w), 0, wia)); r = fc2(f, slash_c2(f, m_i32(xia), taga(wd)), shape_c2(f, m_f64(ria), incG(x))); } else if (xia>7) { From e40728618515a09cdf8df13c85a46743b124593a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 28 Sep 2022 21:56:57 -0400 Subject: [PATCH 13/22] =?UTF-8?q?BMI2-based=20constant=20boolean=20Replica?= =?UTF-8?q?te=20for=20=F0=9D=95=A8<=3D52?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 9374d42d..55319646 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -713,6 +713,34 @@ B slash_c2(B t, B w, B x) { if (xl == 0) { u64* xp = bitarr_ptr(x); u64* rp; r = m_bitarrv(&rp, s); + #if __BMI2__ + if (wv <= 52) { + u64 m = (u64)-1 / (((u64)1<>= (64/wv); + if ((j&(wv-1))==0) xw = xp[++i]; + u64 rw = _pdep_u64(xw, m); + rp[j] = (rw<> (xi%8); + u64 ex = (xw&mt)<>o|(xw&1)); + o += q; + bool oo = o>=wv; xi+=d+oo; o-=wv&-oo; + } + } + } else + #endif if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) } else { BOOL_REP_OVER(wv, xlen) } goto decX_ret; From 29886f355a2825d982df0c2ba6d6cbb811999aff Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 29 Sep 2022 19:58:14 -0400 Subject: [PATCH 14/22] Combine small-replication shuffle vector tables across types --- src/singeli/src/slash.singeli | 53 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index a83958db..a519c2bc 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -72,12 +72,8 @@ def get_shufs{step, wv, len} = { def i = iota{len*step} split{step, (i - i%wv)/wv} } -def get_shuf_data{wv, elbytes, len} = { - def expand{ex}{s} = join{table{+, ex*s, iota{ex}}} - def expand{ex==1}{s} = s - each{expand{elbytes}, get_shufs{32 / elbytes, wv, len}} -} -def get_shuf_data{wv, b} = get_shuf_data{wv, b, wv} +def get_shuf_data{wv, len} = get_shufs{32, wv, len} +def get_shuf_data{wv} = get_shuf_data{wv, wv} def rep_iter_from_sh{sh} = { def l = tuplen{sh} def h = l >> 1 @@ -98,6 +94,24 @@ def get_rep_iter{V==[4]u64, wv} = { def sh = each{base4, get_shufs{step, wv, wv}} {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} } +def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { + def double{x} = { + s:=shuf{[4]u64, x, 4b3120}; s+=s + each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} + } + def doubles{n,tup} = slice{join{each{double,tup}}, 0, n} + def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}} + def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up + def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} } + def ext{e} = { + def m = tlen{2*e}; def n = tlen{e} # m=,64}, replicate{fact_tab==1, fact_inds}} -rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 1, 4}, rcsh4_dom}}} +rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { - shp := *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv} - def double{x} = { s:=shuf{[4]u64, x, 4b3120}; s+=s; each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} } - sh0 := load{shp, 0}; sh1:=sh0; sh2:=sh0; sh3:=sh0 - if (elbytes<=2) sh1=load{shp,1}; else tup{sh0,sh1}=double{sh0}; - if (elbytes<=1) tup{sh2,sh3}=each{bind{load,shp},tup{2,3}}; - else { t:=sh1; tup{sh0,sh1}=double{sh0}; tup{sh2,sh3}=double{t} } - def h = 4; def sh = tup{sh0,sh1,sh2,sh3} + def h = 4 + def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}} def V = [32]i8 def step = vcount{V} # Bytes written def wvb = wv * elbytes @@ -212,9 +220,8 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { specialize{2} if (wv <= tupsel{-1,rcsh_vals}) { ri := wv - tupsel{0,rcsh_vals} - def ti = elbytes-1 - (elbytes==4) - shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri}*3 + ti*wv - rep_const_shuffle_full(wv, x, r, n*elbytes, shp) + shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri} + rep_const_shuffle_full(wv, elbytes, x, r, n*elbytes, shp) } else { rep_const_shuffle_partial4(wv, elbytes, x, r, n) } From 582afe33c983672d2160e19e47be4007c841f16f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 29 Sep 2022 20:12:45 -0400 Subject: [PATCH 15/22] Merge 3 to 7 replicate float with other types: shuffle instead of permute --- src/singeli/src/slash.singeli | 37 +++++++++++++++-------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index a519c2bc..2e1b7c68 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -62,7 +62,7 @@ def makefact{divisor, range} = { def t = table{{a,b}=>0==b%a, divisor, range} fold{+, 1, reverse{scan{|, reverse{t}}}} } -def basic_rep = incl{2,7} +def basic_rep = incl{2, 7} def fact_size = 128 def fact_inds = slice{iota{fact_size},8} def fact_tab = makefact{basic_rep, fact_inds} @@ -108,7 +108,7 @@ def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { if (elbytes<=e) set{slice{iota{n},m}} else slice{sh,0,n} = doubles{n,slice{sh,0,m}} } - set{iota{tlen{4}}}; ext{2}; ext{1} + set{iota{tlen{8}}}; ext{4}; ext{2}; ext{1} sh } @@ -135,7 +135,7 @@ def rep_const_shuffle{V, wv, onreps, xv:*V, rv:*V, n:u64} = { } def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} -def rcsh_vals = incl{3, 7} +def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}} rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { @@ -143,10 +143,6 @@ rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { def st = read_shuf_vecs{wv, elbytes, sh} rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} } -rep_const_shuffle_full(wv:i32, eb:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { - def try{k} = { if (wv==k) rcsh_sub{k}(eb, x, r, n, sh) } - each{try, rcsh_vals} -} def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} @@ -181,6 +177,17 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } +rep_const_shuffle_any(wv:i32, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + if (wv > tupsel{-1,rcsh_vals}) { + return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)} + } + n *= elbytes + ri := wv - tupsel{0,rcsh_vals} + sh := *[32]i8~~rcsh_data + load{rcsh_offs,ri} + def try{k} = { if (wv==k) rcsh_sub{k}(elbytes, x, r, n, sh) } + each{try, rcsh_vals} +} + def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { assert{kv > 0} def V = [256/width{T}]T @@ -215,20 +222,8 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { def specialize{k} = { if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} } - if (wT<=32) { - def elbytes = wT/8 - specialize{2} - if (wv <= tupsel{-1,rcsh_vals}) { - ri := wv - tupsel{0,rcsh_vals} - shp:= *[32]i8~~rcsh_data + load{rcsh_offs,ri} - rep_const_shuffle_full(wv, elbytes, x, r, n*elbytes, shp) - } else { - rep_const_shuffle_partial4(wv, elbytes, x, r, n) - } - } else { - assert{max_shuffle <= tupsel{0, fact_inds}} - each{specialize, basic_rep} - } + specialize{2} + rep_const_shuffle_any(wv, wT/8, x, r, n) } else { kv := wv / vn @unroll (k from (max_shuffle/vn) to 4) { From acc200222d63022d19dddd8ed0277d80aa7374f0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 29 Sep 2022 20:33:09 -0400 Subject: [PATCH 16/22] Move constant Replicate Singeli code to its own file --- makefile | 4 +- src/builtins/constrep.c | 6 + src/builtins/slash.c | 9 +- src/singeli/src/constrep.singeli | 186 +++++++++++++++++++++++++++++++ src/singeli/src/slash.singeli | 186 ------------------------------- 5 files changed, 201 insertions(+), 190 deletions(-) create mode 100644 src/builtins/constrep.c create mode 100644 src/singeli/src/constrep.singeli diff --git a/makefile b/makefile index 38b18583..e4f40c3e 100644 --- a/makefile +++ b/makefile @@ -198,7 +198,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< @@ -224,7 +224,7 @@ preSingeliBin: @${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/constrep.c b/src/builtins/constrep.c new file mode 100644 index 00000000..2d22cfec --- /dev/null +++ b/src/builtins/constrep.c @@ -0,0 +1,6 @@ +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/constrep.c" + #pragma GCC diagnostic pop +#endif diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 55319646..d80dab81 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -111,6 +111,11 @@ extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v #define avx2_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i0==b%a, divisor, range} + fold{+, 1, reverse{scan{|, reverse{t}}}} +} +def basic_rep = incl{2, 7} +def fact_size = 128 +def fact_inds = slice{iota{fact_size},8} +def fact_tab = makefact{basic_rep, fact_inds} +factors:*u8 = fact_tab + +def get_shufs{step, wv, len} = { + def i = iota{len*step} + split{step, (i - i%wv)/wv} +} +def get_shuf_data{wv, len} = get_shufs{32, wv, len} +def get_shuf_data{wv} = get_shuf_data{wv, wv} +def rep_iter_from_sh{sh} = { + def l = tuplen{sh} + def h = l >> 1 + {x, gen} => { + def fs{v, s} = gen{sel{[16]i8, v, s}} + a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}} + if (l%2) fs{x, tupsel{h, sh}} + b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} + } +} +def get_rep_iter{V, wv==2}{x, gen} = { + def s = shuf{[4]u64, x, 4b3120} + each{{q}=>gen{V~~q}, unpackQ{s, s}} +} +def get_rep_iter{V==[4]u64, wv} = { + def step = 4 + def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} } + def sh = each{base4, get_shufs{step, wv, wv}} + {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} +} +def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { + def double{x} = { + s:=shuf{[4]u64, x, 4b3120}; s+=s + each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} + } + def doubles{n,tup} = slice{join{each{double,tup}}, 0, n} + def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}} + def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up + def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} } + def ext{e} = { + def m = tlen{2*e}; def n = tlen{e} # m { + s = v + if (j == e) goto{end} + write{s} + }} + setlabel{end} + q := nr & (step-1) + if (q) maskstoreF{rv, maskOf{V, q}, e, s} + } +} +def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} + +def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially +rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} +rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}} +rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { + def V = [32]i8 + def st = read_shuf_vecs{wv, elbytes, sh} + rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} +} + +def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} +rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} +rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} +rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + def h = 4 + def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}} + def V = [32]i8 + def step = vcount{V} # Bytes written + def wvb = wv * elbytes + def hs = (h*step) / wvb # Actual step size in argument elements + re := r + n*wvb - h*step + i:u64 = 0 + while (r <= re) { + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + @unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}} + i += hs*elbytes + r += hs*wvb + } + re += (h-1)*step + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + s := broadcast{V, 0} + def end = makelabel{} + @unroll (j to h) { + s = sel{[16]i8, a, tupsel{j,sh}} + if (r > re) goto{end} + store{*V~~r, 0, s} + r += step + } + setlabel{end} + q := (re+step) - r + if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} +} + +rep_const_shuffle_any(wv:i32, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + if (wv > tupsel{-1,rcsh_vals}) { + return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)} + } + n *= elbytes + ri := wv - tupsel{0,rcsh_vals} + sh := *[32]i8~~rcsh_data + load{rcsh_offs,ri} + def try{k} = { if (wv==k) rcsh_sub{k}(elbytes, x, r, n, sh) } + each{try, rcsh_vals} +} + +def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { + assert{kv > 0} + def V = [256/width{T}]T + @for (x over n) { + v := broadcast{V, x} + @loop (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} + } +} +rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n} +rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n} + +rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { + assert{wv>=2} + if (wv>=8 and wv<=fact_size) { + k := u32~~wv + fa := promote{u32, load{factors,k-8}} + if (fa > 1) { + fi := promote{u64, k / fa} + def t = *T~~r + (promote{u64,k}-fi)*n + rep_const{T}(fi,x,t,n) + rep_const{T}(fa,t,r,fi*n) + return{} + } + } + def wT = width{T} + def vn = 256/wT + def V = [vn]T + def max_shuffle = 2*vn + if (wv <= max_shuffle) { + def specialize{k} = { + if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} + } + specialize{2} + rep_const_shuffle_any(wv, wT/8, x, r, n) + } else { + kv := wv / vn + @unroll (k from (max_shuffle/vn) to 4) { + if (kv == k) return{rep_const_broadcast{T, k}(wv, x, r, n)} + } + rep_const_broadcast{T}(kv, wv, x, r, n) + } +} + +'constrep_u8' = rep_const{i8 }; 'constrep_u16' = rep_const{i16} +'constrep_u32' = rep_const{i32}; 'constrep_u64' = rep_const{u64} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 2e1b7c68..b0e1638c 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -49,189 +49,3 @@ slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = { 'bmipopc_2slash16' = slash2{comp16, i16} 'bmipopc_1slash8' = slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808} 'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004} - -include './sse3' -include './avx' -include './avx2' -include './mask' -include 'util/tup' -def incl{a,b} = slice{iota{b+1},a} - -# 1+˝∨`⌾⌽0=div|⌜range -def makefact{divisor, range} = { - def t = table{{a,b}=>0==b%a, divisor, range} - fold{+, 1, reverse{scan{|, reverse{t}}}} -} -def basic_rep = incl{2, 7} -def fact_size = 128 -def fact_inds = slice{iota{fact_size},8} -def fact_tab = makefact{basic_rep, fact_inds} -factors:*u8 = fact_tab - -def get_shufs{step, wv, len} = { - def i = iota{len*step} - split{step, (i - i%wv)/wv} -} -def get_shuf_data{wv, len} = get_shufs{32, wv, len} -def get_shuf_data{wv} = get_shuf_data{wv, wv} -def rep_iter_from_sh{sh} = { - def l = tuplen{sh} - def h = l >> 1 - {x, gen} => { - def fs{v, s} = gen{sel{[16]i8, v, s}} - a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}} - if (l%2) fs{x, tupsel{h, sh}} - b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} - } -} -def get_rep_iter{V, wv==2}{x, gen} = { - def s = shuf{[4]u64, x, 4b3120} - each{{q}=>gen{V~~q}, unpackQ{s, s}} -} -def get_rep_iter{V==[4]u64, wv} = { - def step = 4 - def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} } - def sh = each{base4, get_shufs{step, wv, wv}} - {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} -} -def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { - def double{x} = { - s:=shuf{[4]u64, x, 4b3120}; s+=s - each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} - } - def doubles{n,tup} = slice{join{each{double,tup}}, 0, n} - def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}} - def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up - def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} } - def ext{e} = { - def m = tlen{2*e}; def n = tlen{e} # m { - s = v - if (j == e) goto{end} - write{s} - }} - setlabel{end} - q := nr & (step-1) - if (q) maskstoreF{rv, maskOf{V, q}, e, s} - } -} -def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} - -def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially -rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} -rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}} -rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { - def V = [32]i8 - def st = read_shuf_vecs{wv, elbytes, sh} - rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} -} - -def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} -rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} -rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} -rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { - def h = 4 - def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}} - def V = [32]i8 - def step = vcount{V} # Bytes written - def wvb = wv * elbytes - def hs = (h*step) / wvb # Actual step size in argument elements - re := r + n*wvb - h*step - i:u64 = 0 - while (r <= re) { - a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} - @unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}} - i += hs*elbytes - r += hs*wvb - } - re += (h-1)*step - a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} - s := broadcast{V, 0} - def end = makelabel{} - @unroll (j to h) { - s = sel{[16]i8, a, tupsel{j,sh}} - if (r > re) goto{end} - store{*V~~r, 0, s} - r += step - } - setlabel{end} - q := (re+step) - r - if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} -} - -rep_const_shuffle_any(wv:i32, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { - if (wv > tupsel{-1,rcsh_vals}) { - return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)} - } - n *= elbytes - ri := wv - tupsel{0,rcsh_vals} - sh := *[32]i8~~rcsh_data + load{rcsh_offs,ri} - def try{k} = { if (wv==k) rcsh_sub{k}(elbytes, x, r, n, sh) } - each{try, rcsh_vals} -} - -def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { - assert{kv > 0} - def V = [256/width{T}]T - @for (x over n) { - v := broadcast{V, x} - @loop (j to kv) store{*V~~r, j, v} - r += wv - store{*V~~r, -1, v} - } -} -rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n} -rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n} - -rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { - assert{wv>=2} - if (wv>=8 and wv<=fact_size) { - k := u32~~wv - fa := promote{u32, load{factors,k-8}} - if (fa > 1) { - fi := promote{u64, k / fa} - def t = *T~~r + (promote{u64,k}-fi)*n - rep_const{T}(fi,x,t,n) - rep_const{T}(fa,t,r,fi*n) - return{} - } - } - def wT = width{T} - def vn = 256/wT - def V = [vn]T - def max_shuffle = 2*vn - if (wv <= max_shuffle) { - def specialize{k} = { - if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} - } - specialize{2} - rep_const_shuffle_any(wv, wT/8, x, r, n) - } else { - kv := wv / vn - @unroll (k from (max_shuffle/vn) to 4) { - if (kv == k) return{rep_const_broadcast{T, k}(wv, x, r, n)} - } - rep_const_broadcast{T}(kv, wv, x, r, n) - } -} - -'rep_u8' = rep_const{i8 }; 'rep_u16' = rep_const{i16} -'rep_u32' = rep_const{i32}; 'rep_u64' = rep_const{u64} From cad566d547d40699b9ce1bc24e2b3cfadfeb7630 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 29 Sep 2022 20:39:46 -0400 Subject: [PATCH 17/22] Not-equals scan with carry-less multiply --- makefile | 4 ++-- src/builtins/md1.c | 7 +++++++ src/builtins/nescan.c | 6 ++++++ src/singeli/src/neq.singeli | 34 ++++++++++++++++++++++++++++++++++ src/singeli/src/sse3.singeli | 5 ++++- 5 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 src/builtins/nescan.c create mode 100644 src/singeli/src/neq.singeli diff --git a/makefile b/makefile index e4f40c3e..ac53541b 100644 --- a/makefile +++ b/makefile @@ -198,7 +198,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o nescan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< @@ -224,7 +224,7 @@ preSingeliBin: @${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c constrep.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c neq.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/md1.c b/src/builtins/md1.c index be50fdc9..60992716 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -108,8 +108,14 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; return homFil2(f, eachd(f, w, x), wf, xf); } +#if SINGELI +extern void (*const clmul_scan_ne)(uint64_t v0,uint64_t* v1,uint64_t* v2,uint64_t v3); +#endif B scan_ne(u64 p, u64* xp, u64 ia) { u64* rp; B r=m_bitarrv(&rp,ia); +#if SINGELI + clmul_scan_ne(p, xp, rp, BIT_N(ia)); +#else for (usz i = 0; i < BIT_N(ia); i++) { u64 c = xp[i]; u64 r = c ^ (c<<1); @@ -118,6 +124,7 @@ B scan_ne(u64 p, u64* xp, u64 ia) { rp[i] = r; p = -(r>>63); // repeat sign bit } +#endif return r; } diff --git a/src/builtins/nescan.c b/src/builtins/nescan.c new file mode 100644 index 00000000..b0d60751 --- /dev/null +++ b/src/builtins/nescan.c @@ -0,0 +1,6 @@ +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/neq.c" + #pragma GCC diagnostic pop +#endif diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli new file mode 100644 index 00000000..86c60f18 --- /dev/null +++ b/src/singeli/src/neq.singeli @@ -0,0 +1,34 @@ +include './base' +include './sse3' + +def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} +def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b} + +def loadl{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si64', a+n} +def storel{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si64', a+n, v} + +clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { + def V = [2]u64 + m := broadcast{V, mark} + def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total + p := clmul{a, m, i} + t := shr{[16]u8, p, 8} + s := p ^ carry + carry = s ^ t + s + } + xv := *V ~~ x + rv := *V ~~ r + e := words/2; + c := broadcast{V, init} + @for (rv, xv over e) { + rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})} + } + if (words & 1) { + storel{rv,e, clmul{loadl{xv,e}, m, 0} ^ c} + } +} +clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = { + clmul_scan_ne_any(x, r, init, ia, -(u64~~1)) +} +'clmul_scan_ne' = clmul_scan_ne_bit diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 0b4182b2..39d95049 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -97,6 +97,9 @@ def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b} def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} # no 64-bit arithmetic shift :/ +def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} +def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} + def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b} @@ -174,4 +177,4 @@ def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b} # SSE4.2 -def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b} \ No newline at end of file +def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b} From 47cdf02877bac497a45bc67b2a0045d1e21be576 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 30 Sep 2022 11:35:25 -0400 Subject: [PATCH 18/22] Build Singeli files with a filename prefix --- SingeliMake.bqn | 3 ++- src/singeli/src/base.singeli | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/SingeliMake.bqn b/SingeliMake.bqn index 6f19cbd0..48407e30 100755 --- a/SingeliMake.bqn +++ b/SingeliMake.bqn @@ -9,9 +9,10 @@ srcfile ← •file.At srcfileR resfile ← •file.At resfileR tmpfile ← (•file.At buildDir) •file.At filename∾".d.orig" depfile ← (•file.At buildDir) •file.At filename∾".d" +prefix ← "si_"∾ (∧`'.'⊸≠)⊸/ •file.Name srcfile # invoke singeli, which creates the result file -(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ srcfile⟩) •Import siDir •file.At "singeli" +(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ "-n" ⋄ prefix ⋄ srcfile⟩) •Import siDir •file.At "singeli" # output dependency file deps ← siDir⊸•file.At¨ •file.Lines tmpfile diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index fb002cd8..6029454c 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -19,7 +19,7 @@ def bit {k,x} = x & (1< Date: Fri, 30 Sep 2022 11:40:37 -0400 Subject: [PATCH 19/22] Move nescan.c into md1.c and constrep.c into slash.c --- makefile | 2 +- src/builtins/constrep.c | 6 ------ src/builtins/md1.c | 5 ++++- src/builtins/nescan.c | 6 ------ src/builtins/slash.c | 24 ++++++++++++------------ 5 files changed, 17 insertions(+), 26 deletions(-) delete mode 100644 src/builtins/constrep.c delete mode 100644 src/builtins/nescan.c diff --git a/makefile b/makefile index ac53541b..3b448d64 100644 --- a/makefile +++ b/makefile @@ -198,7 +198,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o constrep.o group.o sort.o selfsearch.o nescan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< diff --git a/src/builtins/constrep.c b/src/builtins/constrep.c deleted file mode 100644 index 2d22cfec..00000000 --- a/src/builtins/constrep.c +++ /dev/null @@ -1,6 +0,0 @@ -#if SINGELI - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-variable" - #include "../singeli/gen/constrep.c" - #pragma GCC diagnostic pop -#endif diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 60992716..f3787916 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -109,7 +109,10 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; } #if SINGELI -extern void (*const clmul_scan_ne)(uint64_t v0,uint64_t* v1,uint64_t* v2,uint64_t v3); + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/neq.c" + #pragma GCC diagnostic pop #endif B scan_ne(u64 p, u64* xp, u64 ia) { u64* rp; B r=m_bitarrv(&rp,ia); diff --git a/src/builtins/nescan.c b/src/builtins/nescan.c deleted file mode 100644 index b0d60751..00000000 --- a/src/builtins/nescan.c +++ /dev/null @@ -1,6 +0,0 @@ -#if SINGELI - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-variable" - #include "../singeli/gen/neq.c" - #pragma GCC diagnostic pop -#endif diff --git a/src/builtins/slash.c b/src/builtins/slash.c index d80dab81..f67ca784 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -105,19 +105,19 @@ #endif #if SINGELI -extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); -extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3); -extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); -#define avx2_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i Date: Fri, 30 Sep 2022 20:00:54 +0300 Subject: [PATCH 20/22] update Singeli submodule --- SingeliClone | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingeliClone b/SingeliClone index 4a4cee08..88901dc8 160000 --- a/SingeliClone +++ b/SingeliClone @@ -1 +1 @@ -Subproject commit 4a4cee084d0e0f13593ed1feb678aa85bde02002 +Subproject commit 88901dc87defdb281a1496e402a70afa96b02020 From 82ac059b75ebef3cc81776127bfc7538175ef502 Mon Sep 17 00:00:00 2001 From: dzaima Date: Fri, 30 Sep 2022 20:06:32 +0300 Subject: [PATCH 21/22] =?UTF-8?q?disable=20clmul=20=E2=89=A0`=20when=20pcl?= =?UTF-8?q?mul=20not=20available?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/md1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index f3787916..1b1e4152 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -108,7 +108,7 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; return homFil2(f, eachd(f, w, x), wf, xf); } -#if SINGELI +#if SINGELI && __PCLMUL__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #include "../singeli/gen/neq.c" @@ -116,7 +116,7 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; #endif B scan_ne(u64 p, u64* xp, u64 ia) { u64* rp; B r=m_bitarrv(&rp,ia); -#if SINGELI +#if SINGELI && __PCLMUL__ clmul_scan_ne(p, xp, rp, BIT_N(ia)); #else for (usz i = 0; i < BIT_N(ia); i++) { From ce74e36a19e51a0640296d0c64ece3ae4ed783b0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 30 Sep 2022 20:20:38 -0400 Subject: [PATCH 22/22] Fix lots of missing casts for function calls --- src/singeli/src/constrep.singeli | 17 ++++++++--------- src/singeli/src/neq.singeli | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/singeli/src/constrep.singeli b/src/singeli/src/constrep.singeli index ddac9772..78eb7917 100644 --- a/src/singeli/src/constrep.singeli +++ b/src/singeli/src/constrep.singeli @@ -126,7 +126,7 @@ rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} } -rep_const_shuffle_any(wv:i32, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { +rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { if (wv > tupsel{-1,rcsh_vals}) { return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)} } @@ -150,14 +150,13 @@ def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n} rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n} -rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { +rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { assert{wv>=2} if (wv>=8 and wv<=fact_size) { - k := u32~~wv - fa := promote{u32, load{factors,k-8}} + fa := promote{u64, load{factors,wv-8}} if (fa > 1) { - fi := promote{u64, k / fa} - def t = *T~~r + (promote{u64,k}-fi)*n + fi := wv / fa + def t = *void~~(*T~~r + (promote{u64,wv}-fi)*n) rep_const{T}(fi,x,t,n) rep_const{T}(fa,t,r,fi*n) return{} @@ -172,13 +171,13 @@ rep_const{T}(wv:i32, x:*void, r:*void, n:u64) : void = { if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} } specialize{2} - rep_const_shuffle_any(wv, wT/8, x, r, n) + rep_const_shuffle_any(wv, wT/8, *i8~~x, *i8~~r, n) } else { kv := wv / vn @unroll (k from (max_shuffle/vn) to 4) { - if (kv == k) return{rep_const_broadcast{T, k}(wv, x, r, n)} + if (kv == k) return{rep_const_broadcast{T, k}(wv, *T~~x, *T~~r, n)} } - rep_const_broadcast{T}(kv, wv, x, r, n) + rep_const_broadcast{T}(kv, wv, *T~~x, *T~~r, n) } } diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli index 86c60f18..f258032a 100644 --- a/src/singeli/src/neq.singeli +++ b/src/singeli/src/neq.singeli @@ -29,6 +29,6 @@ clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { } } clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = { - clmul_scan_ne_any(x, r, init, ia, -(u64~~1)) + clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1)) } 'clmul_scan_ne' = clmul_scan_ne_bit