diff --git a/SingeliClone b/SingeliClone index 4a4cee08..88901dc8 160000 --- a/SingeliClone +++ b/SingeliClone @@ -1 +1 @@ -Subproject commit 4a4cee084d0e0f13593ed1feb678aa85bde02002 +Subproject commit 88901dc87defdb281a1496e402a70afa96b02020 diff --git a/SingeliMake.bqn b/SingeliMake.bqn index 6f19cbd0..48407e30 100755 --- a/SingeliMake.bqn +++ b/SingeliMake.bqn @@ -9,9 +9,10 @@ srcfile ← •file.At srcfileR resfile ← •file.At resfileR tmpfile ← (•file.At buildDir) •file.At filename∾".d.orig" depfile ← (•file.At buildDir) •file.At filename∾".d" +prefix ← "si_"∾ (∧`'.'⊸≠)⊸/ •file.Name srcfile # invoke singeli, which creates the result file -(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ srcfile⟩) •Import siDir •file.At "singeli" +(⟨"-d" ⋄ tmpfile ⋄ "-o" ⋄ resfile ⋄ "-n" ⋄ prefix ⋄ srcfile⟩) •Import siDir •file.At "singeli" # output dependency file deps ← siDir⊸•file.At¨ •file.Lines tmpfile diff --git a/makefile b/makefile index 38b18583..3b448d64 100644 --- a/makefile +++ b/makefile @@ -224,7 +224,7 @@ preSingeliBin: @${MAKE} i_singeli=0 singeli=0 force_build_dir=obj/presingeli f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c slash.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c scan.c neq.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 01c7481d..1b1e4152 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -61,7 +61,7 @@ B tbl_c2(Md1D* d, B w, B x) { B f = d->f; BBB2B fc2 = c2fn(f); if (!EACH_FILLS && isFun(f) && isPervasiveDy(f) && TI(w,arrD1)) { - if (TI(x,arrD1) && xia<80 && wia>130) { + if (TI(x,arrD1) && wia>130 && xia<2560>>arrTypeBitsLog(TY(x))) { Arr* wd = arr_shVec(TI(w,slice)(incG(w), 0, wia)); r = fc2(f, slash_c2(f, m_i32(xia), taga(wd)), shape_c2(f, m_f64(ria), incG(x))); } else if (xia>7) { @@ -108,8 +108,17 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; return homFil2(f, eachd(f, w, x), wf, xf); } +#if SINGELI && __PCLMUL__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/neq.c" + #pragma GCC diagnostic pop +#endif B scan_ne(u64 p, u64* xp, u64 ia) { u64* rp; B r=m_bitarrv(&rp,ia); +#if SINGELI && __PCLMUL__ + clmul_scan_ne(p, xp, rp, BIT_N(ia)); +#else for (usz i = 0; i < BIT_N(ia); i++) { u64 c = xp[i]; u64 r = c ^ (c<<1); @@ -118,6 +127,7 @@ B scan_ne(u64 p, u64* xp, u64 ia) { rp[i] = r; p = -(r>>63); // repeat sign bit } +#endif return r; } diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7cb5a879..f67ca784 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -105,14 +105,19 @@ #endif #if SINGELI -extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3); -extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3); -extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3); -#define avx2_scan_pluswrap_u64(V0,V1,V2,V3) for (usz i=k; i>= (64/wv); + if ((j&(wv-1))==0) xw = xp[++i]; + u64 rw = _pdep_u64(xw, m); + rp[j] = (rw<> (xi%8); + u64 ex = (xw&mt)<>o|(xw&1)); + o += q; + bool oo = o>=wv; xi+=d+oo; o-=wv&-oo; + } + } + } else + #endif if (wv <= 256) { BOOL_REP_XOR_SCAN(wv) } else { BOOL_REP_OVER(wv, xlen) } goto decX_ret; @@ -720,7 +753,11 @@ B slash_c2(B t, B w, B x) { u8 xk = xl-3; void* rv = m_tyarrv(&r, 1<>1<<1) == n & n>0} = lb{n>>1}+1 def lb{n==1} = 0 -def tail{n,x} = x - (x>>n << n) # get the n least significant bits -def bit {k,x} = tail{1,x>>k} << k # get the k-th bit +def tail{n,x} = x & ((1<0==b%a, divisor, range} + fold{+, 1, reverse{scan{|, reverse{t}}}} +} +def basic_rep = incl{2, 7} +def fact_size = 128 +def fact_inds = slice{iota{fact_size},8} +def fact_tab = makefact{basic_rep, fact_inds} +factors:*u8 = fact_tab + +def get_shufs{step, wv, len} = { + def i = iota{len*step} + split{step, (i - i%wv)/wv} +} +def get_shuf_data{wv, len} = get_shufs{32, wv, len} +def get_shuf_data{wv} = get_shuf_data{wv, wv} +def rep_iter_from_sh{sh} = { + def l = tuplen{sh} + def h = l >> 1 + {x, gen} => { + def fs{v, s} = gen{sel{[16]i8, v, s}} + a := shuf{[4]u64, x, 4b1010}; each{bind{fs,a}, slice{sh,0,h}} + if (l%2) fs{x, tupsel{h, sh}} + b := shuf{[4]u64, x, 4b3232}; each{bind{fs,b}, slice{sh,-h}} + } +} +def get_rep_iter{V, wv==2}{x, gen} = { + def s = shuf{[4]u64, x, 4b3120} + each{{q}=>gen{V~~q}, unpackQ{s, s}} +} +def get_rep_iter{V==[4]u64, wv} = { + def step = 4 + def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} } + def sh = each{base4, get_shufs{step, wv, wv}} + {x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh} +} +def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { + def double{x} = { + s:=shuf{[4]u64, x, 4b3120}; s+=s + each{bind{~~,[32]i8},unpackQ{s, s+broadcast{type{s},1}}} + } + def doubles{n,tup} = slice{join{each{double,tup}}, 0, n} + def sh = each{{v}=>{r:=v}, copy{l, broadcast{[32]i8, 0}}} + def tlen{e} = (l+(-l)%e)/e # Length for e bytes, rounded up + def set{i} = { tupsel{i,sh} = each{bind{load,shp},i} } + def ext{e} = { + def m = tlen{2*e}; def n = tlen{e} # m { + s = v + if (j == e) goto{end} + write{s} + }} + setlabel{end} + q := nr & (step-1) + if (q) maskstoreF{rv, maskOf{V, q}, e, s} + } +} +def rep_const_shuffle{V, wv, xv:*V, rv:*V, n:u64} = rep_const_shuffle{V, wv, get_rep_iter{V, wv}, xv, rv, n} + +def rcsh_vals = slice{basic_rep, 1} # Handle 2 specially +rcsh_offs:*u8 = shiftright{0, scan{+,rcsh_vals}} +rcsh_data:*i8 = join{join{each{get_shuf_data, rcsh_vals}}} +rcsh_sub{wv}(elbytes:u64, x:*i8, r:*i8, n:u64, sh:*[32]i8) : void = { + def V = [32]i8 + def st = read_shuf_vecs{wv, elbytes, sh} + rep_const_shuffle{V, wv, rep_iter_from_sh{st}, *V~~x, *V~~r, n} +} + +def rcsh4_dom = replicate{bind{>=,64}, replicate{fact_tab==1, fact_inds}} +rcsh4_dat:*i8 = join{join{each{{wv}=>get_shuf_data{wv, 4}, rcsh4_dom}}} +rchs4_lkup:*i8 = shiftright{0, scan{+, fold{|, table{==, rcsh4_dom, iota{64}}}}} +rep_const_shuffle_partial4(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + def h = 4 + def sh = read_shuf_vecs{h, elbytes, *[32]i8~~rcsh4_dat + 4*load{rchs4_lkup,wv}} + def V = [32]i8 + def step = vcount{V} # Bytes written + def wvb = wv * elbytes + def hs = (h*step) / wvb # Actual step size in argument elements + re := r + n*wvb - h*step + i:u64 = 0 + while (r <= re) { + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + @unroll (j to h) store{*V~~r, j, sel{[16]i8, a, tupsel{j,sh}}} + i += hs*elbytes + r += hs*wvb + } + re += (h-1)*step + a := shuf{[4]u64, load{*V~~(x+i),0}, 4b1010} + s := broadcast{V, 0} + def end = makelabel{} + @unroll (j to h) { + s = sel{[16]i8, a, tupsel{j,sh}} + if (r > re) goto{end} + store{*V~~r, 0, s} + r += step + } + setlabel{end} + q := (re+step) - r + if (q) maskstoreF{*V~~r, maskOf{V, q}, 0, s} +} + +rep_const_shuffle_any(wv:u64, elbytes:u64, x:*i8, r:*i8, n:u64) : void = { + if (wv > tupsel{-1,rcsh_vals}) { + return{rep_const_shuffle_partial4(wv, elbytes, x, r, n)} + } + n *= elbytes + ri := wv - tupsel{0,rcsh_vals} + sh := *[32]i8~~rcsh_data + load{rcsh_offs,ri} + def try{k} = { if (wv==k) rcsh_sub{k}(elbytes, x, r, n, sh) } + each{try, rcsh_vals} +} + +def rep_const_broadcast{T, kv, loop, wv:u64, x:*T, r:*T, n:u64} = { + assert{kv > 0} + def V = [256/width{T}]T + @for (x over n) { + v := broadcast{V, x} + @loop (j to kv) store{*V~~r, j, v} + r += wv + store{*V~~r, -1, v} + } +} +rep_const_broadcast{T, kv }(wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, unroll, wv, x, r, n} +rep_const_broadcast{T}(kv:u64, wv:u64, x:*T, r:*T, n:u64) : void = rep_const_broadcast{T, kv, for , wv, x, r, n} + +rep_const{T}(wv:u64, x:*void, r:*void, n:u64) : void = { + assert{wv>=2} + if (wv>=8 and wv<=fact_size) { + fa := promote{u64, load{factors,wv-8}} + if (fa > 1) { + fi := wv / fa + def t = *void~~(*T~~r + (promote{u64,wv}-fi)*n) + rep_const{T}(fi,x,t,n) + rep_const{T}(fa,t,r,fi*n) + return{} + } + } + def wT = width{T} + def vn = 256/wT + def V = [vn]T + def max_shuffle = 2*vn + if (wv <= max_shuffle) { + def specialize{k} = { + if (wv==k) return{rep_const_shuffle{V, k, *V~~x, *V~~r, n}} + } + specialize{2} + rep_const_shuffle_any(wv, wT/8, *i8~~x, *i8~~r, n) + } else { + kv := wv / vn + @unroll (k from (max_shuffle/vn) to 4) { + if (kv == k) return{rep_const_broadcast{T, k}(wv, *T~~x, *T~~r, n)} + } + rep_const_broadcast{T}(kv, wv, *T~~x, *T~~r, n) + } +} + +'constrep_u8' = rep_const{i8 }; 'constrep_u16' = rep_const{i16} +'constrep_u32' = rep_const{i32}; 'constrep_u64' = rep_const{u64} diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli new file mode 100644 index 00000000..f258032a --- /dev/null +++ b/src/singeli/src/neq.singeli @@ -0,0 +1,34 @@ +include './base' +include './sse3' + +def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} +def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b} + +def loadl{a:T, n & w128i{eltype{T} }} = emit{eltype{T}, '_mm_loadu_si64', a+n} +def storel{a:T, n, v & w128i{eltype{T} }} = emit{void, '_mm_storeu_si64', a+n, v} + +clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { + def V = [2]u64 + m := broadcast{V, mark} + def xor64{a, i, carry} = { # carry is 64-bit broadcasted current total + p := clmul{a, m, i} + t := shr{[16]u8, p, 8} + s := p ^ carry + carry = s ^ t + s + } + xv := *V ~~ x + rv := *V ~~ r + e := words/2; + c := broadcast{V, init} + @for (rv, xv over e) { + rv = apply{unpacklo, (@collect (j to 2) xor64{xv, j, c})} + } + if (words & 1) { + storel{rv,e, clmul{loadl{xv,e}, m, 0} ^ c} + } +} +clmul_scan_ne_bit(init:u64, x:*u64, r:*u64, ia:u64) : void = { + clmul_scan_ne_any(*void~~x, *void~~r, init, ia, -(u64~~1)) +} +'clmul_scan_ne' = clmul_scan_ne_bit diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 899dcb6a..b0e1638c 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -48,4 +48,4 @@ slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = { 'bmipopc_2slash8' = slash2{comp8, i8} 'bmipopc_2slash16' = slash2{comp16, i16} 'bmipopc_1slash8' = slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808} -'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004} \ No newline at end of file +'bmipopc_1slash16' = slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004} diff --git a/src/singeli/src/sse3.singeli b/src/singeli/src/sse3.singeli index 0b4182b2..39d95049 100644 --- a/src/singeli/src/sse3.singeli +++ b/src/singeli/src/sse3.singeli @@ -97,6 +97,9 @@ def __shr{a:T,b & w128s{T, 16} & knum{b}} = emit{T, '_mm_srai_epi16', a, b} def __shr{a:T,b & w128s{T, 32} & knum{b}} = emit{T, '_mm_srai_epi32', a, b} # no 64-bit arithmetic shift :/ +def shl{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bslli_si128', x, n} +def shr{S==[16]u8, x:T, n & w128{T}} = T ~~ emit{T, '_mm_bsrli_si128', x, n} + def min{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_min_epi16', a, b}; def max{a:T,b:T & T==[ 8]i16} = emit{T, '_mm_max_epi16', a, b} def min{a:T,b:T & T==[16]u8 } = emit{T, '_mm_min_epu8', a, b}; def max{a:T,b:T & T==[16]u8 } = emit{T, '_mm_max_epu8', a, b} @@ -174,4 +177,4 @@ def __eq{a:T,b:T & T==[ 2]u64} = emit{[ 2]u64, '_mm_cmpeq_epi64', a, b} # SSE4.2 -def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b} \ No newline at end of file +def __gt{a:T,b:T & T==[ 2]i64} = emit{[ 2]u64, '_mm_cmpgt_epi64', a, b}