From f315a6d3ea5efb7c9eb523ee78b0429e8b568bbb Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 11 Jul 2023 10:57:15 -0400 Subject: [PATCH 01/41] Emulate pext instruction in Singeli --- build/src/build.bqn | 2 +- src/builtins/slash.c | 21 ++++++++----- src/singeli/src/base.singeli | 3 ++ src/singeli/src/slash.singeli | 56 ++++++++++++++++++++++++++++++++++- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index 9db098e9..bb4d3d4e 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -638,7 +638,7 @@ cachedBin‿linkerCache ← { "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2a."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", - "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" + "xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 731bd562..b39112e0 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -4,7 +4,7 @@ // Boolean 𝕨 (Where/Compress) general case based on result type width // COULD use AVX-512 // Size 1: pext, or bit-at-a-time -// SHOULD emulate pext if unavailable +// Emulate pext if unavailable // COULD return boolean result from Where // Size 8, 16: pdep/pext, or branchless // SHOULD try vector lookup-shuffle if unavailable or old AMD @@ -70,15 +70,18 @@ #define _pdep_u64 vg_pdep_u64 #else #define vg_loadLUT64(p, i) p[i] - #define rand_popc64(X) POPC(X) #endif static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } - #if SINGELI_AVX2 - #define SINGELI_FILE slash - #include "../utils/includeSingeli.h" - #endif +#endif +#if !USE_VALGRIND + #define rand_popc64(X) POPC(X) +#endif + +#if SINGELI + #define SINGELI_FILE slash + #include "../utils/includeSingeli.h" #endif #if SINGELI_AVX2 || SINGELI_NEON @@ -437,13 +440,17 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { default: r = compress_grouped(wp, x, wia, wsum, xt); break; case 0: { u64* xp = bitarr_ptr(x); u64* rp; - #if defined(__BMI2__) + #if defined(__BMI2__) || SINGELI r = m_bitarrv(&rp,wsum+128); a(r)->ia = wsum; u64 cw = 0; // current word u64 ro = 0; // offset in word where next bit should be written; never 64 for (usz i=0; i 1} = { + def h = k>>1 # Increase size from h to k + {x,z} := build{h} + def low = lowbits{k} # Low bit in each new group + if (k <= 3) { + z0 := z & low + zm := z>>1 & low + if (k == 2) tup{ + x - (x>>1 & z0), + z0 + zm + } else tup{ # Faster 1->3 jump, currently unused + x - ((x>>1&mod{low*3}) & (z|z0<<1)) - (x>>2 & (z & zm)), + (z0 + zm) + (z>>2 & low) + } + } else { + # Shift high x group down by low z, then add halves of z + even:T = mod{low*(1<>sh | (x &~ m) + if (2*sh<=k/2) shift{2*sh, o>>1, s} else s + } + tup{ + (x&even) | shift{1, z, x&~even}, + if (k>4) (z + z>>h)&even else ((z&~even)>>h) + (z&even) + } + } + } + # Finally, compose groups with regular shifts + def g = 8 # 12 performs about the same + {b,z} := build{g} + o := z*lowbits{g} # Offsets by prefix sum + def s = 1<> (o>>(sh-g) & s) + fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} +} + +export{'si_pext_u64', pext{u64}} From 1314859a8085a6cc4604532f054e92d0aa7890d0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 11 Jul 2023 15:36:43 -0400 Subject: [PATCH 02/41] clmul-based pext emulation --- src/singeli/src/slash.singeli | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index ce2473b6..ecffa66c 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -2,6 +2,12 @@ include './base' if (hasarch{'BMI2'}) { include './bmi2' } +if (hasarch{'PCLMUL'}) { + include './sse' # PCLMUL implies SSE4.2 + def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} +} else { + def clmul{...x} = assert{'clmul not supported', show{...x}} +} include 'util/tup' def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} @@ -102,4 +108,23 @@ fn pext{T}(x:T, m:T) { fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} } +fn pext{T & hasarch{'PCLMUL'} & T==u64}(xs:T, ms:T) { + def num = lb{width{T}} + def V = [2]T + m := V**ms + x := V**xs & m + d := ~m << 1 # One bit of the position difference at x + c := V**(1<<64-1) + @unroll (i to num) { + def sh = 1 << i + def shift_at{v, s} = { v = (v&~s) | (v&s)>>sh } + p := clmul{d, c, 0} # xor-scan + d = d &~ p # Remove even bits + p &= m + shift_at{m, p} + shift_at{x, p} + } + extract{x, 0} +} + export{'si_pext_u64', pext{u64}} From 871a81ca5fc1466c4bed204c3370eac7a606f361 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 11 Jul 2023 15:43:45 -0400 Subject: [PATCH 03/41] BMI2 pext not-emulation in Singeli --- src/singeli/src/slash.singeli | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index ecffa66c..cfb50a9c 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -127,4 +127,6 @@ fn pext{T & hasarch{'PCLMUL'} & T==u64}(xs:T, ms:T) { extract{x, 0} } +fn pext{T & hasarch{'BMI2'}}(x:T, m:T) = pext{x, m} + export{'si_pext_u64', pext{u64}} From 03d89193073160cfe556c12f1fd12ca3facb3b8f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 11 Jul 2023 19:52:47 -0400 Subject: [PATCH 04/41] Add has='pclmul' option and avoid using broadcast --- build/src/build.bqn | 4 ++-- src/singeli/src/slash.singeli | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index bb4d3d4e..50d157b3 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -216,7 +216,7 @@ po ← { # parsed options has ⇐ Lowercase GetOpt "has" has ↩ ∧⍷has ∾ (⊑"slow-pdep"<⊸∊has)/⋈"bmi2" - {𝕊: "Error: Unsupported 'has' options; options:"∾1↓∾", "⊸∾¨𝕩}_assert_(∧´has∊⊢) "avx2"‿"bmi2"‿"slow-pdep" + {𝕊: "Error: Unsupported 'has' options; options:"∾1↓∾", "⊸∾¨𝕩}_assert_(∧´has∊⊢) "avx2"‿"bmi2"‿"pclmul"‿"slow-pdep" {𝕊: "Error: Cannot have 'has' options on architecture '"∾arch∾"'; add an argument of "∾compat⊑"arch=x86-64"‿"target_arch=x86-64"}_assert_¬ (arch≢"x86-64") ∧ 0≠≠has avx2 ⇐ (arch≡"x86-64") ∧ singeli ∧ native ∨ ⊑"avx2"<⊸∊has @@ -672,7 +672,7 @@ cachedBin‿linkerCache ← { singeliArgs ← po.singeliFlags∾⟨"-l", "gen="∾AtRoot singeliCache.folder, "-c", "usz=u"∾•Repr po.usz⟩∾{ po.native? ⟨⟩; - "-a" ⋈ ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ∾ ∾ ','⊸∾¨ Uppercase ∊⟜"avx2"‿"bmi2"⊸/po.has + "-a" ⋈ ({"x86-64":"X86_64"; "aarch64":"AARCH64"; "none"} po.arch) ∾ ∾ ','⊸∾¨ Uppercase ∊⟜"avx2"‿"bmi2"‿"pclmul"⊸/po.has } {𝕊: "Singeli args: "∾•Repr singeliArgs} _verboseLog @ singeliObjs ↩ {MakeSingeliInv ⟨singeliArgs, {𝕊:UpdateSubmodule po.singeliDir}, singeliCache, 𝕩, "src/singeli/src/"•file.At 𝕩∾".singeli", (𝕩≡"dyarith")/⟨gaRule⟩⟩}¨ 1⊑¨singeliMap diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index cfb50a9c..2c2ab692 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -110,11 +110,11 @@ fn pext{T}(x:T, m:T) { fn pext{T & hasarch{'PCLMUL'} & T==u64}(xs:T, ms:T) { def num = lb{width{T}} - def V = [2]T - m := V**ms - x := V**xs & m + def vec{s} = make{[2]T, s, 0} + m := vec{ms} + x := vec{xs} & m d := ~m << 1 # One bit of the position difference at x - c := V**(1<<64-1) + c := vec{1<<64-1} @unroll (i to num) { def sh = 1 << i def shift_at{v, s} = { v = (v&~s) | (v&s)>>sh } From 0921fc2c62ed2d2fbad51c8b6460af78e4b6ddd3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 12 Jul 2023 21:26:17 -0400 Subject: [PATCH 05/41] AVX2 compress implementations using permutevar and lookup tables --- src/builtins/slash.c | 20 +++++++++------- src/singeli/src/slash.singeli | 45 ++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index b39112e0..8cde693f 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -486,20 +486,24 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { DENSE; } \ break; } - #if SINGELI_AVX2 && FAST_PDEP - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - #else - case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } + #if SINGELI_AVX2 && FAST_PDEP + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) + case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break; + case 6: if (TI(x,elType)!=el_B) { + WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64)) + } // else follows + #else + case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} + +fn compress32{T==i32}(w:*u64, x:*T, r:*T, l:u64) : void = { + expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}}}} + def V = [8]i32 + @for (w in *u8~~w, x in *V~~x over i to cdiv{l,8}) { + pc := popc{w} + ind := load{itab, w} # pext{base{256,iota{8}}, pdep{promote{u64, w}, base{256,8**1}}*255} + s := sel{[16]i8, V~~[4]u64**ind, expander} + store{*V~~r, 0, sel{V, x, s}} + r+= pc + } +} + +i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} + +fn compress64{T==i64}(wp:*u64, x:*T, r:*T, l:u64) : void = { + def V = [8]u32 + expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>1}}} + trail := make{V, iota{8}%2} + def step{w,i} = { + pc := popc{w} + s := trail | sel{[16]i8, V**load{i64tab,w}, expander} + store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} + r+= pc + } + @for (w in *u8~~wp over i to cdiv{l,8}) { + step{w&0xf, 2*i} + step{w>>4, 2*i+1} + } +} + +if (hasarch{'AVX2'}) { +export{'avx2_2slash32', compress32{i32}} +export{'avx2_2slash64', compress64{i64}} +} + # pext, or boolean compress fn pext{T}(x:T, m:T) { def w = width{T} From 9010f648bb77d14fa03c52a6f829a104a3d423d0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 12 Jul 2023 21:44:38 -0400 Subject: [PATCH 06/41] Unify permutevar-based compress implementations --- src/singeli/src/slash.singeli | 38 +++++++++++++++-------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 95f317b9..62633484 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -67,41 +67,35 @@ export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x08080808080808 export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} } -itab:*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} - -fn compress32{T==i32}(w:*u64, x:*T, r:*T, l:u64) : void = { - expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}}}} - def V = [8]i32 - @for (w in *u8~~w, x in *V~~x over i to cdiv{l,8}) { - pc := popc{w} - ind := load{itab, w} # pext{base{256,iota{8}}, pdep{promote{u64, w}, base{256,8**1}}*255} - s := sel{[16]i8, V~~[4]u64**ind, expander} - store{*V~~r, 0, sel{V, x, s}} - r+= pc - } -} - +itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} -fn compress64{T==i64}(wp:*u64, x:*T, r:*T, l:u64) : void = { +fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = { + def tw = width{T} def V = [8]u32 - expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>1}}} - trail := make{V, iota{8}%2} + expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} + def tab = if (tw==32) itab else i64tab def step{w,i} = { pc := popc{w} - s := trail | sel{[16]i8, V**load{i64tab,w}, expander} + ind := load{tab, w}; def I = type{ind} + s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander} + if (tw==64) s |= make{V, iota{8}%2} store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} r+= pc } @for (w in *u8~~wp over i to cdiv{l,8}) { - step{w&0xf, 2*i} - step{w>>4, 2*i+1} + if (tw==32) { + step{w, i} + } else { + step{w&0xf, 2*i} + step{w>>4, 2*i+1} + } } } if (hasarch{'AVX2'}) { -export{'avx2_2slash32', compress32{i32}} -export{'avx2_2slash64', compress64{i64}} +export{'avx2_2slash32', avx2_compress{i32}} +export{'avx2_2slash64', avx2_compress{i64}} } # pext, or boolean compress From 8f4b1966cb4d8a9a50e003592c2f52a797cf4227 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jul 2023 12:17:47 -0400 Subject: [PATCH 07/41] Non-overwriting AVX2 4-byte and 8-byte compress, using a buffer --- src/builtins/slash.c | 4 ++-- src/singeli/src/slash.singeli | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 8cde693f..619313fe 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -492,9 +492,9 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { #if SINGELI_AVX2 && FAST_PDEP case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break; + case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break; case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64)) + WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum)) } // else follows #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} -fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = { +# Modifies the input variable r +# Assumes iter{} will increment r, by at most write_len +def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { + assert{isreg{r}}; assert{begin==0} + def T = eltype{type{r}}; def tw = width{T} + def ov = write_len-1 + buf := emit{*T, 'alloca', 2*(ov+1)*width{T}/8} + r0 := r + end := r + sum - ov + i:u64 = 0; buf_used:u1 = 0 + def restart = setlabel{} + while (r < end) { + iter{i, vars} + ++i + } + if (not buf_used) { + end += buf - r + ov + if (buf < end) { + r0 = r + r = buf + buf_used = 1; goto{restart} + } + } else { + def vc = 256/tw; + def R = [vc]T + if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + } +} + +fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} @@ -83,7 +114,7 @@ fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = { store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} r+= pc } - @for (w in *u8~~wp over i to cdiv{l,8}) { + @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { if (tw==32) { step{w, i} } else { From d956ba921bb63a06c7c95ec77e0c5316dc3a02c5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jul 2023 13:24:41 -0400 Subject: [PATCH 08/41] Non-overwriting 1-byte and 2-byte compress/where --- src/builtins/slash.c | 33 +++++----- src/singeli/src/slash.singeli | 118 ++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 619313fe..4dbff005 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -163,7 +163,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) { static void where_block_u16(u64* src, u16* dst, usz len, usz sum) { assert(len <= bsp_max); #if SINGELI_AVX2 && FAST_PDEP - if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len); + if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum); #else if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); #endif @@ -236,18 +236,16 @@ static B where(B x, usz xia, u64 s) { usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/8) { - i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); - bmipopc_1slash16(xp, rp, xia); - FINISH_OVERALLOC_A(r, s*2, 16); + i16* rp = m_tyarrv(&r, 2, s, t_i16arr); + bmipopc_1slash16(xp, rp, xia, s); } #else if (s >= xia/4+xia/8) { @@ -278,7 +276,7 @@ static B where(B x, usz xia, u64 s) { } #if SINGELI_AVX2 && FAST_PDEP if (bs >= b/8+b/16) { - bmipopc_1slash16(xp, buf, b); + bmipopc_1slash16(xp, buf, b, bs); for (usz j=0; jxia-i? xia-i : b; - usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2); for (usz j=0; j=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } #if SINGELI_AVX2 && FAST_PDEP - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break; + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); bmipopc_2slash8 (wp, xp, rp, wia, wsum)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); bmipopc_2slash16(wp, xp, rp, wia, wsum)) + case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32 (wp, xp, rp, wia, wsum)) case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum)) + WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64 (wp, xp, rp, wia, wsum)) } // else follows #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i>4} # this runs even if the above step was all that's required, so it'll act on the invalid result of "r+= pc", so we need to overallocate even more to compensate - } -} - -fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = { - xv:= *u64~~x - F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l} -} - -fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = { - x:u64 = iota - F{w, {} => {c:= x; x+= add; c}, r, l} -} - -# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes -if (hasarch{'BMI2'}) { -export{'bmipopc_2slash8', slash2{comp8, i8}} -export{'bmipopc_2slash16', slash2{comp16, i16}} -export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}} -export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} -} - -itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} -i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} - # Modifies the input variable r # Assumes iter{} will increment r, by at most write_len def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { @@ -95,12 +41,70 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { } } else { def vc = 256/tw; - def R = [vc]T - if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } - homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + if (hasarch{'AVX2'} and write_len >= vc) { + def R = [vc]T + if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + } else { + @for (r0, buf over u64~~(end-buf)) r0 = buf + } } } +def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} +def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} + +def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = { + @for_special_buffered{r,8} (w in *u8~~w over sum) { + pc:= popc{w} + storeu{*u64~~r, 0, pext{promote{u64,X{}}, pdep{promote{u64, w}, cast{u64,0x0101010101010101}}*255}} + r+= pc + } +} + +def tab{n,l} = { + def m=n-1; def t=tab{m,l} + def k = (1<>4} + } +} + +fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { + xv:= *u64~~x + F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l, sum} +} + +fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64, sum:u64) : void = { + x:u64 = iota + F{w, {} => {c:= x; x+= add; c}, r, l, sum} +} + +# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes +if (hasarch{'BMI2'}) { +export{'bmipopc_2slash8', slash2{comp8, i8}} +export{'bmipopc_2slash16', slash2{comp16, i16}} +export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}} +export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} +} + +itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} +i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} + fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 From 13b3bcd74b6d5781e1ec412c2ed7fedbab549ec9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jul 2023 16:31:59 -0400 Subject: [PATCH 09/41] Table-less AVX2 1-byte compress --- src/singeli/src/base.singeli | 2 ++ src/singeli/src/slash.singeli | 59 ++++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 86e3c930..a1b4eb97 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -174,6 +174,8 @@ def unpackLo{...x} = assert{'unpackLo not supported', show{...x}} def unpackHi{...x} = assert{'unpackHi not supported', show{...x}} def unpackQ{...x} = assert{'unpackQ not supported', show{...x}} def packQ{...x} = assert{'packQ not supported', show{...x}} +def shl{...x} = assert{'shl not supported', show{...x}} +def shr{...x} = assert{'shr not supported', show{...x}} def __mulhi{...x} = assert{'__mulhi not supported', show{...x}} def fold_addw{...x} = assert{'fold_addw not supported', show{...x}} def vfold{...x} = assert{'vfold not supported', show{...x}} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 2e283656..42971c9e 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -94,12 +94,63 @@ fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64, sum:u64) : void = { F{w, {} => {c:= x; x+= add; c}, r, l, sum} } +def get_comp{T & width{T}==8 } = comp8 +def get_comp{T & width{T}==16} = comp16 +def slash1{T & hasarch{'BMI2'}} = { + def w = width{T} + def n = 64/w + def b = bind{base, 1<fold{+,a}, ... 4**iota{2}} - 4} + def ind4{b} = shiftright{indices{reverse{b}}-iota{fold{+,b}},4**0} + def ind2x2{...b} = base{4, ind4{b}} + itab := mI{flat_table{ind2x2, ... 4**iota{2}}} + + @for_special_buffered{r,32} (w in *u32~~w, x in *[32]T~~x over sum) { + def step{k==1} = { # Unused, ~15% slower + bit := I~~make{[32]u8, 1<<(iota{32}%8)} + sum := I~~(s8{I~~S**w, make{I,iota{32}>>3}}&bit != bit) + tup{sum + shl{[16]u8, sum, 1}, s8{x, io - sum}} + } + def step{k==2} = { + wv := I~~(S**w >> make{S,4*iota{8}}) & I**0xf + sum:= s8{sumtab, wv} + ws := s8{itab, s8{wv, mI{4*(iota{16}%4)}}} + w4 := io + s8{I~~(S~~ws >> make{S,2*(iota{8}%4)}) & I**3, tr4x4} + tup{shl{[16]u8, sum, 3}, s8{x, w4}} + } + def step{k & k>2} = { + {sum, res} := step{k-1} + ik := mI{zlow{k,iota{16}} + (1<<(k-1) - 1)} + ss := s8{sum, ik} + d := io - ss + tup{sum+ss, homBlend{res, s8{res,d}, d > ik}} + } + {_,r16} := step{4} + + store{*[16]T~~r, 0, half{r16, 0}} + store{*[16]T~~(r+popc{w&0xffff}), 0, half{r16, 1}} + r += popc{w} + } +} + # 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes if (hasarch{'BMI2'}) { -export{'bmipopc_2slash8', slash2{comp8, i8}} -export{'bmipopc_2slash16', slash2{comp16, i16}} -export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}} -export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} +export{'bmipopc_2slash8', slash2{i8}} +export{'bmipopc_2slash16', slash2{i16}} +export{'bmipopc_1slash8', slash1{i8}} +export{'bmipopc_1slash16', slash1{i16}} } itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} From 80e93c41c913a2e4e0c6d6eb82b53fd1973fe006 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 14 Jul 2023 18:46:02 -0400 Subject: [PATCH 10/41] Use new undefined{type,len} instead of alloca --- src/singeli/src/slash.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 42971c9e..07fbdace 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -23,7 +23,7 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { assert{isreg{r}}; assert{begin==0} def T = eltype{type{r}}; def tw = width{T} def ov = write_len-1 - buf := emit{*T, 'alloca', 2*(ov+1)*width{T}/8} + buf := undefined{T, 2*(ov+1)} r0 := r end := r + sum - ov i:u64 = 0; buf_used:u1 = 0 From 84a928863b6028f8f67901b22f43b12bc153465c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 14 Jul 2023 19:25:53 -0400 Subject: [PATCH 11/41] AVX-512 compress implementations --- src/singeli/src/slash.singeli | 44 ++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 07fbdace..27f00f37 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -145,18 +145,10 @@ fn slash2{T==i8 & hasarch{'AVX2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = } } -# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes -if (hasarch{'BMI2'}) { -export{'bmipopc_2slash8', slash2{i8}} -export{'bmipopc_2slash16', slash2{i16}} -export{'bmipopc_1slash8', slash1{i8}} -export{'bmipopc_1slash16', slash1{i16}} -} - itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} -fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} @@ -179,9 +171,39 @@ fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = } } +fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { + def f = match { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } + def wt = width{T} + def vl = 512/wt + def V = [vl]T + def wu = max{32,vl} + def load {a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n} + @for (w in *(ty_u{vl})~~w, x in *V~~x over cdiv{l,vl}) { + def I = ty_u{wu} + def emitT{O, name, ...a} = emit{O, merge{'_mm512_',name,'_epi',f{wt}}, ...a} + def to_mask{a} = emit{[vl]u1, merge{'_cvtu',f{wu},'_mask',f{vl}}, a} + m := to_mask{promote{I,w}} + c := popc{w} + # The compress-store instruction performs very poorly on Zen4, + # and is also a lot worse than the following on Tiger Lake + # emitT{void, 'mask_compressstoreu', r, m, x} + cs := cast_i{I,promote{i64,1}<>6 + v := emitT{V, 'mask_compress', x, m, x} + emitT{void, 'mask_storeu', r, to_mask{cs}, v} + r += c + } +} + +if (hasarch{'BMI2'}) { +export{'bmipopc_2slash8', slash2{i8}} +export{'bmipopc_2slash16', slash2{i16}} +export{'bmipopc_1slash8', slash1{i8}} +export{'bmipopc_1slash16', slash1{i16}} +} if (hasarch{'AVX2'}) { -export{'avx2_2slash32', avx2_compress{i32}} -export{'avx2_2slash64', avx2_compress{i64}} +export{'avx2_2slash32', slash2{i32}} +export{'avx2_2slash64', slash2{i64}} } # pext, or boolean compress From f36426031a45254c55d782a860128c66f02a6d9a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 14 Jul 2023 20:44:11 -0400 Subject: [PATCH 12/41] Slightly faster table-less compress method --- src/singeli/src/slash.singeli | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 27f00f37..ec06a9e9 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -118,26 +118,28 @@ fn slash2{T==i8 & hasarch{'AVX2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = itab := mI{flat_table{ind2x2, ... 4**iota{2}}} @for_special_buffered{r,32} (w in *u32~~w, x in *[32]T~~x over sum) { - def step{k==1} = { # Unused, ~15% slower + def step{k==1} = { # Unused, ~10% slower bit := I~~make{[32]u8, 1<<(iota{32}%8)} sum := I~~(s8{I~~S**w, make{I,iota{32}>>3}}&bit != bit) - tup{sum + shl{[16]u8, sum, 1}, s8{x, io - sum}} + tup{sum + shl{[16]u8, sum, 1}, io - sum} } def step{k==2} = { wv := I~~(S**w >> make{S,4*iota{8}}) & I**0xf sum:= s8{sumtab, wv} ws := s8{itab, s8{wv, mI{4*(iota{16}%4)}}} w4 := io + s8{I~~(S~~ws >> make{S,2*(iota{8}%4)}) & I**3, tr4x4} - tup{shl{[16]u8, sum, 3}, s8{x, w4}} + tup{shl{[16]u8, sum, 3}, w4} } def step{k & k>2} = { - {sum, res} := step{k-1} - ik := mI{zlow{k,iota{16}} + (1<<(k-1) - 1)} + def h = k-1 + {sum, res} := step{h} + ik := mI{zlow{k,iota{16}} + (1<>h & 1)} ss := s8{sum, ik} - d := io - ss - tup{sum+ss, homBlend{res, s8{res,d}, d > ik}} + tup{sum+ss, max{res, s8{res & mh, io - ss}}} } - {_,r16} := step{4} + {_,j16} := step{4} + r16 := s8{x, j16} store{*[16]T~~r, 0, half{r16, 0}} store{*[16]T~~(r+popc{w&0xffff}), 0, half{r16, 1}} From f852d12a5c83dab0d8b4160b391c8b5fb9dfafbb Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 15 Jul 2023 11:14:52 -0400 Subject: [PATCH 13/41] Generic Singeli compress implementation --- src/builtins/slash.c | 18 ++++++++++++------ src/singeli/src/slash.singeli | 23 +++++++++++++++-------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 4dbff005..d7953e57 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -477,21 +477,26 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { } \ TFREE(buf) #define COMPRESS_BLOCK(T) COMPRESS_BLOCK_PREP(T, ) + #if SINGELI_AVX2 + #define THR(C) C + #else + #define THR(C) 2 + #endif #define WITH_SPARSE(W, CUTOFF, DENSE) { \ i##W *xp=tyany_ptr(x), *rp; \ - if (wsum=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } - #if SINGELI_AVX2 && FAST_PDEP - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); bmipopc_2slash8 (wp, xp, rp, wia, wsum)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); bmipopc_2slash16(wp, xp, rp, wia, wsum)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32 (wp, xp, rp, wia, wsum)) + #if SINGELI + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); si_2slash8 (wp, xp, rp, wia, wsum)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); si_2slash16(wp, xp, rp, wia, wsum)) + case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); si_2slash32(wp, xp, rp, wia, wsum)) case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64 (wp, xp, rp, wia, wsum)) + WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); si_2slash64(wp, xp, rp, wia, wsum)) } // else follows #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i>6} >> (n&63)) & 1 + @for (x over i to l) { + store{r, 0, x} + r+= bitp_get{w,i} + } +} + def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = { @for_special_buffered{r,8} (w in *u8~~w over sum) { pc:= popc{w} @@ -198,15 +207,13 @@ fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x } if (hasarch{'BMI2'}) { -export{'bmipopc_2slash8', slash2{i8}} -export{'bmipopc_2slash16', slash2{i16}} export{'bmipopc_1slash8', slash1{i8}} export{'bmipopc_1slash16', slash1{i16}} } -if (hasarch{'AVX2'}) { -export{'avx2_2slash32', slash2{i32}} -export{'avx2_2slash64', slash2{i64}} -} +export{'si_2slash8', slash2{i8}} +export{'si_2slash16', slash2{i16}} +export{'si_2slash32', slash2{i32}} +export{'si_2slash64', slash2{i64}} # pext, or boolean compress fn pext{T}(x:T, m:T) { From 824bb04b4d5c52a9888689175e260c772072965e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 15 Jul 2023 11:24:54 -0400 Subject: [PATCH 14/41] Plain SSE2 1-byte compress --- src/singeli/src/slash.singeli | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 55f5b6bf..3c2ca4b6 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -63,6 +63,35 @@ fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { } } +fn slash2{T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { + def U = [16]u8 + k1 := U**1 + @for_special_buffered{r,16} (w in *u16~~w, x0 in *U~~x over sum) { + bm := make{U, 1<<(iota{16}%8)} + rb := make{U, replicate{8,each{bind{cast_i,u8},tup{w,w>>8}}}} + bit := rb&bm == bm # Bits of w expanded to a byte each + x := x0&bit + dif := k1 + bit + # Prefix sum halves of dif + @unroll (k to 3) dif += U~~([2]i64~~dif << (8< 8 - (extract{[8]u16~~dif, j} >> 8), tup{3,7}} + dif = U~~([2]i64~~dif << 8) + # Shift each value in x down by the corresponding one in dif + b := k1 + @unroll (k to 3) { + m := (dif & b) == b # Mask of positions to shift + x = shr{U, x&m, 1< { emit{void, ins, *[8]u8~~r, x}; r+=c }, + tup{'_mm_storel_pi','_mm_storeh_pi'}, + pc + } + } +} + def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = { @for_special_buffered{r,8} (w in *u8~~w over sum) { pc:= popc{w} From f5121f6806f8da64291ceb38f5e9dc07f9db78a4 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 08:19:20 -0400 Subject: [PATCH 15/41] Thresholds for compress methods (versus sparse) defined in Singeli --- src/builtins/slash.c | 25 ++++++++++--------------- src/singeli/src/slash.singeli | 19 +++++++++++++++---- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index d7953e57..179b185e 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -477,36 +477,32 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { } \ TFREE(buf) #define COMPRESS_BLOCK(T) COMPRESS_BLOCK_PREP(T, ) - #if SINGELI_AVX2 - #define THR(C) C - #else - #define THR(C) 2 - #endif #define WITH_SPARSE(W, CUTOFF, DENSE) { \ i##W *xp=tyany_ptr(x), *rp; \ - if (wsum=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ - else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } #if SINGELI - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); si_2slash8 (wp, xp, rp, wia, wsum)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); si_2slash16(wp, xp, rp, wia, wsum)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); si_2slash32(wp, xp, rp, wia, wsum)) + #define DO(W) \ + WITH_SPARSE(W, si_thresh_2slash##W, rp=m_tyarrv(&r,W/8,wsum,xt); si_2slash##W(wp, xp, rp, wia, wsum)) + case 3: DO(8) case 4: DO(16) case 5: DO(32) case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); si_2slash64(wp, xp, rp, wia, wsum)) + DO(64) } // else follows + #undef DO #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ + else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } case 5: BLOCK_OR_GROUPED(i32) break; case 6: if (TI(x,elType)!=el_B) { BLOCK_OR_GROUPED(u64) } + #undef BLOCK_OR_GROUPED #endif #undef WITH_SPARSE - #undef THR else { B xf = getFillR(x); B* xp = arr_bptr(x); @@ -523,7 +519,6 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { } } break; - #undef BLOCK_OR_GROUPED #undef COMPRESS_BLOCK } ur xr = RNK(x); diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 3c2ca4b6..3c6d5286 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -55,6 +55,7 @@ def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # Assumes w is trimmed, so the last 1 appears at index l-1 +def thresh2{T} = 2 fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @for (x over i to l) { @@ -63,6 +64,7 @@ fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { } } +def thresh2{T==i8 & hasarch{'X86_64'}} = 4 fn slash2{T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 @@ -140,8 +142,11 @@ def slash1{T & hasarch{'BMI2'}} = { def b = bind{base, 1< join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} +def thresh2{T==i32 & hasarch{'AVX2'}} = 32 +def thresh2{T==i64 & hasarch{'AVX2'}} = 8 fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 @@ -211,6 +218,10 @@ fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u6 } } +def thresh2{T==i8 & hasarch{'AVX512VBMI2'}} = 256 +def thresh2{T==i16 & hasarch{'AVX512VBMI2'}} = 128 +def thresh2{T==i32 & hasarch{'AVX512F'}} = 64 +def thresh2{T==i64 & hasarch{'AVX512F'}} = 16 fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def f = match { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } def wt = width{T} @@ -239,10 +250,10 @@ if (hasarch{'BMI2'}) { export{'bmipopc_1slash8', slash1{i8}} export{'bmipopc_1slash16', slash1{i16}} } -export{'si_2slash8', slash2{i8}} -export{'si_2slash16', slash2{i16}} -export{'si_2slash32', slash2{i32}} -export{'si_2slash64', slash2{i64}} +export{'si_2slash8' , slash2{i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} +export{'si_2slash16', slash2{i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} +export{'si_2slash32', slash2{i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} +export{'si_2slash64', slash2{i64}}; export{'si_thresh_2slash64', u64~~thresh2{i64}} # pext, or boolean compress fn pext{T}(x:T, m:T) { From b244ba98f54dd364d5c95ab9c4a91cc8e031e41b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 12:40:40 -0400 Subject: [PATCH 16/41] Unify slash1{} and slash2{} --- src/singeli/src/slash.singeli | 54 ++++++++++++++++------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 3c6d5286..988249b8 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -56,7 +56,7 @@ def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # Assumes w is trimmed, so the last 1 appears at index l-1 def thresh2{T} = 2 -fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c==1, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @for (x over i to l) { store{r, 0, x} @@ -65,7 +65,7 @@ fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { } def thresh2{T==i8 & hasarch{'X86_64'}} = 4 -fn slash2{T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c==1, T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 @for_special_buffered{r,16} (w in *u16~~w, x0 in *U~~x over sum) { @@ -123,31 +123,27 @@ def comp16{w:*u64, X, r:*i16, l:u64, sum:u64} = { step{rs+popcRand{h}, w>>4} } } - -fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { - xv:= *u64~~x - F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l, sum} -} - -fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64, sum:u64) : void = { - x:u64 = iota - F{w, {} => {c:= x; x+= add; c}, r, l, sum} -} - def get_comp{T & width{T}==8 } = comp8 def get_comp{T & width{T}==16} = comp16 -def slash1{T & hasarch{'BMI2'}} = { - def w = width{T} - def n = 64/w - def b = bind{base, 1< {c:= loadu{xv}; xv+= 1; c}, r, l, sum} +} + +fn slash{c==0, T & hasarch{'BMI2'}}(w:*u64, r:*T, l:u64, sum:u64) : void = { + def wt = width{T} + def n = 64/wt + def b = bind{base, 1< {c:= x; x+= add; c}, r, l, sum} +} def thresh2{T==i8 & hasarch{'AVX2'}} = 32 -fn slash2{T==i8 & hasarch{'AVX2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c==1, T==i8 & hasarch{'AVX2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def I = [32]i8 def S = [8]u32 def s8 = bind{sel,[16]u8} @@ -195,7 +191,7 @@ i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, def thresh2{T==i32 & hasarch{'AVX2'}} = 32 def thresh2{T==i64 & hasarch{'AVX2'}} = 8 -fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c==1, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} @@ -222,7 +218,7 @@ def thresh2{T==i8 & hasarch{'AVX512VBMI2'}} = 256 def thresh2{T==i16 & hasarch{'AVX512VBMI2'}} = 128 def thresh2{T==i32 & hasarch{'AVX512F'}} = 64 def thresh2{T==i64 & hasarch{'AVX512F'}} = 16 -fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def f = match { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } def wt = width{T} def vl = 512/wt @@ -247,13 +243,13 @@ fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x } if (hasarch{'BMI2'}) { -export{'bmipopc_1slash8', slash1{i8}} -export{'bmipopc_1slash16', slash1{i16}} +export{'bmipopc_1slash8', slash{0, i8}} +export{'bmipopc_1slash16', slash{0, i16}} } -export{'si_2slash8' , slash2{i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} -export{'si_2slash16', slash2{i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} -export{'si_2slash32', slash2{i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} -export{'si_2slash64', slash2{i64}}; export{'si_thresh_2slash64', u64~~thresh2{i64}} +export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} +export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} +export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} +export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh2{i64}} # pext, or boolean compress fn pext{T}(x:T, m:T) { From da5f1faa07beb9fefafd312c1f7b7448970be85c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 17:05:09 -0400 Subject: [PATCH 17/41] Merge BMI2 slash{} cases, with some simplifications --- src/singeli/src/slash.singeli | 65 +++++++++++++++-------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 988249b8..dcebe0a6 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -17,6 +17,8 @@ if (hasarch{'AVX2'}) { include './mask' include 'util/tup' +def arg{c,T} = if (c) *T else tup{} + # Modifies the input variable r # Assumes iter{} will increment r, by at most write_len def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { @@ -94,52 +96,41 @@ fn slash{c==1, T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : } } -def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = { - @for_special_buffered{r,8} (w in *u8~~w over sum) { - pc:= popc{w} - storeu{*u64~~r, 0, pext{promote{u64,X{}}, pdep{promote{u64, w}, cast{u64,0x0101010101010101}}*255}} - r+= pc - } -} - -def tab{n,l} = { - def m=n-1; def t=tab{m,l} +def tab{n,l} = if (n==0) tup{0} else { + def m = n-1 def k = (1<>4} - } -} -def get_comp{T & width{T}==8 } = comp8 -def get_comp{T & width{T}==16} = comp16 - def thresh2{T==i8 & hasarch{'BMI2'}} = 32 def thresh2{T==i16 & hasarch{'BMI2'}} = 16 -fn slash{c==1, T & hasarch{'BMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { - xv:= *u64~~x - get_comp{T}{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l, sum} -} - -fn slash{c==0, T & hasarch{'BMI2'}}(w:*u64, r:*T, l:u64, sum:u64) : void = { +fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def wt = width{T} - def n = 64/wt def b = bind{base, 1< {c:= x; x+= add; c}, r, l, sum} + def X = if (c) { + xv:= *u64~~x + {} => {c:= loadu{xv}; xv+= 1; c} + } else { + def n = 64/wt + x:u64 = b{iota{n}} + def add = b{n**n} + {} => {c:= x; x+= add; c} + } + @for_special_buffered{r,8} (w in *u8~~w over sum) { + pc:= popc{w} + def out{r,e} = storeu{*u64~~r, 0, pext{promote{u64,X{}}, e}} + if (wt == 8) { + out{r, pdep{promote{u64, w}, u64~~b{8**1}}*255} + } else { + def step{r, w} = out{r, vgLoad{c16lut, w}} + h := w&0xf + step{r, h} + step{r+popcRand{h}, w>>4} + } + r+= pc + } } def thresh2{T==i8 & hasarch{'AVX2'}} = 32 From c484f8ee8da64d24c34f8258fc80c4650e73117c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 17:22:26 -0400 Subject: [PATCH 18/41] Avoid UB from shifting by 64 --- src/singeli/src/slash.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index dcebe0a6..9062b1ad 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -225,7 +225,7 @@ fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u # The compress-store instruction performs very poorly on Zen4, # and is also a lot worse than the following on Tiger Lake # emitT{void, 'mask_compressstoreu', r, m, x} - cs := cast_i{I,promote{i64,1}<>6 v := emitT{V, 'mask_compress', x, m, x} emitT{void, 'mask_storeu', r, to_mask{cs}, v} From 4415869496625f71ddf5b516bcbcb16f95c0c2c0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 18:38:27 -0400 Subject: [PATCH 19/41] Generic Singeli where implementation --- src/builtins/slash.c | 36 +++++++++++++++++------------------ src/singeli/src/slash.singeli | 14 +++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 179b185e..4c2020bd 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -162,8 +162,8 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) { static void where_block_u16(u64* src, u16* dst, usz len, usz sum) { assert(len <= bsp_max); - #if SINGELI_AVX2 && FAST_PDEP - if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum); + #if SINGELI + if (sum >= len/si_thresh_1slash16) si_1slash16(src, (i16*)dst, len, sum); #else if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); #endif @@ -235,17 +235,17 @@ static B where(B x, usz xia, u64 s) { u64* xp = bitarr_ptr(x); usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/8) { + #if SINGELI + if (s >= xia/si_thresh_1slash16) { i16* rp = m_tyarrv(&r, 2, s, t_i16arr); - bmipopc_1slash16(xp, rp, xia, s); + si_1slash16(xp, rp, xia, s); } #else if (s >= xia/4+xia/8) { @@ -274,9 +274,9 @@ static B where(B x, usz xia, u64 s) { } else { bs = bit_sum(xp,b); } - #if SINGELI_AVX2 && FAST_PDEP - if (bs >= b/8+b/16) { - bmipopc_1slash16(xp, buf, b, bs); + #if SINGELI + if (bs >= b/si_thresh_1slash16+b/16) { + si_1slash16(xp, buf, b, bs); for (usz j=0; jxia-i? xia-i : b; - usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j>6} >> (n&63)) & 1 - @for (x over i to l) { - store{r, 0, x} + @for (i to l) { + store{r, 0, if (c) load{x,i} else i} r+= bitp_get{w,i} } } @@ -104,6 +105,7 @@ def tab{n,l} = if (n==0) tup{0} else { c16lut:*u64 = tab{4,16} def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i} +def thresh1{T==i16 & hasarch{'BMI2'}} = 8 def thresh2{T==i8 & hasarch{'BMI2'}} = 32 def thresh2{T==i16 & hasarch{'BMI2'}} = 16 fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { @@ -233,10 +235,8 @@ fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u } } -if (hasarch{'BMI2'}) { -export{'bmipopc_1slash8', slash{0, i8}} -export{'bmipopc_1slash16', slash{0, i16}} -} +export{'si_1slash8' , slash{0, i8 }} +export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh1{i16}} export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} From 3bd8d1de68f821fdb60c44fe614ce3eb150fc447 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 20:18:54 -0400 Subject: [PATCH 20/41] Extend all Singeli Compress methods to do Where --- src/builtins/slash.c | 1 + src/singeli/src/slash.singeli | 77 +++++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 4c2020bd..f0e7f133 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -366,6 +366,7 @@ B grade_bool(B x, usz xia, bool up) { B notx = bit_negate(incG(x)); u64* xp0 = bitarr_ptr(notx); u64* xp1 = xp; + u64 q=xia%64; if (q) { usz e=xia/64; u64 m=((u64)1<promote{eltype{T},c},reverse{xs}} + emit{T, mti{'set',T}, ...p} + } + def broadcast{T, v & isvec{T} & 512==width{T}} = { + emit{T, mti{'set1',T}, promote{eltype{T},v}} + } + def __add{a:T,b:T & 512==width{T}} = emit{T, mti{'add',T}, a, b} +} include './mask' include 'util/tup' @@ -57,7 +69,6 @@ def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # Assumes w is trimmed, so the last 1 appears at index l-1 -def thresh1{T} = 2 def thresh2{T} = 2 fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @@ -67,15 +78,29 @@ fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { } } +def getter{c, V, x} = { + if (c) { + i:u64 = 0 + {} => { v:=load{*V~~x, i}; ++i; v } + } else { + def k = vcount{V} + i := make{V, iota{k}} + ii := V**k + {} => { v:=i; i+=ii; v } + } +} + def thresh2{T==i8 & hasarch{'X86_64'}} = 4 -fn slash{c==1, T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c, T==i8 & hasarch{'X86_64'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 - @for_special_buffered{r,16} (w in *u16~~w, x0 in *U~~x over sum) { + def X = getter{c, U, x} + @for_special_buffered{r,16} (w in *u16~~w to sum) { + x := X{} bm := make{U, 1<<(iota{16}%8)} rb := make{U, replicate{8,each{bind{cast_i,u8},tup{w,w>>8}}}} bit := rb&bm == bm # Bits of w expanded to a byte each - x := x0&bit + x &= bit dif := k1 + bit # Prefix sum halves of dif @unroll (k to 3) dif += U~~([2]i64~~dif << (8< { v:=load{*[32]T~~x, i}; ++i; s8{v, j} } + } else { + i := make{I, replicate{16,tup{0,16}}} + ii := I**32 + {j} => { v:=i+j; i+=ii; v } + } + @for_special_buffered{r,32} (w in *u32~~w over sum) { def step{k==1} = { # Unused, ~10% slower bit := I~~make{[32]u8, 1<<(iota{32}%8)} sum := I~~(s8{I~~S**w, make{I,iota{32}>>3}}&bit != bit) @@ -171,7 +204,7 @@ fn slash{c==1, T==i8 & hasarch{'AVX2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : vo tup{sum+ss, max{res, s8{res & mh, io - ss}}} } {_,j16} := step{4} - r16 := s8{x, j16} + r16 := from_ind{j16} store{*[16]T~~r, 0, half{r16, 0}} store{*[16]T~~(r+popc{w&0xffff}), 0, half{r16, 1}} @@ -184,25 +217,26 @@ i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, def thresh2{T==i32 & hasarch{'AVX2'}} = 32 def thresh2{T==i64 & hasarch{'AVX2'}} = 8 -fn slash{c==1, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { +fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 + def X = getter{c, V, x} expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} def tab = if (tw==32) itab else i64tab - def step{w,i} = { + def step{w} = { pc := popc{w} ind := load{tab, w}; def I = type{ind} s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander} if (tw==64) s |= make{V, iota{8}%2} - store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} + store{*V~~r, 0, sel{V, X{}, s}} r+= pc } - @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { + @for_special_buffered{r,8} (w in *u8~~wp to sum) { if (tw==32) { - step{w, i} + step{w} } else { - step{w&0xf, 2*i} - step{w>>4, 2*i+1} + step{w&0xf} + step{w>>4} } } } @@ -211,19 +245,20 @@ def thresh2{T==i8 & hasarch{'AVX512VBMI2'}} = 256 def thresh2{T==i16 & hasarch{'AVX512VBMI2'}} = 128 def thresh2{T==i32 & hasarch{'AVX512F'}} = 64 def thresh2{T==i64 & hasarch{'AVX512F'}} = 16 -fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { - def f = match { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } +fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { + def f = fmtnat def wt = width{T} def vl = 512/wt def V = [vl]T + def X = getter{c, V, x} def wu = max{32,vl} - def load {a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n} - @for (w in *(ty_u{vl})~~w, x in *V~~x over cdiv{l,vl}) { + @for (w in *(ty_u{vl})~~w over cdiv{l,vl}) { def I = ty_u{wu} def emitT{O, name, ...a} = emit{O, merge{'_mm512_',name,'_epi',f{wt}}, ...a} def to_mask{a} = emit{[vl]u1, merge{'_cvtu',f{wu},'_mask',f{vl}}, a} m := to_mask{promote{I,w}} c := popc{w} + x := X{} # The compress-store instruction performs very poorly on Zen4, # and is also a lot worse than the following on Tiger Lake # emitT{void, 'mask_compressstoreu', r, m, x} @@ -236,7 +271,7 @@ fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u } export{'si_1slash8' , slash{0, i8 }} -export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh1{i16}} +export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}} export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} From 8b297ae2dc060cc2f2fd7ebf84c8f1f93b59d99e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 16 Jul 2023 21:32:48 -0400 Subject: [PATCH 21/41] Dedicated 32-bit Singeli where, taking index offset --- src/builtins/slash.c | 13 ++++++------- src/singeli/src/slash.singeli | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index f0e7f133..7dba710a 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -275,9 +275,8 @@ static B where(B x, usz xia, u64 s) { bs = bit_sum(xp,b); } #if SINGELI - if (bs >= b/si_thresh_1slash16+b/16) { - si_1slash16(xp, buf, b, bs); - for (usz j=0; j= b/si_thresh_1slash32) { + si_1slash32(xp, i, rq, b, bs); } #else if (bs >= b/2) { @@ -379,17 +378,17 @@ B grade_bool(B x, usz xia, bool up) { i32* rp0; r = m_i32arrv(&rp0, xia); i32* rp1 = rp0 + l0; if (!up) { i32* t=rp1; rp1=rp0; rp0=t; } - usz b = 256; TALLOC(u8, buf, b); + usz b = 256; u64 xp0[4]; // 4 ≡ b/64 u64* xp1 = xp; for (usz i=0; ixia-i? xia-i : b; - usz s0=bit_sum(xp0,b2); si_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j>6} >> (n&63)) & 1 @for (i to l) { - store{r, 0, if (c) load{x,i} else i} + store{r, 0, if (c) load{x,i} else if (T==i32) cast_i{T,i}+x else i} r+= bitp_get{w,i} } } @@ -85,6 +85,7 @@ def getter{c, V, x} = { } else { def k = vcount{V} i := make{V, iota{k}} + if (eltype{V}==i32) i += V**x ii := V**k {} => { v:=i; i+=ii; v } } @@ -220,15 +221,23 @@ def thresh2{T==i64 & hasarch{'AVX2'}} = 8 fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 - def X = getter{c, V, x} expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} + def from_ind = if (c) { + i:u64 = 0 + {j} => { v:=load{*V~~x, i}; ++i; sel{V, v, j} } + } else if (T==i32) { + def VT = [8]T + i := VT**x + ii := VT**8 + {j} => { v:=i+VT~~j; i+=ii; v } + } def tab = if (tw==32) itab else i64tab def step{w} = { pc := popc{w} ind := load{tab, w}; def I = type{ind} s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander} if (tw==64) s |= make{V, iota{8}%2} - store{*V~~r, 0, sel{V, X{}, s}} + store{*V~~r, 0, from_ind{s}} r+= pc } @for_special_buffered{r,8} (w in *u8~~wp to sum) { @@ -272,6 +281,7 @@ fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, export{'si_1slash8' , slash{0, i8 }} export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}} +export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh2{i32}} export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} From fc187afdf28074027a8ac38d9d5b9da31d892986 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 11:28:04 -0400 Subject: [PATCH 22/41] Extend 1-byte SSE2 Where to 2-byte and 4-byte with unpacked writes --- src/singeli/src/slash.singeli | 62 +++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 1679534b..cf4178be 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -69,7 +69,7 @@ def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # Assumes w is trimmed, so the last 1 appears at index l-1 -def thresh2{T} = 2 +def thresh{c, T} = 2 fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @for (i to l) { @@ -85,18 +85,21 @@ def getter{c, V, x} = { } else { def k = vcount{V} i := make{V, iota{k}} - if (eltype{V}==i32) i += V**x + if (isreg{x}) i += V**cast_i{eltype{V},x} ii := V**k {} => { v:=i; i+=ii; v } } } -def thresh2{T==i8 & hasarch{'X86_64'}} = 4 -fn slash{c, T==i8 & hasarch{'X86_64'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)} = 4 +fn slash{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 def X = getter{c, U, x} - @for_special_buffered{r,16} (w in *u16~~w to sum) { + def make_top{S} = to_el{S,U}**(if (T>width{S}}) + top := each{make_top, replicate{{S}=>S>8}}}} @@ -116,11 +119,22 @@ fn slash{c, T==i8 & hasarch{'X86_64'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) dif = max{dif, shr{U, dif&m, 1< { emit{void, ins, *[8]u8~~r, x}; r+=c }, - tup{'_mm_storel_pi','_mm_storeh_pi'}, - pc + def each_pc{gen, ...par} = each{{...p,c} => { gen{...p}; r+=c }, ...par, pc} + if (T==i8) { # 0==tuplen{top} + def st{ins} = emit{void, ins, *[8]u8~~r, x} + each_pc{st, tup{'_mm_storel_pi','_mm_storeh_pi'}} + } else { + def st{k, v:V} = store{*V~~r, k, v} + def st{v} = if (T==i16) st{0, v} + else each{st, iota{2}, unpack{v, tupsel{1,top}}} + each_pc{st, unpack{[16]i8~~x, tupsel{0,top}}} } + # Increment top vector when i*16 passes width of bottom vector + def inc{{}} = {} + def inc{{t:V, ...ts}} = { + if ((i+1+i_off)%(1<<(elwidth{V}-4)) == 0) { t += V**1; inc{ts} } + } + inc{top} } } @@ -132,8 +146,8 @@ def tab{n,l} = if (n==0) tup{0} else { c16lut:*u64 = tab{4,16} def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i} -def thresh2{T==i8 & hasarch{'BMI2'}} = 32 -def thresh2{T==i16 & hasarch{'BMI2'}} = 16 +def thresh{c, T==i8 & hasarch{'BMI2'}} = 32 +def thresh{c, T==i16 & hasarch{'BMI2'}} = 16 fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def wt = width{T} def b = bind{base, 1< join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} -def thresh2{T==i32 & hasarch{'AVX2'}} = 32 -def thresh2{T==i64 & hasarch{'AVX2'}} = 8 +def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 +def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 @@ -250,10 +264,10 @@ fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64 } } -def thresh2{T==i8 & hasarch{'AVX512VBMI2'}} = 256 -def thresh2{T==i16 & hasarch{'AVX512VBMI2'}} = 128 -def thresh2{T==i32 & hasarch{'AVX512F'}} = 64 -def thresh2{T==i64 & hasarch{'AVX512F'}} = 16 +def thresh{c, T==i8 & hasarch{'AVX512VBMI2'}} = 256 +def thresh{c, T==i16 & hasarch{'AVX512VBMI2'}} = 128 +def thresh{c, T==i32 & hasarch{'AVX512F'}} = 64 +def thresh{c, T==i64 & hasarch{'AVX512F'}} = 16 fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def f = fmtnat def wt = width{T} @@ -280,12 +294,12 @@ fn slash{c, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, } export{'si_1slash8' , slash{0, i8 }} -export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh2{i16}} -export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh2{i32}} -export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} -export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} -export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} -export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh2{i64}} +export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh{0, i16}} +export{'si_1slash32', slash{0, i32}}; export{'si_thresh_1slash32', u64~~thresh{0, i32}} +export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh{1, i8 }} +export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh{1, i16}} +export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh{1, i32}} +export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1, i64}} # pext, or boolean compress fn pext{T}(x:T, m:T) { From 68978f7e107c9235dfdcaec5f491c4e58d430797 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 16:05:06 -0400 Subject: [PATCH 23/41] Table-based 1- and 2-byte Where --- src/builtins/slash.c | 5 +-- src/singeli/src/bmi2.singeli | 3 -- src/singeli/src/slash.singeli | 58 +++++++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7dba710a..28265d19 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -72,12 +72,13 @@ #define vg_loadLUT64(p, i) p[i] #endif - static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } - static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #endif + #if !USE_VALGRIND #define rand_popc64(X) POPC(X) #endif +static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } +static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #if SINGELI #define SINGELI_FILE slash diff --git a/src/singeli/src/bmi2.singeli b/src/singeli/src/bmi2.singeli index 3fdcb7b0..cf13020e 100644 --- a/src/singeli/src/bmi2.singeli +++ b/src/singeli/src/bmi2.singeli @@ -2,6 +2,3 @@ def pdep{x:u64, m:u64} = emit{u64, '_pdep_u64', x, m} def pdep{x:u32, m:u32} = emit{u32, '_pdep_u32', x, m} def pext{x:u64, m:u64} = emit{u64, '_pext_u64', x, m} def pext{x:u32, m:u32} = emit{u32, '_pext_u32', x, m} - -def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones -def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index cf4178be..a5358a6d 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -29,6 +29,26 @@ if (hasarch{'AVX512F'}) { include './mask' include 'util/tup' +def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} +def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} + +def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones +def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} + +def maketab{l,w} = { # Table from l bits to w-bit indices + def lw = l*w + fold{{t,k} => join{each{tup,t,k+(t< join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} -i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} +# TODO avoid calling popcnt if it's not a single instruction +def thresh{c==0, T==i8 & use_table} = 32 +def thresh{c==0, T==i16} = 16 +fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { + def tw = width{T} + def n = 64/tw + def tab = if (tw==8) itab else tab_4_16 + j:u64 = 0 + def inc = base{1<>4} + } + j += inc + r += pc + } +} -def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 -def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 -fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T==i32 & hasarch{'AVX2'} & use_table} = 32 +def thresh{c, T==i64 & hasarch{'AVX2'} } = 8 +fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} From f005169518716c65d165c0ea4b46bf49eb11d9b9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 16:19:05 -0400 Subject: [PATCH 24/41] 1- and 2-byte compress with table and SSSE3 --- src/singeli/src/slash.singeli | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index a5358a6d..f15480f5 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -268,6 +268,23 @@ fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, } } +def thresh{c==1, T==i8 & hasarch{'SSSE3'} & use_table} = 64 +def thresh{c==1, T==i16 & hasarch{'SSSE3'} & use_table} = 32 +fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'} & use_table}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { + def tw = width{T} + def V = [16]i8 + @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { + pc := popc{w} + ind := load{itab, w}; def I = type{ind} + s := V~~make{[2]I, ind,0} + if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} } + res := sel{V, load{*V~~(x+8*i)}, s} + if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}} + else store{*V~~r, 0, res} + r+= pc + } +} + def thresh{c, T==i32 & hasarch{'AVX2'} & use_table} = 32 def thresh{c, T==i64 & hasarch{'AVX2'} } = 8 fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { From 8add95b7ab398571fb216ec982d892c8bba130b1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 16:20:30 -0400 Subject: [PATCH 25/41] Remove BMI2 where/compress methods --- src/singeli/src/slash.singeli | 37 ----------------------------------- 1 file changed, 37 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index f15480f5..82fbc340 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -155,43 +155,6 @@ fn slash{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)}(w:*u64, x:arg{c,T}, } } -def tab{n,l} = if (n==0) tup{0} else { - def m = n-1 - def k = (1< {c:= loadu{xv}; xv+= 1; c} - } else { - def n = 64/wt - x:u64 = b{iota{n}} - def add = b{n**n} - {} => {c:= x; x+= add; c} - } - @for_special_buffered{r,8} (w in *u8~~w over sum) { - pc:= popc{w} - def out{r,e} = storeu{*u64~~r, 0, pext{promote{u64,X{}}, e}} - if (wt == 8) { - out{r, pdep{promote{u64, w}, u64~~b{8**1}}*255} - } else { - def step{r, w} = out{r, vgLoad{c16lut, w}} - h := w&0xf - step{r, h} - step{r+popcRand{h}, w>>4} - } - r+= pc - } -} - def thresh{c, T==i8 & hasarch{'AVX2'}} = 32 fn slash{c, T==i8 & hasarch{'AVX2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def I = [32]i8 From 2f6a8e55a178ab34bbd2c67c5ab06e688a980ada Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 16:31:01 -0400 Subject: [PATCH 26/41] Always use vectors for for_special_buffered buffer copy --- src/singeli/src/slash.singeli | 6 +++--- src/singeli/src/sse.singeli | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 82fbc340..8af48000 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -74,10 +74,10 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { buf_used = 1; goto{restart} } } else { - def vc = 256/tw; - if (hasarch{'AVX2'} and write_len >= vc) { + if (hasarch{'X86_64'} | hasarch{'AARCH64'}) { + def vc = (if (hasarch{'AVX2'}) 256 else 128)/tw; def R = [vc]T - if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + @unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} } else { @for (r0, buf over u64~~(end-buf)) r0 = buf diff --git a/src/singeli/src/sse.singeli b/src/singeli/src/sse.singeli index b48ee71a..e8c7b895 100644 --- a/src/singeli/src/sse.singeli +++ b/src/singeli/src/sse.singeli @@ -47,8 +47,8 @@ def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,32}} = T ~~ emit{[4]f32, '_mm_ble def topBlend{f:T, t:T, m:M & w128{T} & w128i{M,64}} = T ~~ emit{[2]f64, '_mm_blendv_pd', v2d{f}, v2d{t}, v2d{m}} def topBlend{f:T, t:T, m:M & w128{T} & w128i{M, 8}} = T ~~ emit{[16]i8, '_mm_blendv_epi8', v2i{f}, v2i{t}, v2i{m}} # assumes all bits are the same in each mask item -def homBlend{f:T, t:T, m:M & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m} -def homBlend{f:T, t:T, m:M & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m} +def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M} & elwidth{M}!=16} = topBlend{f, t, m} +def homBlend{f:T, t:T, m:M & hasarch{'SSE4.1'} & w128{T} & w128{M,16}} = topBlend{f, t, [16]i8~~m} From 6df283c3795bac58ccfa9a86b2e3fbdf073fdefd Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 17:52:59 -0400 Subject: [PATCH 27/41] Always increment by full popcount instead of two halves when splitting byte --- src/singeli/src/slash.singeli | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 8af48000..2bc8e3f2 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -264,21 +264,22 @@ fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, {j} => { v:=i+VT~~j; i+=ii; v } } def tab = if (tw==32) itab else i64tab - def step{w} = { - pc := popc{w} + def step{r, w} = { ind := load{tab, w}; def I = type{ind} s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander} if (tw==64) s |= make{V, iota{8}%2} store{*V~~r, 0, from_ind{s}} - r+= pc } @for_special_buffered{r,8} (w in *u8~~wp to sum) { + pc := popc{w} if (tw==32) { - step{w} + step{r,w} } else { - step{w&0xf} - step{w>>4} + h := w&0xf + step{r, h} + step{r+popcRand{h}, w>>4} } + r += pc } } From 93e1262864a9a7a7d4bdc60f4277dff7ddc6ee3b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 18 Jul 2023 16:52:08 -0400 Subject: [PATCH 28/41] Avoid using popcount primitive in Singeli if POPCNT isn't present --- src/singeli/src/slash.singeli | 45 +++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 2bc8e3f2..11c6a324 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -35,12 +35,19 @@ def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} -def maketab{l,w} = { # Table from l bits to w-bit indices - def lw = l*w - fold{{t,k} => join{each{tup,t,k+(t< join{each{tup, t, k + t<>(64-w) + 1) & (1<>4} + def rn = if (has_popc) r+popc{w} else 0 + def step{w} = { + i := load{tab, w} + storeu{*u64~~r, 0, j + i} + r += popc_alt{w, i, tw} + j += inc } - j += inc - r += pc + if (tw==8) { step{w} } + else { step{w&0xf}; step{w>>4} } + if (has_popc) r = rn # Shorter dependency chain } } @@ -237,9 +246,9 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'} & use_table}(wp:*u64, x:arg{c,T}, r def tw = width{T} def V = [16]i8 @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { - pc := popc{w} - ind := load{itab, w}; def I = type{ind} - s := V~~make{[2]I, ind,0} + ind := load{itab, w} + pc := popc_alt{w, ind, 8} + s := V~~make{[2]u64, ind,0} if (T==i16) { s+=s; s = V~~unpackLo{s, s+V**1} } res := sel{V, load{*V~~(x+8*i)}, s} if (T==i8) store{*u64~~r, 0, extract{[2]u64~~res, 0}} From b0e0f210c65cbf2eb58991209632fb5100f47e23 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 18 Jul 2023 21:10:39 -0400 Subject: [PATCH 29/41] 2- and 4-byte Where with unpack and table --- src/singeli/src/slash.singeli | 56 ++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 11c6a324..d81e2ef7 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -46,15 +46,13 @@ def maketab{l,w} = { def top = (fold{bind{flat_table,+}, l**iota{2}} - 1)%(1<>(64-w) + 1) & (1<>width{S}}) + top := each{make_top, replicate{{S}=>S>width{S}}) - top := each{make_top, replicate{{S}=>S Date: Tue, 18 Jul 2023 21:35:08 -0400 Subject: [PATCH 30/41] Never use generic dense compress methods if use_table: where is faster --- src/builtins/slash.c | 13 +++++++++---- src/singeli/src/slash.singeli | 13 ++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 28265d19..3cb921ee 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -479,10 +479,15 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { TFREE(buf) #define COMPRESS_BLOCK(T) COMPRESS_BLOCK_PREP(T, ) #define WITH_SPARSE(W, CUTOFF, DENSE) { \ - i##W *xp=tyany_ptr(x), *rp; \ - if (wsum=wia/8 && groups_lt(wp,wia, wia/W)) r = compress_grouped(wp, x, wia, wsum, xt); \ + else { rp=m_tyarrv(&r,W/8,wsum,xt); COMPRESS_BLOCK(i##W); } \ + } \ break; } #if SINGELI #define DO(W) \ diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index d81e2ef7..54b9dfa6 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -95,7 +95,8 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { } # Assumes w is trimmed, so the last 1 appears at index l-1 -def thresh{c, T} = 2 +def fast_where = hasarch{'X86_64'} & use_table +def thresh{c, T} = if (fast_where) 1 else 2 fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @for (i to l) { @@ -133,7 +134,9 @@ def topper{T, U, k, x} = { tup{top, inc} } -def thresh{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)} = 4 +def thresh{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)} = { + if (fast_where) 1 else 4 +} fn slash{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 @@ -248,9 +251,9 @@ fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, } } -def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32 -def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16 -fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==0, T==i16 & hasarch{'X86_64'} & use_table} = 32 +def thresh{c==0, T==i32 & hasarch{'X86_64'} & use_table} = 16 +fn slash{c==0, T & hasarch{'X86_64'} & use_table & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def I = [16]i8 j := I**(if (T==i16) 0 else cast_i{i8,x}) def {top, inctop} = topper{T, I, 8, x} From ba837ba01bc98284ff08afb9353033b329870a11 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 19 Jul 2023 07:22:55 -0400 Subject: [PATCH 31/41] Remove table-less where/compress methods, as they're not competitive --- src/singeli/src/slash.singeli | 130 +++++----------------------------- 1 file changed, 16 insertions(+), 114 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 54b9dfa6..d568a1a9 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -46,12 +46,8 @@ def maketab{l,w} = { def top = (fold{bind{flat_table,+}, l**iota{2}} - 1)%(1<>6} >> (n&63)) & 1 @for (i to l) { @@ -134,104 +130,10 @@ def topper{T, U, k, x} = { tup{top, inc} } -def thresh{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)} = { - if (fast_where) 1 else 4 -} -fn slash{c, T & hasarch{'X86_64'} & T<=(if (c) i8 else i32)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { - def U = [16]u8 - k1 := U**1 - def X = getter{c, U, x} - def {top, inctop} = topper{T, U, 16, x} - @for_special_buffered{r,16} (w in *u16~~w over i to sum) { - x := X{} - bm := make{U, 1<<(iota{16}%8)} - rb := make{U, replicate{8,each{bind{cast_i,u8},tup{w,w>>8}}}} - bit := rb&bm == bm # Bits of w expanded to a byte each - x &= bit - dif := k1 + bit - # Prefix sum halves of dif - @unroll (k to 3) dif += U~~([2]i64~~dif << (8< 8 - (extract{[8]u16~~dif, j} >> 8), tup{3,7}} - dif = U~~([2]i64~~dif << 8) - # Shift each value in x down by the corresponding one in dif - b := k1 - @unroll (k to 3) { - m := (dif & b) == b # Mask of positions to shift - y := shr{U, x&m, 1< { gen{...p}; r+=c }, ...par, pc} - if (T==i8) { # 0==tuplen{top} - def st{ins} = emit{void, ins, *[8]u8~~r, x} - each_pc{st, tup{'_mm_storel_pi','_mm_storeh_pi'}} - } else { - def st{k, v:V} = store{*V~~r, k, v} - def st{v} = if (T==i16) st{0, v} - else each{st, iota{2}, unpack{v, tupsel{1,top}}} - each_pc{st, unpack{[16]i8~~x, tupsel{0,top}}} - } - inctop{i, top} - } -} - -def thresh{c, T==i8 & hasarch{'AVX2'}} = 32 -fn slash{c, T==i8 & hasarch{'AVX2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { - def I = [32]i8 - def S = [8]u32 - def s8 = bind{sel,[16]u8} - def mI{t} = make{I, merge{t,t}} - io := mI{iota{16}} - tr4x4 := mI{join{flip{split{4,iota{16}}}}} - - sumtab := mI{flat_table{{...a}=>fold{+,a}, ... 4**iota{2}} - 4} - def ind4{b} = shiftright{indices{reverse{b}}-iota{fold{+,b}},4**0} - def ind2x2{...b} = base{4, ind4{b}} - itab := mI{flat_table{ind2x2, ... 4**iota{2}}} - - def from_ind = if (c) { - i:u64 = 0 - {j} => { v:=load{*[32]T~~x, i}; ++i; s8{v, j} } - } else { - i := make{I, replicate{16,tup{0,16}}} - ii := I**32 - {j} => { v:=i+j; i+=ii; v } - } - @for_special_buffered{r,32} (w in *u32~~w over sum) { - def step{k==1} = { # Unused, ~10% slower - bit := I~~make{[32]u8, 1<<(iota{32}%8)} - sum := I~~(s8{I~~S**w, make{I,iota{32}>>3}}&bit != bit) - tup{sum + shl{[16]u8, sum, 1}, io - sum} - } - def step{k==2} = { - wv := I~~(S**w >> make{S,4*iota{8}}) & I**0xf - sum:= s8{sumtab, wv} - ws := s8{itab, s8{wv, mI{4*(iota{16}%4)}}} - w4 := io + s8{I~~(S~~ws >> make{S,2*(iota{8}%4)}) & I**3, tr4x4} - tup{shl{[16]u8, sum, 3}, w4} - } - def step{k & k>2} = { - def h = k-1 - {sum, res} := step{h} - ik := mI{zlow{k,iota{16}} + (1<>h & 1)} - ss := s8{sum, ik} - tup{sum+ss, max{res, s8{res & mh, io - ss}}} - } - {_,j16} := step{4} - r16 := from_ind{j16} - - store{*[16]T~~r, 0, half{r16, 0}} - store{*[16]T~~(r+popc{w&0xffff}), 0, half{r16, 1}} - r += popc{w} - } -} - itab_4_16:*u64 = maketab{4,16} -def thresh{c==0, T==i8 & use_table} = 32 -def thresh{c==0, T==i16} = 16 -fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==0, T==i8 } = 32 +def thresh{c==0, T==i16} = 16 +fn slash{c==0, T & T<=i16}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def n = 64/tw def tab = if (tw==8) itab else itab_4_16 @@ -251,9 +153,9 @@ fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, } } -def thresh{c==0, T==i16 & hasarch{'X86_64'} & use_table} = 32 -def thresh{c==0, T==i32 & hasarch{'X86_64'} & use_table} = 16 -fn slash{c==0, T & hasarch{'X86_64'} & use_table & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==0, T==i16 & hasarch{'X86_64'}} = 32 +def thresh{c==0, T==i32 & hasarch{'X86_64'}} = 16 +fn slash{c==0, T & hasarch{'X86_64'} & i16<=T & T<=i32}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def I = [16]i8 j := I**(if (T==i16) 0 else cast_i{i8,x}) def {top, inctop} = topper{T, I, 8, x} @@ -270,9 +172,9 @@ fn slash{c==0, T & hasarch{'X86_64'} & use_table & i16<=T & T<=i32}(w:*u64, x:ar } } -def thresh{c==1, T==i8 & hasarch{'SSSE3'} & use_table} = 64 -def thresh{c==1, T==i16 & hasarch{'SSSE3'} & use_table} = 32 -fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'} & use_table}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c==1, T==i8 & hasarch{'SSSE3'}} = 64 +def thresh{c==1, T==i16 & hasarch{'SSSE3'}} = 32 +fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [16]i8 @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { @@ -288,9 +190,9 @@ fn slash{c==1, T & T<=i16 & hasarch{'SSSE3'} & use_table}(wp:*u64, x:arg{c,T}, r } i64tab:*u32 = (maketab{4,8}*2)%(1<<32) -def thresh{c, T==i32 & hasarch{'AVX2'} & use_table} = 32 -def thresh{c, T==i64 & hasarch{'AVX2'} } = 8 -fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 +def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 +fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} From c4364aefa6e01db8ca1b9210831f578c0245d050 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 19 Jul 2023 08:15:53 -0400 Subject: [PATCH 32/41] SSSE3 4-byte compress with half-width table --- src/singeli/src/slash.singeli | 38 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index d568a1a9..480f73e2 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -35,23 +35,24 @@ def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} -# Table from l bits to w-bit indices -def maketab{l,w} = { +# Table from l bits to w-bit indices, shifted left by s +def maketab{l,w,s} = { def bot = fold{ {t,k} => join{each{tup, t, k + t<>(64-w) + 1) & (1<>(width{I}-w) + 1) & (1<>2} + trail := make{V, tail{2,iota{16}}} + def step{w,i} = { + ind := load{i32tab, w} + pc := popc_alt{w, ind, 6} + s := sel{[16]i8, V~~make{[4]u32, ind, ... 3**0}, expander} | trail + res := sel{V, load{*V~~(x+4*i)}, s} + store{*V~~r, 0, res} + r+= pc + } + @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { + def rn = if (has_popc) r+popc{w} else 0 + step{w&0xf, 2*i} + step{w>>4, 2*i+1} + if (has_popc) r = rn + } +} + +i64tab:*u32 = maketab{4,8,1} def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { From 301a23fb1f91d8d063f849ab1913490bc1e2ab6b Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 19 Jul 2023 09:35:40 -0400 Subject: [PATCH 33/41] Update where/compress implementation comments --- src/builtins/slash.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 3cb921ee..effcd0ae 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -2,12 +2,16 @@ // In the notes 𝕨 might indicate 𝕩 for Indices too // Boolean 𝕨 (Where/Compress) general case based on result type width -// COULD use AVX-512 // Size 1: pext, or bit-at-a-time // Emulate pext if unavailable // COULD return boolean result from Where -// Size 8, 16: pdep/pext, or branchless -// SHOULD try vector lookup-shuffle if unavailable or old AMD +// Size 8, 16, 32, 64: mostly table-based +// Where: direct table lookup, widening for 16 and 32 if available +// Compress: table lookup plus shuffle +// AVX2 permutevar8x32 for 32 and 64 if available +// Sparse method using table-based Where fills in if no shuffle +// SHOULD implement for NEON +// AVX-512: compress instruction, separate store not compressstore // Size 32, 64: 16-bit indices from where_block_u16 // Other sizes: always used grouped code // Adaptivity based on 𝕨 statistics From 77188b8e7375869af13ef17c3cfbdad937990650 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 19 Jul 2023 09:36:50 -0400 Subject: [PATCH 34/41] Add clmul.singeli --- src/singeli/src/base.singeli | 1 + src/singeli/src/clmul.singeli | 2 ++ src/singeli/src/neq.singeli | 2 +- src/singeli/src/slash.singeli | 19 ++++--------------- 4 files changed, 8 insertions(+), 16 deletions(-) create mode 100644 src/singeli/src/clmul.singeli diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index a1b4eb97..e9854284 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -209,6 +209,7 @@ def extract{...x} = assert{'extract not supported', show{...x}} def abs{...x} = assert{'abs not supported', show{...x}} def homBlend{...x} = assert{'homBlend not supported', show{...x}} def zip{...x} = assert{'zip not supported', show{...x}} +def clmul{...x} = assert{'clmul not supported', show{...x}} def andnot{a, b & anyNum{a} & anyNum{b}} = a & ~b oper &~ andnot infix none 35 diff --git a/src/singeli/src/clmul.singeli b/src/singeli/src/clmul.singeli new file mode 100644 index 00000000..ea5f9123 --- /dev/null +++ b/src/singeli/src/clmul.singeli @@ -0,0 +1,2 @@ +def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} +def clmul{a, b} = clmul{a, b, 0} diff --git a/src/singeli/src/neq.singeli b/src/singeli/src/neq.singeli index 480645e8..cbf6afed 100644 --- a/src/singeli/src/neq.singeli +++ b/src/singeli/src/neq.singeli @@ -1,7 +1,7 @@ include './base' include './sse' +include './clmul' -def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} def unpacklo{a:T,b:T & T==[2]u64} = emit{T, '_mm_unpacklo_epi64', a, b} fn clmul_scan_ne_any(x:*void, r:*void, init:u64, words:u64, mark:u64) : void = { diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 480f73e2..6d7a89cb 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -1,19 +1,8 @@ include './base' -if (hasarch{'BMI2'}) { - include './bmi2' -} -if (hasarch{'X86_64'}) { - include './sse' -} -if (hasarch{'PCLMUL'}) { - def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm} -} else { - def clmul{...x} = assert{'clmul not supported', show{...x}} -} -if (hasarch{'AVX2'}) { - include './avx' - include './avx2' -} +if (hasarch{'X86_64'}) include './sse' +if (hasarch{'PCLMUL'}) include './clmul' +if (hasarch{'AVX2'}) { include './avx'; include './avx2' } +if (hasarch{'BMI2'}) include './bmi2' if (hasarch{'AVX512F'}) { local def mti{s,T} = merge{'_mm512_',s,'_epi',fmtnat{elwidth{T}}} def load{a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n} From 6930bc34b69e62752ac362107417134a9cade936 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 16:51:14 +0300 Subject: [PATCH 35/41] further update slash.c implementation comments --- src/builtins/slash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index effcd0ae..648fa4f3 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -2,8 +2,8 @@ // In the notes 𝕨 might indicate 𝕩 for Indices too // Boolean 𝕨 (Where/Compress) general case based on result type width -// Size 1: pext, or bit-at-a-time -// Emulate pext if unavailable +// Size 1: pext +// Emulate if unavailable // COULD return boolean result from Where // Size 8, 16, 32, 64: mostly table-based // Where: direct table lookup, widening for 16 and 32 if available From 03c21112c250d533d0b0481cae953c5f879fa198 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 16:58:35 +0300 Subject: [PATCH 36/41] make NEON builds work --- src/singeli/src/slash.singeli | 1 + 1 file changed, 1 insertion(+) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 6d7a89cb..1dd1aea4 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -3,6 +3,7 @@ if (hasarch{'X86_64'}) include './sse' if (hasarch{'PCLMUL'}) include './clmul' if (hasarch{'AVX2'}) { include './avx'; include './avx2' } if (hasarch{'BMI2'}) include './bmi2' +if (hasarch{'AARCH64'}) include './neon' if (hasarch{'AVX512F'}) { local def mti{s,T} = merge{'_mm512_',s,'_epi',fmtnat{elwidth{T}}} def load{a:T, n & 512==width{eltype{T}}} = emit{eltype{T}, '_mm512_loadu_si512', a+n} From a0797318caaa7eb7e64343eddec4f0f972eb084b Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 17:03:11 +0300 Subject: [PATCH 37/41] use has_simd & arch_defvw --- src/singeli/src/slash.singeli | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 1dd1aea4..be001d32 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -70,8 +70,8 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { buf_used = 1; goto{restart} } } else { - if (hasarch{'X86_64'} | hasarch{'AARCH64'}) { - def vc = (if (hasarch{'AVX2'}) 256 else 128)/tw; + if (has_simd) { + def vc = arch_defvw/tw; def R = [vc]T @unroll ((ov/vc)>>0) if (end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} From 48111d48734a6d1e6156d4e3ef68af9c8f11abac Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 17:24:20 +0300 Subject: [PATCH 38/41] extract iota{} for AVX-512 --- src/singeli/src/slash.singeli | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index be001d32..d3a00a0d 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -11,6 +11,7 @@ if (hasarch{'AVX512F'}) { def p = each{{c}=>promote{eltype{T},c},reverse{xs}} emit{T, mti{'set',T}, ...p} } + def iota{T & isvec{T} & 512==width{T}} = make{T, iota{vcount{T}}} def broadcast{T, v & isvec{T} & 512==width{T}} = { emit{T, mti{'set1',T}, promote{eltype{T},v}} } @@ -97,10 +98,9 @@ def getter{c, V, x} = { i:u64 = 0 {} => { v:=load{*V~~x, i}; ++i; v } } else { - def k = vcount{V} - i := make{V, iota{k}} + i := iota{V} if (isreg{x}) i += V**cast_i{eltype{V},x} - ii := V**k + ii := V**vcount{V} {} => { v:=i; i+=ii; v } } } From ff9bb258c25828319e7106ffef73ac10aa8c2a3a Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 18:50:01 +0300 Subject: [PATCH 39/41] more comments in slash.singeli --- src/singeli/src/slash.singeli | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index d3a00a0d..c4c71aeb 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -38,15 +38,20 @@ def maketab{l,w,s} = { top<<(l*w-w+s) | bot # Overlaps for all-1 value only } def maketab{l,w} = maketab{l,w,0} -# 2KB table shared by many methods -itab:*u64 = maketab{8,8} + +itab:*u64 = maketab{8,8} # 256 elts, 2KB; shared by many methods # Recover popcount, for when POPCNT isn't there def has_popc = hasarch{'POPCNT'} def tab_popc{i:I, w} = (i>>(width{I}-w) + 1) & (1<=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { @@ -237,6 +247,7 @@ fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: } } +# everything; 512 bits/iter; AVX-512 compress def thresh{c, T==i8 & hasarch{'AVX512VBMI2'}} = 256 def thresh{c, T==i16 & hasarch{'AVX512VBMI2'}} = 128 def thresh{c, T==i32 & hasarch{'AVX512F'}} = 64 From 9945711b5e4e2273eb5c1b67abe46458fc80530c Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 19:58:41 +0300 Subject: [PATCH 40/41] AVX2 i32 & i64 slash{} via widening load --- src/singeli/src/slash.singeli | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index c4c71aeb..ca645a0d 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -26,10 +26,10 @@ def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} -# Table from l bits to w-bit indices, shifted left by s -def maketab{l,w,s} = { +# Table from l bits to w-bit indices, shifted left by s, and G applied afterwards +def maketab{l,w,s,G} = { def bot = fold{ - {t,k} => join{each{tup, t, k + t< join{each{tup, t, G{k} + t<x} def maketab{l,w} = maketab{l,w,0} itab:*u64 = maketab{8,8} # 256 elts, 2KB; shared by many methods @@ -211,7 +212,7 @@ fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: } # i32 & i64 w/x & x+/w; 256 bits/step, 8 elts/iter; [8]i32 shuffle -i64tab:*u32 = maketab{4,8,1} # 16 elts, 64B +i64tab:*u64 = maketab{4,16,1,{x}=>(1+x)*0x100 + x} # 16 elts, 64B def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { @@ -229,9 +230,7 @@ fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: } def tab = if (tw==32) itab else i64tab def step{r, w} = { - ind := load{tab, w}; def I = type{ind} - s := sel{[16]i8, V~~[width{V}/width{I}]I**ind, expander} - if (tw==64) s |= make{V, iota{8}%2} + s:= loadBatch{*u8~~(tab+w), 0, V} store{*V~~r, 0, from_ind{s}} } @for_special_buffered{r,8} (w in *u8~~wp to sum) { From 19b35e1074ba71a193b359e06f33f3efe0c6c618 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 19 Jul 2023 20:11:14 +0300 Subject: [PATCH 41/41] fix size comment --- src/singeli/src/slash.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index ca645a0d..3044972b 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -212,7 +212,7 @@ fn slash{c==1, T==i32 & hasarch{'SSSE3'}}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum: } # i32 & i64 w/x & x+/w; 256 bits/step, 8 elts/iter; [8]i32 shuffle -i64tab:*u64 = maketab{4,16,1,{x}=>(1+x)*0x100 + x} # 16 elts, 64B +i64tab:*u64 = maketab{4,16,1,{x}=>(1+x)*0x100 + x} # 16 elts, 128B def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 fn slash{c, T & hasarch{'AVX2'} & T>=i32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {