From e09d385df5a04aee536b085b86cdf473903de8e8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 1 Aug 2023 21:31:20 -0400 Subject: [PATCH 01/13] Simplify pairwise pext implementation; drop 12-bit as no longer competitive --- src/singeli/src/slash.singeli | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 32819d95..08396a54 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -299,33 +299,30 @@ fn pext{T}(x:T, m:T) { def h = k>>1 # Increase size from h to k {x,z} := build{h} def low = lowbits{k} # Low bit in each new group - if (k <= 3) { + if (k == 2) { z0 := z & low zm := z>>1 & low - if (k == 2) tup{ - x - (x>>1 & z0), - z0 + zm - } else tup{ # Faster 1->3 jump, currently unused - x - ((x>>1&mod{low*3}) & (z|z0<<1)) - (x>>2 & (z & zm)), - (z0 + zm) + (z>>2 & low) - } + tup{ x - (x>>1 & z0), zm + z0 } } else { - # Shift high x group down by low z, then add halves of z - even:T = mod{low*(1<>sh | (x &~ m) - if (2*sh<=k/2) shift{2*sh, o>>1, s} else s + if (2*sh>1, s} else s } + # Shift high x group down by low z, then add halves of z + odd:T = mod{low*(1<0 move tup{ - (x&even) | shift{1, z, x&~even}, - if (k>4) (z + z>>h)&even else ((z&~even)>>h) + (z&even) + (x&~move) | shift{1, z1, x&move}>>1, + (z&odd)>>h + ze } } } # Finally, compose groups with regular shifts - def g = 8 # 12 performs about the same + def g = 8 {b,z} := build{g} o := z*lowbits{g} # Offsets by prefix sum def s = 1< Date: Fri, 4 Aug 2023 21:26:20 -0400 Subject: [PATCH 02/13] Remove boolean Compress overallocation: not needed with w masked off --- src/builtins/slash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index d35b3f15..d8c828ad 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -441,7 +441,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { case 0: { u64* xp = bitarr_ptr(x); u64* rp; #if defined(__BMI2__) || SINGELI - r = m_bitarrv(&rp,wsum+128); a(r)->ia = wsum; + r = m_bitarrv(&rp,wsum); u64 cw = 0; // current word u64 ro = 0; // offset in word where next bit should be written; never 64 for (usz i=0; i Date: Fri, 4 Aug 2023 21:30:06 -0400 Subject: [PATCH 03/13] Move pext-based compress loop from C to Singeli --- src/builtins/slash.c | 27 ++++----------------------- src/singeli/src/slash.singeli | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index d8c828ad..829dafab 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -439,30 +439,11 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { switch(xl) { default: r = compress_grouped(wp, x, wia, wsum, xt); break; case 0: { - u64* xp = bitarr_ptr(x); u64* rp; - #if defined(__BMI2__) || SINGELI - r = m_bitarrv(&rp,wsum); - u64 cw = 0; // current word - u64 ro = 0; // offset in word where next bit should be written; never 64 - for (usz i=0; i=64) { - *(rp++) = cw; - cw = ro? v>>(64-ro) : 0; - } - ro = ro2&63; - } - if (ro) *rp = cw; + u64* xp = bitarr_ptr(x); + u64* rp; r = m_bitarrv(&rp,wsum); + #if SINGELI + si_compress_bool(wp, xp, rp, wia); #else - r = m_bitarrv(&rp,wsum); for (usz i=0, ri=0; i= 64) { + store{r, 0, cw}; ++r + cw = 0; if (ro>0) cw = v>>(64-ro) + } + ro = ro2%64 + } + if (ro > 0) store{r, 0, cw} +} + +export{'si_compress_bool', compress_bool} From 66c0fe041cd1424b1b14c935ce7065e7788858a1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 4 Aug 2023 21:43:17 -0400 Subject: [PATCH 04/13] Get pext and popcount together, since generic pext emulation includes popc --- src/singeli/src/slash.singeli | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 995f4b22..caa87b5d 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -287,7 +287,7 @@ export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh{1 export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1, i64}} # pext, or boolean compress -fn pext{T}(x:T, m:T) { +def pext_popc{x:T, m:T} = { def w = width{T} def mod{a} = a % (1<> (o>>(sh-g) & s) - fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} + pe := fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} + tup{pe, w - o>>(w-g)} } -fn pext{T & hasarch{'PCLMUL'} & T==u64}(xs:T, ms:T) { +def pext_popc{xs:T, ms:T & hasarch{'PCLMUL'} & T==u64} = { def num = lb{width{T}} def vec{s} = make{[2]T, s, 0} m := vec{ms} @@ -346,18 +347,17 @@ fn pext{T & hasarch{'PCLMUL'} & T==u64}(xs:T, ms:T) { shift_at{m, p} shift_at{x, p} } - extract{x, 0} + tup{extract{x, 0}, popc{ms}} } -fn pext{T & hasarch{'BMI2'}}(x:T, m:T) = pext{x, m} +def pext_popc{x:T, m:T & hasarch{'BMI2'}} = tup{pext{x, m}, popc{m}} fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { cw:u64 = 0; # current word ro:u64 = 0; # offset in word where next bit should be written; never 64 @for (w, x over i to cdiv{n,64}) { - v := pext{u64}(x, w) - c := cast_i{u64, popcRand{w}} - cw|= v<= 64) { store{r, 0, cw}; ++r From 371aa6f2ef33e1e73e2d120a198e94ed0032adf6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 5 Aug 2023 21:50:22 -0400 Subject: [PATCH 05/13] AVX2 boolean Compress using variable shifts --- src/singeli/src/slash.singeli | 48 +++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index caa87b5d..bffca005 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -288,7 +288,8 @@ export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1 # pext, or boolean compress def pext_popc{x:T, m:T} = { - def w = width{T} + def w = if (isvec{T}) elwidth{T} else width{T} + def scal{v} = if (isvec{T}) T**v else v def mod{a} = a % (1< 1} = { def h = k>>1 # Increase size from h to k {x,z} := build{h} - def low = lowbits{k} # Low bit in each new group + def low_s = lowbits{k} # Low bit in each new group + def low = scal{low_s} if (k == 2) { z0 := z & low zm := z>>1 & low tup{ x - (x>>1 & z0), zm + z0 } + } else if (hasarch{'AVX2'} and isvec{T} and k >= 32) { + # We have variable shifts at these sizes + lh := scal{low_s*(1<> S~~zl) | (x&lh), T~~(S~~z >> h) + zl} } else { # SWAR shifter: shift x by sh*o, in length-k groups def shift{sh, o, x} = { @@ -311,9 +319,9 @@ def pext_popc{x:T, m:T} = { if (2*sh>1, s} else s } # Shift high x group down by low z, then add halves of z - odd:T = mod{low*(1<0 move tup{ (x&~move) | shift{1, z1, x&move}>>1, @@ -323,12 +331,16 @@ def pext_popc{x:T, m:T} = { } # Finally, compose groups with regular shifts def g = 8 - {b,z} := build{g} - o := z*lowbits{g} # Offsets by prefix sum - def s = 1<> (o>>(sh-g) & s) - pe := fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} - tup{pe, w - o>>(w-g)} + def build{k & ~isvec{T} & k > g} = { + {x,z} := build{g} + o := z*lowbits{g} # Offsets by prefix sum + def s = 1<> (o>>(sh-g) & s) + pe := fold{|, x&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} + tup{pe, o>>(w-g)} + } + def {pe, z} = build{w} + tup{pe, scal{w} - z} } def pext_popc{xs:T, ms:T & hasarch{'PCLMUL'} & T==u64} = { @@ -350,13 +362,12 @@ def pext_popc{xs:T, ms:T & hasarch{'PCLMUL'} & T==u64} = { tup{extract{x, 0}, popc{ms}} } -def pext_popc{x:T, m:T & hasarch{'BMI2'}} = tup{pext{x, m}, popc{m}} +def pext_popc{x:T, m:T & hasarch{'BMI2'} & T==u64} = tup{pext{x, m}, popc{m}} fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { cw:u64 = 0; # current word ro:u64 = 0; # offset in word where next bit should be written; never 64 - @for (w, x over i to cdiv{n,64}) { - {v, c} := pext_popc{x, w} + def add_bits{{v, c}} = { cw |= v<= 64) { @@ -365,6 +376,17 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { } ro = ro2%64 } + if (hasarch{'AVX2'}) { + def V = [4]u64 + nv := n/256 + @for (w in *V~~w, x in *V~~x over i to nv) { + vc := pext_popc{x, w} + @unroll (j to 4) add_bits{each{extract{., j}, vc}} + } + @for (w, x over i from nv*4 to cdiv{n,64}) add_bits{pext_popc{x, w}} + } else { + @for (w, x over i to cdiv{n,64}) add_bits{pext_popc{x, w}} + } if (ro > 0) store{r, 0, cw} } From 07ace41d6cd49e8fff1db6bebdc8941df9f2305d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 6 Aug 2023 07:36:05 -0400 Subject: [PATCH 06/13] Vectorized version of the clmul boolean Compress --- src/singeli/src/slash.singeli | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index bffca005..81c32141 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -286,9 +286,11 @@ export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh{1 export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh{1, i32}} export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1, i64}} +def scalwidth{T} = if (isvec{T}) elwidth{T} else width{T} + # pext, or boolean compress def pext_popc{x:T, m:T} = { - def w = if (isvec{T}) elwidth{T} else width{T} + def w = scalwidth{T} def scal{v} = if (isvec{T}) T**v else v def mod{a} = a % (1<>sh } - p := clmul{d, c, 0} # xor-scan + p := clmul{d, c} # xor-scan d = d &~ p # Remove even bits p &= m shift_at{m, p} shift_at{x, p} } - tup{extract{x, 0}, popc{ms}} + if (vt) tup{x, @collect (j to 2) popc{extract{ms,j}}} + else tup{extract{x, 0}, popc{ms}} } def pext_popc{x:T, m:T & hasarch{'BMI2'} & T==u64} = tup{pext{x, m}, popc{m}} @@ -376,14 +383,16 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { } ro = ro2%64 } - if (hasarch{'AVX2'}) { - def V = [4]u64 - nv := n/256 + def extract{t, i & istup{t}} = tupsel{i,t} + if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) { + def v = if (hasarch{'AVX2'}) 4 else 2 + def V = [v]u64 + nv := n/(v*64) @for (w in *V~~w, x in *V~~x over i to nv) { vc := pext_popc{x, w} - @unroll (j to 4) add_bits{each{extract{., j}, vc}} + @unroll (j to v) add_bits{each{extract{., j}, vc}} } - @for (w, x over i from nv*4 to cdiv{n,64}) add_bits{pext_popc{x, w}} + @for (w, x over i from nv*v to cdiv{n,64}) add_bits{pext_popc{x, w}} } else { @for (w, x over i to cdiv{n,64}) add_bits{pext_popc{x, w}} } From 9046dd6b5375c6ed101bb44536c722522a26cb57 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 6 Aug 2023 08:03:14 -0400 Subject: [PATCH 07/13] Handle boolean Compress SIMD tail with conditional write, not scalar loop --- src/singeli/src/slash.singeli | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 81c32141..e1f9b89a 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -345,18 +345,13 @@ def pext_popc{x:T, m:T} = { tup{pe, scal{w} - z} } -def pext_popc{xs:T, ms:T & hasarch{'PCLMUL'} & width{T}<=128} = { - def vt = isvec{T} - def V = if (vt) T else [2]T - def vec{s} = if (vt) s else make{V, s, 0} - def clmul{a, b} = { - if (vt) zipLo{...@collect (j to 2) clmul{a,b,j}} else clmul{a, b, 0} - } - m := vec{ms} - x := vec{xs} & m +def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { + def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} + m := m0 + x := x0 & m d := ~m << 1 # One bit of the position difference at x c := V**(1<<64-1) - @unroll (i to lb{scalwidth{T}}) { + @unroll (i to lb{scalwidth{V}}) { def sh = 1 << i def shift_at{v, s} = { v = (v&~s) | (v&s)>>sh } p := clmul{d, c} # xor-scan @@ -365,8 +360,7 @@ def pext_popc{xs:T, ms:T & hasarch{'PCLMUL'} & width{T}<=128} = { shift_at{m, p} shift_at{x, p} } - if (vt) tup{x, @collect (j to 2) popc{extract{ms,j}}} - else tup{extract{x, 0}, popc{ms}} + tup{x, @collect (j to 2) popc{extract{m0,j}}} } def pext_popc{x:T, m:T & hasarch{'BMI2'} & T==u64} = tup{pext{x, m}, popc{m}} @@ -387,12 +381,19 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) { def v = if (hasarch{'AVX2'}) 4 else 2 def V = [v]u64 - nv := n/(v*64) - @for (w in *V~~w, x in *V~~x over i to nv) { + d := cdiv{n,64}; e := d/v + @for (w in *V~~w, x in *V~~x over i to cdiv{d,v}) { vc := pext_popc{x, w} - @unroll (j to v) add_bits{each{extract{., j}, vc}} + def add{j} = add_bits{each{extract{., j}, vc}} + if (i < e) { + @unroll (j to v) add{j} + } else { + # last write: between 1 and v-1 words + m := d%v + def ar{j} = { add{j}; def jn=j+1; if (jn Date: Sun, 6 Aug 2023 08:09:08 -0400 Subject: [PATCH 08/13] Fix disabled BMI2 boolean Compress; cleaner SIMD width system --- src/singeli/src/slash.singeli | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index e1f9b89a..94d2349e 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -289,6 +289,7 @@ export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1 def scalwidth{T} = if (isvec{T}) elwidth{T} else width{T} # pext, or boolean compress +def pext_width{} = if (hasarch{'AVX2'}) 4 else 1 def pext_popc{x:T, m:T} = { def w = scalwidth{T} def scal{v} = if (isvec{T}) T**v else v @@ -345,6 +346,7 @@ def pext_popc{x:T, m:T} = { tup{pe, scal{w} - z} } +def pext_width{..._ & hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} m := m0 @@ -363,6 +365,7 @@ def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { tup{x, @collect (j to 2) popc{extract{m0,j}}} } +def pext_width{..._ & hasarch{'BMI2'}} = 1 def pext_popc{x:T, m:T & hasarch{'BMI2'} & T==u64} = tup{pext{x, m}, popc{m}} fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { @@ -378,8 +381,8 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { ro = ro2%64 } def extract{t, i & istup{t}} = tupsel{i,t} - if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) { - def v = if (hasarch{'AVX2'}) 4 else 2 + def v = pext_width{} + if (v > 1) { def V = [v]u64 d := cdiv{n,64}; e := d/v @for (w in *V~~w, x in *V~~x over i to cdiv{d,v}) { From 120df2681421016280eba89296757a6d2a5fdcac Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Aug 2023 11:22:40 -0400 Subject: [PATCH 09/13] mullo works for both signed and unsigned --- src/singeli/src/avx2.singeli | 12 ++++++------ src/singeli/src/sse2.singeli | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 65a52263..d1f1f30a 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -69,12 +69,12 @@ def packQQ{{a, b}} = packQQ{a, b} # arith -def __mul{a:T,b:T & [16]i16==T} = emit{T, '_mm256_mullo_epi16', a, b} -def mulHi{a:T,b:T & [16]i16==T} = emit{T, '_mm256_mulhi_epi16', a, b} -def mulHi{a:T,b:T & [16]u16==T} = emit{T, '_mm256_mulhi_epu16', a, b} -def __mul{a:T,b:T & [ 8]i32==T} = emit{T, '_mm256_mullo_epi32', a, b} -def mul32{a:T,b:T & [ 4]i64==T} = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments -def mul32{a:T,b:T & [ 4]u64==T} = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments +def __mul{a:T,b:T & w256i{T, 16}} = emit{T, '_mm256_mullo_epi16', a, b} +def mulHi{a:T,b:T & [16]i16==T } = emit{T, '_mm256_mulhi_epi16', a, b} +def mulHi{a:T,b:T & [16]u16==T } = emit{T, '_mm256_mulhi_epu16', a, b} +def __mul{a:T,b:T & w256i{T, 32}} = emit{T, '_mm256_mullo_epi32', a, b} +def mul32{a:T,b:T & [ 4]i64==T } = emit{T, '_mm256_mul_epi32', a, b} # reads only low 32 bits of arguments +def mul32{a:T,b:T & [ 4]u64==T } = emit{T, '_mm256_mul_epu32', a, b} # reads only low 32 bits of arguments def abs{a:T & w256i{T,8 }} = emit{T, '_mm256_abs_epi8', a} def abs{a:T & w256i{T,16}} = emit{T, '_mm256_abs_epi16', a} diff --git a/src/singeli/src/sse2.singeli b/src/singeli/src/sse2.singeli index 31c665cd..b7193df7 100644 --- a/src/singeli/src/sse2.singeli +++ b/src/singeli/src/sse2.singeli @@ -119,7 +119,7 @@ def __sub{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_sub_epi16', a, b} def __sub{a:T,b:T & w128i{T, 32}} = emit{T, '_mm_sub_epi32', a, b} def __sub{a:T,b:T & w128i{T, 64}} = emit{T, '_mm_sub_epi64', a, b} -def __mul{a:T,b:T & [8]i16==T} = emit{T, '_mm_mullo_epi16', a, b} +def __mul{a:T,b:T & w128i{T, 16}} = emit{T, '_mm_mullo_epi16', a, b} def mulHi{a:T,b:T & [8]i16==T} = emit{T, '_mm_mulhi_epi16', a, b} def mulHi{a:T,b:T & [8]u16==T} = emit{T, '_mm_mulhi_epu16', a, b} def mul32{a:T,b:T & [2]u64==T} = emit{T, '_mm_mul_epu32', a, b} # reads only low 32 bits of arguments From bbad98bce11fcb49eedd03d7a15d6d32bb0840e1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Aug 2023 11:32:02 -0400 Subject: [PATCH 10/13] In AVX2 boolean Compress, jump from 8 to 32 with 32-bit shifts --- src/singeli/src/slash.singeli | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 94d2349e..4ccc1be0 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -293,8 +293,7 @@ def pext_width{} = if (hasarch{'AVX2'}) 4 else 1 def pext_popc{x:T, m:T} = { def w = scalwidth{T} def scal{v} = if (isvec{T}) T**v else v - def mod{a} = a % (1< 1} = { def h = k>>1 # Increase size from h to k {x,z} := build{h} - def low_s = lowbits{k} # Low bit in each new group + def low_s = lowbits{w,k} # Low bit in each new group def low = scal{low_s} if (k == 2) { z0 := z & low @@ -322,7 +321,7 @@ def pext_popc{x:T, m:T} = { if (2*sh>1, s} else s } # Shift high x group down by low z, then add halves of z - odd:T = scal{mod{low_s*(1<0 move @@ -332,16 +331,25 @@ def pext_popc{x:T, m:T} = { } } } - # Finally, compose groups with regular shifts - def g = 8 - def build{k & ~isvec{T} & k > g} = { - {x,z} := build{g} - o := z*lowbits{g} # Offsets by prefix sum + # Compose k/g groups with k/g-1 regular shifts + def multi_shift{x, z, g, k, sc} = { + o := z * sc{lowbits{k,g}} # Offsets by prefix sum def s = 1<> (o>>(sh-g) & s) - pe := fold{|, x&s, each{gr, g*slice{iota{cdiv{w,g}},1}}} - tup{pe, o>>(w-g)} + def s0 = sc{s} + def oo{sh} = if (sh==g) z else o>>(sh-g) # Offset for group + def gr{sh} = (x & sc{s<> (oo{sh} & s0) # Shifted group + pe := fold{|, x&s0, each{gr, g*slice{iota{k/g},1}}} + tup{pe, o>>(k-g)} } + def build{k==32 & hasarch{'AVX2'} & isvec{T}} = { + def S = re_el{ty_u{k}, T} + def c{T,vs} = each{{v}=>T~~v, vs} + c{T, multi_shift{...c{S, build{8}}, 8, k, {s}=>S**s}} + } + def build{k & ~isvec{T} & k > 8} = { + multi_shift{...build{8}, 8, k, {s}=>s} + } + # Final result def {pe, z} = build{w} tup{pe, scal{w} - z} } From a7908a10b0833929017d402192b7f5ed66def2e2 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Aug 2023 12:26:40 -0400 Subject: [PATCH 11/13] Faster, sparse, non-Singeli boolean Compress --- src/builtins/slash.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 829dafab..47dcf595 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -444,7 +444,15 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { #if SINGELI si_compress_bool(wp, xp, rp, wia); #else - for (usz i=0, ri=0; i>1 | (x>>CTZ(v))<<63; + ++j; if (j%64==0) rp[j/64-1] = o; + } + } + usz q=(-j)%64; if (q) rp[j/64] = o>>q; #endif break; } From 4e9745688b2118e28bc48a49b3b98bdca7382511 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Aug 2023 13:44:52 -0400 Subject: [PATCH 12/13] Use generic boolean Compress as the sparse case with Singeli --- src/builtins/slash.c | 7 ++++--- src/singeli/src/slash.singeli | 10 +++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 47dcf595..be0aca20 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -442,8 +442,10 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { u64* xp = bitarr_ptr(x); u64* rp; r = m_bitarrv(&rp,wsum); #if SINGELI - si_compress_bool(wp, xp, rp, wia); - #else + if (wsum>=wia/si_thresh_compress_bool) { + si_compress_bool(wp, xp, rp, wia); break; + } + #endif u64 o = 0; usz j = 0; for (usz i=0; i>q; - #endif break; } #define COMPRESS_BLOCK_PREP(T, PREP) \ diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 4ccc1be0..e73ccbc2 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -289,7 +289,8 @@ export{'si_2slash64', slash{1, i64}}; export{'si_thresh_2slash64', u64~~thresh{1 def scalwidth{T} = if (isvec{T}) elwidth{T} else width{T} # pext, or boolean compress -def pext_width{} = if (hasarch{'AVX2'}) 4 else 1 +def pext_width{} = if (hasarch{'AVX2'}) 4 else 1 +def thresh_bool{} = if (hasarch{'AVX2'}) 128 else 16 def pext_popc{x:T, m:T} = { def w = scalwidth{T} def scal{v} = if (isvec{T}) T**v else v @@ -354,7 +355,8 @@ def pext_popc{x:T, m:T} = { tup{pe, scal{w} - z} } -def pext_width{..._ & hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 +def pext_width {..._ & hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 2 +def thresh_bool{..._ & hasarch{'PCLMUL'} > hasarch{'AVX2'}} = 32 def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { def clmul{a, b} = zipLo{...@collect (j to 2) clmul{a,b,j}} m := m0 @@ -373,7 +375,8 @@ def pext_popc{x0:V, m0:V & hasarch{'PCLMUL'} & V==[2]u64} = { tup{x, @collect (j to 2) popc{extract{m0,j}}} } -def pext_width{..._ & hasarch{'BMI2'}} = 1 +def pext_width {..._ & hasarch{'BMI2'}} = 1 +def thresh_bool{..._ & hasarch{'BMI2'}} = 512 def pext_popc{x:T, m:T & hasarch{'BMI2'} & T==u64} = tup{pext{x, m}, popc{m}} fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { @@ -412,3 +415,4 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = { } export{'si_compress_bool', compress_bool} +export{'si_thresh_compress_bool', u64~~thresh_bool{}} From 2dd3a0fdfa1e1fcb48aae29498512ee1f0eddd0e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Aug 2023 15:33:37 -0400 Subject: [PATCH 13/13] Boolean compress implementation comments --- src/builtins/slash.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index be0aca20..ff8c78ea 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -2,8 +2,11 @@ // In the notes 𝕨 might indicate 𝕩 for Indices too // Boolean 𝕨 (Where/Compress) general case based on result type width -// Size 1: pext -// Emulate if unavailable +// Size 1: compress 64-bit units, possibly SIMD +// pext if BMI2 is present +// Pairwise combination, SIMD if AVX2 +// SIMD shift-by-offset if there's CLMUL but no AVX2 +// SHOULD use with polynomial multiply in NEON // COULD return boolean result from Where // Size 8, 16, 32, 64: mostly table-based // Where: direct table lookup, widening for 16 and 32 if available @@ -18,7 +21,7 @@ // None for 8-bit Where, too short // COULD try per-block adaptivity for 16-bit Compress // Sparse if +´𝕨 is small, branchless unless it's very small -// Chosen per-argument for 8, 16 and per-block for larger +// Chosen per-argument for 1, 8, 16 and per-block for larger // Careful when benchmarking, branch predictor has a long memory // Grouped if +Β΄Β»βŠΈβ‰ π•¨ is small, always branching // Chosen per-argument with a threshold that gives up early