From 94e6b6cb3a538778c13164c8dbfa827948bd5ed9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 23 Feb 2025 08:27:47 -0500 Subject: [PATCH] =?UTF-8?q?Better=20formula=20for=20within-word=20masked?= =?UTF-8?q?=20=E2=88=A7`=20and=20=E2=88=A8`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/scan.singeli | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 55be386c..9b384a59 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -342,6 +342,10 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, nl:usz, l:usz) : void = { def qand = not id assert{l > 0} nw := cdiv{nl, 64} + def scan_mask{x:T, m:T} = { + if (qand) { p:= (x &~ m) >> 1; (x - p) ^ p } + else { p:= (x | m) >> 1; (p - x) ^ p } + } def res_m1{x,c,m} = { # result word with carry c, popc{m}<=1 if (qand) x &~ ((x+c) & (x+m)) else x | ((-x-c) &~ (x-m)) @@ -354,25 +358,15 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, nl:usz, l:usz) : void = { } } else { m:u64 = aligned_spaced_mask{l} - t := m << (l-1) - @for (r in dst, x in src over nw) { - r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) - else x | ~((t&~x) ^ ((x|t) - m))) - } + @for (r in dst, x in src over nw) r = scan_mask{x, m} } # could use for l>=8; not much faster and takes up space # def rowwise{T} = @for (r in *T~~dst, x in *T~~src over (64/width{T})*nw) r = x &~ (x+1) } else if (hasarch{'AVX2'}) { - def scan_words{x, m:V} = { - mb:= m | V**1 - p:= if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 - a:= if (qand) p + (mb & x) else p + (mb &~ x) - if (qand) p ^ a else ~(p ^ a) - } def apply_carry{s, c, f} = { - if (qand) s & (~f | c) else s | (f & c) + if (qand) s & (~f | c) else s | (f & c) } - avx2_loop_with_unaligned_mask{src, dst, nw, l, scan_words, apply_carry} + avx2_loop_with_unaligned_mask{src, dst, nw, l, scan_mask, apply_carry} } else { loop_with_unaligned_mask{src, dst, nw, l, {x, c, m} => { s:= (if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 )