Use a lookup table for evenly-spaced masks, getting division results from those

This commit is contained in:
Marshall Lochbaum 2024-06-15 18:30:46 -04:00
parent e6f1e04de2
commit f0f130c42e
7 changed files with 46 additions and 27 deletions

View File

@ -17,6 +17,8 @@
#include "../utils/calls.h"
#if SINGELI
extern uint64_t* const si_spaced_masks;
#define get_spaced_mask(i) si_spaced_masks[i-1]
#define SINGELI_FILE fold
#include "../utils/includeSingeli.h"
#endif

View File

@ -17,6 +17,8 @@ B sub_c2(B, B, B);
B mul_c2(B, B, B);
#if SINGELI
extern uint64_t* const si_spaced_masks;
#define get_spaced_mask(i) si_spaced_masks[i-1]
#define SINGELI_FILE scan
#include "../utils/includeSingeli.h"
#endif

View File

@ -777,7 +777,11 @@ B slash_c2(B t, B w, B x) {
u64* rp; r = m_bitarrv(&rp, s);
#if FAST_PDEP
if (wv <= 52) {
u64 m = (u64)-1 / (((u64)1<<wv)-1); // TODO table lookup
#if SINGELI
u64 m = si_spaced_masks[wv-1];
#else
u64 m = (u64)-1 / (((u64)1<<wv)-1);
#endif
u64 xw = 0;
usz d = POPC(m); // == 64/wv
if (m & 1) { // Power of two

View File

@ -1,6 +1,7 @@
include './base'
include './mask'
if_inline (hasarch{'BMI2'}) include './bmi2'
include './spaced'
include 'util/tup'
@ -123,12 +124,6 @@ fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, eq:u1) : void = {
flush_bits{n, fixout}
}
def unaligned_mask{l} = {
def d = 64 % l
def m = (~u64~~0 >> d) / ((u64~~1 << l)-1)
tup{m<<l | 1, d}
}
def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
nw := cdiv{n*l,64}
xx := -promote{u64, op_and}
@ -173,9 +168,9 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
}
if (op_and) loop{{x} => x & ((x&~t) + m)}
else loop{{x} => x | ((x| t) - m)}
} else { # odd row length
d:usz = 64 % l
e := ((~u64~~0 >> d) / ((u64~~1 << l)-1)) << (l-1)
} else {
{e0, d} := unaligned_spaced_mask_mod{l}
e := e0 << (l-1)
r:u64 = 0
rh := *u32~~rp
ri:ux = 0
@ -193,8 +188,7 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
}
}
} else {
dm:= 64/l
e0:= e<<1 | 1
dm:= cast_i{usz, popc{e}}
c:u64 = 0
def loop{...par} = {
@for (xo in xp over nw) {
@ -226,15 +220,15 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
} else if (l < 8) {
assert{l > 4}
ld:= l-1; lld:= l*ld
{mult0, _} := unaligned_mask{ld}
{mult0, _} := unaligned_spaced_mask_mod{ld}
mult0 &= u64~~1<<lld - 1
{mult1, _} := unaligned_mask{lld}
{mult1, _} := unaligned_spaced_mask_mod{lld}
ll:= l*l
{tk, tkd} := unaligned_mask{ll}; tk <<= tkd
{tk, tkd} := unaligned_spaced_mask_mod{ll}; tk <<= tkd
topk := tk - tk>>l; topk|= topk<<ll | topk>>ll
loop{tup{mult0,mult1}, tup{topk}}
} else {
{mult, _} := unaligned_mask{l-1}
{mult, _} := unaligned_spaced_mask_mod{l-1}
loop{tup{mult}, tup{}}
}
}

View File

@ -5,6 +5,7 @@ if_inline (hasarch{'X86_64'}) {
}
include './mask'
include './f64'
include './spaced'
include 'util/tup'
include './scan_common'
@ -300,14 +301,8 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}}
# Row-wise boolean scan
def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1)
def unaligned_mask{l} = {
def d = 64 % l
def m = (~u64~~0 >> d) / ((u64~~1 << l)-1)
tup{m<<l | 1, d}
}
def loop_with_unaligned_mask{x, r, nw, l, step} = {
{m, d} := unaligned_mask{l}
{m, d} := unaligned_spaced_mask_mod{l}
c:u64 = 0 # carry (initial value never matters)
@for (x, r over nw) {
match (step{x, c, m}) {
@ -318,7 +313,7 @@ def loop_with_unaligned_mask{x, r, nw, l, step} = {
}
}
def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
{ms, d} := unaligned_mask{l}
{ms, d} := unaligned_spaced_mask_mod{l}
def V = [4]u64
d4:usz = width{V} % l
m:= make{V, scan{{a,_} => a>>d | a<<(l-d), tup{ms, ...iota{3}}}}
@ -347,7 +342,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = {
r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10))
}
} else {
m:u64 = aligned_mask{l}
m:u64 = aligned_spaced_mask{l}
t := m << (l-1)
@for (r in dst, x in src over nw) {
r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m))
@ -422,7 +417,7 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = {
nw := cdiv{nl, 64}
if (l < 64) {
if ((l & (l-1)) == 0) {
m:u64 = aligned_mask{l}
m:u64 = aligned_spaced_mask{l}
@for (r, x over nw) {
s:= scan_word{x}
b:= s<<1 & m # last bit of previous row
@ -472,7 +467,7 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = {
nw := cdiv{nl, 64}
if (l < 64) {
if ((l & (l-1)) == 0) {
m:u64 = aligned_mask{l}
m:u64 = aligned_spaced_mask{l}
@for (r, x over nw) { b:= x & m; r = b<<l - b }
} else if (hasarch{'AVX2'}) {
def scan_words{x, m} = { b:= x&m; b<<l - b }

View File

@ -400,3 +400,11 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = {
export{'si_compress_bool', compress_bool}
export{'si_thresh_compress_bool', u64~~thresh_bool{}}
# Spaced boolean masks, such as 2b000001000001...0000010000 for 6
# Mask i is the smallest possible mask containing 1 every i bits:
# there would be a 1 just past the top bit.
# The number of trailing zeros is 64%i , and the popcount is 64/i .
def get_spaced_masks{i} = (1<<64 - 1<<(64%i)) / (1<<i - 1)
spaced_masks:*u64 = get_spaced_masks{1 + iota{64}}
export{'si_spaced_masks', spaced_masks}

View File

@ -0,0 +1,14 @@
def mask_of{l} = emit{u64, 'get_spaced_mask', l} # see slash.singeli
def aligned_spaced_mask{l} = {
assert{l <= 64}
assert{l&(l-1) == 0}
mask_of{l}
}
def unaligned_spaced_mask_mod{l:T} = {
assert{l < 64}
def m = mask_of{l}
def d = cast_i{T, ctz{m}} # = 64%l
tup{m>>d | m<<(l-d), d}
}