Use a lookup table for evenly-spaced masks, getting division results from those
This commit is contained in:
parent
e6f1e04de2
commit
f0f130c42e
@ -17,6 +17,8 @@
|
||||
#include "../utils/calls.h"
|
||||
|
||||
#if SINGELI
|
||||
extern uint64_t* const si_spaced_masks;
|
||||
#define get_spaced_mask(i) si_spaced_masks[i-1]
|
||||
#define SINGELI_FILE fold
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
@ -17,6 +17,8 @@ B sub_c2(B, B, B);
|
||||
B mul_c2(B, B, B);
|
||||
|
||||
#if SINGELI
|
||||
extern uint64_t* const si_spaced_masks;
|
||||
#define get_spaced_mask(i) si_spaced_masks[i-1]
|
||||
#define SINGELI_FILE scan
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
@ -777,7 +777,11 @@ B slash_c2(B t, B w, B x) {
|
||||
u64* rp; r = m_bitarrv(&rp, s);
|
||||
#if FAST_PDEP
|
||||
if (wv <= 52) {
|
||||
u64 m = (u64)-1 / (((u64)1<<wv)-1); // TODO table lookup
|
||||
#if SINGELI
|
||||
u64 m = si_spaced_masks[wv-1];
|
||||
#else
|
||||
u64 m = (u64)-1 / (((u64)1<<wv)-1);
|
||||
#endif
|
||||
u64 xw = 0;
|
||||
usz d = POPC(m); // == 64/wv
|
||||
if (m & 1) { // Power of two
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
include './base'
|
||||
include './mask'
|
||||
if_inline (hasarch{'BMI2'}) include './bmi2'
|
||||
include './spaced'
|
||||
|
||||
include 'util/tup'
|
||||
|
||||
@ -123,12 +124,6 @@ fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, eq:u1) : void = {
|
||||
flush_bits{n, fixout}
|
||||
}
|
||||
|
||||
def unaligned_mask{l} = {
|
||||
def d = 64 % l
|
||||
def m = (~u64~~0 >> d) / ((u64~~1 << l)-1)
|
||||
tup{m<<l | 1, d}
|
||||
}
|
||||
|
||||
def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
|
||||
nw := cdiv{n*l,64}
|
||||
xx := -promote{u64, op_and}
|
||||
@ -173,9 +168,9 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
|
||||
}
|
||||
if (op_and) loop{{x} => x & ((x&~t) + m)}
|
||||
else loop{{x} => x | ((x| t) - m)}
|
||||
} else { # odd row length
|
||||
d:usz = 64 % l
|
||||
e := ((~u64~~0 >> d) / ((u64~~1 << l)-1)) << (l-1)
|
||||
} else {
|
||||
{e0, d} := unaligned_spaced_mask_mod{l}
|
||||
e := e0 << (l-1)
|
||||
r:u64 = 0
|
||||
rh := *u32~~rp
|
||||
ri:ux = 0
|
||||
@ -193,8 +188,7 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
dm:= 64/l
|
||||
e0:= e<<1 | 1
|
||||
dm:= cast_i{usz, popc{e}}
|
||||
c:u64 = 0
|
||||
def loop{...par} = {
|
||||
@for (xo in xp over nw) {
|
||||
@ -226,15 +220,15 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = {
|
||||
} else if (l < 8) {
|
||||
assert{l > 4}
|
||||
ld:= l-1; lld:= l*ld
|
||||
{mult0, _} := unaligned_mask{ld}
|
||||
{mult0, _} := unaligned_spaced_mask_mod{ld}
|
||||
mult0 &= u64~~1<<lld - 1
|
||||
{mult1, _} := unaligned_mask{lld}
|
||||
{mult1, _} := unaligned_spaced_mask_mod{lld}
|
||||
ll:= l*l
|
||||
{tk, tkd} := unaligned_mask{ll}; tk <<= tkd
|
||||
{tk, tkd} := unaligned_spaced_mask_mod{ll}; tk <<= tkd
|
||||
topk := tk - tk>>l; topk|= topk<<ll | topk>>ll
|
||||
loop{tup{mult0,mult1}, tup{topk}}
|
||||
} else {
|
||||
{mult, _} := unaligned_mask{l-1}
|
||||
{mult, _} := unaligned_spaced_mask_mod{l-1}
|
||||
loop{tup{mult}, tup{}}
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ if_inline (hasarch{'X86_64'}) {
|
||||
}
|
||||
include './mask'
|
||||
include './f64'
|
||||
include './spaced'
|
||||
include 'util/tup'
|
||||
include './scan_common'
|
||||
|
||||
@ -300,14 +301,8 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}}
|
||||
|
||||
|
||||
# Row-wise boolean scan
|
||||
def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1)
|
||||
def unaligned_mask{l} = {
|
||||
def d = 64 % l
|
||||
def m = (~u64~~0 >> d) / ((u64~~1 << l)-1)
|
||||
tup{m<<l | 1, d}
|
||||
}
|
||||
def loop_with_unaligned_mask{x, r, nw, l, step} = {
|
||||
{m, d} := unaligned_mask{l}
|
||||
{m, d} := unaligned_spaced_mask_mod{l}
|
||||
c:u64 = 0 # carry (initial value never matters)
|
||||
@for (x, r over nw) {
|
||||
match (step{x, c, m}) {
|
||||
@ -318,7 +313,7 @@ def loop_with_unaligned_mask{x, r, nw, l, step} = {
|
||||
}
|
||||
}
|
||||
def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = {
|
||||
{ms, d} := unaligned_mask{l}
|
||||
{ms, d} := unaligned_spaced_mask_mod{l}
|
||||
def V = [4]u64
|
||||
d4:usz = width{V} % l
|
||||
m:= make{V, scan{{a,_} => a>>d | a<<(l-d), tup{ms, ...iota{3}}}}
|
||||
@ -347,7 +342,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = {
|
||||
r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10))
|
||||
}
|
||||
} else {
|
||||
m:u64 = aligned_mask{l}
|
||||
m:u64 = aligned_spaced_mask{l}
|
||||
t := m << (l-1)
|
||||
@for (r in dst, x in src over nw) {
|
||||
r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m))
|
||||
@ -422,7 +417,7 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = {
|
||||
nw := cdiv{nl, 64}
|
||||
if (l < 64) {
|
||||
if ((l & (l-1)) == 0) {
|
||||
m:u64 = aligned_mask{l}
|
||||
m:u64 = aligned_spaced_mask{l}
|
||||
@for (r, x over nw) {
|
||||
s:= scan_word{x}
|
||||
b:= s<<1 & m # last bit of previous row
|
||||
@ -472,7 +467,7 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = {
|
||||
nw := cdiv{nl, 64}
|
||||
if (l < 64) {
|
||||
if ((l & (l-1)) == 0) {
|
||||
m:u64 = aligned_mask{l}
|
||||
m:u64 = aligned_spaced_mask{l}
|
||||
@for (r, x over nw) { b:= x & m; r = b<<l - b }
|
||||
} else if (hasarch{'AVX2'}) {
|
||||
def scan_words{x, m} = { b:= x&m; b<<l - b }
|
||||
|
||||
@ -400,3 +400,11 @@ fn compress_bool(w:*u64, x:*u64, r:*u64, n:u64) : void = {
|
||||
|
||||
export{'si_compress_bool', compress_bool}
|
||||
export{'si_thresh_compress_bool', u64~~thresh_bool{}}
|
||||
|
||||
# Spaced boolean masks, such as 2b000001000001...0000010000 for 6
|
||||
# Mask i is the smallest possible mask containing 1 every i bits:
|
||||
# there would be a 1 just past the top bit.
|
||||
# The number of trailing zeros is 64%i , and the popcount is 64/i .
|
||||
def get_spaced_masks{i} = (1<<64 - 1<<(64%i)) / (1<<i - 1)
|
||||
spaced_masks:*u64 = get_spaced_masks{1 + iota{64}}
|
||||
export{'si_spaced_masks', spaced_masks}
|
||||
|
||||
14
src/singeli/src/spaced.singeli
Normal file
14
src/singeli/src/spaced.singeli
Normal file
@ -0,0 +1,14 @@
|
||||
def mask_of{l} = emit{u64, 'get_spaced_mask', l} # see slash.singeli
|
||||
|
||||
def aligned_spaced_mask{l} = {
|
||||
assert{l <= 64}
|
||||
assert{l&(l-1) == 0}
|
||||
mask_of{l}
|
||||
}
|
||||
|
||||
def unaligned_spaced_mask_mod{l:T} = {
|
||||
assert{l < 64}
|
||||
def m = mask_of{l}
|
||||
def d = cast_i{T, ctz{m}} # = 64%l
|
||||
tup{m>>d | m<<(l-d), d}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user