Table-based 1- and 2-byte Where

This commit is contained in:
Marshall Lochbaum 2023-07-17 16:05:06 -04:00
parent fc187afdf2
commit 68978f7e10
3 changed files with 51 additions and 15 deletions

View File

@ -72,12 +72,13 @@
#define vg_loadLUT64(p, i) p[i]
#endif
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#endif
#if !USE_VALGRIND
#define rand_popc64(X) POPC(X)
#endif
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#if SINGELI
#define SINGELI_FILE slash

View File

@ -2,6 +2,3 @@ def pdep{x:u64, m:u64} = emit{u64, '_pdep_u64', x, m}
def pdep{x:u32, m:u32} = emit{u32, '_pdep_u32', x, m}
def pext{x:u64, m:u64} = emit{u64, '_pext_u64', x, m}
def pext{x:u32, m:u32} = emit{u32, '_pext_u32', x, m}
def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones
def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x}

View File

@ -29,6 +29,26 @@ if (hasarch{'AVX512F'}) {
include './mask'
include 'util/tup'
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones
def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x}
def maketab{l,w} = { # Table from l bits to w-bit indices
def lw = l*w
fold{{t,k} => join{each{tup,t,k+(t<<w)%(1<<lw)}}, tup{base{1<<w,l**l}}, reverse{iota{l}}}
}
# 16-element tables
i64tab :*u32 = maketab{4,8}*2
tab_4_16:*u64 = maketab{4,16}
if (1) {
def use_table = 1
itab :*u64 = maketab{8,8}
} else {
def use_table = 0
}
def arg{c,T} = if (c) *T else if (T==i32) T else tup{}
# Modifies the input variable r
@ -65,9 +85,6 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
}
}
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
# Assumes w is trimmed, so the last 1 appears at index l-1
def thresh{c, T} = 2
fn slash{c, T}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
@ -148,7 +165,7 @@ def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i}
def thresh{c, T==i8 & hasarch{'BMI2'}} = 32
def thresh{c, T==i16 & hasarch{'BMI2'}} = 16
fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
fn slash{c, T & T<=i16 & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def wt = width{T}
def b = bind{base, 1<<wt}
def X = if (c) {
@ -175,7 +192,7 @@ fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : voi
}
}
def thresh{c, T==i8 & hasarch{'AVX2'}} = 32
def thresh{c, T==i8 & hasarch{'AVX2'}} = 32
fn slash{c, T==i8 & hasarch{'AVX2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def I = [32]i8
def S = [8]u32
@ -227,12 +244,33 @@ fn slash{c, T==i8 & hasarch{'AVX2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) :
}
}
itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
# TODO avoid calling popcnt if it's not a single instruction
def thresh{c==0, T==i8 & use_table} = 32
def thresh{c==0, T==i16} = 16
fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def tw = width{T}
def n = 64/tw
def tab = if (tw==8) itab else tab_4_16
j:u64 = 0
def inc = base{1<<tw, n**n}
@for_special_buffered{r,8} (w in *u8~~w over sum) {
pc:= popc{w}
def step{r, w} = storeu{*u64~~r, 0, j + load{tab, w}}
if (tw==8) {
step{r,w}
} else {
h := w&0xf
step{r, h}; j += inc
step{r+popcRand{h}, w>>4}
}
j += inc
r += pc
}
}
def thresh{c, T==i32 & hasarch{'AVX2'}} = 32
def thresh{c, T==i64 & hasarch{'AVX2'}} = 8
fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def thresh{c, T==i32 & hasarch{'AVX2'} & use_table} = 32
def thresh{c, T==i64 & hasarch{'AVX2'} } = 8
fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = {
def tw = width{T}
def V = [8]u32
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}