From 68978f7e107c9235dfdcaec5f491c4e58d430797 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jul 2023 16:05:06 -0400 Subject: [PATCH] Table-based 1- and 2-byte Where --- src/builtins/slash.c | 5 +-- src/singeli/src/bmi2.singeli | 3 -- src/singeli/src/slash.singeli | 58 +++++++++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 7dba710a..28265d19 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -72,12 +72,13 @@ #define vg_loadLUT64(p, i) p[i] #endif - static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } - static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #endif + #if !USE_VALGRIND #define rand_popc64(X) POPC(X) #endif +static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); } +static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; } #if SINGELI #define SINGELI_FILE slash diff --git a/src/singeli/src/bmi2.singeli b/src/singeli/src/bmi2.singeli index 3fdcb7b0..cf13020e 100644 --- a/src/singeli/src/bmi2.singeli +++ b/src/singeli/src/bmi2.singeli @@ -2,6 +2,3 @@ def pdep{x:u64, m:u64} = emit{u64, '_pdep_u64', x, m} def pdep{x:u32, m:u32} = emit{u32, '_pdep_u32', x, m} def pext{x:u64, m:u64} = emit{u64, '_pext_u64', x, m} def pext{x:u32, m:u32} = emit{u32, '_pext_u32', x, m} - -def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones -def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index cf4178be..a5358a6d 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -29,6 +29,26 @@ if (hasarch{'AVX512F'}) { include './mask' include 'util/tup' +def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} +def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} + +def popcRand{x:T & isint{T} & width{T}==64} = emit{u8, 'rand_popc64', x} # under valgrind, return a random result in the range of possible ones +def popcRand{x:T & isint{T} & width{T}<=32} = emit{u8, 'rand_popc64', x} + +def maketab{l,w} = { # Table from l bits to w-bit indices + def lw = l*w + fold{{t,k} => join{each{tup,t,k+(t< join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} -i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} +# TODO avoid calling popcnt if it's not a single instruction +def thresh{c==0, T==i8 & use_table} = 32 +def thresh{c==0, T==i16} = 16 +fn slash{c==0, T & (if (T==i8) use_table else T==i16)}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { + def tw = width{T} + def n = 64/tw + def tab = if (tw==8) itab else tab_4_16 + j:u64 = 0 + def inc = base{1<>4} + } + j += inc + r += pc + } +} -def thresh{c, T==i32 & hasarch{'AVX2'}} = 32 -def thresh{c, T==i64 & hasarch{'AVX2'}} = 8 -fn slash{c, T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { +def thresh{c, T==i32 & hasarch{'AVX2'} & use_table} = 32 +def thresh{c, T==i64 & hasarch{'AVX2'} } = 8 +fn slash{c, T & hasarch{'AVX2'} & (if (T==i32) use_table else T==i64)}(wp:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}