Emulate pext instruction in Singeli

This commit is contained in:
Marshall Lochbaum 2023-07-11 10:57:15 -04:00
parent 3946f2cc87
commit f315a6d3ea
4 changed files with 73 additions and 9 deletions

View File

@ -638,7 +638,7 @@ cachedBin‿linkerCache ← {
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",
"2a.""src/builtins/slash.c""constrep", "2..""src/builtins/scan.c""neq",
"2..""src/builtins/slash.c""slash", "2..""src/builtins/slash.c""count"
"xag""src/builtins/slash.c""slash", "2..""src/builtins/slash.c""count"
objs

View File

@ -4,7 +4,7 @@
// Boolean 𝕨 (Where/Compress) general case based on result type width
// COULD use AVX-512
// Size 1: pext, or bit-at-a-time
// SHOULD emulate pext if unavailable
// Emulate pext if unavailable
// COULD return boolean result from Where
// Size 8, 16: pdep/pext, or branchless
// SHOULD try vector lookup-shuffle if unavailable or old AMD
@ -70,15 +70,18 @@
#define _pdep_u64 vg_pdep_u64
#else
#define vg_loadLUT64(p, i) p[i]
#define rand_popc64(X) POPC(X)
#endif
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
#if SINGELI_AVX2
#define SINGELI_FILE slash
#include "../utils/includeSingeli.h"
#endif
#endif
#if !USE_VALGRIND
#define rand_popc64(X) POPC(X)
#endif
#if SINGELI
#define SINGELI_FILE slash
#include "../utils/includeSingeli.h"
#endif
#if SINGELI_AVX2 || SINGELI_NEON
@ -437,13 +440,17 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
default: r = compress_grouped(wp, x, wia, wsum, xt); break;
case 0: {
u64* xp = bitarr_ptr(x); u64* rp;
#if defined(__BMI2__)
#if defined(__BMI2__) || SINGELI
r = m_bitarrv(&rp,wsum+128); a(r)->ia = wsum;
u64 cw = 0; // current word
u64 ro = 0; // offset in word where next bit should be written; never 64
for (usz i=0; i<BIT_N(wia); i++) {
u64 wv = wp[i];
#if defined(__BMI2__)
u64 v = _pext_u64(xp[i], wv);
#else
u64 v = si_pext_u64(xp[i], wv);
#endif
u64 c = rand_popc64(wv);
cw|= v<<ro;
u64 ro2 = ro+c;

View File

@ -160,6 +160,9 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
# base cases
def pdep{...x} = assert{'pdep not supported', show{...x}}
def pext{...x} = assert{'pext not supported', show{...x}}
def popcRand{...x} = assert{'popcRand not supported', show{...x}}
def andnz{...x} = assert{'andnz not supported', show{...x}}
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
def topMask{...x} = assert{'topMask not supported', show{...x}}

View File

@ -1,5 +1,8 @@
include './base'
include './bmi2'
if (hasarch{'BMI2'}) {
include './bmi2'
}
include 'util/tup'
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
@ -45,7 +48,58 @@ fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
}
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
if (hasarch{'BMI2'}) {
export{'bmipopc_2slash8', slash2{comp8, i8}}
export{'bmipopc_2slash16', slash2{comp16, i16}}
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
}
# pext, or boolean compress
fn pext{T}(x:T, m:T) {
def w = width{T}
def mod{a} = a % (1<<w)
def lowbits{k} = base{1<<k, cdiv{w,k}**1}
# At each step, x and z are split into groups of length k
# - z tells how many bits in the group are NOT used
# - x contains the bits, with z zeros above
def build{k==1} = tup{x&m, ~m}
def build{k & k > 1} = {
def h = k>>1 # Increase size from h to k
{x,z} := build{h}
def low = lowbits{k} # Low bit in each new group
if (k <= 3) {
z0 := z & low
zm := z>>1 & low
if (k == 2) tup{
x - (x>>1 & z0),
z0 + zm
} else tup{ # Faster 1->3 jump, currently unused
x - ((x>>1&mod{low*3}) & (z|z0<<1)) - (x>>2 & (z & zm)),
(z0 + zm) + (z>>2 & low)
}
} else {
# Shift high x group down by low z, then add halves of z
even:T = mod{low*(1<<h - 1)}
# SWAR shifter: shift x by sh*o, in length-k groups
def shift{sh, o, x} = {
l := o & low; m := l<<k - l
s := (x & m)>>sh | (x &~ m)
if (2*sh<=k/2) shift{2*sh, o>>1, s} else s
}
tup{
(x&even) | shift{1, z, x&~even},
if (k>4) (z + z>>h)&even else ((z&~even)>>h) + (z&even)
}
}
}
# Finally, compose groups with regular shifts
def g = 8 # 12 performs about the same
{b,z} := build{g}
o := z*lowbits{g} # Offsets by prefix sum
def s = 1<<g - 1
def gr{sh} = (b & mod{s<<sh}) >> (o>>(sh-g) & s)
fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}}
}
export{'si_pext_u64', pext{u64}}