Emulate pext instruction in Singeli
This commit is contained in:
parent
3946f2cc87
commit
f315a6d3ea
@ -638,7 +638,7 @@ cachedBin‿linkerCache ← {
|
||||
|
||||
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
||||
"2a."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq",
|
||||
"2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
||||
"xag"‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"count"
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
// Boolean 𝕨 (Where/Compress) general case based on result type width
|
||||
// COULD use AVX-512
|
||||
// Size 1: pext, or bit-at-a-time
|
||||
// SHOULD emulate pext if unavailable
|
||||
// Emulate pext if unavailable
|
||||
// COULD return boolean result from Where
|
||||
// Size 8, 16: pdep/pext, or branchless
|
||||
// SHOULD try vector lookup-shuffle if unavailable or old AMD
|
||||
@ -70,15 +70,18 @@
|
||||
#define _pdep_u64 vg_pdep_u64
|
||||
#else
|
||||
#define vg_loadLUT64(p, i) p[i]
|
||||
#define rand_popc64(X) POPC(X)
|
||||
#endif
|
||||
|
||||
static void storeu_u64(u64* p, u64 v) { memcpy(p, &v, 8); }
|
||||
static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, p, 8); return v; }
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_FILE slash
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
#endif
|
||||
#if !USE_VALGRIND
|
||||
#define rand_popc64(X) POPC(X)
|
||||
#endif
|
||||
|
||||
#if SINGELI
|
||||
#define SINGELI_FILE slash
|
||||
#include "../utils/includeSingeli.h"
|
||||
#endif
|
||||
|
||||
#if SINGELI_AVX2 || SINGELI_NEON
|
||||
@ -437,13 +440,17 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
default: r = compress_grouped(wp, x, wia, wsum, xt); break;
|
||||
case 0: {
|
||||
u64* xp = bitarr_ptr(x); u64* rp;
|
||||
#if defined(__BMI2__)
|
||||
#if defined(__BMI2__) || SINGELI
|
||||
r = m_bitarrv(&rp,wsum+128); a(r)->ia = wsum;
|
||||
u64 cw = 0; // current word
|
||||
u64 ro = 0; // offset in word where next bit should be written; never 64
|
||||
for (usz i=0; i<BIT_N(wia); i++) {
|
||||
u64 wv = wp[i];
|
||||
#if defined(__BMI2__)
|
||||
u64 v = _pext_u64(xp[i], wv);
|
||||
#else
|
||||
u64 v = si_pext_u64(xp[i], wv);
|
||||
#endif
|
||||
u64 c = rand_popc64(wv);
|
||||
cw|= v<<ro;
|
||||
u64 ro2 = ro+c;
|
||||
|
||||
@ -160,6 +160,9 @@ def minvalue{T & issigned{T}} = - (1<<(width{T}-1))
|
||||
def maxvalue{T & issigned{T}} = (1<<(width{T}-1))-1
|
||||
|
||||
# base cases
|
||||
def pdep{...x} = assert{'pdep not supported', show{...x}}
|
||||
def pext{...x} = assert{'pext not supported', show{...x}}
|
||||
def popcRand{...x} = assert{'popcRand not supported', show{...x}}
|
||||
def andnz{...x} = assert{'andnz not supported', show{...x}}
|
||||
def topBlend{...x} = assert{'topBlend not supported', show{...x}}
|
||||
def topMask{...x} = assert{'topMask not supported', show{...x}}
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
include './base'
|
||||
include './bmi2'
|
||||
if (hasarch{'BMI2'}) {
|
||||
include './bmi2'
|
||||
}
|
||||
include 'util/tup'
|
||||
|
||||
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
@ -45,7 +48,58 @@ fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
|
||||
}
|
||||
|
||||
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
|
||||
if (hasarch{'BMI2'}) {
|
||||
export{'bmipopc_2slash8', slash2{comp8, i8}}
|
||||
export{'bmipopc_2slash16', slash2{comp16, i16}}
|
||||
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
|
||||
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
|
||||
}
|
||||
|
||||
# pext, or boolean compress
|
||||
fn pext{T}(x:T, m:T) {
|
||||
def w = width{T}
|
||||
def mod{a} = a % (1<<w)
|
||||
def lowbits{k} = base{1<<k, cdiv{w,k}**1}
|
||||
# At each step, x and z are split into groups of length k
|
||||
# - z tells how many bits in the group are NOT used
|
||||
# - x contains the bits, with z zeros above
|
||||
def build{k==1} = tup{x&m, ~m}
|
||||
def build{k & k > 1} = {
|
||||
def h = k>>1 # Increase size from h to k
|
||||
{x,z} := build{h}
|
||||
def low = lowbits{k} # Low bit in each new group
|
||||
if (k <= 3) {
|
||||
z0 := z & low
|
||||
zm := z>>1 & low
|
||||
if (k == 2) tup{
|
||||
x - (x>>1 & z0),
|
||||
z0 + zm
|
||||
} else tup{ # Faster 1->3 jump, currently unused
|
||||
x - ((x>>1&mod{low*3}) & (z|z0<<1)) - (x>>2 & (z & zm)),
|
||||
(z0 + zm) + (z>>2 & low)
|
||||
}
|
||||
} else {
|
||||
# Shift high x group down by low z, then add halves of z
|
||||
even:T = mod{low*(1<<h - 1)}
|
||||
# SWAR shifter: shift x by sh*o, in length-k groups
|
||||
def shift{sh, o, x} = {
|
||||
l := o & low; m := l<<k - l
|
||||
s := (x & m)>>sh | (x &~ m)
|
||||
if (2*sh<=k/2) shift{2*sh, o>>1, s} else s
|
||||
}
|
||||
tup{
|
||||
(x&even) | shift{1, z, x&~even},
|
||||
if (k>4) (z + z>>h)&even else ((z&~even)>>h) + (z&even)
|
||||
}
|
||||
}
|
||||
}
|
||||
# Finally, compose groups with regular shifts
|
||||
def g = 8 # 12 performs about the same
|
||||
{b,z} := build{g}
|
||||
o := z*lowbits{g} # Offsets by prefix sum
|
||||
def s = 1<<g - 1
|
||||
def gr{sh} = (b & mod{s<<sh}) >> (o>>(sh-g) & s)
|
||||
fold{|, b&s, each{gr, g*slice{iota{cdiv{w,g}},1}}}
|
||||
}
|
||||
|
||||
export{'si_pext_u64', pext{u64}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user