Select from <=128 booleans with shuffles

This commit is contained in:
Marshall Lochbaum 2022-11-29 22:03:37 -05:00
parent fdfeb67e8f
commit 534c92f38a
3 changed files with 43 additions and 1 deletions

View File

@ -69,6 +69,12 @@ B select_c2(B t, B w, B x) {
#if SINGELI
#define CPUSEL(W, NEXT) \
if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds");
#define BOOL_USE_SIMD (xia<=128)
#define BOOL_SPECIAL(W) \
if (sizeof(W)==1 && BOOL_USE_SIMD) { \
if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \
goto dec_ret; \
}
#else
#define CPUSEL(W, NEXT) \
if (sizeof(W) >= 4) { \
@ -90,12 +96,15 @@ B select_c2(B t, B w, B x) {
} \
if (wt) TFREE(wt); \
}
#define BOOL_USE_SIMD 0
#define BOOL_SPECIAL(W)
#endif
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
u64* rp; r = m_bitarrc(&rp, w); \
BOOL_SPECIAL(W) \
u64 b=0; \
for (usz i = wia; ; ) { \
i--; \
@ -121,7 +130,7 @@ B select_c2(B t, B w, B x) {
for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)))); \
decG(x); return withFill(HARR_FCD(r,w),xf); \
}
if (xe==el_bit && wia>=256 && wia/4>=xia && we!=el_bit) {
if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) {
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
}
if (we==el_bit) {

View File

@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}

View File

@ -4,8 +4,11 @@ include './sse3'
include './avx'
include './avx2'
include './mask'
include './bitops'
include 'util/tup'
oper &~ andnot infix none 35
# def:T - masked original content
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
# idx - actual (unscaled) index list
@ -135,3 +138,32 @@ def selects = join{table{select, tup{i8, i16, i32}, # indices
tup{u8, u16, u32, u64}}} # values
avx2_select_tab:*type{tupsel{0,selects}} = selects
'avx2_select_tab' = avx2_select_tab
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def TI = i8
def VI = [32]TI
w:= *VI ~~ w0
r:= *u32 ~~ r0
xlf:= broadcast{VI, cast_i{TI, xl}}
if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
store{r, i, getmask{sel{[16]i8, xb, cw}}}
}}
} else {
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
low:= broadcast{VI, 7}
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
mask:= sel{[16]i8, b, cw & low}
store{r, i, getmask{(mask & byte) == mask}}
}}
}
1
}
'avx2_select_bool128' = avx2_select_bool128