From 534c92f38a746f5e5cdeb73598918e443b6da191 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 29 Nov 2022 22:03:37 -0500 Subject: [PATCH] Select from <=128 booleans with shuffles --- src/builtins/select.c | 11 ++++++++++- src/singeli/src/avx.singeli | 1 + src/singeli/src/select.singeli | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index 8997a82c..30bd3694 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -69,6 +69,12 @@ B select_c2(B t, B w, B x) { #if SINGELI #define CPUSEL(W, NEXT) \ if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("โŠ: Indexing out-of-bounds"); + #define BOOL_USE_SIMD (xia<=128) + #define BOOL_SPECIAL(W) \ + if (sizeof(W)==1 && BOOL_USE_SIMD) { \ + if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("โŠ: Indexing out-of-bounds"); \ + goto dec_ret; \ + } #else #define CPUSEL(W, NEXT) \ if (sizeof(W) >= 4) { \ @@ -90,12 +96,15 @@ B select_c2(B t, B w, B x) { } \ if (wt) TFREE(wt); \ } + #define BOOL_USE_SIMD 0 + #define BOOL_SPECIAL(W) #endif #define CASE(S, E) case S: for (usz i=i0; i=256 && wia/4>=xia && we!=el_bit) { + if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) { return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x))))); } if (we==el_bit) { diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 91064a3d..006ce927 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}} def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}} +def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 46c496bc..d4a6635e 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -4,8 +4,11 @@ include './sse3' include './avx' include './avx2' include './mask' +include './bitops' include 'util/tup' +oper &~ andnot infix none 35 + # def:T - masked original content # b:B - pointer to data to index; if width{B}32 and xl<=16) { + xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + store{r, i, getmask{sel{[16]i8, xb, cw}}} + }} + } else { + x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} + low:= broadcast{VI, 7} + b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)} + mask:= sel{[16]i8, b, cw & low} + store{r, i, getmask{(mask & byte) == mask}} + }} + } + 1 +} +'avx2_select_bool128' = avx2_select_bool128