Select from <=128 booleans with shuffles
This commit is contained in:
parent
fdfeb67e8f
commit
534c92f38a
@ -69,6 +69,12 @@ B select_c2(B t, B w, B x) {
|
|||||||
#if SINGELI
|
#if SINGELI
|
||||||
#define CPUSEL(W, NEXT) \
|
#define CPUSEL(W, NEXT) \
|
||||||
if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds");
|
if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds");
|
||||||
|
#define BOOL_USE_SIMD (xia<=128)
|
||||||
|
#define BOOL_SPECIAL(W) \
|
||||||
|
if (sizeof(W)==1 && BOOL_USE_SIMD) { \
|
||||||
|
if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \
|
||||||
|
goto dec_ret; \
|
||||||
|
}
|
||||||
#else
|
#else
|
||||||
#define CPUSEL(W, NEXT) \
|
#define CPUSEL(W, NEXT) \
|
||||||
if (sizeof(W) >= 4) { \
|
if (sizeof(W) >= 4) { \
|
||||||
@ -90,12 +96,15 @@ B select_c2(B t, B w, B x) {
|
|||||||
} \
|
} \
|
||||||
if (wt) TFREE(wt); \
|
if (wt) TFREE(wt); \
|
||||||
}
|
}
|
||||||
|
#define BOOL_USE_SIMD 0
|
||||||
|
#define BOOL_SPECIAL(W)
|
||||||
#endif
|
#endif
|
||||||
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
|
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
|
||||||
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
|
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
|
||||||
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
|
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
|
||||||
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
|
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
|
||||||
u64* rp; r = m_bitarrc(&rp, w); \
|
u64* rp; r = m_bitarrc(&rp, w); \
|
||||||
|
BOOL_SPECIAL(W) \
|
||||||
u64 b=0; \
|
u64 b=0; \
|
||||||
for (usz i = wia; ; ) { \
|
for (usz i = wia; ; ) { \
|
||||||
i--; \
|
i--; \
|
||||||
@ -121,7 +130,7 @@ B select_c2(B t, B w, B x) {
|
|||||||
for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)))); \
|
for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)))); \
|
||||||
decG(x); return withFill(HARR_FCD(r,w),xf); \
|
decG(x); return withFill(HARR_FCD(r,w),xf); \
|
||||||
}
|
}
|
||||||
if (xe==el_bit && wia>=256 && wia/4>=xia && we!=el_bit) {
|
if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) {
|
||||||
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
|
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
|
||||||
}
|
}
|
||||||
if (we==el_bit) {
|
if (we==el_bit) {
|
||||||
|
|||||||
@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b
|
|||||||
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
|
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
|
||||||
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
|
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
|
||||||
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
|
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
|
||||||
|
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
|
||||||
|
|
||||||
# float comparison
|
# float comparison
|
||||||
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
||||||
|
|||||||
@ -4,8 +4,11 @@ include './sse3'
|
|||||||
include './avx'
|
include './avx'
|
||||||
include './avx2'
|
include './avx2'
|
||||||
include './mask'
|
include './mask'
|
||||||
|
include './bitops'
|
||||||
include 'util/tup'
|
include 'util/tup'
|
||||||
|
|
||||||
|
oper &~ andnot infix none 35
|
||||||
|
|
||||||
# def:T - masked original content
|
# def:T - masked original content
|
||||||
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
|
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
|
||||||
# idx - actual (unscaled) index list
|
# idx - actual (unscaled) index list
|
||||||
@ -135,3 +138,32 @@ def selects = join{table{select, tup{i8, i16, i32}, # indices
|
|||||||
tup{u8, u16, u32, u64}}} # values
|
tup{u8, u16, u32, u64}}} # values
|
||||||
avx2_select_tab:*type{tupsel{0,selects}} = selects
|
avx2_select_tab:*type{tupsel{0,selects}} = selects
|
||||||
'avx2_select_tab' = avx2_select_tab
|
'avx2_select_tab' = avx2_select_tab
|
||||||
|
|
||||||
|
|
||||||
|
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||||
|
def TI = i8
|
||||||
|
def VI = [32]TI
|
||||||
|
w:= *VI ~~ w0
|
||||||
|
r:= *u32 ~~ r0
|
||||||
|
xlf:= broadcast{VI, cast_i{TI, xl}}
|
||||||
|
|
||||||
|
if (wl>32 and xl<=16) {
|
||||||
|
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
|
||||||
|
maskedLoop{32, wl, {i, M} => {
|
||||||
|
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
||||||
|
store{r, i, getmask{sel{[16]i8, xb, cw}}}
|
||||||
|
}}
|
||||||
|
} else {
|
||||||
|
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
|
||||||
|
low:= broadcast{VI, 7}
|
||||||
|
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||||
|
maskedLoop{32, wl, {i, M} => {
|
||||||
|
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
||||||
|
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
|
||||||
|
mask:= sel{[16]i8, b, cw & low}
|
||||||
|
store{r, i, getmask{(mask & byte) == mask}}
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
1
|
||||||
|
}
|
||||||
|
'avx2_select_bool128' = avx2_select_bool128
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user