Merge pull request #64 from mlochbaum/avx2sel

AVX2 selection
This commit is contained in:
dzaima 2022-12-02 16:42:12 +02:00 committed by GitHub
commit ae6e3a6504
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 239 additions and 71 deletions

View File

@ -278,7 +278,7 @@ endif
@${MAKE} i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c fold.c scan.c neq.c slash.c constrep.c bits.c}
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c}
@echo $(postmsg)
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
@echo $< | cut -c 17- | sed 's/^/ /'

View File

@ -1,14 +1,52 @@
// First Cell and Select (⊏)
// First Cell is just a slice
// Complications in Select mostly come from range checks and negative 𝕨
// Atom 𝕨 and any rank 𝕩: slice
// Rank-1 𝕩:
// Empty 𝕨: no selection
// Small 𝕩 with Singeli: use shuffles
// Boolean 𝕨: use bit_sel for blend or similar
// Boolean 𝕩 and larger 𝕨: convert to i8, select, convert back
// Boolean 𝕩 otherwise: select/shift bytes, reversed for fast writing
// TRIED pext, doesn't seem faster (mask built with shifts anyway)
// SHOULD squeeze 𝕨 if not ≤i32 to get to optimized cases
// Integer 𝕨 with Singeli: fused wrap, range-check, and gather
// COULD try selecting from boolean with gather
// COULD detect <Skylake where gather is slow
// i32 𝕨: wrap, check, select one index at a time
// i8 and i16 𝕨: separate range check in blocks to auto-vectorize
// SHOULD optimize simple 𝕨 based on cell size for any rank 𝕩
// SHOULD implement nested 𝕨
// Under Select ⌾(i⊸⊏)
// Specialized for rank-1 numeric 𝕩
// SHOULD apply to characters as well
// No longer needs to range-check but indices can be negative
// COULD convert negative indices before selection
// Must check collisions if CHECK_VALID; uses a byte set
// Sparse initialization if 𝕨 is much smaller than 𝕩
// COULD call Mark Firsts (∊) for very short 𝕨 to avoid allocation
#include "../core.h"
#include "../utils/talloc.h"
#include "../utils/mut.h"
#include "../builtins.h"
// #if SINGELI
// #pragma GCC diagnostic push
// #pragma GCC diagnostic ignored "-Wunused-variable"
// #include "../singeli/gen/select.c"
// #pragma GCC diagnostic pop
// #endif
#if SINGELI
#include <xmmintrin.h>
#if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32
static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); }
static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); }
#define _mm_loadu_si32 custom_loadu_si32
#define _mm_storeu_si32 custom_storeu_si32
#endif
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#include "../singeli/gen/select.c"
#pragma GCC diagnostic pop
#endif
extern B rt_select;
B select_c1(B t, B x) {
@ -67,31 +105,16 @@ B select_c2(B t, B w, B x) {
u8 xe = TI(x,elType);
u8 we = TI(w,elType);
#if SINGELI
// if (we==el_i8 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i8_32 (i8any_ptr (w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
// if (we==el_i16 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i16_32(i16any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
// if (we==el_i32 && xe==el_i8 ) { i8* rp; r = m_i8arrc (&rp, w); if (!avx2_select_i32_8 (i32any_ptr(w), i8any_ptr (x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
// if (we==el_i32 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i32_32(i32any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
// if (we==el_i32 && xe==el_f64) { f64* rp; r = m_f64arrc(&rp, w); if (!avx2_select_i32_64(i32any_ptr(w), f64any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
#endif
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
u64* rp; r = m_bitarrc(&rp, w); \
u64 b=0; \
for (usz i = wia; ; ) { \
i--; \
usz n = WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)); \
b <<= 1; \
b |= (-(xp[n/64] & (1ull<<(n%64)))) >> 63; \
if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \
} \
goto dec_ret; \
} \
if (xe!=el_B) { \
usz xw = elWidth(xe); \
void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \
void* xp = tyany_ptr(x); \
#define CPUSEL(W, NEXT) \
if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds");
#define BOOL_USE_SIMD (xia<=128)
#define BOOL_SPECIAL(W) \
if (sizeof(W)==1 && BOOL_USE_SIMD) { \
if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \
goto dec_ret; \
}
#else
#define CPUSEL(W, NEXT) \
if (sizeof(W) >= 4) { \
switch(xw) { default:UD; CASEW(1,u8); CASEW(2,u16); CASEW(4,u32); CASEW(8,f64); } \
} else { \
@ -110,9 +133,32 @@ B select_c2(B t, B w, B x) {
switch(xw) { default:UD; CASE(1,u8); CASE(2,u16); CASE(4,u32); CASE(8,f64); } \
} \
if (wt) TFREE(wt); \
}
#define BOOL_USE_SIMD 0
#define BOOL_SPECIAL(W)
#endif
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
u64* rp; r = m_bitarrc(&rp, w); \
BOOL_SPECIAL(W) \
u64 b=0; \
for (usz i = wia; ; ) { \
i--; \
usz n = WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)); \
b = 2*b + ((((u8*)xp)[n/8] >> (n%8)) & 1); \
if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \
} \
goto dec_ret; \
} \
if (xe!=el_B) { \
usz xw = elWidth(xe); \
void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \
void* xp = tyany_ptr(x); \
CPUSEL(W, NEXT) \
goto dec_ret; \
} \
M_HARR(r, wia); \
if (TY(x)==t_harr || TY(x)==t_hslice) { \
B* xp = hany_ptr(x); \
@ -122,7 +168,7 @@ B select_c2(B t, B w, B x) {
for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)))); \
decG(x); return withFill(HARR_FCD(r,w),xf); \
}
if (xe==el_bit && wia>=256 && wia/4>=xia && we!=el_bit) {
if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) {
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
}
if (we==el_bit) {
@ -210,11 +256,17 @@ B select_ucw(B t, B o, B w, B x) {
if (isAtm(rep) || !eqShape(w, rep)) thrF("𝔽⌾(a⊸⊏)𝕩: Result of 𝔽 must have the same shape as 'a' (expected %H, got %H)", w, rep);
#if CHECK_VALID
TALLOC(bool, set, xia);
for (i64 i = 0; i < xia; i++) set[i] = false;
bool sparse = wia < xia/64;
if (!sparse) for (i64 i = 0; i < xia; i++) set[i] = false;
#define SPARSE_INIT(WI) \
if (sparse) for (usz i = 0; i < wia; i++) { \
i64 cw = WI; if (RARE(cw<0)) cw+= (i64)xia; set[cw] = false; \
}
#define EQ(F) if (set[cw] && (F)) thrM("𝔽⌾(a⊸⊏): Incompatible result elements"); set[cw] = true;
#define FREE_CHECK TFREE(set)
SLOWIF(xia>100 && wia<xia/20) SLOW2("⌾(𝕨⊸⊏)𝕩 because CHECK_VALID", w, x);
#else
#define SPARSE_INIT(GET)
#define EQ(F)
#define FREE_CHECK
#endif
@ -226,6 +278,7 @@ B select_ucw(B t, B o, B w, B x) {
if (elInt(we)) {
w = toI32Any(w);
i32* wp = i32any_ptr(w);
SPARSE_INIT(wp[i])
if (elNum(re) && elNum(xe)) {
u8 me = xe>re?xe:re;
bool reuse = reusable(x);
@ -319,6 +372,7 @@ B select_ucw(B t, B o, B w, B x) {
MUTG_INIT(r);
mut_copyG(r, 0, x, 0, xia);
SGet(rep)
SPARSE_INIT(o2i64G(GetU(w, i)))
for (usz i = 0; i < wia; i++) {
i64 cw = o2i64G(GetU(w, i)); if (RARE(cw<0)) cw+= (i64)xia;
B cr = Get(rep, i);
@ -328,6 +382,7 @@ B select_ucw(B t, B o, B w, B x) {
}
decG(w); decG(rep); FREE_CHECK;
return mut_fcd(r, x);
#undef SPARSE_INIT
#undef EQ
#undef FREE_CHECK
}

View File

@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
# float comparison
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
@ -102,6 +103,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
# conversion
def half{x:T, i & w256{T} & knum{i}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
def half{x:T, i==0 & w256{T}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}

View File

@ -107,8 +107,8 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void,
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
@ -186,4 +186,4 @@ def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO che
def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}

View File

@ -16,6 +16,9 @@ def lb{n==1} = 0
def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
def bit {k,x} = x & (1<<k) # get the k-th bit
# Convert tuple to number in little-endian base b
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
def assert{x & x==0} = assert{'failed assertion'}
def assert{x & x==1} = 1

View File

@ -39,8 +39,7 @@ def get_rep_iter{V, wv==2}{x, gen} = {
}
def get_rep_iter{V==[4]u64, wv} = {
def step = 4
def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} }
def sh = each{base4, get_shufs{step, wv, wv}}
def sh = each{bind{base,4}, get_shufs{step, wv, wv}}
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
}
def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = {

View File

@ -7,7 +7,6 @@ include './mask'
def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}}
def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}}
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}}
# Fill last 4 bytes with last element, in each lane

View File

@ -4,9 +4,10 @@ include './sse3'
include './avx'
include './avx2'
include './mask'
include './bitops'
include 'util/tup'
def minBulk{w, A, B & width{A}< width{B}} = w/width{B}
def minBulk{w, A, B & width{A}>=width{B}} = w/width{A}
oper &~ andnot infix none 35
# def:T - masked original content
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
@ -20,36 +21,149 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *i64~~b, idx, width{eltype{B}}/8}
}
def wrapChk{cw0, VI,xlf, M} = {
cw:= cw0 + (xlf & VI~~(cw0<broadcast{VI, 0}))
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
cw
}
def storeExp{dst, ind, val, M, ext, rd, wl} = {
def s{M} = storeBatch{dst, ind, val, M}
if (ext==1 or not M{0}) s{M}
else if (ind*rd+rd <= wl) s{maskNone}
else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} }
}
def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI
def ext = ri/rd
xlf:= broadcast{VI, cast_i{TI, xl}}
maskedLoop{ri, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i)
def se{e, c, o} = {
c2:= shuf{[4]u64, c+c, 4b3120}
each{
{c,o} => se{e*2, VI~~c, o},
unpackQ{c2, c2+broadcast{VI,1}},
2*o + iota{2}
}
}
def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl}
se{1, cw, 0}
}}
}
def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
def VI = [ri]TI
def ext = ri/rd
xlf:= broadcast{VI, cast_i{TI, xl}}
maskedLoop{ri, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
is:= (if (ext>1) i<<lb{ext}; else i)
def part{o} = cvt{i8, [8]i32, shuf{[4]u64, cw, 4b3210+o}}
def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl}
each{se, iota{ext}}
}}
}
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m}
def bblendn{m}{tf} = bblend{m}{reverse{tf}}
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})
def bs{b, c, x} = cshuf{x, c}
def bs{b, c, x & tuplen{b}>0} = {
tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}}
}
def i = iota{logv}
def vs = each{bind{broadcast,VI}, nsel<<reverse{i}}
{c} => VD~~bs{each{bb{c},i==0,vs}, c, xd}
}
def makeshuf{VI, VD, x0, logv} = {
x:= *VD~~x0
def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}}
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)}
def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010}
def readx{l==1,o} = halves{load{x, o}}
xd:= readx{logv, 0}
makeselx{VI,VD,16,xd,logv, bind{sel,[16]i8}}
}
def makeperm{VI, VD, x0, logv} = {
x:= *VD~~x0
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-1)}
def readx{l==0,o} = load{x, o}
makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}}
}
select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def TIE = i32
def TDE = tern{width{TD}<32, u32, TD}
def bulk = minBulk{rw,TIE,TDE}
def TIF = [bulk]TIE
def TDF = [bulk]TDE
def xlf = broadcast{TIF, cast_i{eltype{TIF}, xl}}
w:= *TI ~~ w0
x:= *TD ~~ x0
r:= *TD ~~ r0
maskedLoop{bulk, wl, {i, M} => {
cw0:= loadBatch{w, i, TIF}
cw1:= cw0+xlf
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
got:= gather{broadcast{TDF,0}, x, cw, M}
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
storeBatch{r, i, got, M}
}}
def wd = width{TD}; def rd = rw/wd
def wi = width{TI}; def ri = rw/wi
def reg_select{sel,make}{l} = {
sel{ri, rd, TI, w, r, wl, xl, make{[ri]TI,[rd]TD, x,l}}
}
def shuf_select = reg_select{shuf_select, makeshuf}
def perm_select = reg_select{perm_select, makeperm}
if (wi==8 and wd==32 and xl*wd<=256 ) perm_select{0}
else if (wi==8 and wd==32 and xl*wd<=256<<1) perm_select{1}
else if (wi==8 and wd<=16 and xl*wd<=128 ) shuf_select{0}
else if (wi==8 and wd<=16 and xl*wd<=128<<1) shuf_select{1}
else if (wi==8 and wd<=16 and xl*wd<=128<<2) shuf_select{2}
else if (wi==8 and wd<= 8 and xl*wd<=128<<3) shuf_select{3}
else {
def TIE = i32
def TDE = tern{wd<32, u32, TD}
def bulk = rw / width{TDE}
def VI = [bulk]TIE
def VD = [bulk]TDE
def xlf = broadcast{VI, cast_i{TIE, xl}}
maskedLoop{bulk, wl, {i, M} => {
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
got:= gather{broadcast{VD,0}, x, cw, M}
if (TDE!=TD) got&= broadcast{VD, (1<<wd)-1}
storeBatch{r, i, got, M}
}}
}
1
}
def select{TI, TD} = select{256, TI, TD}
def selects = join{table{select, tup{i8, i16, i32}, # indices
tup{u8, u16, u32, u64}}} # values
avx2_select_tab:*type{tupsel{0,selects}} = selects
'avx2_select_tab' = avx2_select_tab
'avx2_select_i32_8' = select{256, i32, u8}
'avx2_select_i32_16' = select{256, i32, u16}
'avx2_select_i32_32' = select{256, i32, u32}
'avx2_select_i32_64' = select{256, i32, u64}
'avx2_select_i8_32' = select{256, i8, u32}
'avx2_select_i16_32' = select{256, i16, u32}
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
def TI = i8
def VI = [32]TI
w:= *VI ~~ w0
r:= *u32 ~~ r0
xlf:= broadcast{VI, cast_i{TI, xl}}
if (wl>32 and xl<=16) {
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
store{r, i, getmask{sel{[16]i8, xb, cw}}}
}}
} else {
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
low:= broadcast{VI, 7}
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
maskedLoop{32, wl, {i, M} => {
cw:= wrapChk{load{w, i}, VI,xlf, M}
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
mask:= sel{[16]i8, b, cw & low}
store{r, i, getmask{(mask & byte) == mask}}
}}
}
1
}
'avx2_select_bool128' = avx2_select_bool128

View File

@ -220,12 +220,8 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
#if SINGELI
#include <xmmintrin.h>
#if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32
__m128i custom_loadu_si32(void* p) {
return (__m128i) _mm_load_ss(p);
}
void custom_storeu_si32(void* p, __m128i x) {
_mm_store_ss(p, _mm_castsi128_ps(x));
}
static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); }
static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); }
#define _mm_loadu_si32 custom_loadu_si32
#define _mm_storeu_si32 custom_storeu_si32
#endif