commit
ae6e3a6504
2
makefile
2
makefile
@ -278,7 +278,7 @@ endif
|
||||
@${MAKE} i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c
|
||||
|
||||
|
||||
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c fold.c scan.c neq.c slash.c constrep.c bits.c}
|
||||
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c}
|
||||
@echo $(postmsg)
|
||||
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
||||
@echo $< | cut -c 17- | sed 's/^/ /'
|
||||
|
||||
@ -1,14 +1,52 @@
|
||||
// First Cell and Select (⊏)
|
||||
|
||||
// First Cell is just a slice
|
||||
|
||||
// Complications in Select mostly come from range checks and negative 𝕨
|
||||
// Atom 𝕨 and any rank 𝕩: slice
|
||||
// Rank-1 𝕩:
|
||||
// Empty 𝕨: no selection
|
||||
// Small 𝕩 with Singeli: use shuffles
|
||||
// Boolean 𝕨: use bit_sel for blend or similar
|
||||
// Boolean 𝕩 and larger 𝕨: convert to i8, select, convert back
|
||||
// Boolean 𝕩 otherwise: select/shift bytes, reversed for fast writing
|
||||
// TRIED pext, doesn't seem faster (mask built with shifts anyway)
|
||||
// SHOULD squeeze 𝕨 if not ≤i32 to get to optimized cases
|
||||
// Integer 𝕨 with Singeli: fused wrap, range-check, and gather
|
||||
// COULD try selecting from boolean with gather
|
||||
// COULD detect <Skylake where gather is slow
|
||||
// i32 𝕨: wrap, check, select one index at a time
|
||||
// i8 and i16 𝕨: separate range check in blocks to auto-vectorize
|
||||
// SHOULD optimize simple 𝕨 based on cell size for any rank 𝕩
|
||||
// SHOULD implement nested 𝕨
|
||||
|
||||
// Under Select ⌾(i⊸⊏)
|
||||
// Specialized for rank-1 numeric 𝕩
|
||||
// SHOULD apply to characters as well
|
||||
// No longer needs to range-check but indices can be negative
|
||||
// COULD convert negative indices before selection
|
||||
// Must check collisions if CHECK_VALID; uses a byte set
|
||||
// Sparse initialization if 𝕨 is much smaller than 𝕩
|
||||
// COULD call Mark Firsts (∊) for very short 𝕨 to avoid allocation
|
||||
|
||||
#include "../core.h"
|
||||
#include "../utils/talloc.h"
|
||||
#include "../utils/mut.h"
|
||||
#include "../builtins.h"
|
||||
|
||||
// #if SINGELI
|
||||
// #pragma GCC diagnostic push
|
||||
// #pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
// #include "../singeli/gen/select.c"
|
||||
// #pragma GCC diagnostic pop
|
||||
// #endif
|
||||
#if SINGELI
|
||||
#include <xmmintrin.h>
|
||||
#if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32
|
||||
static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); }
|
||||
static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); }
|
||||
#define _mm_loadu_si32 custom_loadu_si32
|
||||
#define _mm_storeu_si32 custom_storeu_si32
|
||||
#endif
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#include "../singeli/gen/select.c"
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
extern B rt_select;
|
||||
B select_c1(B t, B x) {
|
||||
@ -67,31 +105,16 @@ B select_c2(B t, B w, B x) {
|
||||
u8 xe = TI(x,elType);
|
||||
u8 we = TI(w,elType);
|
||||
#if SINGELI
|
||||
// if (we==el_i8 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i8_32 (i8any_ptr (w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
|
||||
// if (we==el_i16 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i16_32(i16any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
|
||||
// if (we==el_i32 && xe==el_i8 ) { i8* rp; r = m_i8arrc (&rp, w); if (!avx2_select_i32_8 (i32any_ptr(w), i8any_ptr (x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
|
||||
// if (we==el_i32 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i32_32(i32any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
|
||||
// if (we==el_i32 && xe==el_f64) { f64* rp; r = m_f64arrc(&rp, w); if (!avx2_select_i32_64(i32any_ptr(w), f64any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; }
|
||||
#endif
|
||||
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
|
||||
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
|
||||
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
|
||||
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
|
||||
u64* rp; r = m_bitarrc(&rp, w); \
|
||||
u64 b=0; \
|
||||
for (usz i = wia; ; ) { \
|
||||
i--; \
|
||||
usz n = WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)); \
|
||||
b <<= 1; \
|
||||
b |= (-(xp[n/64] & (1ull<<(n%64)))) >> 63; \
|
||||
if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \
|
||||
} \
|
||||
goto dec_ret; \
|
||||
} \
|
||||
if (xe!=el_B) { \
|
||||
usz xw = elWidth(xe); \
|
||||
void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \
|
||||
void* xp = tyany_ptr(x); \
|
||||
#define CPUSEL(W, NEXT) \
|
||||
if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds");
|
||||
#define BOOL_USE_SIMD (xia<=128)
|
||||
#define BOOL_SPECIAL(W) \
|
||||
if (sizeof(W)==1 && BOOL_USE_SIMD) { \
|
||||
if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \
|
||||
goto dec_ret; \
|
||||
}
|
||||
#else
|
||||
#define CPUSEL(W, NEXT) \
|
||||
if (sizeof(W) >= 4) { \
|
||||
switch(xw) { default:UD; CASEW(1,u8); CASEW(2,u16); CASEW(4,u32); CASEW(8,f64); } \
|
||||
} else { \
|
||||
@ -110,9 +133,32 @@ B select_c2(B t, B w, B x) {
|
||||
switch(xw) { default:UD; CASE(1,u8); CASE(2,u16); CASE(4,u32); CASE(8,f64); } \
|
||||
} \
|
||||
if (wt) TFREE(wt); \
|
||||
}
|
||||
#define BOOL_USE_SIMD 0
|
||||
#define BOOL_SPECIAL(W)
|
||||
#endif
|
||||
#define CASE(S, E) case S: for (usz i=i0; i<i1; i++) ((E*)rp)[i] = ((E*)xp+off)[ip[i]]; break
|
||||
#define CASEW(S, E) case S: for (usz i=0; i<wia; i++) ((E*)rp)[i] = ((E*)xp)[WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia))]; break
|
||||
#define TYPE(W, NEXT) { W* wp = W##any_ptr(w); \
|
||||
if (xe==el_bit) { u64* xp=bitarr_ptr(x); \
|
||||
u64* rp; r = m_bitarrc(&rp, w); \
|
||||
BOOL_SPECIAL(W) \
|
||||
u64 b=0; \
|
||||
for (usz i = wia; ; ) { \
|
||||
i--; \
|
||||
usz n = WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)); \
|
||||
b = 2*b + ((((u8*)xp)[n/8] >> (n%8)) & 1); \
|
||||
if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \
|
||||
} \
|
||||
goto dec_ret; \
|
||||
} \
|
||||
if (xe!=el_B) { \
|
||||
usz xw = elWidth(xe); \
|
||||
void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \
|
||||
void* xp = tyany_ptr(x); \
|
||||
CPUSEL(W, NEXT) \
|
||||
goto dec_ret; \
|
||||
} \
|
||||
M_HARR(r, wia); \
|
||||
if (TY(x)==t_harr || TY(x)==t_hslice) { \
|
||||
B* xp = hany_ptr(x); \
|
||||
@ -122,7 +168,7 @@ B select_c2(B t, B w, B x) {
|
||||
for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)))); \
|
||||
decG(x); return withFill(HARR_FCD(r,w),xf); \
|
||||
}
|
||||
if (xe==el_bit && wia>=256 && wia/4>=xia && we!=el_bit) {
|
||||
if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) {
|
||||
return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x)))));
|
||||
}
|
||||
if (we==el_bit) {
|
||||
@ -210,11 +256,17 @@ B select_ucw(B t, B o, B w, B x) {
|
||||
if (isAtm(rep) || !eqShape(w, rep)) thrF("𝔽⌾(a⊸⊏)𝕩: Result of 𝔽 must have the same shape as 'a' (expected %H, got %H)", w, rep);
|
||||
#if CHECK_VALID
|
||||
TALLOC(bool, set, xia);
|
||||
for (i64 i = 0; i < xia; i++) set[i] = false;
|
||||
bool sparse = wia < xia/64;
|
||||
if (!sparse) for (i64 i = 0; i < xia; i++) set[i] = false;
|
||||
#define SPARSE_INIT(WI) \
|
||||
if (sparse) for (usz i = 0; i < wia; i++) { \
|
||||
i64 cw = WI; if (RARE(cw<0)) cw+= (i64)xia; set[cw] = false; \
|
||||
}
|
||||
#define EQ(F) if (set[cw] && (F)) thrM("𝔽⌾(a⊸⊏): Incompatible result elements"); set[cw] = true;
|
||||
#define FREE_CHECK TFREE(set)
|
||||
SLOWIF(xia>100 && wia<xia/20) SLOW2("⌾(𝕨⊸⊏)𝕩 because CHECK_VALID", w, x);
|
||||
#else
|
||||
#define SPARSE_INIT(GET)
|
||||
#define EQ(F)
|
||||
#define FREE_CHECK
|
||||
#endif
|
||||
@ -226,6 +278,7 @@ B select_ucw(B t, B o, B w, B x) {
|
||||
if (elInt(we)) {
|
||||
w = toI32Any(w);
|
||||
i32* wp = i32any_ptr(w);
|
||||
SPARSE_INIT(wp[i])
|
||||
if (elNum(re) && elNum(xe)) {
|
||||
u8 me = xe>re?xe:re;
|
||||
bool reuse = reusable(x);
|
||||
@ -319,6 +372,7 @@ B select_ucw(B t, B o, B w, B x) {
|
||||
MUTG_INIT(r);
|
||||
mut_copyG(r, 0, x, 0, xia);
|
||||
SGet(rep)
|
||||
SPARSE_INIT(o2i64G(GetU(w, i)))
|
||||
for (usz i = 0; i < wia; i++) {
|
||||
i64 cw = o2i64G(GetU(w, i)); if (RARE(cw<0)) cw+= (i64)xia;
|
||||
B cr = Get(rep, i);
|
||||
@ -328,6 +382,7 @@ B select_ucw(B t, B o, B w, B x) {
|
||||
}
|
||||
decG(w); decG(rep); FREE_CHECK;
|
||||
return mut_fcd(r, x);
|
||||
#undef SPARSE_INIT
|
||||
#undef EQ
|
||||
#undef FREE_CHECK
|
||||
}
|
||||
|
||||
@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b
|
||||
def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}}
|
||||
def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}}
|
||||
def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}}
|
||||
def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}}
|
||||
|
||||
# float comparison
|
||||
local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n}
|
||||
@ -102,6 +103,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a}
|
||||
|
||||
# conversion
|
||||
def half{x:T, i & w256{T} & knum{i}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}
|
||||
def half{x:T, i==0 & w256{T}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}}
|
||||
def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_mm256_setr_m128i', a, b}
|
||||
def pair{x} = pair{tupsel{0,x},tupsel{1,x}}
|
||||
|
||||
|
||||
@ -107,8 +107,8 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void,
|
||||
def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}}
|
||||
def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x}
|
||||
|
||||
def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n}
|
||||
def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n}
|
||||
|
||||
def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m}
|
||||
def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m}
|
||||
@ -186,4 +186,4 @@ def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO che
|
||||
|
||||
|
||||
def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x}
|
||||
def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
|
||||
def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x}
|
||||
|
||||
@ -16,6 +16,9 @@ def lb{n==1} = 0
|
||||
def tail{n,x} = x & ((1<<n) - 1) # get the n least significant bits
|
||||
def bit {k,x} = x & (1<<k) # get the k-th bit
|
||||
|
||||
# Convert tuple to number in little-endian base b
|
||||
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
|
||||
|
||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||
def assert{x & x==0} = assert{'failed assertion'}
|
||||
def assert{x & x==1} = 1
|
||||
|
||||
@ -39,8 +39,7 @@ def get_rep_iter{V, wv==2}{x, gen} = {
|
||||
}
|
||||
def get_rep_iter{V==[4]u64, wv} = {
|
||||
def step = 4
|
||||
def base4{l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+4*base4{slice{l,1}} }
|
||||
def sh = each{base4, get_shufs{step, wv, wv}}
|
||||
def sh = each{bind{base,4}, get_shufs{step, wv, wv}}
|
||||
{x, gen} => each{{s}=>gen{shuf{V, x, s}}, sh}
|
||||
}
|
||||
def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = {
|
||||
|
||||
@ -7,7 +7,6 @@ include './mask'
|
||||
def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}}
|
||||
def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}}
|
||||
|
||||
def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} }
|
||||
def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}}
|
||||
|
||||
# Fill last 4 bytes with last element, in each lane
|
||||
|
||||
@ -4,9 +4,10 @@ include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
include './bitops'
|
||||
include 'util/tup'
|
||||
|
||||
def minBulk{w, A, B & width{A}< width{B}} = w/width{B}
|
||||
def minBulk{w, A, B & width{A}>=width{B}} = w/width{A}
|
||||
oper &~ andnot infix none 35
|
||||
|
||||
# def:T - masked original content
|
||||
# b:B - pointer to data to index; if width{B}<width{eltype{T}}, padding bytes are garbage read after wanted position
|
||||
@ -20,36 +21,149 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
||||
else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *i64~~b, idx, width{eltype{B}}/8}
|
||||
}
|
||||
|
||||
def wrapChk{cw0, VI,xlf, M} = {
|
||||
cw:= cw0 + (xlf & VI~~(cw0<broadcast{VI, 0}))
|
||||
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
cw
|
||||
}
|
||||
|
||||
def storeExp{dst, ind, val, M, ext, rd, wl} = {
|
||||
def s{M} = storeBatch{dst, ind, val, M}
|
||||
if (ext==1 or not M{0}) s{M}
|
||||
else if (ind*rd+rd <= wl) s{maskNone}
|
||||
else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} }
|
||||
}
|
||||
|
||||
def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = {
|
||||
def VI = [ri]TI
|
||||
def ext = ri/rd
|
||||
xlf:= broadcast{VI, cast_i{TI, xl}}
|
||||
maskedLoop{ri, wl, {i, M} => {
|
||||
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
||||
is:= (if (ext>1) i<<lb{ext}; else i)
|
||||
def se{e, c, o} = {
|
||||
c2:= shuf{[4]u64, c+c, 4b3120}
|
||||
each{
|
||||
{c,o} => se{e*2, VI~~c, o},
|
||||
unpackQ{c2, c2+broadcast{VI,1}},
|
||||
2*o + iota{2}
|
||||
}
|
||||
}
|
||||
def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl}
|
||||
se{1, cw, 0}
|
||||
}}
|
||||
}
|
||||
|
||||
def perm_select{ri, rd, TI, w, r, wl, xl, selx} = {
|
||||
def VI = [ri]TI
|
||||
def ext = ri/rd
|
||||
xlf:= broadcast{VI, cast_i{TI, xl}}
|
||||
maskedLoop{ri, wl, {i, M} => {
|
||||
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
||||
is:= (if (ext>1) i<<lb{ext}; else i)
|
||||
def part{o} = cvt{i8, [8]i32, shuf{[4]u64, cw, 4b3210+o}}
|
||||
def se{o} = storeExp{r, is+o, selx{part{o}}, M, ext, rd, wl}
|
||||
each{se, iota{ext}}
|
||||
}}
|
||||
}
|
||||
|
||||
def makeselx{VI, VD, nsel, xd, logv, cshuf} = {
|
||||
def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m}
|
||||
def bblendn{m}{tf} = bblend{m}{reverse{tf}}
|
||||
def bb{c}{f, v} = (if (f) bblendn{c<v}; else bblend{(c&v)==v})
|
||||
|
||||
def bs{b, c, x} = cshuf{x, c}
|
||||
def bs{b, c, x & tuplen{b}>0} = {
|
||||
tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}}
|
||||
}
|
||||
|
||||
def i = iota{logv}
|
||||
def vs = each{bind{broadcast,VI}, nsel<<reverse{i}}
|
||||
{c} => VD~~bs{each{bb{c},i==0,vs}, c, xd}
|
||||
}
|
||||
def makeshuf{VI, VD, x0, logv} = {
|
||||
x:= *VD~~x0
|
||||
def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}}
|
||||
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)}
|
||||
def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010}
|
||||
def readx{l==1,o} = halves{load{x, o}}
|
||||
xd:= readx{logv, 0}
|
||||
makeselx{VI,VD,16,xd,logv, bind{sel,[16]i8}}
|
||||
}
|
||||
def makeperm{VI, VD, x0, logv} = {
|
||||
x:= *VD~~x0
|
||||
def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-1)}
|
||||
def readx{l==0,o} = load{x, o}
|
||||
makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}}
|
||||
}
|
||||
|
||||
select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
def TIE = i32
|
||||
def TDE = tern{width{TD}<32, u32, TD}
|
||||
def bulk = minBulk{rw,TIE,TDE}
|
||||
def TIF = [bulk]TIE
|
||||
def TDF = [bulk]TDE
|
||||
def xlf = broadcast{TIF, cast_i{eltype{TIF}, xl}}
|
||||
|
||||
w:= *TI ~~ w0
|
||||
x:= *TD ~~ x0
|
||||
r:= *TD ~~ r0
|
||||
|
||||
maskedLoop{bulk, wl, {i, M} => {
|
||||
cw0:= loadBatch{w, i, TIF}
|
||||
cw1:= cw0+xlf
|
||||
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
|
||||
if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0}
|
||||
got:= gather{broadcast{TDF,0}, x, cw, M}
|
||||
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
|
||||
storeBatch{r, i, got, M}
|
||||
}}
|
||||
def wd = width{TD}; def rd = rw/wd
|
||||
def wi = width{TI}; def ri = rw/wi
|
||||
def reg_select{sel,make}{l} = {
|
||||
sel{ri, rd, TI, w, r, wl, xl, make{[ri]TI,[rd]TD, x,l}}
|
||||
}
|
||||
def shuf_select = reg_select{shuf_select, makeshuf}
|
||||
def perm_select = reg_select{perm_select, makeperm}
|
||||
if (wi==8 and wd==32 and xl*wd<=256 ) perm_select{0}
|
||||
else if (wi==8 and wd==32 and xl*wd<=256<<1) perm_select{1}
|
||||
else if (wi==8 and wd<=16 and xl*wd<=128 ) shuf_select{0}
|
||||
else if (wi==8 and wd<=16 and xl*wd<=128<<1) shuf_select{1}
|
||||
else if (wi==8 and wd<=16 and xl*wd<=128<<2) shuf_select{2}
|
||||
else if (wi==8 and wd<= 8 and xl*wd<=128<<3) shuf_select{3}
|
||||
else {
|
||||
def TIE = i32
|
||||
def TDE = tern{wd<32, u32, TD}
|
||||
def bulk = rw / width{TDE}
|
||||
def VI = [bulk]TIE
|
||||
def VD = [bulk]TDE
|
||||
def xlf = broadcast{VI, cast_i{TIE, xl}}
|
||||
|
||||
maskedLoop{bulk, wl, {i, M} => {
|
||||
cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M}
|
||||
got:= gather{broadcast{VD,0}, x, cw, M}
|
||||
if (TDE!=TD) got&= broadcast{VD, (1<<wd)-1}
|
||||
storeBatch{r, i, got, M}
|
||||
}}
|
||||
}
|
||||
1
|
||||
}
|
||||
def select{TI, TD} = select{256, TI, TD}
|
||||
|
||||
def selects = join{table{select, tup{i8, i16, i32}, # indices
|
||||
tup{u8, u16, u32, u64}}} # values
|
||||
avx2_select_tab:*type{tupsel{0,selects}} = selects
|
||||
'avx2_select_tab' = avx2_select_tab
|
||||
|
||||
|
||||
'avx2_select_i32_8' = select{256, i32, u8}
|
||||
'avx2_select_i32_16' = select{256, i32, u16}
|
||||
'avx2_select_i32_32' = select{256, i32, u32}
|
||||
'avx2_select_i32_64' = select{256, i32, u64}
|
||||
'avx2_select_i8_32' = select{256, i8, u32}
|
||||
'avx2_select_i16_32' = select{256, i16, u32}
|
||||
|
||||
avx2_select_bool128(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = {
|
||||
def TI = i8
|
||||
def VI = [32]TI
|
||||
w:= *VI ~~ w0
|
||||
r:= *u32 ~~ r0
|
||||
xlf:= broadcast{VI, cast_i{TI, xl}}
|
||||
|
||||
if (wl>32 and xl<=16) {
|
||||
xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010}
|
||||
maskedLoop{32, wl, {i, M} => {
|
||||
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
||||
store{r, i, getmask{sel{[16]i8, xb, cw}}}
|
||||
}}
|
||||
} else {
|
||||
x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010}
|
||||
low:= broadcast{VI, 7}
|
||||
b := VI~~make{[32]u8, 1 << (iota{32} & 7)}
|
||||
maskedLoop{32, wl, {i, M} => {
|
||||
cw:= wrapChk{load{w, i}, VI,xlf, M}
|
||||
byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)}
|
||||
mask:= sel{[16]i8, b, cw & low}
|
||||
store{r, i, getmask{(mask & byte) == mask}}
|
||||
}}
|
||||
}
|
||||
1
|
||||
}
|
||||
'avx2_select_bool128' = avx2_select_bool128
|
||||
|
||||
@ -220,12 +220,8 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x
|
||||
#if SINGELI
|
||||
#include <xmmintrin.h>
|
||||
#if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32
|
||||
__m128i custom_loadu_si32(void* p) {
|
||||
return (__m128i) _mm_load_ss(p);
|
||||
}
|
||||
void custom_storeu_si32(void* p, __m128i x) {
|
||||
_mm_store_ss(p, _mm_castsi128_ps(x));
|
||||
}
|
||||
static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); }
|
||||
static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); }
|
||||
#define _mm_loadu_si32 custom_loadu_si32
|
||||
#define _mm_storeu_si32 custom_storeu_si32
|
||||
#endif
|
||||
|
||||
Loading…
Reference in New Issue
Block a user