diff --git a/makefile b/makefile index 42857a08..0e83c6b1 100644 --- a/makefile +++ b/makefile @@ -278,7 +278,7 @@ endif @${MAKE} i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c fold.c scan.c neq.c slash.c constrep.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/select.c b/src/builtins/select.c index 9b56f275..5aa251a4 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -1,14 +1,52 @@ +// First Cell and Select (⊏) + +// First Cell is just a slice + +// Complications in Select mostly come from range checks and negative 𝕨 +// Atom 𝕨 and any rank 𝕩: slice +// Rank-1 𝕩: +// Empty 𝕨: no selection +// Small 𝕩 with Singeli: use shuffles +// Boolean 𝕨: use bit_sel for blend or similar +// Boolean 𝕩 and larger 𝕨: convert to i8, select, convert back +// Boolean 𝕩 otherwise: select/shift bytes, reversed for fast writing +// TRIED pext, doesn't seem faster (mask built with shifts anyway) +// SHOULD squeeze 𝕨 if not ≀i32 to get to optimized cases +// Integer 𝕨 with Singeli: fused wrap, range-check, and gather +// COULD try selecting from boolean with gather +// COULD detect + #if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32 + static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); } + static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); } + #define _mm_loadu_si32 custom_loadu_si32 + #define _mm_storeu_si32 custom_storeu_si32 + #endif + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/select.c" + #pragma GCC diagnostic pop +#endif extern B rt_select; B select_c1(B t, B x) { @@ -67,31 +105,16 @@ B select_c2(B t, B w, B x) { u8 xe = TI(x,elType); u8 we = TI(w,elType); #if SINGELI - // if (we==el_i8 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i8_32 (i8any_ptr (w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i16 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i16_32(i16any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_i8 ) { i8* rp; r = m_i8arrc (&rp, w); if (!avx2_select_i32_8 (i32any_ptr(w), i8any_ptr (x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i32_32(i32any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_f64) { f64* rp; r = m_f64arrc(&rp, w); if (!avx2_select_i32_64(i32any_ptr(w), f64any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - #endif - #define CASE(S, E) case S: for (usz i=i0; i> 63; \ - if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \ - } \ - goto dec_ret; \ - } \ - if (xe!=el_B) { \ - usz xw = elWidth(xe); \ - void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \ - void* xp = tyany_ptr(x); \ + #define CPUSEL(W, NEXT) \ + if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); + #define BOOL_USE_SIMD (xia<=128) + #define BOOL_SPECIAL(W) \ + if (sizeof(W)==1 && BOOL_USE_SIMD) { \ + if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \ + goto dec_ret; \ + } + #else + #define CPUSEL(W, NEXT) \ if (sizeof(W) >= 4) { \ switch(xw) { default:UD; CASEW(1,u8); CASEW(2,u16); CASEW(4,u32); CASEW(8,f64); } \ } else { \ @@ -110,9 +133,32 @@ B select_c2(B t, B w, B x) { switch(xw) { default:UD; CASE(1,u8); CASE(2,u16); CASE(4,u32); CASE(8,f64); } \ } \ if (wt) TFREE(wt); \ + } + #define BOOL_USE_SIMD 0 + #define BOOL_SPECIAL(W) + #endif + #define CASE(S, E) case S: for (usz i=i0; i> (n%8)) & 1); \ + if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \ } \ goto dec_ret; \ } \ + if (xe!=el_B) { \ + usz xw = elWidth(xe); \ + void* rp = m_tyarrc(&r, xw, w, el2t(xe)); \ + void* xp = tyany_ptr(x); \ + CPUSEL(W, NEXT) \ + goto dec_ret; \ + } \ M_HARR(r, wia); \ if (TY(x)==t_harr || TY(x)==t_hslice) { \ B* xp = hany_ptr(x); \ @@ -122,7 +168,7 @@ B select_c2(B t, B w, B x) { for (usz i=0; i < wia; i++) HARR_ADD(r, i, Get(x, WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%iβˆŠπ•¨, %s≑≠𝕩)", wp[i], xia)))); \ decG(x); return withFill(HARR_FCD(r,w),xf); \ } - if (xe==el_bit && wia>=256 && wia/4>=xia && we!=el_bit) { + if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) { return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x))))); } if (we==el_bit) { @@ -210,11 +256,17 @@ B select_ucw(B t, B o, B w, B x) { if (isAtm(rep) || !eqShape(w, rep)) thrF("π”½βŒΎ(a⊸⊏)𝕩: Result of 𝔽 must have the same shape as 'a' (expected %H, got %H)", w, rep); #if CHECK_VALID TALLOC(bool, set, xia); - for (i64 i = 0; i < xia; i++) set[i] = false; + bool sparse = wia < xia/64; + if (!sparse) for (i64 i = 0; i < xia; i++) set[i] = false; + #define SPARSE_INIT(WI) \ + if (sparse) for (usz i = 0; i < wia; i++) { \ + i64 cw = WI; if (RARE(cw<0)) cw+= (i64)xia; set[cw] = false; \ + } #define EQ(F) if (set[cw] && (F)) thrM("π”½βŒΎ(a⊸⊏): Incompatible result elements"); set[cw] = true; #define FREE_CHECK TFREE(set) SLOWIF(xia>100 && wiare?xe:re; bool reuse = reusable(x); @@ -319,6 +372,7 @@ B select_ucw(B t, B o, B w, B x) { MUTG_INIT(r); mut_copyG(r, 0, x, 0, xia); SGet(rep) + SPARSE_INIT(o2i64G(GetU(w, i))) for (usz i = 0; i < wia; i++) { i64 cw = o2i64G(GetU(w, i)); if (RARE(cw<0)) cw+= (i64)xia; B cr = Get(rep, i); @@ -328,6 +382,7 @@ B select_ucw(B t, B o, B w, B x) { } decG(w); decG(rep); FREE_CHECK; return mut_fcd(r, x); + #undef SPARSE_INIT #undef EQ #undef FREE_CHECK } diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index f27c7878..006ce927 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}} def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}} +def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} @@ -102,6 +103,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion def half{x:T, i & w256{T} & knum{i}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} +def half{x:T, i==0 & w256{T}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} def pair{x} = pair{tupsel{0,x},tupsel{1,x}} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 468f0dff..57d4214b 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -107,8 +107,8 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}} def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x} -def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} -def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} +def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} +def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m} def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} @@ -186,4 +186,4 @@ def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO che def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} -def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} \ No newline at end of file +def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 6029454c..e1a2ceec 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -16,6 +16,9 @@ def lb{n==1} = 0 def tail{n,x} = x & ((1< each{{s}=>gen{shuf{V, x, s}}, sh} } def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 235c4c13..0d4fb034 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -7,7 +7,6 @@ include './mask' def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}} def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} -def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} } def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} # Fill last 4 bytes with last element, in each lane diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 88320c5a..d4a6635e 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -4,9 +4,10 @@ include './sse3' include './avx' include './avx2' include './mask' +include './bitops' +include 'util/tup' -def minBulk{w, A, B & width{A}< width{B}} = w/width{B} -def minBulk{w, A, B & width{A}>=width{B}} = w/width{A} +oper &~ andnot infix none 35 # def:T - masked original content # b:B - pointer to data to index; if width{B}= ty_u{xlf}}}) return{0} + cw +} + +def storeExp{dst, ind, val, M, ext, rd, wl} = { + def s{M} = storeBatch{dst, ind, val, M} + if (ext==1 or not M{0}) s{M} + else if (ind*rd+rd <= wl) s{maskNone} + else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } +} + +def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { + def VI = [ri]TI + def ext = ri/rd + xlf:= broadcast{VI, cast_i{TI, xl}} + maskedLoop{ri, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + is:= (if (ext>1) i< se{e*2, VI~~c, o}, + unpackQ{c2, c2+broadcast{VI,1}}, + 2*o + iota{2} + } + } + def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl} + se{1, cw, 0} + }} +} + +def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { + def VI = [ri]TI + def ext = ri/rd + xlf:= broadcast{VI, cast_i{TI, xl}} + maskedLoop{ri, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + is:= (if (ext>1) i<0} = { + tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}} + } + + def i = iota{logv} + def vs = each{bind{broadcast,VI}, nsel< VD~~bs{each{bb{c},i==0,vs}, c, xd} +} +def makeshuf{VI, VD, x0, logv} = { + x:= *VD~~x0 + def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}} + def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)} + def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010} + def readx{l==1,o} = halves{load{x, o}} + xd:= readx{logv, 0} + makeselx{VI,VD,16,xd,logv, bind{sel,[16]i8}} +} +def makeperm{VI, VD, x0, logv} = { + x:= *VD~~x0 + def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-1)} + def readx{l==0,o} = load{x, o} + makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}} +} select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { - def TIE = i32 - def TDE = tern{width{TD}<32, u32, TD} - def bulk = minBulk{rw,TIE,TDE} - def TIF = [bulk]TIE - def TDF = [bulk]TDE - def xlf = broadcast{TIF, cast_i{eltype{TIF}, xl}} - w:= *TI ~~ w0 x:= *TD ~~ x0 r:= *TD ~~ r0 - maskedLoop{bulk, wl, {i, M} => { - cw0:= loadBatch{w, i, TIF} - cw1:= cw0+xlf - cw:= blendF{cw0, cw1, cw0= ty_u{xlf}}}) return{0} - got:= gather{broadcast{TDF,0}, x, cw, M} - if (TDE!=TD) got&= broadcast{TDF, (1< { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + got:= gather{broadcast{VD,0}, x, cw, M} + if (TDE!=TD) got&= broadcast{VD, (1<32 and xl<=16) { + xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + store{r, i, getmask{sel{[16]i8, xb, cw}}} + }} + } else { + x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} + low:= broadcast{VI, 7} + b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)} + mask:= sel{[16]i8, b, cw & low} + store{r, i, getmask{(mask & byte) == mask}} + }} + } + 1 +} +'avx2_select_bool128' = avx2_select_bool128 diff --git a/src/utils/mut.c b/src/utils/mut.c index e795f9e4..158017c4 100644 --- a/src/utils/mut.c +++ b/src/utils/mut.c @@ -220,12 +220,8 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x #if SINGELI #include #if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32 - __m128i custom_loadu_si32(void* p) { - return (__m128i) _mm_load_ss(p); - } - void custom_storeu_si32(void* p, __m128i x) { - _mm_store_ss(p, _mm_castsi128_ps(x)); - } + static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); } + static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); } #define _mm_loadu_si32 custom_loadu_si32 #define _mm_storeu_si32 custom_storeu_si32 #endif