From 7cb8cf57729e86bccaac20f6cb83400ea0a8c47f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 26 Nov 2022 20:30:11 -0500 Subject: [PATCH 01/13] Enable selection with AVX2 gather for i8 to i32 indices and 1- to 8-byte values --- makefile | 2 +- src/builtins/select.c | 60 +++++++++++++++++----------------- src/singeli/src/select.singeli | 14 ++++---- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/makefile b/makefile index 42857a08..0e83c6b1 100644 --- a/makefile +++ b/makefile @@ -278,7 +278,7 @@ endif @${MAKE} i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c fold.c scan.c neq.c slash.c constrep.c bits.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/select.c b/src/builtins/select.c index 9b56f275..1d6acdad 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -3,12 +3,12 @@ #include "../utils/mut.h" #include "../builtins.h" -// #if SINGELI -// #pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wunused-variable" -// #include "../singeli/gen/select.c" -// #pragma GCC diagnostic pop -// #endif +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/select.c" + #pragma GCC diagnostic pop +#endif extern B rt_select; B select_c1(B t, B x) { @@ -67,11 +67,29 @@ B select_c2(B t, B w, B x) { u8 xe = TI(x,elType); u8 we = TI(w,elType); #if SINGELI - // if (we==el_i8 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i8_32 (i8any_ptr (w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i16 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i16_32(i16any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_i8 ) { i8* rp; r = m_i8arrc (&rp, w); if (!avx2_select_i32_8 (i32any_ptr(w), i8any_ptr (x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_i32) { i32* rp; r = m_i32arrc(&rp, w); if (!avx2_select_i32_32(i32any_ptr(w), i32any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } - // if (we==el_i32 && xe==el_f64) { f64* rp; r = m_f64arrc(&rp, w); if (!avx2_select_i32_64(i32any_ptr(w), f64any_ptr(x), rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); goto dec_ret; } + #define CPUSEL(W, NEXT) \ + if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); + #else + #define CPUSEL(W, NEXT) \ + if (sizeof(W) >= 4) { \ + switch(xw) { default:UD; CASEW(1,u8); CASEW(2,u16); CASEW(4,u32); CASEW(8,f64); } \ + } else { \ + W* wt = NULL; \ + for (usz bl=(1<<14)/sizeof(W), i0=0, i1=0; i0wia) i1=wia; \ + W min=wp[i0], max=min; for (usz i=i0+1; imax) max=e; if (e=(i64)xia) thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", max, xia); \ + W* ip=wp; usz off=xia; \ + if (max>=0) { off=0; if (RARE(min<0)) { \ + if (RARE(xia > (1ULL<<(sizeof(W)*8-1)))) { w=taga(NEXT(w)); mm_free(v(r)); return select_c2(m_f64(0), w, x); } \ + if (!wt) {wt=TALLOCP(W,i1-i0);} ip=wt-i0; \ + for (usz i=i0; i= 4) { \ - switch(xw) { default:UD; CASEW(1,u8); CASEW(2,u16); CASEW(4,u32); CASEW(8,f64); } \ - } else { \ - W* wt = NULL; \ - for (usz bl=(1<<14)/sizeof(W), i0=0, i1=0; i0wia) i1=wia; \ - W min=wp[i0], max=min; for (usz i=i0+1; imax) max=e; if (e=(i64)xia) thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", max, xia); \ - W* ip=wp; usz off=xia; \ - if (max>=0) { off=0; if (RARE(min<0)) { \ - if (RARE(xia > (1ULL<<(sizeof(W)*8-1)))) { w=taga(NEXT(w)); mm_free(v(r)); return select_c2(m_f64(0), w, x); } \ - if (!wt) {wt=TALLOCP(W,i1-i0);} ip=wt-i0; \ - for (usz i=i0; i=width{B}} = w/width{A} @@ -44,12 +45,9 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { }} 1 } +def select{TI, TD} = select{256, TI, TD} - -'avx2_select_i32_8' = select{256, i32, u8} -'avx2_select_i32_16' = select{256, i32, u16} -'avx2_select_i32_32' = select{256, i32, u32} -'avx2_select_i32_64' = select{256, i32, u64} -'avx2_select_i8_32' = select{256, i8, u32} -'avx2_select_i16_32' = select{256, i16, u32} - +def selects = join{table{select, tup{i8, i16, i32}, # indices + tup{u8, u16, u32, u64}}} # values +avx2_select_tab:*type{tupsel{0,selects}} = selects +'avx2_select_tab' = avx2_select_tab From e09d555eba1dc9bb0bb63aa6e5097907bc236c37 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 26 Nov 2022 20:39:18 -0500 Subject: [PATCH 02/13] Refactoring and renaming in select.singeli --- src/singeli/src/select.singeli | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 44e74270..5ee83680 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -6,9 +6,6 @@ include './avx2' include './mask' include 'util/tup' -def minBulk{w, A, B & width{A}< width{B}} = w/width{B} -def minBulk{w, A, B & width{A}>=width{B}} = w/width{A} - # def:T - masked original content # b:B - pointer to data to index; if width{B} { - cw0:= loadBatch{w, i, TIF} + cw0:= loadBatch{w, i, VI} cw1:= cw0+xlf - cw:= blendF{cw0, cw1, cw0= ty_u{xlf}}}) return{0} - got:= gather{broadcast{TDF,0}, x, cw, M} - if (TDE!=TD) got&= broadcast{TDF, (1< Date: Sun, 27 Nov 2022 21:03:58 -0500 Subject: [PATCH 03/13] 1-byte select with <=16 elements using shuffle --- src/singeli/src/select.singeli | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 5ee83680..a2fe4dbe 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -19,6 +19,26 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = { } select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { + w:= *TI ~~ w0 + x:= *TD ~~ x0 + r:= *TD ~~ r0 + + def wd = width{TD} + if (width{TI}==8 and wd==8 and xl*wd<=128) { + def VI = [32]TI + def VD = [rw/wd]TD + def xlf = broadcast{VI, cast_i{TI, xl}} + xd := shuf{[4]u64, load{*VD~~x}, 4b1010} + maskedLoop{rw/8, wl, {i, M} => { + cw0:= loadBatch{w, i, VI} + cw1:= cw0+xlf + cw:= blendF{cw0, cw1, cw0= ty_u{xlf}}}) return{0} + storeBatch{r, i, VD~~sel{[16]i8, xd, cw}, M} + }} + return{1} + } + def TIE = i32 def TDE = tern{width{TD}<32, u32, TD} def bulk = rw / width{TDE} @@ -26,10 +46,6 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { def VD = [bulk]TDE def xlf = broadcast{VI, cast_i{TIE, xl}} - w:= *TI ~~ w0 - x:= *TD ~~ x0 - r:= *TD ~~ r0 - maskedLoop{bulk, wl, {i, M} => { cw0:= loadBatch{w, i, VI} cw1:= cw0+xlf From 43fba7cd781a3d4878024422c61a1ef7cb2ea2b6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 28 Nov 2022 11:28:29 -0500 Subject: [PATCH 04/13] Extend shuffle selection to >1 byte data --- src/singeli/src/select.singeli | 38 ++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index a2fe4dbe..ff8a3a7a 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -23,18 +23,36 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { x:= *TD ~~ x0 r:= *TD ~~ r0 + def wrap{cw0, VI,xlf} = cw0 + (xlf & VI~~(cw0 { - cw0:= loadBatch{w, i, VI} - cw1:= cw0+xlf - cw:= blendF{cw0, cw1, cw0 { + cw:= wrap{loadBatch{w, i, VI}, VI,xlf} if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0} - storeBatch{r, i, VD~~sel{[16]i8, xd, cw}, M} + is := (if (wd>wi) i< se{w*2, VI~~c, o}, + unpackQ{c2, c2+broadcast{VI,1}}, + 2*o + iota{2} + } + } + def se{w==wd, c, o} = { + io:= is+o + got:= VD~~sel{[16]i8, xd, c} + def s{M} = storeBatch{r, io, got, M} + if (wd==wi or not M{0}) s{M} + else if (io*rd+rd <= wl) s{maskNone} + else if (io*rd < wl) s{maskAfter{wl & (rd-1)}} + } + se{wi, cw, 0} }} return{1} } @@ -47,9 +65,7 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { def xlf = broadcast{VI, cast_i{TIE, xl}} maskedLoop{bulk, wl, {i, M} => { - cw0:= loadBatch{w, i, VI} - cw1:= cw0+xlf - cw:= blendF{cw0, cw1, cw0= ty_u{xlf}}}) return{0} got:= gather{broadcast{VD,0}, x, cw, M} if (TDE!=TD) got&= broadcast{VD, (1< Date: Mon, 28 Nov 2022 12:14:30 -0500 Subject: [PATCH 05/13] Move base{} utility to base.singeli --- src/singeli/src/base.singeli | 3 +++ src/singeli/src/constrep.singeli | 3 +-- src/singeli/src/scan.singeli | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 6029454c..e1a2ceec 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -16,6 +16,9 @@ def lb{n==1} = 0 def tail{n,x} = x & ((1< each{{s}=>gen{shuf{V, x, s}}, sh} } def read_shuf_vecs{l, elbytes:u64, shp:*[32]i8} = { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 235c4c13..0d4fb034 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -7,7 +7,6 @@ include './mask' def sel8{v, t} = sel{[16]u8, v, make{[32]i8, t}} def sel8{v, t & istup{t} & tuplen{t}==16} = sel8{v, merge{t,t}} -def base{b,l} = { if (0==tuplen{l}) 0; else tupsel{0,l}+b*base{b,slice{l,1}} } def shuf{T, v, n & istup{n}} = shuf{T, v, base{4,n}} # Fill last 4 bytes with last element, in each lane From dc6e49b289c24728044b6e75a5e9389f8b536796 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 28 Nov 2022 21:38:36 -0500 Subject: [PATCH 06/13] Shuffle selection using >1 data register --- src/singeli/src/select.singeli | 125 ++++++++++++++++++++------------- 1 file changed, 78 insertions(+), 47 deletions(-) diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index ff8a3a7a..c7c41d53 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -18,59 +18,90 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = { else T ~~ emit{[4]i64, '_mm256_i32gather_epi64', *i64~~b, idx, width{eltype{B}}/8} } +def wrapChk{cw0, VI,xlf, M} = { + cw:= cw0 + (xlf & VI~~(cw0= ty_u{xlf}}}) return{0} + cw +} + +def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { + def VI = [ri]TI + def ext = ri/rd + xlf:= broadcast{VI, cast_i{TI, xl}} + maskedLoop{ri, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + is:= (if (ext>1) i< se{e*2, VI~~c, o}, + unpackQ{c2, c2+broadcast{VI,1}}, + 2*o + iota{2} + } + } + def se{e==ext, c, o} = { + io:= is+o + got:= selx{c} + def s{M} = storeBatch{r, io, got, M} + if (ext==1 or not M{0}) s{M} + else if (io*rd+rd <= wl) s{maskNone} + else { if (io*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } + } + se{1, cw, 0} + }} +} + +def makesel{VI,VD, x0,logv} = { + x:= *VD~~x0 + def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}} + def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)} + def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010} + def readx{l==1,o} = halves{load{x, o}} + xd:= readx{logv, 0} + + def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m} + def bblendn{m}{tf} = bblend{m}{reverse{tf}} + def bb{c}{f, v} = (if (f) bblendn{c0} = { + tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}} + } + + def i = iota{logv} + def vs = each{bind{broadcast,VI}, 16< VD~~bs{each{bb{c},i==0,vs}, c, xd} +} + select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { w:= *TI ~~ w0 x:= *TD ~~ x0 r:= *TD ~~ r0 - def wrap{cw0, VI,xlf} = cw0 + (xlf & VI~~(cw0 { - cw:= wrap{loadBatch{w, i, VI}, VI,xlf} - if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0} - is := (if (wd>wi) i< se{w*2, VI~~c, o}, - unpackQ{c2, c2+broadcast{VI,1}}, - 2*o + iota{2} - } - } - def se{w==wd, c, o} = { - io:= is+o - got:= VD~~sel{[16]i8, xd, c} - def s{M} = storeBatch{r, io, got, M} - if (wd==wi or not M{0}) s{M} - else if (io*rd+rd <= wl) s{maskNone} - else if (io*rd < wl) s{maskAfter{wl & (rd-1)}} - } - se{wi, cw, 0} - }} - return{1} + def wd = width{TD}; def rd = rw/wd + def wi = width{TI}; def ri = rw/wi + def shuf_select{l} = { + shuf_select{ri, rd, TI, w, r, wl, xl, makesel{[ri]TI,[rd]TD, x,l}} + } + if (wi==8 and wd<=32 and xl*wd<=128 ) { shuf_select{0} } + else if (wi==8 and wd<=16 and xl*wd<=128<<1) { shuf_select{1} } + else if (wi==8 and wd<=16 and xl*wd<=128<<2) { shuf_select{2} } + else if (wi==8 and wd<= 8 and xl*wd<=128<<3) { shuf_select{3} } + else { + def TIE = i32 + def TDE = tern{wd<32, u32, TD} + def bulk = rw / width{TDE} + def VI = [bulk]TIE + def VD = [bulk]TDE + def xlf = broadcast{VI, cast_i{TIE, xl}} + + maskedLoop{bulk, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + got:= gather{broadcast{VD,0}, x, cw, M} + if (TDE!=TD) got&= broadcast{VD, (1< { - cw:= wrap{loadBatch{w, i, VI}, VI,xlf} - if (any{M{ty_u{cw} >= ty_u{xlf}}}) return{0} - got:= gather{broadcast{VD,0}, x, cw, M} - if (TDE!=TD) got&= broadcast{VD, (1< Date: Tue, 29 Nov 2022 11:29:25 -0500 Subject: [PATCH 07/13] Faster bit selection --- src/builtins/select.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index 1d6acdad..8997a82c 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -100,8 +100,7 @@ B select_c2(B t, B w, B x) { for (usz i = wia; ; ) { \ i--; \ usz n = WRAP(wp[i], xia, thrF("⊏: Indexing out-of-bounds (%i∊𝕨, %s≡≠𝕩)", wp[i], xia)); \ - b <<= 1; \ - b |= (-(xp[n/64] & (1ull<<(n%64)))) >> 63; \ + b = 2*b + ((((u8*)xp)[n/8] >> (n%8)) & 1); \ if (i%64 == 0) { rp[i/64]=b; if (!i) break; } \ } \ goto dec_ret; \ From 41f464795f292aa156a52bd80a3cbe775c529131 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 29 Nov 2022 17:48:24 -0500 Subject: [PATCH 08/13] Selection with permutevar8x32 --- src/singeli/src/avx.singeli | 1 + src/singeli/src/avx2.singeli | 6 +++--- src/singeli/src/select.singeli | 37 +++++++++++++++++++++++++--------- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index f27c7878..91064a3d 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -102,6 +102,7 @@ def ceil{a:[4]f64} = emit{[4]f64, '_mm256_ceil_pd', a} # conversion def half{x:T, i & w256{T} & knum{i}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i} +def half{x:T, i==0 & w256{T}} = [vcount{T}/2](eltype{T}) ~~ emit{[8]i16, '_mm256_castsi256_si128', v2i{x}} def pair{a:T,b:T & width{T}==128} = [vcount{T}*2](eltype{T}) ~~ emit{[8]i32, '_mm256_setr_m128i', a, b} def pair{x} = pair{tupsel{0,x},tupsel{1,x}} diff --git a/src/singeli/src/avx2.singeli b/src/singeli/src/avx2.singeli index 468f0dff..57d4214b 100644 --- a/src/singeli/src/avx2.singeli +++ b/src/singeli/src/avx2.singeli @@ -107,8 +107,8 @@ def maskstore{a:T, m:M, n, v & w256{eltype{T}, 64} & w256i{M, 64}} = emit{void, def maskstoreF{p, m, n, x:T} = store{p, n, blendF{load{p,n}, x, m}} def maskstoreF{p, m, n, x:T & width{eltype{T}}>=32} = maskstore{p,m,n,x} -def shl{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} -def shr{S==[16]u8, x:T, n & w256{T}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} +def shl{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bslli_epi128', x, n} +def shr{S==[16]u8, x:T, n & w256{T} & knum{n}} = T ~~ emit{T, '_mm256_bsrli_epi128', x, n} def blend{L==[8]u16, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[16]i16, '_mm256_blend_epi16', v2i{a}, v2i{b}, m} def blend{L==[8]u32, a:T, b:T, m & w256{T} & knum{m}} = T ~~ emit{[ 8]i32, '_mm256_blend_epi32', v2i{a}, v2i{b}, m} @@ -186,4 +186,4 @@ def ucvt{T, x:X & w256{X} & width{T}==width{eltype{X}}} = to_el{T, x} # TODO che def cvt2{T, x:X & T==i32 & X==[4]f64} = emit{[4]i32, '_mm256_cvtpd_epi32', x} -def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} \ No newline at end of file +def cvt2{T, x:X & T==f64 & X==[4]i32} = emit{[4]f64, '_mm256_cvtepi32_pd', x} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index c7c41d53..67f59d26 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -24,6 +24,13 @@ def wrapChk{cw0, VI,xlf, M} = { cw } +def storeExp{dst, ind, val, M, ext, rd, wl} = { + def s{M} = storeBatch{dst, ind, val, M} + if (ext==1 or not M{0}) s{M} + else if (ind*rd+rd <= wl) s{maskNone} + else { if (ind*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } +} + def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { def VI = [ri]TI def ext = ri/rd @@ -39,18 +46,24 @@ def shuf_select{ri, rd, TI, w, r, wl, xl, selx} = { 2*o + iota{2} } } - def se{e==ext, c, o} = { - io:= is+o - got:= selx{c} - def s{M} = storeBatch{r, io, got, M} - if (ext==1 or not M{0}) s{M} - else if (io*rd+rd <= wl) s{maskNone} - else { if (io*rd < wl) s{maskAfter{wl & (rd-1)}}; return{1} } - } + def se{e==ext, c, o} = storeExp{r, is+o, selx{c}, M, ext, rd, wl} se{1, cw, 0} }} } +def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { + def VI = [ri]TI + def ext = ri/rd + xlf:= broadcast{VI, cast_i{TI, xl}} + maskedLoop{ri, wl, {i, M} => { + cw:= wrapChk{loadBatch{w, i, VI}, VI,xlf, M} + is:= (if (ext>1) i<sel{VD, xd, c}} + } + if (wi==8 and wd==32 and xl*wd<=256 ) { perm_select{ } } + else if (wi==8 and wd<=16 and xl*wd<=128 ) { shuf_select{0} } else if (wi==8 and wd<=16 and xl*wd<=128<<1) { shuf_select{1} } else if (wi==8 and wd<=16 and xl*wd<=128<<2) { shuf_select{2} } else if (wi==8 and wd<= 8 and xl*wd<=128<<3) { shuf_select{3} } From fdfeb67e8f4ebe9da5e47061521641eb96b56f82 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 29 Nov 2022 20:27:45 -0500 Subject: [PATCH 09/13] 2-register permutevar8x32 --- src/singeli/src/select.singeli | 58 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 67f59d26..46c496bc 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -64,26 +64,34 @@ def perm_select{ri, rd, TI, w, r, wl, xl, selx} = { }} } -def makesel{VI,VD, x0,logv} = { +def makeselx{VI, VD, nsel, xd, logv, cshuf} = { + def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m} + def bblendn{m}{tf} = bblend{m}{reverse{tf}} + def bb{c}{f, v} = (if (f) bblendn{c0} = { + tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}} + } + + def i = iota{logv} + def vs = each{bind{broadcast,VI}, nsel< VD~~bs{each{bb{c},i==0,vs}, c, xd} +} +def makeshuf{VI, VD, x0, logv} = { x:= *VD~~x0 def halves{v} = each{bind{shuf, [4]u64, v}, tup{4b1010, 4b3232}} def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-2)} def readx{l==0,o} = shuf{[4]u64, load{x}, 4b1010} def readx{l==1,o} = halves{load{x, o}} xd:= readx{logv, 0} - - def bblend {m}{ft} = blend{tupsel{0,ft}, tupsel{1,ft}, m} - def bblendn{m}{tf} = bblend{m}{reverse{tf}} - def bb{c}{f, v} = (if (f) bblendn{c0} = { - tupsel{0,b}{each{bind{bs, slice{b,1}, c}, x}} - } - - def i = iota{logv} - def vs = each{bind{broadcast,VI}, 16< VD~~bs{each{bb{c},i==0,vs}, c, xd} + makeselx{VI,VD,16,xd,logv, bind{sel,[16]i8}} +} +def makeperm{VI, VD, x0, logv} = { + x:= *VD~~x0 + def readx{l,o} = each{bind{readx,l-1}, o + iota{2}<<(l-1)} + def readx{l==0,o} = load{x, o} + makeselx{[8]i32,VD,8, readx{logv, 0}, logv, bind{sel,[8]i32}} } select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { @@ -93,19 +101,17 @@ select{rw, TI, TD}(w0:*void, x0:*void, r0:*void, wl:u64, xl:u64) : u1 = { def wd = width{TD}; def rd = rw/wd def wi = width{TI}; def ri = rw/wi - def shuf_select{l} = { - shuf_select{ri, rd, TI, w, r, wl, xl, makesel{[ri]TI,[rd]TD, x,l}} + def reg_select{sel,make}{l} = { + sel{ri, rd, TI, w, r, wl, xl, make{[ri]TI,[rd]TD, x,l}} } - def perm_select{} = { - def VD = [rd]TD - xd:= load{*VD~~x} - perm_select{ri, rd, TI, w, r, wl, xl, {c}=>sel{VD, xd, c}} - } - if (wi==8 and wd==32 and xl*wd<=256 ) { perm_select{ } } - else if (wi==8 and wd<=16 and xl*wd<=128 ) { shuf_select{0} } - else if (wi==8 and wd<=16 and xl*wd<=128<<1) { shuf_select{1} } - else if (wi==8 and wd<=16 and xl*wd<=128<<2) { shuf_select{2} } - else if (wi==8 and wd<= 8 and xl*wd<=128<<3) { shuf_select{3} } + def shuf_select = reg_select{shuf_select, makeshuf} + def perm_select = reg_select{perm_select, makeperm} + if (wi==8 and wd==32 and xl*wd<=256 ) perm_select{0} + else if (wi==8 and wd==32 and xl*wd<=256<<1) perm_select{1} + else if (wi==8 and wd<=16 and xl*wd<=128 ) shuf_select{0} + else if (wi==8 and wd<=16 and xl*wd<=128<<1) shuf_select{1} + else if (wi==8 and wd<=16 and xl*wd<=128<<2) shuf_select{2} + else if (wi==8 and wd<= 8 and xl*wd<=128<<3) shuf_select{3} else { def TIE = i32 def TDE = tern{wd<32, u32, TD} From 534c92f38a746f5e5cdeb73598918e443b6da191 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 29 Nov 2022 22:03:37 -0500 Subject: [PATCH 10/13] Select from <=128 booleans with shuffles --- src/builtins/select.c | 11 ++++++++++- src/singeli/src/avx.singeli | 1 + src/singeli/src/select.singeli | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index 8997a82c..30bd3694 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -69,6 +69,12 @@ B select_c2(B t, B w, B x) { #if SINGELI #define CPUSEL(W, NEXT) \ if (!avx2_select_tab[4*(we-el_i8)+CTZ(xw)](wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); + #define BOOL_USE_SIMD (xia<=128) + #define BOOL_SPECIAL(W) \ + if (sizeof(W)==1 && BOOL_USE_SIMD) { \ + if (!avx2_select_bool128(wp, xp, rp, wia, xia)) thrM("⊏: Indexing out-of-bounds"); \ + goto dec_ret; \ + } #else #define CPUSEL(W, NEXT) \ if (sizeof(W) >= 4) { \ @@ -90,12 +96,15 @@ B select_c2(B t, B w, B x) { } \ if (wt) TFREE(wt); \ } + #define BOOL_USE_SIMD 0 + #define BOOL_SPECIAL(W) #endif #define CASE(S, E) case S: for (usz i=i0; i=256 && wia/4>=xia && we!=el_bit) { + if (xe==el_bit && wia>=256 && !BOOL_USE_SIMD && wia/4>=xia && we!=el_bit) { return taga(cpyBitArr(select_c2(m_f64(0), w, taga(cpyI8Arr(x))))); } if (we==el_bit) { diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index 91064a3d..006ce927 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -65,6 +65,7 @@ def __xor{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_xor_ps', v2f{a}, v2f{b def __and{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_and_ps', v2f{a}, v2f{b}} def __or {a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_or_ps', v2f{a}, v2f{b}} def __not{a:T & w256u{T}} = a ^ broadcast{T, ~cast{eltype{T},0}} +def andnot{a:T, b:T & w256{T}} = T ~~ emit{[8]f32, '_mm256_andnot_ps', v2f{b}, v2f{a}} # float comparison local def f32cmpAVX{a,b,n} = [8]u32 ~~ emit{[8]f32, '_mm256_cmp_ps', a, b, n} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index 46c496bc..d4a6635e 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -4,8 +4,11 @@ include './sse3' include './avx' include './avx2' include './mask' +include './bitops' include 'util/tup' +oper &~ andnot infix none 35 + # def:T - masked original content # b:B - pointer to data to index; if width{B}32 and xl<=16) { + xb:= shuf{[4]u64, spreadBits{[32]u8, load{*u32~~x0}}, 4b1010} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + store{r, i, getmask{sel{[16]i8, xb, cw}}} + }} + } else { + x:= shuf{[4]u64, load{*VI ~~ x0}, 4b1010} + low:= broadcast{VI, 7} + b := VI~~make{[32]u8, 1 << (iota{32} & 7)} + maskedLoop{32, wl, {i, M} => { + cw:= wrapChk{load{w, i}, VI,xlf, M} + byte:= sel{[16]i8, x, VI~~(([8]u32~~andnot{cw, low})>>3)} + mask:= sel{[16]i8, b, cw & low} + store{r, i, getmask{(mask & byte) == mask}} + }} + } + 1 +} +'avx2_select_bool128' = avx2_select_bool128 From d7a06befb5913aff7521a8ff0e5073ad8c1324f0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 30 Nov 2022 16:15:50 -0500 Subject: [PATCH 11/13] Select implementation comments --- src/builtins/select.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/builtins/select.c b/src/builtins/select.c index 30bd3694..637401f9 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -1,3 +1,34 @@ +// First Cell and Select (⊏) + +// First Cell is just a slice + +// Complications in Select mostly come from range checks and negative 𝕨 +// Atom 𝕨 and any rank 𝕩: slice +// Rank-1 𝕩: +// Empty 𝕨: no selection +// Small 𝕩 with Singeli: use shuffles +// Boolean 𝕨: use bit_sel for blend or similar +// Boolean 𝕩 and larger 𝕨: convert to i8, select, convert back +// Boolean 𝕩 otherwise: select/shift bytes, reversed for fast writing +// TRIED pext, doesn't seem faster (mask built with shifts anyway) +// SHOULD squeeze 𝕨 if not ≤i32 to get to optimized cases +// Integer 𝕨 with Singeli: fused wrap, range-check, and gather +// COULD try selecting from boolean with gather +// COULD detect Date: Wed, 30 Nov 2022 16:31:32 -0500 Subject: [PATCH 12/13] =?UTF-8?q?Sparse=20initialization=20for=20=E2=8C=BE?= =?UTF-8?q?(i=E2=8A=B8=E2=8A=8F)=20byte=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/select.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index 637401f9..03d45868 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -26,8 +26,8 @@ // No longer needs to range-check but indices can be negative // COULD convert negative indices before selection // Must check collisions if CHECK_VALID; uses a byte set -// SHOULD do sparse initialization if 𝕨 is much smaller than 𝕩 -// COULD call Mark Firsts (∊) for very short 𝕨 +// Sparse initialization if 𝕨 is much smaller than 𝕩 +// COULD call Mark Firsts (∊) for very short 𝕨 to avoid allocation #include "../core.h" #include "../utils/talloc.h" @@ -249,11 +249,17 @@ B select_ucw(B t, B o, B w, B x) { if (isAtm(rep) || !eqShape(w, rep)) thrF("𝔽⌾(a⊸⊏)𝕩: Result of 𝔽 must have the same shape as 'a' (expected %H, got %H)", w, rep); #if CHECK_VALID TALLOC(bool, set, xia); - for (i64 i = 0; i < xia; i++) set[i] = false; + bool sparse = wia < xia/64; + if (!sparse) for (i64 i = 0; i < xia; i++) set[i] = false; + #define SPARSE_INIT(WI) \ + if (sparse) for (usz i = 0; i < wia; i++) { \ + i64 cw = WI; if (RARE(cw<0)) cw+= (i64)xia; set[cw] = false; \ + } #define EQ(F) if (set[cw] && (F)) thrM("𝔽⌾(a⊸⊏): Incompatible result elements"); set[cw] = true; #define FREE_CHECK TFREE(set) SLOWIF(xia>100 && wiare?xe:re; bool reuse = reusable(x); @@ -358,6 +365,7 @@ B select_ucw(B t, B o, B w, B x) { MUTG_INIT(r); mut_copyG(r, 0, x, 0, xia); SGet(rep) + SPARSE_INIT(o2i64G(GetU(w, i))) for (usz i = 0; i < wia; i++) { i64 cw = o2i64G(GetU(w, i)); if (RARE(cw<0)) cw+= (i64)xia; B cr = Get(rep, i); @@ -367,6 +375,7 @@ B select_ucw(B t, B o, B w, B x) { } decG(w); decG(rep); FREE_CHECK; return mut_fcd(r, x); + #undef SPARSE_INIT #undef EQ #undef FREE_CHECK } From 7c6676a4929605d4a550af3944f7c481f67aedc3 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 1 Dec 2022 13:37:01 +0200 Subject: [PATCH 13/13] copy over custom _mm_loadu_si32 for old gcc --- src/builtins/select.c | 7 +++++++ src/utils/mut.c | 8 ++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/builtins/select.c b/src/builtins/select.c index 03d45868..5aa251a4 100644 --- a/src/builtins/select.c +++ b/src/builtins/select.c @@ -35,6 +35,13 @@ #include "../builtins.h" #if SINGELI + #include + #if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32 + static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); } + static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); } + #define _mm_loadu_si32 custom_loadu_si32 + #define _mm_storeu_si32 custom_storeu_si32 + #endif #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #include "../singeli/gen/select.c" diff --git a/src/utils/mut.c b/src/utils/mut.c index e795f9e4..158017c4 100644 --- a/src/utils/mut.c +++ b/src/utils/mut.c @@ -220,12 +220,8 @@ DEF_G(void, copy, B, (void* a, usz ms, B x, usz xs, usz l), ms, x, x #if SINGELI #include #if __GNUC__ && !__clang__ // old gcc versions don't define _mm_loadu_si32 & _mm_storeu_si32 - __m128i custom_loadu_si32(void* p) { - return (__m128i) _mm_load_ss(p); - } - void custom_storeu_si32(void* p, __m128i x) { - _mm_store_ss(p, _mm_castsi128_ps(x)); - } + static __m128i custom_loadu_si32(void* p) { return (__m128i) _mm_load_ss(p); } + static void custom_storeu_si32(void* p, __m128i x) { _mm_store_ss(p, _mm_castsi128_ps(x)); } #define _mm_loadu_si32 custom_loadu_si32 #define _mm_storeu_si32 custom_storeu_si32 #endif