From 49ff5a04fff2043643f0c6baf2c749f32429418c Mon Sep 17 00:00:00 2001 From: dzaima Date: Sat, 2 Apr 2022 16:58:35 +0300 Subject: [PATCH] =?UTF-8?q?singeli=20=E2=89=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- makefile | 2 +- src/builtins/arithd.c | 9 --- src/core.h | 4 ++ src/core/stuff.c | 77 ++++++++++++++++----- src/singeli/c/cmp.c | 5 -- src/singeli/src/avx.singeli | 23 +++++-- src/singeli/src/avx2.singeli | 8 +-- src/singeli/src/base.singeli | 20 ++++-- src/singeli/src/dyarith.singeli | 15 +---- src/singeli/src/equal.singeli | 115 ++++++++++++++++++++++++++++++++ src/singeli/src/mask.singeli | 35 ++++++++++ 11 files changed, 253 insertions(+), 60 deletions(-) create mode 100644 src/singeli/src/equal.singeli create mode 100644 src/singeli/src/mask.singeli diff --git a/makefile b/makefile index 3ffae6ea..670b9a4c 100644 --- a/makefile +++ b/makefile @@ -136,7 +136,7 @@ preSingeliBin: @mv BQN obj/presingeli/BQN -gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c} +gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/arithd.c b/src/builtins/arithd.c index f814daf5..8c8d3278 100644 --- a/src/builtins/arithd.c +++ b/src/builtins/arithd.c @@ -6,11 +6,6 @@ #define BCALL(N, X) N(b(X)) #define interp_f64(X) b(X).f -static i8 mask8[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; -static i16 mask16[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; -static i32 mask32[] = {-1,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0}; -static i64 mask64[] = {-1,-1,-1,0,0,0}; - #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" #include "../singeli/gen/dyarith.c" @@ -235,10 +230,6 @@ static f64 pfmod(f64 a, f64 b) { if (we==el_f64) { Rf64(w) PF(w) DOF(EXPR,x,wp[i],x.f) dec(w); return num_squeeze(r); } #if SINGELI - static void* tyany_ptr(B x) { // TODO extract to some header file - u8 t = v(x)->type; - return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a; - } #define SI_AA(N,S,BASE) R##S(x); usz rlen=avx2_##N##AA##_##S((void*)wp, (void*)xp, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r; #define SI_SA_I(N,S,W,BASE) R##S(x); usz rlen=avx2_##N##SA##_##S((W).u, (void*)xp, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r; #define SI_AS_I(N,S,X,BASE) R##S(w); usz rlen=avx2_##N##AS##_##S((void*)wp, (X).u, (void*)rp, ia); if(RARE(rlen!=ia)) { dec(r); goto BASE; } dec(w);dec(x);return r; diff --git a/src/core.h b/src/core.h index d834fab2..3489d219 100644 --- a/src/core.h +++ b/src/core.h @@ -33,6 +33,10 @@ static B* arr_bptr(B x) { assert(isArr(x)); if (v(x)->type==t_fillslice) return c(FillSlice,x)->a; return NULL; } +static void* tyany_ptr(B x) { + u8 t = v(x)->type; + return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a; +} typedef struct BFn { diff --git a/src/core/stuff.c b/src/core/stuff.c index 9dab8e4e..21bdc3c1 100644 --- a/src/core/stuff.c +++ b/src/core/stuff.c @@ -402,6 +402,41 @@ NOINLINE bool atomEqualR(B w, B x) { { dec(wd);dec(xd); return false; } dec(wd);dec(xd); return true; } + +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/equal.c" + #pragma GCC diagnostic pop + + typedef bool (*EqFn)(u8* a, u8* b, u64 l, u64 data); + bool notEq(u8* a, u8* b, u64 l, u64 data) { return false; } + + #define F(X) avx2_equal_##X + EqFn eqFns[] = { + F(1_1), F(1_8), F(1_16), F(1_32), F(1_f64), notEq, notEq, notEq, + F(1_8), F(8_8), F(s8_16), F(s8_32), F(s8_f64), notEq, notEq, notEq, + F(1_16), F(s8_16), F(8_8), F(s16_32), F(s16_f64), notEq, notEq, notEq, + F(1_32), F(s8_32), F(s16_32), F(8_8), F(s32_f64), notEq, notEq, notEq, + F(1_f64), F(s8_f64), F(s16_f64), F(s32_f64), F(f64_f64), notEq, notEq, notEq, + notEq, notEq, notEq, notEq, notEq, F(8_8), F(u8_16), F(u8_32), + notEq, notEq, notEq, notEq, notEq, F(u8_16), F(8_8), F(u16_32), + notEq, notEq, notEq, notEq, notEq, F(u8_32), F(u16_32), F(8_8), + }; + #undef F + static const u8 n = 99; + u8 eqFnData[] = { // for the main diagonal, amount to shift length by; otherwise, whether to swap arguments + 0,0,0,0,0,n,n,n, + 1,0,0,0,0,n,n,n, + 1,1,1,0,0,n,n,n, + 1,1,1,2,0,n,n,n, + 1,1,1,1,0,n,n,n, + n,n,n,n,n,0,0,0, + n,n,n,n,n,1,1,0, + n,n,n,n,n,1,1,2, + }; +#endif + NOINLINE bool equal(B w, B x) { // doesn't consume bool wa = isAtm(w); bool xa = isAtm(x); @@ -409,24 +444,36 @@ NOINLINE bool equal(B w, B x) { // doesn't consume if (wa) return atomEqual(w, x); if (!eqShape(w,x)) return false; usz ia = a(x)->ia; + if (ia==0) return true; u8 we = TI(w,elType); u8 xe = TI(x,elType); - if (((we==el_f64 | we==el_i32) && (xe==el_f64 | xe==el_i32))) { - if (we==el_i32) { i32* wp = i32any_ptr(w); - if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } - else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } - } else { f64* wp = f64any_ptr(w); - if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } - else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } + + #if SINGELI + if (we<=el_c32 && xe<=el_c32) { + u8* wp = tyany_ptr(w); + u8* xp = tyany_ptr(x); + u64 idx = we*8 + xe; + return eqFns[idx](wp, xp, ia, eqFnData[idx]); } - return true; - } - if (we==el_c32 && xe==el_c32) { - u32* wp = c32any_ptr(w); - u32* xp = c32any_ptr(x); - for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; - return true; - } + #else + if (((we==el_f64 | we==el_i32) && (xe==el_f64 | xe==el_i32))) { + if (we==el_i32) { i32* wp = i32any_ptr(w); + if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } + else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } + } else { f64* wp = f64any_ptr(w); + if(xe==el_i32) { i32* xp = i32any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } + else { f64* xp = f64any_ptr(x); for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; } + } + return true; + } + if (we==el_c32 && xe==el_c32) { + u32* wp = c32any_ptr(w); + u32* xp = c32any_ptr(x); + for (usz i = 0; i < ia; i++) if(wp[i]!=xp[i]) return false; + return true; + } + #endif + SLOW2("equal", w, x); SGetU(x) SGetU(w) for (usz i = 0; i < ia; i++) if(!equal(GetU(w,i),GetU(x,i))) return false; diff --git a/src/singeli/c/cmp.c b/src/singeli/c/cmp.c index b6de82ae..f4550717 100644 --- a/src/singeli/c/cmp.c +++ b/src/singeli/c/cmp.c @@ -36,11 +36,6 @@ FN_LUT(avx2, ge, AS); FN_LUT(avx2, ge, AA); FN_LUT(avx2, lt, AS); FN_LUT(avx2, le, AS); -static void* tyany_ptr(B x) { - u8 t = v(x)->type; - return IS_SLICE(t)? c(TySlice,x)->a : c(TyArr,x)->a; -} - #define AL(X) u64* rp; B r = m_bitarrc(&rp, X); usz ria=a(r)->ia #define CMP_IMPL(CHR, NAME, RNAME, PNAME, L, R, OP, FC, CF, BX) \ if (isF64(w)&isF64(x)) return m_i32(w.f OP x.f); \ diff --git a/src/singeli/src/avx.singeli b/src/singeli/src/avx.singeli index df01e10b..3f5279e8 100644 --- a/src/singeli/src/avx.singeli +++ b/src/singeli/src/avx.singeli @@ -1,15 +1,18 @@ # various utilities +def w256{T} = 0 +def w256{T & isvec{T}} = width{T}==256 + def wgen256{F} = { def r{T} = 0 - def r{T & width{T}==256} = F{eltype{T}} + def r{T & w256{T}} = F{eltype{T}} def r{T,w} = 0 - def r{T,w & width{T}==256} = F{eltype{T}} & (width{eltype{T}}==w) + def r{T,w & w256{T}} = F{eltype{T}} & (width{eltype{T}}==w) + def r{T & ~isvec{T}} = 0 r } -def w256{T} = width{T}==256 -def w256{T, w} = 0 -def w256{T, w & width{T}==256 & width{eltype{T}}==w} = 1 +def w256{T,w} = 0 +def w256{T,w & w256{T}} = width{eltype{T}}==w def w256i = wgen256{{T} => isint{T}} def w256s = wgen256{{T} => isint{T} & issigned{T}} def w256u = wgen256{{T} => isint{T} & isunsigned{T}} @@ -24,8 +27,9 @@ def v2i{x:T & w256{T}} = cast_v{[32]u8, x} # for compact casting for the annoyin def v2f{x:T & w256{T}} = cast_v{[8]f32, x} def v2d{x:T & w256{T}} = cast_v{[4]f64, x} -def ty_vu{T & w256s{T}} = [vcount{T}](ty_iu{eltype{T}}) -def ty_vs{T & w256u{T}} = [vcount{T}](ty_is{eltype{T}}) +def ty_vu{T & w256s{T}} = [vcount{T}](ty_u{eltype{T}}) +def ty_vs{T & w256u{T}} = [vcount{T}](ty_s{eltype{T}}) +def to_el{E, x:T} = cast_v{[width{T}/width{E}]E, x} def forv{T & w256{T}} = forc{{v}=>cast_vp{T,v}} # load & store @@ -122,6 +126,10 @@ def insert{x:T, i, v & w256i{T,16} & knum{i}} = emit{T, '_mm256_insert_epi16', x def insert{x:T, i, v & w256i{T,32} & knum{i}} = emit{T, '_mm256_insert_epi32', x, v, i} def insert{x:T, i, v & w256i{T,64} & knum{i}} = emit{T, '_mm256_insert_epi64', x, v, i} +# blend by sign bit +def blend{f:T, t:T, m:M & w256{T} & w256i{M,32}} = cast_v{T, emit{[8]f32, '_mm256_blendv_ps', v2f{f}, v2f{t}, v2f{m}}} +def blend{f:T, t:T, m:M & w256{T} & w256i{M,64}} = cast_v{T, emit{[4]f64, '_mm256_blendv_pd', v2d{f}, v2d{t}, v2d{m}}} + # mixed-width operations def half{x:T, i & w256{T} & knum{i}} = cast_v{[vcount{T}/2](eltype{T}), emit{[8]i16, '_mm256_extracti128_si256', v2i{x}, i}} def pair{a:T,b:T & width{T}==128} = cast_v{[vcount{T}*2](eltype{T}), emit{[8]i32, '_mm256_setr_m128i', a, b}} @@ -130,6 +138,7 @@ def pair{x} = pair{tupsel{0,x},tupsel{1,x}} # mask stuff def getmask{x:T & w256{T, 32}} = emit{u8, '_mm256_movemask_ps', v2f{x}} def getmask{x:T & w256{T, 64}} = emit{u8, '_mm256_movemask_pd', v2d{x}} +def andIsZero{x:T, y:T & w256i{T}} = emit{u1, '_mm256_testz_si256', x, y} def any{x:T & w256i{T}} = getmask{x} != 0 # assumes elements of x all have equal bits (avx2 utilizes this for 16 bits) def all{x:T & w256i{T}} = getmask{x} == (1<0 & n=32} = maskstore{p,m,n,x} def maskstoreF{p, m, n, x:T} = store{p, n, blendf{load{p,n}, x, m}} -def anyne{x:T, y:T, M & M{0}==0} = ~all{x==y} -def anyne{x:T, y:T, M & M{0}==1} = any{M{x!=y}} - # + & - def arithChk1{F, M, w:T, x:T, r:T & match{F,__add}} = anyneg{M{(w^r) & (x^r)}} def arithChk1{F, M, w:T, x:T, r:T & match{F,__sub}} = anyneg{M{(w^x) & (w^r)}} @@ -94,10 +85,10 @@ def arithAny{VT, F, W, X, r, len} = { @forv{VT} (r over i from 0 to vv) r = arithChk3{F, {x}=>x, W{i}, X{i}, i*bam} left:= len&(bam-1) if (left!=0) { - m:= genmask{VT, left} + m:= maskOf{VT, left} def mask{x:X} = x&cast_v{X,m} def mask{x==0} = 1 - rv:= arithChk3{F, mask, W{vv}, X{vv}, vv*bam} + rv:= arithChk3{F, maskAfter{left}, W{vv}, X{vv}, vv*bam} maskstoreF{cast_p{VT,r}, m, vv, rv} } len diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli new file mode 100644 index 00000000..893ae743 --- /dev/null +++ b/src/singeli/src/equal.singeli @@ -0,0 +1,115 @@ +include './base' +include './cbqnDefs' +include './f64' +include './sse3' +include './avx' +include './avx2' +include './mask' + +def cvt{F==i8, T==[8]i16, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi16', a} +def cvt{F==i8, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi8_epi32', a} +def cvt{F==i16, T==[4]i32, a:A & w128i{A}} = emit{T, '_mm_cvtepi16_epi32', a} + + +def cvt{F==u8, T==[16]u16, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi16', a} +def cvt{F==u8, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu8_epi32', a} +def cvt{F==u16, T==[8]u32, a:A & w128i{A}} = emit{T, '_mm256_cvtepu16_epi32', a} + +def cvt{F==i8, T==[16]i16, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi16', a} +def cvt{F==i8, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi8_epi32', a} +def cvt{F==i16, T==[8]i32, a:A & w128i{A}} = emit{T, '_mm256_cvtepi16_epi32', a} + +def cvt{F , T==[4]f64, a:A & w128i{A}} = cvt{i32, T, cvt{F, [4]i32, a}} +def cvt{F==i32, T==[4]f64, a:A & w128i{A}} = emit{T, '_mm256_cvtepi32_pd', a} + +def cvt{F, T, a & eltype{T}==F} = reinterpret{T, a} + +def cif{C, T, F} = { + if(C) T + else F +} +def swap{w,x} = { + t:= w + w = x + x = t +} + +def maskedLoop{bulk, l, step} = { + m:u64 = l/bulk + @for (i from 0 to m) step{i, maskNone} + + left:= l & (bulk-1) + if (left!=0) step{m, maskAfter{left}} +} + +equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = { + def bulk = 256 / width{X} + if (W!=X) if (d!=0) swap{w,x} + if (W==u1) { + if (X==u1) { # bitarr ≡ bitarr + maskedLoop{256, l, {i, M} => { + cw:= load{cast_p{[32]u8, w}, i} + cx:= load{cast_p{[32]u8, x}, i} + if (anyneBit{cw,cx,M}) return{0} + }} + } else if (X==f64) { # bitarr ≡ f64arr + def T = [4]f64 + def bulk = 4 + f0:= broadcast{T, 0.0} + f1:= broadcast{T, 1.0} + maskedLoop{4, l, {i, M} => { + cw:= load{cast_p{u8,w}, i>>1} >> cast_i{u8, 4*(i&1)} + cx:= load{cast_p{T, x}, i} + wu:= blend{f0, f1, broadcast{[4]u64, cw} << make{[4]u64,63,62,61,60}} + if (anyne{wu, cx, M}) return{0} + }} + } else { # bitarr ≡ i8/i16/i32arr + def T = [256/width{X}]X + def sh{c} = c << (width{X}-1) + def sh{c & X==u8} = cast_v{T, to_el{u16,c}<<7} + + # TODO compare with doing the comparison in vector registers + badBits:= broadcast{T, ~cast{X,1}} + maskedLoop{bulk, l, {i, M} => { + cw:= load{cast_p{ty_u{bulk}, w}, i} + cx:= load{cast_p{T,x}, i} + if (~andIsZero{M{cx}, badBits}) return{0} + if (anyne{promote{u64,getmask{sh{cx}}}, promote{u64,cw}, M}) return{0} + }} + 1 + } + } else { # everything not involving bitarrs + if (W==i8 and X==i8) l<<= d + def ww{gw, E} = [gw/width{E}]E + def fac = width{X}/width{W} + + maskedLoop{bulk, l, {i, M} => { + cw:= load{cast_p{ww{cif{fac==1, 256, 128}, u8}, w + i*32/fac}, 0} + cx:= load{cast_p{ww{256, X}, x}, i} + cwc:= cvt{W, ww{256, X}, cw} + if (anyne{cwc,cx,M}) return{0} + }} + } + 1 +} + +'avx2_equal_1_1' = equal{u1, u1} # todo maybe unify with 8_8 and make the non-bit diagonal shift by 3 +'avx2_equal_1_8' = equal{u1, u8} +'avx2_equal_1_16' = equal{u1, u16} +'avx2_equal_1_32' = equal{u1, u32} +'avx2_equal_1_f64' = equal{u1, f64} + +'avx2_equal_8_8' = equal{i8, i8} + +'avx2_equal_s8_16' = equal{i8, i16} +'avx2_equal_s8_32' = equal{i8, i32} +'avx2_equal_s16_32' = equal{i16, i32} + +'avx2_equal_s8_f64' = equal{i8, f64} +'avx2_equal_s16_f64'= equal{i16, f64} +'avx2_equal_s32_f64'= equal{i32, f64} +'avx2_equal_f64_f64'= equal{f64, f64} + +'avx2_equal_u8_16' = equal{u8, u16} +'avx2_equal_u8_32' = equal{u8, u32} +'avx2_equal_u16_32' = equal{u16, u32} diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli new file mode 100644 index 00000000..67575bcc --- /dev/null +++ b/src/singeli/src/mask.singeli @@ -0,0 +1,35 @@ +# get mask of first n items; n>0 & n { + merge{broadcast{31, 255}, (1<>3)^31 + 64*(n&7)}, 0} + +def anyne{x:T, y:T, M & M{0}==0 & isvec{T}} = ~all{x==y} +def anyne{x:T, y:T, M & M{0}==1 & isvec{T}} = any{M{x!=y}} +def anyne{x:T, y:T, M & M{0}==0 & anyInt{x}} = x!=y +def anyne{x:T, y:T, M & M{0}==1 & anyInt{x}} = M{x^y} != 0 +def anyneBit{x:T, y:T, M} = ~M{x^y, 'all bits zeroes'} + +def maskNone{x} = x +def maskNone{x, mode=='all bits zeroes'} = andIsZero{x, x} +def maskAfter{n} = { + def mask{x:X & isvec{X}} = x&cast_v{X,maskOf{X,n}} + def mask{x:X & anyInt{x}} = x & ((1<