From 113f72e86dfef012ab553ff1c2eef638d975d425 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 21 Apr 2022 18:27:20 +0300 Subject: [PATCH] singeli bitarr conversion to wider arrays --- makefile | 2 +- src/core/tyarr.c | 12 +++++++++- src/singeli/src/avx.singeli | 8 +++---- src/singeli/src/base.singeli | 16 ++++++++----- src/singeli/src/bitops.singeli | 15 +++++++----- src/singeli/src/cbqnDefs.singeli | 1 - src/singeli/src/cmp.singeli | 2 +- src/singeli/src/equal.singeli | 8 ------- src/singeli/src/expand.singeli | 40 ++++++++++++++++++++++++++++++++ src/singeli/src/mask.singeli | 35 +++++++++++++++++++++++++++- src/singeli/src/select.singeli | 35 ++-------------------------- 11 files changed, 112 insertions(+), 62 deletions(-) create mode 100644 src/singeli/src/expand.singeli diff --git a/makefile b/makefile index b369434a..6d91f47c 100644 --- a/makefile +++ b/makefile @@ -140,7 +140,7 @@ preSingeliBin: @${MAKE} singeli=0 postmsg="singeli sources:" t=presingeli f='-O1' OUTPUT=obj/presingeli/BQN c -gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c} +gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c expand.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/core/tyarr.c b/src/core/tyarr.c index 91995580..c5e87368 100644 --- a/src/core/tyarr.c +++ b/src/core/tyarr.c @@ -36,12 +36,22 @@ NOINLINE B m_str32(u32* s) { return r; } +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/expand.c" + #pragma GCC diagnostic pop + #define BIT_ICPY(E) avx2_expand_1_##E(xp, rp, ia); +#else + #define BIT_ICPY(E) for(usz i=0; iia; \ E* rp; Arr* r = m_##E##arrp(&rp, ia); \ arr_shCopy(r, x); \ u8 xe = TI(x,elType); \ - if (xe==el_bit) { u64* xp = bitarr_ptr(x); for(usz i=0; iext{eltype{T},c}, x}}} -def make{T,a,b,c,d & w256i{T}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}} -def make{T,a,b,c,d,e,f,g,h & w256i{T}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}} -def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}} -def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}} +def make{T,a,b,c,d & w256i{T,64}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}} +def make{T,a,b,c,d,e,f,g,h & w256i{T,32}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}} +def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T,16}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}} +def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T,8}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}} def make{T,x & w256{T} & istup{x}} = apply{make, merge{tup{T}, x}} def iota{T & w256{T,64}} = make{T,0,1,2,3} diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 8a66efb3..14ade689 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -1,5 +1,6 @@ include 'skin/c' include 'arch/c' +def Size = u64 oper infix right ~~ reinterpret 55 # TODO move these to a more base file @@ -16,8 +17,8 @@ def cdiv{a,b} = (a+b-1)/b def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1 def lb{n==1} = 0 -# get the n least significant bits -def lsbs{x, n} = x - (x>>n << n) +def tail{n,x} = x - (x>>n << n) # get the n least significant bits +def bit {k,x} = tail{1,x>>k} << k # get the k-th bit def rare{x:u1} = emit{u1, '__builtin_expect', x, 0} def isunsigned{T} = isint{T} & ~issigned{T} @@ -51,6 +52,9 @@ def ty_s{T==u16} = i16; def ty_s{w==16} = i16 def ty_s{T==u32} = i32; def ty_s{w==32} = i32 def ty_s{T==u64} = i64; def ty_s{w==64} = i64 +def ty_u{T==f64} = u64; def ty_u{T==f32} = u32; +def ty_s{T==f64} = i64; def ty_s{T==f32} = i32; + def ty_vu{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}}) def ty_vs{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}}) def ty_vs{x:T} = ty_vs{T}~~x @@ -111,7 +115,7 @@ def broadcast{n, v & knum{n}} = @collect(n) v # debug stuff -def print{x & match{kind{x},'symbol'}} = emit{void, 'printf', '"%s"', merge{'"', x, '"'}} -def print{x & anyNum{x}} = emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x} -def print{x:T & anyNum{x} & width{T}==64} = emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x} -def println{x} = { print{x}; print{'\n'} } \ No newline at end of file +def print{x & match{kind{x},'symbol'}} = { emit{void, 'printf', '"%s"', merge{'"', x, '"'}}; x } +def print{x & anyNum{x}} = { emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x}; x } +def print{x:T & anyNum{x} & width{T}==64} = { emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x}; x } +def println{x} = { print{x}; print{'\n'}; x } \ No newline at end of file diff --git a/src/singeli/src/bitops.singeli b/src/singeli/src/bitops.singeli index 74ab9242..7d780208 100644 --- a/src/singeli/src/bitops.singeli +++ b/src/singeli/src/bitops.singeli @@ -4,6 +4,9 @@ def b_get{x:*u64, n:(Size)} = { ((load{x,n>>6}>>(n&63)) & 1) != 0 } +def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15 +def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n} + def b_set{x:*u64, n:(Size), v:u1} = { m:u64 = cast{u64,1}<<(n&63) p:u64 = load{x,n>>6} @@ -11,7 +14,7 @@ def b_set{x:*u64, n:(Size), v:u1} = { else store{x,n>>6,p & ~m} } -def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v} +def b_setBatch{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v} am:u64 = 64/sz; w:u64 = load{x,n/am} sh:u64 = (n&(am-1)) * sz @@ -20,7 +23,7 @@ def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v} store{x, n/am, w} } -def b_set{sz, x:*u64, n:(Size), v & sz==4} = { +def b_setBatch{sz, x:*u64, n:(Size), v & sz==4} = { x8:= cast_p{u8, x} #w:u64 = cast_i{u64, load{x8,n/2}} @@ -39,7 +42,7 @@ def b_set{sz, x:*u64, n:(Size), v & sz==4} = { store{x8, n/2, cast_i{u8,w}} } -def b_set{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}} -def b_set{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}} -def b_set{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}} -def b_set{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}} +def b_setBatch{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}} +def b_setBatch{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}} +def b_setBatch{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}} +def b_setBatch{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}} diff --git a/src/singeli/src/cbqnDefs.singeli b/src/singeli/src/cbqnDefs.singeli index 731dd1f7..1d604b16 100644 --- a/src/singeli/src/cbqnDefs.singeli +++ b/src/singeli/src/cbqnDefs.singeli @@ -1,4 +1,3 @@ -def Size = u64 def bcall{T, f, x} = emit{T, 'BCALL', f, x} def from_B{T, x & T==f64} = bcall{T, 'o2fu', x} def from_B{T, x & T<=i32 & issigned{T}} = bcall{T, 'o2iu', x} diff --git a/src/singeli/src/cmp.singeli b/src/singeli/src/cmp.singeli index b6e1bd5e..b4f2cdf7 100644 --- a/src/singeli/src/cmp.singeli +++ b/src/singeli/src/cmp.singeli @@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = { while (ri < am) { r:u64 = 0 @unroll (j from 0 to unr) r = r | (ext{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT})) - b_set{bam, dst, ri, r} + b_setBatch{bam, dst, ri, r} xi = xi+unr ri = ri+1 } diff --git a/src/singeli/src/equal.singeli b/src/singeli/src/equal.singeli index a6e73086..d2840c09 100644 --- a/src/singeli/src/equal.singeli +++ b/src/singeli/src/equal.singeli @@ -12,14 +12,6 @@ def swap{w,x} = { x = t } -def maskedLoop{bulk, l, step} = { - m:u64 = l/bulk - @for (i from 0 to m) step{i, maskNone} - - left:= l & (bulk-1) - if (left!=0) step{m, maskAfter{left}} -} - equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = { def bulk = 256 / width{X} if (W!=X) if (d!=0) swap{w,x} diff --git a/src/singeli/src/expand.singeli b/src/singeli/src/expand.singeli new file mode 100644 index 00000000..a305d826 --- /dev/null +++ b/src/singeli/src/expand.singeli @@ -0,0 +1,40 @@ +include './base' +include './sse3' +include './avx' +include './avx2' +include './mask' +include './bitops' + + + +def bitspread{T==[32]u8, a:u32} = { + def idxs = iota{32} + b:= broadcast{[8]u32, a} + c:= [32]u8~~b + d:= sel{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}} + e:= make{[32]u8, 1< { + x:= b_getBatch{bulk, xp, i} # TODO unroll f64 by two to make b_getBatch not sad + y:= bitspread{TU, x} + r:= y & TU~~broadcast{T, 1} + storeBatch{rp, i, T~~r, M} + }} +} + +'avx2_expand_1_i8' = bitexpand{[32]i8} +'avx2_expand_1_i16' = bitexpand{[16]i16} +'avx2_expand_1_i32' = bitexpand{[ 8]i32} +'avx2_expand_1_f64' = bitexpand{[ 4]f64} \ No newline at end of file diff --git a/src/singeli/src/mask.singeli b/src/singeli/src/mask.singeli index fba0732b..6d58b1ce 100644 --- a/src/singeli/src/mask.singeli +++ b/src/singeli/src/mask.singeli @@ -43,4 +43,37 @@ def maskAfter{n} = { def mask{X, mode=='to sign bits'} = maskOf{X,n} def mask{mode=='count'} = n def mask{x==0} = 1 -} \ No newline at end of file +} + + + +# store low packed elements of x to P; TODO the w parameter isn't doing much? +def storeLow{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x} +def storeLow{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}} + +# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M +def storeBatch{ptr:P, n, x:T, M} = { + def rpos = ptr + n*vcount{T} + def E0 = eltype{P} + xu:= ucvt{E0, x} + def TF = to_el{E0, T} + if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu} + else storeLow{rpos, vcount{T}*width{E0}, xu} +} + +# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T +def loadBatch{ptr:P, n, T} = { + def rpos = ptr + n*vcount{T} + def E0 = eltype{P} + + if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0} + else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}} +} + +def maskedLoop{bulk, l, step} = { + m:u64 = l/bulk + @for (i from 0 to m) step{i, maskNone} + + left:= l & (bulk-1) + if (left!=0) step{m, maskAfter{left}} +} diff --git a/src/singeli/src/select.singeli b/src/singeli/src/select.singeli index e4ac1028..ec982dbb 100644 --- a/src/singeli/src/select.singeli +++ b/src/singeli/src/select.singeli @@ -5,14 +5,6 @@ include './avx' include './avx2' include './mask' -def maskedLoop{bulk, l, step} = { - m:u64 = l/bulk - @for (i from 0 to m) step{i, maskNone} - - left:= l & (bulk-1) - if (left!=0) step{m, maskAfter{left}} -} - def min{a, b & knum{a} & knum{b}} = tern{a { - cw0:= loadx{w, i, TIF} + cw0:= loadBatch{w, i, TIF} cw1:= cw0+xlf cw:= blendF{cw0, cw1, cw0= ty_vu{xlf}}}) return{0} got:= gather{broadcast{TDF,0}, x, cw, M} if (TDE!=TD) got&= broadcast{TDF, (1<