singeli bitarr conversion to wider arrays

This commit is contained in:
dzaima 2022-04-21 18:27:20 +03:00
parent b1fd85621a
commit 113f72e86d
11 changed files with 112 additions and 62 deletions

View File

@ -140,7 +140,7 @@ preSingeliBin:
@${MAKE} singeli=0 postmsg="singeli sources:" t=presingeli f='-O1' OUTPUT=obj/presingeli/BQN c
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c}
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c expand.c}
@echo $(postmsg)
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
@echo $< | cut -c 17- | sed 's/^/ /'

View File

@ -36,12 +36,22 @@ NOINLINE B m_str32(u32* s) {
return r;
}
#if SINGELI
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#include "../singeli/gen/expand.c"
#pragma GCC diagnostic pop
#define BIT_ICPY(E) avx2_expand_1_##E(xp, rp, ia);
#else
#define BIT_ICPY(E) for(usz i=0; i<ia; i++) rp[i]=bitp_get(xp,i);
#endif
#define MAKE_ICPY(T,E) T##Arr* cpy##T##Arr(B x) { \
usz ia = a(x)->ia; \
E* rp; Arr* r = m_##E##arrp(&rp, ia); \
arr_shCopy(r, x); \
u8 xe = TI(x,elType); \
if (xe==el_bit) { u64* xp = bitarr_ptr(x); for(usz i=0; i<ia; i++) rp[i]=bitp_get(xp,i); } \
if (xe==el_bit) { u64* xp = bitarr_ptr(x); BIT_ICPY(E) } \
else if (xe==el_i8 ) { i8* xp = i8any_ptr (x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \
else if (xe==el_i16) { i16* xp = i16any_ptr(x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \
else if (xe==el_i32) { i32* xp = i32any_ptr(x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \

View File

@ -57,10 +57,10 @@ def make{T==[4]f64,a,b,c,d} = emit{T,'_mm256_setr_pd',a,b,c,d}
def make{T==[8]f32,a,b,c,d,e,f,g,h} = emit{T,'_mm256_setr_ps',a,b,c,d,e,f,g,h}
def makeGen{T,s,x} = apply{emit, merge{tup{T,s}, each{{c}=>ext{eltype{T},c}, x}}}
def make{T,a,b,c,d & w256i{T}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}}
def make{T,a,b,c,d,e,f,g,h & w256i{T}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}}
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}}
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}}
def make{T,a,b,c,d & w256i{T,64}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}}
def make{T,a,b,c,d,e,f,g,h & w256i{T,32}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}}
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T,16}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}}
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T,8}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}}
def make{T,x & w256{T} & istup{x}} = apply{make, merge{tup{T}, x}}
def iota{T & w256{T,64}} = make{T,0,1,2,3}

View File

@ -1,5 +1,6 @@
include 'skin/c'
include 'arch/c'
def Size = u64
oper infix right ~~ reinterpret 55
# TODO move these to a more base file
@ -16,8 +17,8 @@ def cdiv{a,b} = (a+b-1)/b
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
def lb{n==1} = 0
# get the n least significant bits
def lsbs{x, n} = x - (x>>n << n)
def tail{n,x} = x - (x>>n << n) # get the n least significant bits
def bit {k,x} = tail{1,x>>k} << k # get the k-th bit
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
def isunsigned{T} = isint{T} & ~issigned{T}
@ -51,6 +52,9 @@ def ty_s{T==u16} = i16; def ty_s{w==16} = i16
def ty_s{T==u32} = i32; def ty_s{w==32} = i32
def ty_s{T==u64} = i64; def ty_s{w==64} = i64
def ty_u{T==f64} = u64; def ty_u{T==f32} = u32;
def ty_s{T==f64} = i64; def ty_s{T==f32} = i32;
def ty_vu{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
def ty_vs{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
def ty_vs{x:T} = ty_vs{T}~~x
@ -111,7 +115,7 @@ def broadcast{n, v & knum{n}} = @collect(n) v
# debug stuff
def print{x & match{kind{x},'symbol'}} = emit{void, 'printf', '"%s"', merge{'"', x, '"'}}
def print{x & anyNum{x}} = emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x}
def print{x:T & anyNum{x} & width{T}==64} = emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x}
def println{x} = { print{x}; print{'\n'} }
def print{x & match{kind{x},'symbol'}} = { emit{void, 'printf', '"%s"', merge{'"', x, '"'}}; x }
def print{x & anyNum{x}} = { emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x}; x }
def print{x:T & anyNum{x} & width{T}==64} = { emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x}; x }
def println{x} = { print{x}; print{'\n'}; x }

View File

@ -4,6 +4,9 @@ def b_get{x:*u64, n:(Size)} = {
((load{x,n>>6}>>(n&63)) & 1) != 0
}
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15
def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
def b_set{x:*u64, n:(Size), v:u1} = {
m:u64 = cast{u64,1}<<(n&63)
p:u64 = load{x,n>>6}
@ -11,7 +14,7 @@ def b_set{x:*u64, n:(Size), v:u1} = {
else store{x,n>>6,p & ~m}
}
def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
def b_setBatch{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
am:u64 = 64/sz;
w:u64 = load{x,n/am}
sh:u64 = (n&(am-1)) * sz
@ -20,7 +23,7 @@ def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
store{x, n/am, w}
}
def b_set{sz, x:*u64, n:(Size), v & sz==4} = {
def b_setBatch{sz, x:*u64, n:(Size), v & sz==4} = {
x8:= cast_p{u8, x}
#w:u64 = cast_i{u64, load{x8,n/2}}
@ -39,7 +42,7 @@ def b_set{sz, x:*u64, n:(Size), v & sz==4} = {
store{x8, n/2, cast_i{u8,w}}
}
def b_set{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}}
def b_set{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}}
def b_set{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}}
def b_set{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}}
def b_setBatch{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}}

View File

@ -1,4 +1,3 @@
def Size = u64
def bcall{T, f, x} = emit{T, 'BCALL', f, x}
def from_B{T, x & T==f64} = bcall{T, 'o2fu', x}
def from_B{T, x & T<=i32 & issigned{T}} = bcall{T, 'o2iu', x}

View File

@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
while (ri < am) {
r:u64 = 0
@unroll (j from 0 to unr) r = r | (ext{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}))
b_set{bam, dst, ri, r}
b_setBatch{bam, dst, ri, r}
xi = xi+unr
ri = ri+1
}

View File

@ -12,14 +12,6 @@ def swap{w,x} = {
x = t
}
def maskedLoop{bulk, l, step} = {
m:u64 = l/bulk
@for (i from 0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}
equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = {
def bulk = 256 / width{X}
if (W!=X) if (d!=0) swap{w,x}

View File

@ -0,0 +1,40 @@
include './base'
include './sse3'
include './avx'
include './avx2'
include './mask'
include './bitops'
def bitspread{T==[32]u8, a:u32} = {
def idxs = iota{32}
b:= broadcast{[8]u32, a}
c:= [32]u8~~b
d:= sel{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
e:= make{[32]u8, 1<<tail{3, idxs}}
e == (d&e)
}
def bitspread{T, a & vcount{T} <= width{eltype{T}} & w256u{T}} = {
b:= make{T, 1<<iota{vcount{T}}}
b == (b & broadcast{T, a})
}
bitexpand{T}(xp: *u64, rp:*eltype{T}, l:u64) : void = {
def bulk = vcount{T}
def TU = ty_vu{T}
maskedLoop{vcount{T}, l, {i, M} => {
x:= b_getBatch{bulk, xp, i} # TODO unroll f64 by two to make b_getBatch not sad
y:= bitspread{TU, x}
r:= y & TU~~broadcast{T, 1}
storeBatch{rp, i, T~~r, M}
}}
}
'avx2_expand_1_i8' = bitexpand{[32]i8}
'avx2_expand_1_i16' = bitexpand{[16]i16}
'avx2_expand_1_i32' = bitexpand{[ 8]i32}
'avx2_expand_1_f64' = bitexpand{[ 4]f64}

View File

@ -43,4 +43,37 @@ def maskAfter{n} = {
def mask{X, mode=='to sign bits'} = maskOf{X,n}
def mask{mode=='count'} = n
def mask{x==0} = 1
}
}
# store low packed elements of x to P; TODO the w parameter isn't doing much?
def storeLow{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x}
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}}
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
def storeBatch{ptr:P, n, x:T, M} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
xu:= ucvt{E0, x}
def TF = to_el{E0, T}
if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu}
else storeLow{rpos, vcount{T}*width{E0}, xu}
}
# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T
def loadBatch{ptr:P, n, T} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0}
else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}}
}
def maskedLoop{bulk, l, step} = {
m:u64 = l/bulk
@for (i from 0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}

View File

@ -5,14 +5,6 @@ include './avx'
include './avx2'
include './mask'
def maskedLoop{bulk, l, step} = {
m:u64 = l/bulk
@for (i from 0 to m) step{i, maskNone}
left:= l & (bulk-1)
if (left!=0) step{m, maskAfter{left}}
}
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def minBulk{w, A, B & width{A}< width{B}} = w/width{B}
@ -31,29 +23,6 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
}
# (sign/zero)-extend n'th batch to T
def loadx{ptr:P, n, T} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0}
else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}}
}
# store low packed elements of x to P
def storeL{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x}
def storeL{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}}
# store extended data x to n'th batch in ptr
def storex{ptr:P, n, x:T, M} = {
def rpos = ptr + n*vcount{T}
def E0 = eltype{P}
xu:= ucvt{eltype{P}, x}
def TF = to_el{E0, T}
if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu}
else storeL{rpos, vcount{T}*width{E0}, xu}
}
select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
def TIE = i32
def TDE = tern{width{TD}<32, u32, TD}
@ -67,13 +36,13 @@ select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
r:=cast_p{TD,r0}
maskedLoop{bulk, wl, {i, M} => {
cw0:= loadx{w, i, TIF}
cw0:= loadBatch{w, i, TIF}
cw1:= cw0+xlf
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
if (any{M{ty_vu{cw} >= ty_vu{xlf}}}) return{0}
got:= gather{broadcast{TDF,0}, x, cw, M}
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
storex{r, i, got, M}
storeBatch{r, i, got, M}
}}
1
}