singeli bitarr conversion to wider arrays
This commit is contained in:
parent
b1fd85621a
commit
113f72e86d
2
makefile
2
makefile
@ -140,7 +140,7 @@ preSingeliBin:
|
||||
@${MAKE} singeli=0 postmsg="singeli sources:" t=presingeli f='-O1' OUTPUT=obj/presingeli/BQN c
|
||||
|
||||
|
||||
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c}
|
||||
gen-singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c slash.c equal.c scan.c expand.c}
|
||||
@echo $(postmsg)
|
||||
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
||||
@echo $< | cut -c 17- | sed 's/^/ /'
|
||||
|
||||
@ -36,12 +36,22 @@ NOINLINE B m_str32(u32* s) {
|
||||
return r;
|
||||
}
|
||||
|
||||
#if SINGELI
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#include "../singeli/gen/expand.c"
|
||||
#pragma GCC diagnostic pop
|
||||
#define BIT_ICPY(E) avx2_expand_1_##E(xp, rp, ia);
|
||||
#else
|
||||
#define BIT_ICPY(E) for(usz i=0; i<ia; i++) rp[i]=bitp_get(xp,i);
|
||||
#endif
|
||||
|
||||
#define MAKE_ICPY(T,E) T##Arr* cpy##T##Arr(B x) { \
|
||||
usz ia = a(x)->ia; \
|
||||
E* rp; Arr* r = m_##E##arrp(&rp, ia); \
|
||||
arr_shCopy(r, x); \
|
||||
u8 xe = TI(x,elType); \
|
||||
if (xe==el_bit) { u64* xp = bitarr_ptr(x); for(usz i=0; i<ia; i++) rp[i]=bitp_get(xp,i); } \
|
||||
if (xe==el_bit) { u64* xp = bitarr_ptr(x); BIT_ICPY(E) } \
|
||||
else if (xe==el_i8 ) { i8* xp = i8any_ptr (x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \
|
||||
else if (xe==el_i16) { i16* xp = i16any_ptr(x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \
|
||||
else if (xe==el_i32) { i32* xp = i32any_ptr(x); for(usz i=0; i<ia; i++) rp[i]=xp[i]; } \
|
||||
|
||||
@ -57,10 +57,10 @@ def make{T==[4]f64,a,b,c,d} = emit{T,'_mm256_setr_pd',a,b,c,d}
|
||||
def make{T==[8]f32,a,b,c,d,e,f,g,h} = emit{T,'_mm256_setr_ps',a,b,c,d,e,f,g,h}
|
||||
|
||||
def makeGen{T,s,x} = apply{emit, merge{tup{T,s}, each{{c}=>ext{eltype{T},c}, x}}}
|
||||
def make{T,a,b,c,d & w256i{T}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}}
|
||||
def make{T,a,b,c,d,e,f,g,h & w256i{T}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}}
|
||||
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}}
|
||||
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}}
|
||||
def make{T,a,b,c,d & w256i{T,64}} = makeGen{T, '_mm256_setr_epi64x', tup{a,b,c,d}}
|
||||
def make{T,a,b,c,d,e,f,g,h & w256i{T,32}} = makeGen{T, '_mm256_setr_epi32', tup{a,b,c,d,e,f,g,h}}
|
||||
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p & w256i{T,16}} = makeGen{T, '_mm256_setr_epi16', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}}
|
||||
def make{T,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P & w256i{T,8}} = makeGen{T, '_mm256_setr_epi8', tup{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P}}
|
||||
def make{T,x & w256{T} & istup{x}} = apply{make, merge{tup{T}, x}}
|
||||
|
||||
def iota{T & w256{T,64}} = make{T,0,1,2,3}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
include 'skin/c'
|
||||
include 'arch/c'
|
||||
def Size = u64
|
||||
oper infix right ~~ reinterpret 55
|
||||
|
||||
# TODO move these to a more base file
|
||||
@ -16,8 +17,8 @@ def cdiv{a,b} = (a+b-1)/b
|
||||
def lb{n & knum{n} & (n>>1<<1) == n & n>0} = lb{n>>1}+1
|
||||
def lb{n==1} = 0
|
||||
|
||||
# get the n least significant bits
|
||||
def lsbs{x, n} = x - (x>>n << n)
|
||||
def tail{n,x} = x - (x>>n << n) # get the n least significant bits
|
||||
def bit {k,x} = tail{1,x>>k} << k # get the k-th bit
|
||||
|
||||
def rare{x:u1} = emit{u1, '__builtin_expect', x, 0}
|
||||
def isunsigned{T} = isint{T} & ~issigned{T}
|
||||
@ -51,6 +52,9 @@ def ty_s{T==u16} = i16; def ty_s{w==16} = i16
|
||||
def ty_s{T==u32} = i32; def ty_s{w==32} = i32
|
||||
def ty_s{T==u64} = i64; def ty_s{w==64} = i64
|
||||
|
||||
def ty_u{T==f64} = u64; def ty_u{T==f32} = u32;
|
||||
def ty_s{T==f64} = i64; def ty_s{T==f32} = i32;
|
||||
|
||||
def ty_vu{T & isvec{T}} = [vcount{T}](ty_u{eltype{T}})
|
||||
def ty_vs{T & isvec{T}} = [vcount{T}](ty_s{eltype{T}})
|
||||
def ty_vs{x:T} = ty_vs{T}~~x
|
||||
@ -111,7 +115,7 @@ def broadcast{n, v & knum{n}} = @collect(n) v
|
||||
|
||||
|
||||
# debug stuff
|
||||
def print{x & match{kind{x},'symbol'}} = emit{void, 'printf', '"%s"', merge{'"', x, '"'}}
|
||||
def print{x & anyNum{x}} = emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x}
|
||||
def print{x:T & anyNum{x} & width{T}==64} = emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x}
|
||||
def println{x} = { print{x}; print{'\n'} }
|
||||
def print{x & match{kind{x},'symbol'}} = { emit{void, 'printf', '"%s"', merge{'"', x, '"'}}; x }
|
||||
def print{x & anyNum{x}} = { emit{void, 'printf', tern{issigned{x}, '"%d"', '"%u"'}, x}; x }
|
||||
def print{x:T & anyNum{x} & width{T}==64} = { emit{void, 'printf', tern{issigned{x}, '"%ld"', '"%lu"'}, x}; x }
|
||||
def println{x} = { print{x}; print{'\n'}; x }
|
||||
@ -4,6 +4,9 @@ def b_get{x:*u64, n:(Size)} = {
|
||||
((load{x,n>>6}>>(n&63)) & 1) != 0
|
||||
}
|
||||
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz==4} = (load{*u8~~x, n>>1} >> cast_i{u8, (n&1)*4}) & 15
|
||||
def b_getBatch{sz, x:*u64, n:(Size) & sz>=8} = load{*ty_u{sz}~~x, n}
|
||||
|
||||
def b_set{x:*u64, n:(Size), v:u1} = {
|
||||
m:u64 = cast{u64,1}<<(n&63)
|
||||
p:u64 = load{x,n>>6}
|
||||
@ -11,7 +14,7 @@ def b_set{x:*u64, n:(Size), v:u1} = {
|
||||
else store{x,n>>6,p & ~m}
|
||||
}
|
||||
|
||||
def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
|
||||
am:u64 = 64/sz;
|
||||
w:u64 = load{x,n/am}
|
||||
sh:u64 = (n&(am-1)) * sz
|
||||
@ -20,7 +23,7 @@ def b_set{sz, x:*u64, n:(Size), v} = { vc:u64 = ext{u64,v}
|
||||
store{x, n/am, w}
|
||||
}
|
||||
|
||||
def b_set{sz, x:*u64, n:(Size), v & sz==4} = {
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v & sz==4} = {
|
||||
x8:= cast_p{u8, x}
|
||||
|
||||
#w:u64 = cast_i{u64, load{x8,n/2}}
|
||||
@ -39,7 +42,7 @@ def b_set{sz, x:*u64, n:(Size), v & sz==4} = {
|
||||
|
||||
store{x8, n/2, cast_i{u8,w}}
|
||||
}
|
||||
def b_set{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}}
|
||||
def b_set{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}}
|
||||
def b_set{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}}
|
||||
def b_set{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v & sz== 8} = store{cast_p{u8, x}, n, cast_i{u8, v}}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v & sz==16} = store{cast_p{u16, x}, n, cast_i{u16,v}}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v & sz==32} = store{cast_p{u32, x}, n, cast_i{u32,v}}
|
||||
def b_setBatch{sz, x:*u64, n:(Size), v & sz==64} = store{x, n, cast_i{u64,v}}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
def Size = u64
|
||||
def bcall{T, f, x} = emit{T, 'BCALL', f, x}
|
||||
def from_B{T, x & T==f64} = bcall{T, 'o2fu', x}
|
||||
def from_B{T, x & T<=i32 & issigned{T}} = bcall{T, 'o2iu', x}
|
||||
|
||||
@ -77,7 +77,7 @@ def any2bit{VT, unr, op, wS, wV, xS, xV, dst:*u64, len:(Size)} = {
|
||||
while (ri < am) {
|
||||
r:u64 = 0
|
||||
@unroll (j from 0 to unr) r = r | (ext{u64, getmask{op{wV{xi+j}, xV{xi+j}}}} << (j*vcount{VT}))
|
||||
b_set{bam, dst, ri, r}
|
||||
b_setBatch{bam, dst, ri, r}
|
||||
xi = xi+unr
|
||||
ri = ri+1
|
||||
}
|
||||
|
||||
@ -12,14 +12,6 @@ def swap{w,x} = {
|
||||
x = t
|
||||
}
|
||||
|
||||
def maskedLoop{bulk, l, step} = {
|
||||
m:u64 = l/bulk
|
||||
@for (i from 0 to m) step{i, maskNone}
|
||||
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) step{m, maskAfter{left}}
|
||||
}
|
||||
|
||||
equal{W, X}(w:*u8, x:*u8, l:u64, d:u64) : u1 = {
|
||||
def bulk = 256 / width{X}
|
||||
if (W!=X) if (d!=0) swap{w,x}
|
||||
|
||||
40
src/singeli/src/expand.singeli
Normal file
40
src/singeli/src/expand.singeli
Normal file
@ -0,0 +1,40 @@
|
||||
include './base'
|
||||
include './sse3'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
include './bitops'
|
||||
|
||||
|
||||
|
||||
def bitspread{T==[32]u8, a:u32} = {
|
||||
def idxs = iota{32}
|
||||
b:= broadcast{[8]u32, a}
|
||||
c:= [32]u8~~b
|
||||
d:= sel{[16]u8, c, make{[32]i8, idxs>>3 + bit{4, idxs}}}
|
||||
e:= make{[32]u8, 1<<tail{3, idxs}}
|
||||
e == (d&e)
|
||||
}
|
||||
def bitspread{T, a & vcount{T} <= width{eltype{T}} & w256u{T}} = {
|
||||
b:= make{T, 1<<iota{vcount{T}}}
|
||||
b == (b & broadcast{T, a})
|
||||
}
|
||||
|
||||
|
||||
|
||||
bitexpand{T}(xp: *u64, rp:*eltype{T}, l:u64) : void = {
|
||||
def bulk = vcount{T}
|
||||
def TU = ty_vu{T}
|
||||
|
||||
maskedLoop{vcount{T}, l, {i, M} => {
|
||||
x:= b_getBatch{bulk, xp, i} # TODO unroll f64 by two to make b_getBatch not sad
|
||||
y:= bitspread{TU, x}
|
||||
r:= y & TU~~broadcast{T, 1}
|
||||
storeBatch{rp, i, T~~r, M}
|
||||
}}
|
||||
}
|
||||
|
||||
'avx2_expand_1_i8' = bitexpand{[32]i8}
|
||||
'avx2_expand_1_i16' = bitexpand{[16]i16}
|
||||
'avx2_expand_1_i32' = bitexpand{[ 8]i32}
|
||||
'avx2_expand_1_f64' = bitexpand{[ 4]f64}
|
||||
@ -43,4 +43,37 @@ def maskAfter{n} = {
|
||||
def mask{X, mode=='to sign bits'} = maskOf{X,n}
|
||||
def mask{mode=='count'} = n
|
||||
def mask{x==0} = 1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
# store low packed elements of x to P; TODO the w parameter isn't doing much?
|
||||
def storeLow{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x}
|
||||
def storeLow{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}}
|
||||
|
||||
# store vcount{T} items into the n'th batch of ptr elements, compressing the items if needed; masked by M
|
||||
def storeBatch{ptr:P, n, x:T, M} = {
|
||||
def rpos = ptr + n*vcount{T}
|
||||
def E0 = eltype{P}
|
||||
xu:= ucvt{E0, x}
|
||||
def TF = to_el{E0, T}
|
||||
if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu}
|
||||
else storeLow{rpos, vcount{T}*width{E0}, xu}
|
||||
}
|
||||
|
||||
# (sign/zero)-extend n'th batch of vcount{T} elements of P into elements of T
|
||||
def loadBatch{ptr:P, n, T} = {
|
||||
def rpos = ptr + n*vcount{T}
|
||||
def E0 = eltype{P}
|
||||
|
||||
if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0}
|
||||
else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}}
|
||||
}
|
||||
|
||||
def maskedLoop{bulk, l, step} = {
|
||||
m:u64 = l/bulk
|
||||
@for (i from 0 to m) step{i, maskNone}
|
||||
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) step{m, maskAfter{left}}
|
||||
}
|
||||
|
||||
@ -5,14 +5,6 @@ include './avx'
|
||||
include './avx2'
|
||||
include './mask'
|
||||
|
||||
def maskedLoop{bulk, l, step} = {
|
||||
m:u64 = l/bulk
|
||||
@for (i from 0 to m) step{i, maskNone}
|
||||
|
||||
left:= l & (bulk-1)
|
||||
if (left!=0) step{m, maskAfter{left}}
|
||||
}
|
||||
|
||||
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
|
||||
|
||||
def minBulk{w, A, B & width{A}< width{B}} = w/width{B}
|
||||
@ -31,29 +23,6 @@ def gather{def:T, b:B, idx:[4]i32, M & w256{T,64}} = {
|
||||
}
|
||||
|
||||
|
||||
# (sign/zero)-extend n'th batch to T
|
||||
def loadx{ptr:P, n, T} = {
|
||||
def rpos = ptr + n*vcount{T}
|
||||
def E0 = eltype{P}
|
||||
|
||||
if (width{eltype{T}} == width{E0}) load{cast_p{T, rpos}, 0}
|
||||
else cvt{E0, T, load{cast_p{[16]u8, rpos}, 0}}
|
||||
}
|
||||
|
||||
# store low packed elements of x to P
|
||||
def storeL{ptr:P, w, x:T & width{eltype{P}} == width{eltype{T}}} = store{*T~~ptr, 0, x}
|
||||
def storeL{ptr:P, w, x:T & w256{T} & w==64} = store{*u64~~ptr, 0, extract{[4]u64~~x, 0}}
|
||||
|
||||
# store extended data x to n'th batch in ptr
|
||||
def storex{ptr:P, n, x:T, M} = {
|
||||
def rpos = ptr + n*vcount{T}
|
||||
def E0 = eltype{P}
|
||||
xu:= ucvt{eltype{P}, x}
|
||||
def TF = to_el{E0, T}
|
||||
if (M{0}) maskstoreF{cast_p{TF, rpos}, M{TF, 'to sign bits'}, 0, xu}
|
||||
else storeL{rpos, vcount{T}*width{E0}, xu}
|
||||
}
|
||||
|
||||
select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
|
||||
def TIE = i32
|
||||
def TDE = tern{width{TD}<32, u32, TD}
|
||||
@ -67,13 +36,13 @@ select{rw, TI, TD}(w0:*u8, x0:*u8, r0:*u8, wl:u64, xl:u64) : u1 = {
|
||||
r:=cast_p{TD,r0}
|
||||
|
||||
maskedLoop{bulk, wl, {i, M} => {
|
||||
cw0:= loadx{w, i, TIF}
|
||||
cw0:= loadBatch{w, i, TIF}
|
||||
cw1:= cw0+xlf
|
||||
cw:= blendF{cw0, cw1, cw0<broadcast{TIF, 0}} # TODO this is utilizing clang optimizing out the comparison
|
||||
if (any{M{ty_vu{cw} >= ty_vu{xlf}}}) return{0}
|
||||
got:= gather{broadcast{TDF,0}, x, cw, M}
|
||||
if (TDE!=TD) got&= broadcast{TDF, (1<<width{TD})-1}
|
||||
storex{r, i, got, M}
|
||||
storeBatch{r, i, got, M}
|
||||
}}
|
||||
1
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user