AVX2 compress implementations using permutevar and lookup tables
This commit is contained in:
parent
03d8919307
commit
0921fc2c62
@ -486,20 +486,24 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
|
||||
else { DENSE; } \
|
||||
break; }
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
|
||||
#else
|
||||
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
case 4: WITH_SPARSE(16, 2, rp=m_tyarrv(&r,2,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
#endif
|
||||
#undef WITH_SPARSE
|
||||
#define BLOCK_OR_GROUPED(T) \
|
||||
if (wsum>=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \
|
||||
else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); }
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
|
||||
case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break;
|
||||
case 6: if (TI(x,elType)!=el_B) {
|
||||
WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64))
|
||||
} // else follows
|
||||
#else
|
||||
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
case 4: WITH_SPARSE(16, 2, rp=m_tyarrv(&r,2,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
case 5: BLOCK_OR_GROUPED(i32) break;
|
||||
case 6:
|
||||
if (TI(x,elType)!=el_B) { BLOCK_OR_GROUPED(u64) }
|
||||
#endif
|
||||
#undef WITH_SPARSE
|
||||
else {
|
||||
B xf = getFillR(x);
|
||||
B* xp = arr_bptr(x);
|
||||
|
||||
@ -2,12 +2,18 @@ include './base'
|
||||
if (hasarch{'BMI2'}) {
|
||||
include './bmi2'
|
||||
}
|
||||
if (hasarch{'PCLMUL'}) {
|
||||
if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) {
|
||||
include './sse' # PCLMUL implies SSE4.2
|
||||
}
|
||||
if (hasarch{'PCLMUL'}) {
|
||||
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
|
||||
} else {
|
||||
def clmul{...x} = assert{'clmul not supported', show{...x}}
|
||||
}
|
||||
if (hasarch{'AVX2'}) {
|
||||
include './avx'
|
||||
include './avx2'
|
||||
}
|
||||
include 'util/tup'
|
||||
|
||||
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
@ -61,6 +67,43 @@ export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x08080808080808
|
||||
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
|
||||
}
|
||||
|
||||
itab:*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
|
||||
|
||||
fn compress32{T==i32}(w:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}}}}
|
||||
def V = [8]i32
|
||||
@for (w in *u8~~w, x in *V~~x over i to cdiv{l,8}) {
|
||||
pc := popc{w}
|
||||
ind := load{itab, w} # pext{base{256,iota{8}}, pdep{promote{u64, w}, base{256,8**1}}*255}
|
||||
s := sel{[16]i8, V~~[4]u64**ind, expander}
|
||||
store{*V~~r, 0, sel{V, x, s}}
|
||||
r+= pc
|
||||
}
|
||||
}
|
||||
|
||||
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
|
||||
|
||||
fn compress64{T==i64}(wp:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
def V = [8]u32
|
||||
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>1}}}
|
||||
trail := make{V, iota{8}%2}
|
||||
def step{w,i} = {
|
||||
pc := popc{w}
|
||||
s := trail | sel{[16]i8, V**load{i64tab,w}, expander}
|
||||
store{*V~~r, 0, sel{V, load{*V~~x,i}, s}}
|
||||
r+= pc
|
||||
}
|
||||
@for (w in *u8~~wp over i to cdiv{l,8}) {
|
||||
step{w&0xf, 2*i}
|
||||
step{w>>4, 2*i+1}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasarch{'AVX2'}) {
|
||||
export{'avx2_2slash32', compress32{i32}}
|
||||
export{'avx2_2slash64', compress64{i64}}
|
||||
}
|
||||
|
||||
# pext, or boolean compress
|
||||
fn pext{T}(x:T, m:T) {
|
||||
def w = width{T}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user