AVX2 compress implementations using permutevar and lookup tables

This commit is contained in:
Marshall Lochbaum 2023-07-12 21:26:17 -04:00
parent 03d8919307
commit 0921fc2c62
2 changed files with 56 additions and 9 deletions

View File

@ -486,20 +486,24 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { DENSE; } \
break; }
#if SINGELI_AVX2 && FAST_PDEP
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
#else
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
case 4: WITH_SPARSE(16, 2, rp=m_tyarrv(&r,2,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
#endif
#undef WITH_SPARSE
#define BLOCK_OR_GROUPED(T) \
if (wsum>=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); }
#if SINGELI_AVX2 && FAST_PDEP
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break;
case 6: if (TI(x,elType)!=el_B) {
WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64))
} // else follows
#else
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
case 4: WITH_SPARSE(16, 2, rp=m_tyarrv(&r,2,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
case 5: BLOCK_OR_GROUPED(i32) break;
case 6:
if (TI(x,elType)!=el_B) { BLOCK_OR_GROUPED(u64) }
#endif
#undef WITH_SPARSE
else {
B xf = getFillR(x);
B* xp = arr_bptr(x);

View File

@ -2,12 +2,18 @@ include './base'
if (hasarch{'BMI2'}) {
include './bmi2'
}
if (hasarch{'PCLMUL'}) {
if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) {
include './sse' # PCLMUL implies SSE4.2
}
if (hasarch{'PCLMUL'}) {
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
} else {
def clmul{...x} = assert{'clmul not supported', show{...x}}
}
if (hasarch{'AVX2'}) {
include './avx'
include './avx2'
}
include 'util/tup'
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
@ -61,6 +67,43 @@ export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x08080808080808
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
}
itab:*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
fn compress32{T==i32}(w:*u64, x:*T, r:*T, l:u64) : void = {
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}}}}
def V = [8]i32
@for (w in *u8~~w, x in *V~~x over i to cdiv{l,8}) {
pc := popc{w}
ind := load{itab, w} # pext{base{256,iota{8}}, pdep{promote{u64, w}, base{256,8**1}}*255}
s := sel{[16]i8, V~~[4]u64**ind, expander}
store{*V~~r, 0, sel{V, x, s}}
r+= pc
}
}
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
fn compress64{T==i64}(wp:*u64, x:*T, r:*T, l:u64) : void = {
def V = [8]u32
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>1}}}
trail := make{V, iota{8}%2}
def step{w,i} = {
pc := popc{w}
s := trail | sel{[16]i8, V**load{i64tab,w}, expander}
store{*V~~r, 0, sel{V, load{*V~~x,i}, s}}
r+= pc
}
@for (w in *u8~~wp over i to cdiv{l,8}) {
step{w&0xf, 2*i}
step{w>>4, 2*i+1}
}
}
if (hasarch{'AVX2'}) {
export{'avx2_2slash32', compress32{i32}}
export{'avx2_2slash64', compress64{i64}}
}
# pext, or boolean compress
fn pext{T}(x:T, m:T) {
def w = width{T}