Generic Singeli compress implementation

This commit is contained in:
Marshall Lochbaum 2023-07-15 11:14:52 -04:00
parent f36426031a
commit f852d12a5c
2 changed files with 27 additions and 14 deletions

View File

@ -477,21 +477,26 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
} \
TFREE(buf)
#define COMPRESS_BLOCK(T) COMPRESS_BLOCK_PREP(T, )
#if SINGELI_AVX2
#define THR(C) C
#else
#define THR(C) 2
#endif
#define WITH_SPARSE(W, CUTOFF, DENSE) { \
i##W *xp=tyany_ptr(x), *rp; \
if (wsum<wia/CUTOFF) { rp=m_tyarrv(&r,W/8,wsum,xt); COMPRESS_BLOCK(i##W); } \
if (wsum<wia/THR(CUTOFF)) { rp=m_tyarrv(&r,W/8,wsum,xt); COMPRESS_BLOCK(i##W); } \
else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { DENSE; } \
break; }
#define BLOCK_OR_GROUPED(T) \
if (wsum>=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \
else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); }
#if SINGELI_AVX2 && FAST_PDEP
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); bmipopc_2slash8 (wp, xp, rp, wia, wsum))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); bmipopc_2slash16(wp, xp, rp, wia, wsum))
case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32 (wp, xp, rp, wia, wsum))
#if SINGELI
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); si_2slash8 (wp, xp, rp, wia, wsum))
case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); si_2slash16(wp, xp, rp, wia, wsum))
case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); si_2slash32(wp, xp, rp, wia, wsum))
case 6: if (TI(x,elType)!=el_B) {
WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64 (wp, xp, rp, wia, wsum))
WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); si_2slash64(wp, xp, rp, wia, wsum))
} // else follows
#else
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
@ -501,6 +506,7 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
if (TI(x,elType)!=el_B) { BLOCK_OR_GROUPED(u64) }
#endif
#undef WITH_SPARSE
#undef THR
else {
B xf = getFillR(x);
B* xp = arr_bptr(x);

View File

@ -2,8 +2,8 @@ include './base'
if (hasarch{'BMI2'}) {
include './bmi2'
}
if (hasarch{'PCLMUL'} or hasarch{'AVX2'}) {
include './sse' # PCLMUL implies SSE4.2
if (hasarch{'X86_64'}) {
include './sse'
}
if (hasarch{'PCLMUL'}) {
def clmul{a:T, b:T, imm & w128i{T}} = emit{T, '_mm_clmulepi64_si128', a, b, imm}
@ -54,6 +54,15 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
# Assumes w is trimmed, so the last 1 appears at index l-1
fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = {
def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1
@for (x over i to l) {
store{r, 0, x}
r+= bitp_get{w,i}
}
}
def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = {
@for_special_buffered{r,8} (w in *u8~~w over sum) {
pc:= popc{w}
@ -198,15 +207,13 @@ fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x
}
if (hasarch{'BMI2'}) {
export{'bmipopc_2slash8', slash2{i8}}
export{'bmipopc_2slash16', slash2{i16}}
export{'bmipopc_1slash8', slash1{i8}}
export{'bmipopc_1slash16', slash1{i16}}
}
if (hasarch{'AVX2'}) {
export{'avx2_2slash32', slash2{i32}}
export{'avx2_2slash64', slash2{i64}}
}
export{'si_2slash8', slash2{i8}}
export{'si_2slash16', slash2{i16}}
export{'si_2slash32', slash2{i32}}
export{'si_2slash64', slash2{i64}}
# pext, or boolean compress
fn pext{T}(x:T, m:T) {