Non-overwriting AVX2 4-byte and 8-byte compress, using a buffer
This commit is contained in:
parent
9010f648bb
commit
8f4b1966cb
@ -492,9 +492,9 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
|
||||
case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break;
|
||||
case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break;
|
||||
case 6: if (TI(x,elType)!=el_B) {
|
||||
WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64))
|
||||
WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum))
|
||||
} // else follows
|
||||
#else
|
||||
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
|
||||
@ -14,6 +14,7 @@ if (hasarch{'AVX2'}) {
|
||||
include './avx'
|
||||
include './avx2'
|
||||
}
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
@ -70,7 +71,37 @@ export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x00040004000
|
||||
itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
|
||||
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
|
||||
|
||||
fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
# Modifies the input variable r
|
||||
# Assumes iter{} will increment r, by at most write_len
|
||||
def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
|
||||
assert{isreg{r}}; assert{begin==0}
|
||||
def T = eltype{type{r}}; def tw = width{T}
|
||||
def ov = write_len-1
|
||||
buf := emit{*T, 'alloca', 2*(ov+1)*width{T}/8}
|
||||
r0 := r
|
||||
end := r + sum - ov
|
||||
i:u64 = 0; buf_used:u1 = 0
|
||||
def restart = setlabel{}
|
||||
while (r < end) {
|
||||
iter{i, vars}
|
||||
++i
|
||||
}
|
||||
if (not buf_used) {
|
||||
end += buf - r + ov
|
||||
if (buf < end) {
|
||||
r0 = r
|
||||
r = buf
|
||||
buf_used = 1; goto{restart}
|
||||
}
|
||||
} else {
|
||||
def vc = 256/tw;
|
||||
def R = [vc]T
|
||||
if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc }
|
||||
homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}}
|
||||
}
|
||||
}
|
||||
|
||||
fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = {
|
||||
def tw = width{T}
|
||||
def V = [8]u32
|
||||
expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}}
|
||||
@ -83,7 +114,7 @@ fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
store{*V~~r, 0, sel{V, load{*V~~x,i}, s}}
|
||||
r+= pc
|
||||
}
|
||||
@for (w in *u8~~wp over i to cdiv{l,8}) {
|
||||
@for_special_buffered{r,8} (w in *u8~~wp over i to sum) {
|
||||
if (tw==32) {
|
||||
step{w, i}
|
||||
} else {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user