Non-overwriting 1-byte and 2-byte compress/where
This commit is contained in:
parent
8f4b1966cb
commit
d956ba921b
@ -163,7 +163,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
static void where_block_u16(u64* src, u16* dst, usz len, usz sum) {
|
||||
assert(len <= bsp_max);
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len);
|
||||
if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum);
|
||||
#else
|
||||
if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0);
|
||||
#endif
|
||||
@ -236,18 +236,16 @@ static B where(B x, usz xia, u64 s) {
|
||||
usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<<q) - 1;
|
||||
if (xia <= 128) {
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
i8* rp = m_tyarrvO(&r, 1, s, t_i8arr, 8);
|
||||
bmipopc_1slash8(xp, rp, xia);
|
||||
FINISH_OVERALLOC_A(r, s, 8);
|
||||
i8* rp = m_tyarrv(&r, 1, s, t_i8arr);
|
||||
bmipopc_1slash8(xp, rp, xia, s);
|
||||
#else
|
||||
i8* rp; r=m_i8arrv(&rp,s); WHERE_SPARSE(xp,rp,s,0,);
|
||||
#endif
|
||||
} else if (xia <= 32768) {
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (s >= xia/8) {
|
||||
i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16);
|
||||
bmipopc_1slash16(xp, rp, xia);
|
||||
FINISH_OVERALLOC_A(r, s*2, 16);
|
||||
i16* rp = m_tyarrv(&r, 2, s, t_i16arr);
|
||||
bmipopc_1slash16(xp, rp, xia, s);
|
||||
}
|
||||
#else
|
||||
if (s >= xia/4+xia/8) {
|
||||
@ -278,7 +276,7 @@ static B where(B x, usz xia, u64 s) {
|
||||
}
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
if (bs >= b/8+b/16) {
|
||||
bmipopc_1slash16(xp, buf, b);
|
||||
bmipopc_1slash16(xp, buf, b, bs);
|
||||
for (usz j=0; j<bs; j++) rq[j] = i+buf[j];
|
||||
}
|
||||
#else
|
||||
@ -370,10 +368,9 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
u64* xp1 = xp;
|
||||
if (!up) { u64* t=xp1; xp1=xp0; xp0=t; }
|
||||
#define BMI_GRADE(W) \
|
||||
i##W* rp = m_tyarrvO(&r, W/8, xia, t_i##W##arr, W); \
|
||||
bmipopc_1slash##W(xp0, rp , xia); \
|
||||
bmipopc_1slash##W(xp1, rp+l0, xia); \
|
||||
FINISH_OVERALLOC_A(r, xia*(W/8), W);
|
||||
i##W* rp = m_tyarrv(&r, W/8, xia, t_i##W##arr); \
|
||||
bmipopc_1slash##W(xp0, rp , xia, l0 ); \
|
||||
bmipopc_1slash##W(xp1, rp+l0, xia, xia-l0);
|
||||
if (xia <= 128) { BMI_GRADE(8) } else { BMI_GRADE(16) }
|
||||
#undef BMI_GRADE
|
||||
decG(notx);
|
||||
@ -387,8 +384,8 @@ B grade_bool(B x, usz xia, bool up) {
|
||||
for (usz i=0; i<xia; i+=b) {
|
||||
for (usz j=0; j<BIT_N(b); j++) xp0[j] = ~xp1[j];
|
||||
usz b2 = b>xia-i? xia-i : b;
|
||||
usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
||||
usz s1=b2-s0; bmipopc_1slash8(xp1, (i8*)buf, b2); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
||||
usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j<s0; j++) *rp0++ = i+buf[j];
|
||||
usz s1=b2-s0; bmipopc_1slash8(xp1, (i8*)buf, b2, s1); for (usz j=0; j<s1; j++) *rp1++ = i+buf[j];
|
||||
xp1+= b2/64;
|
||||
}
|
||||
TFREE(buf);
|
||||
@ -490,11 +487,11 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) {
|
||||
if (wsum>=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \
|
||||
else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); }
|
||||
#if SINGELI_AVX2 && FAST_PDEP
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16))
|
||||
case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break;
|
||||
case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); bmipopc_2slash8 (wp, xp, rp, wia, wsum))
|
||||
case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); bmipopc_2slash16(wp, xp, rp, wia, wsum))
|
||||
case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32 (wp, xp, rp, wia, wsum))
|
||||
case 6: if (TI(x,elType)!=el_B) {
|
||||
WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum))
|
||||
WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64 (wp, xp, rp, wia, wsum))
|
||||
} // else follows
|
||||
#else
|
||||
case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i<wia; i++) { *rp = xp[i]; rp+= bitp_get(wp,i); })
|
||||
|
||||
@ -17,60 +17,6 @@ if (hasarch{'AVX2'}) {
|
||||
include './mask'
|
||||
include 'util/tup'
|
||||
|
||||
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
|
||||
def comp8{w:*u64, X, r:*i8, l:u64} = {
|
||||
@for(w in *u8~~w over i to cdiv{l,8}) {
|
||||
pc:= popc{w}
|
||||
storeu{*u64~~r, 0, pext{promote{u64,X{}}, pdep{promote{u64, w}, cast{u64,0x0101010101010101}}*255}}
|
||||
r+= pc
|
||||
}
|
||||
}
|
||||
|
||||
def tab{n,l} = {
|
||||
def m=n-1; def t=tab{m,l}
|
||||
def k = (1<<l - 1) << (m*l)
|
||||
merge{t, k+t}
|
||||
}
|
||||
def tab{n==0,l} = tup{0}
|
||||
c16lut:*u64 = tab{4,16}
|
||||
|
||||
def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i}
|
||||
|
||||
def comp16{w:*u64, X, r:*i16, l:u64} = {
|
||||
@for(w in *u8~~w over i to cdiv{l,8}) {
|
||||
def step{w} = {
|
||||
pc:= popcRand{w}
|
||||
storeu{*u64~~r, 0, pext{promote{u64,X{}}, vgLoad{c16lut, w}}}
|
||||
r+= pc
|
||||
}
|
||||
step{w&15}
|
||||
step{w>>4} # this runs even if the above step was all that's required, so it'll act on the invalid result of "r+= pc", so we need to overallocate even more to compensate
|
||||
}
|
||||
}
|
||||
|
||||
fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = {
|
||||
xv:= *u64~~x
|
||||
F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l}
|
||||
}
|
||||
|
||||
fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = {
|
||||
x:u64 = iota
|
||||
F{w, {} => {c:= x; x+= add; c}, r, l}
|
||||
}
|
||||
|
||||
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
|
||||
if (hasarch{'BMI2'}) {
|
||||
export{'bmipopc_2slash8', slash2{comp8, i8}}
|
||||
export{'bmipopc_2slash16', slash2{comp16, i16}}
|
||||
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
|
||||
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
|
||||
}
|
||||
|
||||
itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
|
||||
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
|
||||
|
||||
# Modifies the input variable r
|
||||
# Assumes iter{} will increment r, by at most write_len
|
||||
def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
|
||||
@ -95,12 +41,70 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = {
|
||||
}
|
||||
} else {
|
||||
def vc = 256/tw;
|
||||
def R = [vc]T
|
||||
if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc }
|
||||
homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}}
|
||||
if (hasarch{'AVX2'} and write_len >= vc) {
|
||||
def R = [vc]T
|
||||
if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc }
|
||||
homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}}
|
||||
} else {
|
||||
@for (r0, buf over u64~~(end-buf)) r0 = buf
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v}
|
||||
def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p}
|
||||
|
||||
def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = {
|
||||
@for_special_buffered{r,8} (w in *u8~~w over sum) {
|
||||
pc:= popc{w}
|
||||
storeu{*u64~~r, 0, pext{promote{u64,X{}}, pdep{promote{u64, w}, cast{u64,0x0101010101010101}}*255}}
|
||||
r+= pc
|
||||
}
|
||||
}
|
||||
|
||||
def tab{n,l} = {
|
||||
def m=n-1; def t=tab{m,l}
|
||||
def k = (1<<l - 1) << (m*l)
|
||||
merge{t, k+t}
|
||||
}
|
||||
def tab{n==0,l} = tup{0}
|
||||
c16lut:*u64 = tab{4,16}
|
||||
|
||||
def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i}
|
||||
|
||||
def comp16{w:*u64, X, r:*i16, l:u64, sum:u64} = {
|
||||
@for_special_buffered{r,8} (w in *u8~~w over sum) {
|
||||
def step{r, w} = {
|
||||
storeu{*u64~~r, 0, pext{promote{u64,X{}}, vgLoad{c16lut, w}}}
|
||||
}
|
||||
rs:= r; r+= popc{w} # Measured slow incrementing at the end
|
||||
h := w&0xf
|
||||
step{rs, h}
|
||||
step{rs+popcRand{h}, w>>4}
|
||||
}
|
||||
}
|
||||
|
||||
fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = {
|
||||
xv:= *u64~~x
|
||||
F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l, sum}
|
||||
}
|
||||
|
||||
fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64, sum:u64) : void = {
|
||||
x:u64 = iota
|
||||
F{w, {} => {c:= x; x+= add; c}, r, l, sum}
|
||||
}
|
||||
|
||||
# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes
|
||||
if (hasarch{'BMI2'}) {
|
||||
export{'bmipopc_2slash8', slash2{comp8, i8}}
|
||||
export{'bmipopc_2slash16', slash2{comp16, i16}}
|
||||
export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}}
|
||||
export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}}
|
||||
}
|
||||
|
||||
itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}}
|
||||
i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}}
|
||||
|
||||
fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = {
|
||||
def tw = width{T}
|
||||
def V = [8]u32
|
||||
|
||||
Loading…
Reference in New Issue
Block a user