From d956ba921bb63a06c7c95ec77e0c5316dc3a02c5 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jul 2023 13:24:41 -0400 Subject: [PATCH] Non-overwriting 1-byte and 2-byte compress/where --- src/builtins/slash.c | 33 +++++----- src/singeli/src/slash.singeli | 118 ++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 619313fe..4dbff005 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -163,7 +163,7 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) { static void where_block_u16(u64* src, u16* dst, usz len, usz sum) { assert(len <= bsp_max); #if SINGELI_AVX2 && FAST_PDEP - if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len); + if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum); #else if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); #endif @@ -236,18 +236,16 @@ static B where(B x, usz xia, u64 s) { usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/8) { - i16* rp = m_tyarrvO(&r, 2, s, t_i16arr, 16); - bmipopc_1slash16(xp, rp, xia); - FINISH_OVERALLOC_A(r, s*2, 16); + i16* rp = m_tyarrv(&r, 2, s, t_i16arr); + bmipopc_1slash16(xp, rp, xia, s); } #else if (s >= xia/4+xia/8) { @@ -278,7 +276,7 @@ static B where(B x, usz xia, u64 s) { } #if SINGELI_AVX2 && FAST_PDEP if (bs >= b/8+b/16) { - bmipopc_1slash16(xp, buf, b); + bmipopc_1slash16(xp, buf, b, bs); for (usz j=0; jxia-i? xia-i : b; - usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2); for (usz j=0; j=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } #if SINGELI_AVX2 && FAST_PDEP - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break; + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); bmipopc_2slash8 (wp, xp, rp, wia, wsum)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); bmipopc_2slash16(wp, xp, rp, wia, wsum)) + case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32 (wp, xp, rp, wia, wsum)) case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum)) + WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64 (wp, xp, rp, wia, wsum)) } // else follows #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i>4} # this runs even if the above step was all that's required, so it'll act on the invalid result of "r+= pc", so we need to overallocate even more to compensate - } -} - -fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64) : void = { - xv:= *u64~~x - F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l} -} - -fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64) : void = { - x:u64 = iota - F{w, {} => {c:= x; x+= add; c}, r, l} -} - -# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes -if (hasarch{'BMI2'}) { -export{'bmipopc_2slash8', slash2{comp8, i8}} -export{'bmipopc_2slash16', slash2{comp16, i16}} -export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}} -export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} -} - -itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} -i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} - # Modifies the input variable r # Assumes iter{} will increment r, by at most write_len def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { @@ -95,12 +41,70 @@ def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { } } else { def vc = 256/tw; - def R = [vc]T - if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } - homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + if (hasarch{'AVX2'} and write_len >= vc) { + def R = [vc]T + if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + } else { + @for (r0, buf over u64~~(end-buf)) r0 = buf + } } } +def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} +def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} + +def comp8{w:*u64, X, r:*i8, l:u64, sum:u64} = { + @for_special_buffered{r,8} (w in *u8~~w over sum) { + pc:= popc{w} + storeu{*u64~~r, 0, pext{promote{u64,X{}}, pdep{promote{u64, w}, cast{u64,0x0101010101010101}}*255}} + r+= pc + } +} + +def tab{n,l} = { + def m=n-1; def t=tab{m,l} + def k = (1<>4} + } +} + +fn slash2{F, T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { + xv:= *u64~~x + F{w, {} => {c:= loadu{xv}; xv+= 1; c}, r, l, sum} +} + +fn slash1{F, T, iota, add}(w:*u64, r:*T, l:u64, sum:u64) : void = { + x:u64 = iota + F{w, {} => {c:= x; x+= add; c}, r, l, sum} +} + +# 8-bit writes ~8 bytes of garbage past end, 16-bit writes ~16 bytes +if (hasarch{'BMI2'}) { +export{'bmipopc_2slash8', slash2{comp8, i8}} +export{'bmipopc_2slash16', slash2{comp16, i16}} +export{'bmipopc_1slash8', slash1{comp8, i8, 0x0706050403020100, 0x0808080808080808}} +export{'bmipopc_1slash16', slash1{comp16, i16, 0x0003000200010000, 0x0004000400040004}} +} + +itab :*u64 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} +i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} + fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32