From 8f4b1966cb4d8a9a50e003592c2f52a797cf4227 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jul 2023 12:17:47 -0400 Subject: [PATCH] Non-overwriting AVX2 4-byte and 8-byte compress, using a buffer --- src/builtins/slash.c | 4 ++-- src/singeli/src/slash.singeli | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 8cde693f..619313fe 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -492,9 +492,9 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { #if SINGELI_AVX2 && FAST_PDEP case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break; + case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); avx2_2slash32(wp, xp, rp, wia, wsum)) break; case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64)) + WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); avx2_2slash64(wp, xp, rp, wia, wsum)) } // else follows #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} -fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = { +# Modifies the input variable r +# Assumes iter{} will increment r, by at most write_len +def for_special_buffered{r, write_len}{vars,begin,sum,iter} = { + assert{isreg{r}}; assert{begin==0} + def T = eltype{type{r}}; def tw = width{T} + def ov = write_len-1 + buf := emit{*T, 'alloca', 2*(ov+1)*width{T}/8} + r0 := r + end := r + sum - ov + i:u64 = 0; buf_used:u1 = 0 + def restart = setlabel{} + while (r < end) { + iter{i, vars} + ++i + } + if (not buf_used) { + end += buf - r + ov + if (buf < end) { + r0 = r + r = buf + buf_used = 1; goto{restart} + } + } else { + def vc = 256/tw; + def R = [vc]T + if (ov>vc and end-buf>vc) { store{*R~~r0, 0, load{*R~~buf}}; r0+=vc; buf+=vc } + homMaskStoreF{*R~~r0, maskOf{R, end-buf}, load{*R~~buf}} + } +} + +fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>lb{tw/32}}}} @@ -83,7 +114,7 @@ fn avx2_compress{T & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64) : void = { store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} r+= pc } - @for (w in *u8~~wp over i to cdiv{l,8}) { + @for_special_buffered{r,8} (w in *u8~~wp over i to sum) { if (tw==32) { step{w, i} } else {