From 0921fc2c62ed2d2fbad51c8b6460af78e4b6ddd3 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 12 Jul 2023 21:26:17 -0400 Subject: [PATCH] AVX2 compress implementations using permutevar and lookup tables --- src/builtins/slash.c | 20 +++++++++------- src/singeli/src/slash.singeli | 45 ++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index b39112e0..8cde693f 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -486,20 +486,24 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { else if (groups_lt(wp,wia, wia/128)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { DENSE; } \ break; } - #if SINGELI_AVX2 && FAST_PDEP - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) - #else - case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } + #if SINGELI_AVX2 && FAST_PDEP + case 3: WITH_SPARSE( 8, 32, rp=m_tyarrvO(&r,1,wsum,xt, 8); bmipopc_2slash8 (wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 8)) + case 4: WITH_SPARSE(16, 16, rp=m_tyarrvO(&r,2,wsum,xt, 16); bmipopc_2slash16(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum*2, 16)) + case 5: WITH_SPARSE(32, 32, rp=m_tyarrvO(&r,4,wsum,xt, 32); avx2_2slash32(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 32)) break; + case 6: if (TI(x,elType)!=el_B) { + WITH_SPARSE(64, 16, rp=m_tyarrvO(&r,8,wsum,xt, 64); avx2_2slash64(wp, xp, rp, wia); FINISH_OVERALLOC_A(r, wsum, 64)) + } // else follows + #else + case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} + +fn compress32{T==i32}(w:*u64, x:*T, r:*T, l:u64) : void = { + expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}}}} + def V = [8]i32 + @for (w in *u8~~w, x in *V~~x over i to cdiv{l,8}) { + pc := popc{w} + ind := load{itab, w} # pext{base{256,iota{8}}, pdep{promote{u64, w}, base{256,8**1}}*255} + s := sel{[16]i8, V~~[4]u64**ind, expander} + store{*V~~r, 0, sel{V, x, s}} + r+= pc + } +} + +i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} + +fn compress64{T==i64}(wp:*u64, x:*T, r:*T, l:u64) : void = { + def V = [8]u32 + expander := make{[32]u8, merge{...each{{i}=>tup{i, ... 3**128}, iota{8}>>1}}} + trail := make{V, iota{8}%2} + def step{w,i} = { + pc := popc{w} + s := trail | sel{[16]i8, V**load{i64tab,w}, expander} + store{*V~~r, 0, sel{V, load{*V~~x,i}, s}} + r+= pc + } + @for (w in *u8~~wp over i to cdiv{l,8}) { + step{w&0xf, 2*i} + step{w>>4, 2*i+1} + } +} + +if (hasarch{'AVX2'}) { +export{'avx2_2slash32', compress32{i32}} +export{'avx2_2slash64', compress64{i64}} +} + # pext, or boolean compress fn pext{T}(x:T, m:T) { def w = width{T}