diff --git a/src/builtins/slash.c b/src/builtins/slash.c index d7953e57..179b185e 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -477,36 +477,32 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { } \ TFREE(buf) #define COMPRESS_BLOCK(T) COMPRESS_BLOCK_PREP(T, ) - #if SINGELI_AVX2 - #define THR(C) C - #else - #define THR(C) 2 - #endif #define WITH_SPARSE(W, CUTOFF, DENSE) { \ i##W *xp=tyany_ptr(x), *rp; \ - if (wsum=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ - else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } #if SINGELI - case 3: WITH_SPARSE( 8, 32, rp=m_tyarrv(&r,1,wsum,xt); si_2slash8 (wp, xp, rp, wia, wsum)) - case 4: WITH_SPARSE(16, 16, rp=m_tyarrv(&r,2,wsum,xt); si_2slash16(wp, xp, rp, wia, wsum)) - case 5: WITH_SPARSE(32, 32, rp=m_tyarrv(&r,4,wsum,xt); si_2slash32(wp, xp, rp, wia, wsum)) + #define DO(W) \ + WITH_SPARSE(W, si_thresh_2slash##W, rp=m_tyarrv(&r,W/8,wsum,xt); si_2slash##W(wp, xp, rp, wia, wsum)) + case 3: DO(8) case 4: DO(16) case 5: DO(32) case 6: if (TI(x,elType)!=el_B) { - WITH_SPARSE(64, 16, rp=m_tyarrv(&r,8,wsum,xt); si_2slash64(wp, xp, rp, wia, wsum)) + DO(64) } // else follows + #undef DO #else case 3: WITH_SPARSE( 8, 2, rp=m_tyarrv(&r,1,wsum,xt); for (usz i=0; i=wia/8 && groups_lt(wp,wia, wia/16)) r = compress_grouped(wp, x, wia, wsum, xt); \ + else { T* xp=tyany_ptr(x); T* rp=m_tyarrv(&r,sizeof(T),wsum,xt); COMPRESS_BLOCK(T); } case 5: BLOCK_OR_GROUPED(i32) break; case 6: if (TI(x,elType)!=el_B) { BLOCK_OR_GROUPED(u64) } + #undef BLOCK_OR_GROUPED #endif #undef WITH_SPARSE - #undef THR else { B xf = getFillR(x); B* xp = arr_bptr(x); @@ -523,7 +519,6 @@ static B compress(B w, B x, usz wia, u8 xl, u8 xt) { } } break; - #undef BLOCK_OR_GROUPED #undef COMPRESS_BLOCK } ur xr = RNK(x); diff --git a/src/singeli/src/slash.singeli b/src/singeli/src/slash.singeli index 3c2ca4b6..3c6d5286 100644 --- a/src/singeli/src/slash.singeli +++ b/src/singeli/src/slash.singeli @@ -55,6 +55,7 @@ def storeu{p:T, i, v:eltype{T} & *u64==T} = emit{void, 'storeu_u64', p+i, v} def loadu{p:T & *u64==T} = emit{eltype{T}, 'loadu_u64', p} # Assumes w is trimmed, so the last 1 appears at index l-1 +def thresh2{T} = 2 fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def bitp_get{arr, n} = (load{arr,n>>6} >> (n&63)) & 1 @for (x over i to l) { @@ -63,6 +64,7 @@ fn slash2{T}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { } } +def thresh2{T==i8 & hasarch{'X86_64'}} = 4 fn slash2{T==i8 & hasarch{'X86_64'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def U = [16]u8 k1 := U**1 @@ -140,8 +142,11 @@ def slash1{T & hasarch{'BMI2'}} = { def b = bind{base, 1< join{each{tup,t,k+(t<<8)%(1<<64)}}, tup{0x8080808080808080}, reverse{iota{8}}} i64tab:*u32 = fold{{t,k} => join{each{tup,t,k+(t<<8)%(1<<32)}}, tup{0x80808080}, reverse{2*iota{4}}} +def thresh2{T==i32 & hasarch{'AVX2'}} = 32 +def thresh2{T==i64 & hasarch{'AVX2'}} = 8 fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def tw = width{T} def V = [8]u32 @@ -211,6 +218,10 @@ fn slash2{T & hasarch{'AVX2'} & width{T}>=32}(wp:*u64, x:*T, r:*T, l:u64, sum:u6 } } +def thresh2{T==i8 & hasarch{'AVX512VBMI2'}} = 256 +def thresh2{T==i16 & hasarch{'AVX512VBMI2'}} = 128 +def thresh2{T==i32 & hasarch{'AVX512F'}} = 64 +def thresh2{T==i64 & hasarch{'AVX512F'}} = 16 fn slash2{T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u64, x:*T, r:*T, l:u64, sum:u64) : void = { def f = match { {_==8}=>'8'; {_==16}=>'16'; {_==32}=>'32'; {_==64}=>'64' } def wt = width{T} @@ -239,10 +250,10 @@ if (hasarch{'BMI2'}) { export{'bmipopc_1slash8', slash1{i8}} export{'bmipopc_1slash16', slash1{i16}} } -export{'si_2slash8', slash2{i8}} -export{'si_2slash16', slash2{i16}} -export{'si_2slash32', slash2{i32}} -export{'si_2slash64', slash2{i64}} +export{'si_2slash8' , slash2{i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} +export{'si_2slash16', slash2{i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} +export{'si_2slash32', slash2{i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}} +export{'si_2slash64', slash2{i64}}; export{'si_thresh_2slash64', u64~~thresh2{i64}} # pext, or boolean compress fn pext{T}(x:T, m:T) {