diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 179b185e..4c2020bd 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -162,8 +162,8 @@ static void bsp_u16(u64* src, u16* dst, usz len, usz sum) { static void where_block_u16(u64* src, u16* dst, usz len, usz sum) { assert(len <= bsp_max); - #if SINGELI_AVX2 && FAST_PDEP - if (sum >= len/8) bmipopc_1slash16(src, (i16*)dst, len, sum); + #if SINGELI + if (sum >= len/si_thresh_1slash16) si_1slash16(src, (i16*)dst, len, sum); #else if (sum >= len/4+len/8) WHERE_DENSE(src, dst, len, 0); #endif @@ -235,17 +235,17 @@ static B where(B x, usz xia, u64 s) { u64* xp = bitarr_ptr(x); usz q=xia%64; if (q) xp[xia/64] &= ((u64)1<= xia/8) { + #if SINGELI + if (s >= xia/si_thresh_1slash16) { i16* rp = m_tyarrv(&r, 2, s, t_i16arr); - bmipopc_1slash16(xp, rp, xia, s); + si_1slash16(xp, rp, xia, s); } #else if (s >= xia/4+xia/8) { @@ -274,9 +274,9 @@ static B where(B x, usz xia, u64 s) { } else { bs = bit_sum(xp,b); } - #if SINGELI_AVX2 && FAST_PDEP - if (bs >= b/8+b/16) { - bmipopc_1slash16(xp, buf, b, bs); + #if SINGELI + if (bs >= b/si_thresh_1slash16+b/16) { + si_1slash16(xp, buf, b, bs); for (usz j=0; jxia-i? xia-i : b; - usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2, s0); for (usz j=0; j>6} >> (n&63)) & 1 - @for (x over i to l) { - store{r, 0, x} + @for (i to l) { + store{r, 0, if (c) load{x,i} else i} r+= bitp_get{w,i} } } @@ -104,6 +105,7 @@ def tab{n,l} = if (n==0) tup{0} else { c16lut:*u64 = tab{4,16} def vgLoad{p:T, i & T == *u64} = emit{eltype{T}, 'vg_loadLUT64', p, i} +def thresh1{T==i16 & hasarch{'BMI2'}} = 8 def thresh2{T==i8 & hasarch{'BMI2'}} = 32 def thresh2{T==i16 & hasarch{'BMI2'}} = 16 fn slash{c, T & hasarch{'BMI2'}}(w:*u64, x:arg{c,T}, r:*T, l:u64, sum:u64) : void = { @@ -233,10 +235,8 @@ fn slash{c==1, T & hasarch{if (width{T}>=32) 'AVX512F' else 'AVX512VBMI2'}}(w:*u } } -if (hasarch{'BMI2'}) { -export{'bmipopc_1slash8', slash{0, i8}} -export{'bmipopc_1slash16', slash{0, i16}} -} +export{'si_1slash8' , slash{0, i8 }} +export{'si_1slash16', slash{0, i16}}; export{'si_thresh_1slash16', u64~~thresh1{i16}} export{'si_2slash8' , slash{1, i8 }}; export{'si_thresh_2slash8' , u64~~thresh2{i8 }} export{'si_2slash16', slash{1, i16}}; export{'si_thresh_2slash16', u64~~thresh2{i16}} export{'si_2slash32', slash{1, i32}}; export{'si_thresh_2slash32', u64~~thresh2{i32}}