Dedicated generic k/bool for divisors of 8; factor; remove pdep emulation
This commit is contained in:
parent
13ec029d9f
commit
3e5dbdbf8d
@ -1,5 +1,4 @@
|
|||||||
include './base'
|
include './base'
|
||||||
if_inline (hasarch{'BMI2'}) include './bmi2'
|
|
||||||
include './mask'
|
include './mask'
|
||||||
include './spaced'
|
include './spaced'
|
||||||
|
|
||||||
@ -291,53 +290,60 @@ exportT{'si_constrep', each{rep_const, dat_types}}
|
|||||||
|
|
||||||
# Constant replicate on boolean
|
# Constant replicate on boolean
|
||||||
fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = {
|
fn rep_const_bool{}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 = {
|
||||||
def has_pdep = 0 # Obselete, kept here for descriptiveness
|
if (wv > 64) return{0}
|
||||||
if (wv > 32) return{0}
|
|
||||||
m:u64 = spaced_mask_of{wv}
|
|
||||||
xw:u64 = 0
|
|
||||||
d := cast_i{usz, popc{m}} # == 64/wv
|
|
||||||
nw := cdiv{rlen, 64}
|
nw := cdiv{rlen, 64}
|
||||||
if (wv&1 != 0) {
|
if (wv&1 == 0) {
|
||||||
rep_const_bool_generic_odd{wv, x, r, nw, m, d}
|
p := ctz{wv | 8} # Power of two for second replicate
|
||||||
} else if (not has_pdep and wv <= 8) {
|
wf := wv>>p
|
||||||
return{0}
|
if (wf == 1) {
|
||||||
} else if (m&1 != 0) { # Power of two
|
rep_const_bool_div8{wv, x, r, nw}
|
||||||
i := -usz~~1
|
} else {
|
||||||
def expand = if (has_pdep) pdep{., m} else {
|
tlen := rlen>>p
|
||||||
mult:u64 = spaced_mask_of{wv-1} >> d
|
wq := usz~~1<<p
|
||||||
xm := (u64~~1 << d) - 1
|
if (p == 1 or (p == 2 and wv>=52)) { # Expanding odd second is faster
|
||||||
{xw} => ((xw&xm)*mult) & m
|
tlen = rlen / wf
|
||||||
}
|
t:=wf; wf=wq; wq=t
|
||||||
@for (r over j to nw) {
|
}
|
||||||
xw >>= d
|
t := r + cdiv{rlen, 64} - cdiv{tlen, 64}
|
||||||
if ((j&(wv-1))==0) { ++i; xw = load{x, i} }
|
rep_const_bool{}(wf, x, t, tlen)
|
||||||
rw := expand{xw}
|
rep_const_bool{}(wq, t, r, rlen)
|
||||||
r = (rw<<wv) - rw
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
q := cast_i{usz, ctz{m}} # == 64%wv
|
m:u64 = spaced_mask_of{wv}
|
||||||
m = m<<(wv-q) | 1
|
d := cast_i{usz, popc{m}} # == 64/wv
|
||||||
mt := u64~~1 << (d+1) # Bit d+1 may be needed, isn't pdep-ed
|
rep_const_bool_generic_odd{wv, x, r, nw, m, d}
|
||||||
tsh := d*wv-(d+1)
|
|
||||||
xb := *u8~~x
|
|
||||||
xi:usz=0; o:usz=0
|
|
||||||
def expand = if (has_pdep) pdep{., m} else {
|
|
||||||
{mult, _} := unaligned_spaced_mask_mod{wv-1}
|
|
||||||
xm := mt - 1
|
|
||||||
{xw} => ((xw&xm)*mult) & m
|
|
||||||
}
|
|
||||||
@for (r over j to nw) {
|
|
||||||
xw = loadu{*u64~~(xb + xi/8)} >> (xi%8)
|
|
||||||
ex := (xw & mt) << tsh
|
|
||||||
rw := expand{xw}
|
|
||||||
r = ((rw-ex)<<(wv-o)) - (rw>>o|(xw&1))
|
|
||||||
o += q
|
|
||||||
oo := o>=wv; xi+=d+promote{usz,oo}; o-=wv&-oo
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
1
|
1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def rep_const_bool_div8{wv, x, r, nw} = {
|
||||||
|
def run{k} = {
|
||||||
|
# 2 -> 64w0x33, 12 -> 64w0x000f, etc.
|
||||||
|
def getm{sh} = base{2, iota{64}&sh == 0}
|
||||||
|
def osh{v, s} = v | v<<s
|
||||||
|
def expand = match (k) {
|
||||||
|
{2} => fold{
|
||||||
|
{v, sh} => osh{v, sh} & getm{sh},
|
||||||
|
., 1 << reverse{iota{5}}
|
||||||
|
}
|
||||||
|
{4} => fold{
|
||||||
|
{v, sh} => osh{osh{v, sh}, 2*sh} & getm{sh},
|
||||||
|
., tup{12, 3}
|
||||||
|
}
|
||||||
|
{8} => {
|
||||||
|
def mult = base{1<<7, 8**1}
|
||||||
|
{x} => (x | ((x&~1) * mult)) & 64w0x01
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@for (xt in *ty_u{64/k}~~x, r over nw) {
|
||||||
|
def v = expand{promote{u64, xt}}
|
||||||
|
r = v<<k - v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def cases{k} = if (wv==k) run{k} else if (k<8) cases{2*k}
|
||||||
|
cases{2}
|
||||||
|
}
|
||||||
|
|
||||||
def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = {
|
def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = {
|
||||||
# Every-k-bits mask
|
# Every-k-bits mask
|
||||||
mask_sh := cast_i{usz, ctz{m}} # == 64%k
|
mask_sh := cast_i{usz, ctz{m}} # == 64%k
|
||||||
@ -350,7 +356,7 @@ def rep_const_bool_generic_odd{k, xp, rp, nw, m, d} = {
|
|||||||
def bits = (k*iota{1<<lw} >> merge{0, replicate{1<<i, i}}) & 1
|
def bits = (k*iota{1<<lw} >> merge{0, replicate{1<<i, i}}) & 1
|
||||||
~u64~~base{2, bits}
|
~u64~~base{2, bits}
|
||||||
}
|
}
|
||||||
swtab:*u64 = each{swdat{6,.}, 1+2*iota{16}}
|
swtab:*u64 = each{swdat{6,.}, 1+2*iota{32}}
|
||||||
def swap_lens = reverse{2 << iota{5}}
|
def swap_lens = reverse{2 << iota{5}}
|
||||||
swap_data := load{swtab, k>>1}
|
swap_data := load{swtab, k>>1}
|
||||||
swsel:u64 = ~u64~~0
|
swsel:u64 = ~u64~~0
|
||||||
@ -441,7 +447,7 @@ fn rep_const_bool{if hasarch{'SSSE3'}}(wv:usz, x:*u64, r:*u64, rlen:usz) : u1 =
|
|||||||
if (wv&1 == 0) {
|
if (wv&1 == 0) {
|
||||||
p := ctz{wv | 8} # Power of two for second replicate
|
p := ctz{wv | 8} # Power of two for second replicate
|
||||||
if (wv>>p == 1) {
|
if (wv>>p == 1) {
|
||||||
rep_const_bool_ssse3_div8{wv, x, r, rlen}
|
rep_const_bool_div8{wv, x, r, rlen}
|
||||||
} else {
|
} else {
|
||||||
tlen := rlen>>p
|
tlen := rlen>>p
|
||||||
t := r + cdiv{rlen, 64} - cdiv{tlen, 64}
|
t := r + cdiv{rlen, 64} - cdiv{tlen, 64}
|
||||||
@ -482,7 +488,7 @@ def get_boolvec_writer{V, r, rlen} = {
|
|||||||
tup{output, flush}
|
tup{output, flush}
|
||||||
}
|
}
|
||||||
|
|
||||||
def rep_const_bool_ssse3_div8{wv, x, r, rlen} = { # wv in 2,4,8
|
def rep_const_bool_div8{wv, x, r, rlen if hasarch{'SSSE3'}} = { # wv in 2,4,8
|
||||||
oper // ({a,b}=>floor{a/b}) infix left 40
|
oper // ({a,b}=>floor{a/b}) infix left 40
|
||||||
def avx2 = hasarch{'AVX2'}
|
def avx2 = hasarch{'AVX2'}
|
||||||
def vl = if (avx2) 32 else 16
|
def vl = if (avx2) 32 else 16
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user