Refactoring, simplification, comments
This commit is contained in:
parent
a8b036ad08
commit
5e3cc6de81
@ -76,70 +76,67 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = {
|
|||||||
export{'si_sum_f64', fold_assoc_0{f64,+}}
|
export{'si_sum_f64', fold_assoc_0{f64,+}}
|
||||||
|
|
||||||
|
|
||||||
|
# Short-row boolean folds: main challenge is bit packing
|
||||||
def fold_rows_bit_lt64{
|
def fold_rows_bit_lt64{
|
||||||
op, run_loop2, run_loop4, pext_res, mult_in, xx, rx, rxs,
|
op, run_loop2, run_loop4, pext_res, mult_in,
|
||||||
|
xx, rx, # input and output xor for cases not specialized to individual functions
|
||||||
|
rxs, # output xor only, where and/or are specialized
|
||||||
xp, rp, n, l
|
xp, rp, n, l
|
||||||
} = {
|
} = {
|
||||||
nw := cdiv{n*l,64}
|
nw := cdiv{n*l,64}
|
||||||
def loop_pext{T, get, b} = {
|
def loop_T{T, proc_word} = {
|
||||||
@for (x in xp, r in *T~~rp over nw) r = cast_i{T, rxs{pext{get{x}, b}}}
|
@for (x in xp, r in *T~~rp over nw) r = cast_i{T, rxs{proc_word{x}}}
|
||||||
}
|
}
|
||||||
if (l == 2) {
|
if (l == 2) {
|
||||||
t:u64 = 64w2b10
|
def extract = if (fast_BMI2{}) pext{., u64~~64w2b01} else {
|
||||||
run_loop2{
|
# repeated shift-or-mask
|
||||||
if (fast_BMI2{}) {
|
def step{x, {m, sh}} = { def a=x&m; a|a>>sh }
|
||||||
{op} => loop_pext{u32, {x}=>op{x, x<<1}, t}
|
def ss = 1<<iota{5}
|
||||||
} else {
|
def ms = (1<<64 - 1) / (1 + 1<<ss)
|
||||||
def all = 1<<64-1
|
fold{step, ., each{tup, ms, ss}}
|
||||||
ms:5**u64 = reverse{slice{scan{{x,s}=>x^((x<<s)&all), all, reverse{1<<range{6}}}, 1}}
|
}
|
||||||
{op} => {
|
run_loop2{{op} => loop_T{u32, {x} => extract{op{x, x>>1}}}}
|
||||||
def T = u32
|
|
||||||
@for (x in xp, r in *T~~rp over nw) {
|
|
||||||
a := op{x, x>>1}
|
|
||||||
each{{m, sh} => { a &= m; a |= a>>sh }, ms, 1<<range{5}}
|
|
||||||
r = cast_i{T, rxs{a}}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (l == 4) {
|
} else if (l == 4) {
|
||||||
m:u64 = 64w2b0001; t := m<<3
|
m:u64 = 64w2b0001; t := m<<3
|
||||||
run_loop4{m, t,
|
def extract = if (fast_BMI2{}) pext{., t} else {
|
||||||
if (fast_BMI2{}) {
|
{x} => {
|
||||||
loop_pext{u16, ., t}
|
a:= x & t
|
||||||
} else {
|
a = (a * 4r2b001) & 64w0xf000
|
||||||
{get} => {
|
a = (a * 4r0x001) >> (64-16)
|
||||||
def T = u16
|
|
||||||
@for (x in xp, r in *T~~rp over nw) {
|
|
||||||
a := get{x} & t
|
|
||||||
a = (a * 4r2b001) & (64w0xf000)
|
|
||||||
a = (a * fold{+, 1<<(12*iota{4})}) >> (3*16)
|
|
||||||
r = cast_i{T, rxs{a}}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
run_loop4{m, t, {get} => loop_T{u16, {x} => extract{get{x}}}}
|
||||||
|
} else { # generic width<64
|
||||||
{e0, d} := unaligned_spaced_mask_mod{l}
|
{e0, d} := unaligned_spaced_mask_mod{l}
|
||||||
e := e0 << (l-1)
|
e := e0 << (l-1) # ending bit of each row
|
||||||
r:u64 = 0
|
c:u64 = 0 # carry, use depends on algorithm
|
||||||
rh := *u32~~rp
|
def {write_bits, flush_bits} = {
|
||||||
ri:ux = 0
|
r:u64 = 0
|
||||||
if (fast_BMI2{}) {
|
rh := *u32~~rp
|
||||||
c:u64 = 0
|
ri:ux = 0
|
||||||
@for (x in xp over i to nw) {
|
def write{rb, n_bits} = {
|
||||||
r |= pext_res{xx{x}, e, c} << ri
|
r |= rb << ri
|
||||||
ri += popc{e}
|
ri += promote{ux, n_bits}
|
||||||
e = e>>d | e<<(l-d)
|
|
||||||
if (ri >= 32) {
|
if (ri >= 32) {
|
||||||
store{rh, 0, cast_i{u32,rx{r}}}; ++rh;
|
store{rh, 0, cast_i{u32,rx{r}}}; ++rh;
|
||||||
r >>= 32; ri -= 32
|
r >>= 32; ri -= 32
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
def flush{} = if (ri > 0) store{rh, 0, cast_i{u32,rx{r}}}
|
||||||
|
tup{write, flush}
|
||||||
|
}
|
||||||
|
def finish_step{...a} = {
|
||||||
|
write_bits{...a}
|
||||||
|
e = e>>d | e<<(l-d) # update end mask for next iteration
|
||||||
|
}
|
||||||
|
if (fast_BMI2{}) {
|
||||||
|
@for (x in xp over nw) finish_step{pext_res{xx{x}, e, c}, popc{e}}
|
||||||
} else {
|
} else {
|
||||||
dm:= cast_i{usz, popc{e}}
|
# Emulate pext with 1, 2, or 3 multiply/mask steps.
|
||||||
e0 |= -promote{u64, l & (l-1) == 0}
|
# To move size-a groups spaced at distance b together,
|
||||||
c:u64 = 0
|
# the multiplier has up to b/a bits spaced by b-a.
|
||||||
|
dm:= cast_i{usz, popc{e}} # minimum output bits per word
|
||||||
|
dm-= promote{usz, l&(l-1) == 0} # for divisors of 64, e0 effectively overflows; subtract 1 to correct
|
||||||
def loop{...par} = {
|
def loop{...par} = {
|
||||||
@for (xo in xp over nw) {
|
@for (xo in xp over nw) {
|
||||||
x:= xx{xo}
|
x:= xx{xo}
|
||||||
@ -152,18 +149,13 @@ def fold_rows_bit_lt64{
|
|||||||
mult_in{x, e, c}
|
mult_in{x, e, c}
|
||||||
}
|
}
|
||||||
nb:= dm + promote{usz, e>=e0} # = popc{e}
|
nb:= dm + promote{usz, e>=e0} # = popc{e}
|
||||||
|
# the multiply-mask sequence (last "mask" is the shift by (64 - nb))
|
||||||
def extract = match {
|
def extract = match {
|
||||||
{{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q
|
|
||||||
{ {q}, {}} => (s & e) * (q << clz{e})
|
{ {q}, {}} => (s & e) * (q << clz{e})
|
||||||
|
{{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q
|
||||||
}
|
}
|
||||||
rb:= extract{...par} >> (64 - nb)
|
rb:= extract{...par} >> (64 - nb)
|
||||||
r |= mod_rb{rb} << ri
|
finish_step{mod_rb{rb}, nb}
|
||||||
ri += promote{ux, nb}
|
|
||||||
if (ri >= 32) {
|
|
||||||
store{rh, 0, cast_i{u32,rx{r}}}; ++rh;
|
|
||||||
r >>= 32; ri -= 32
|
|
||||||
}
|
|
||||||
e = e>>d | e<<(l-d)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (l == 3) {
|
if (l == 3) {
|
||||||
@ -187,7 +179,7 @@ def fold_rows_bit_lt64{
|
|||||||
loop{tup{mult}, tup{}}
|
loop{tup{mult}, tup{}}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (ri > 0) store{rh, 0, cast_i{u32,rx{r}}}
|
flush_bits{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user