Version of count_i32_i32 without large blocks
This commit is contained in:
parent
3b103aadd0
commit
11117fcc67
@ -54,7 +54,7 @@ fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = {
|
|||||||
dt := promote{u64, fold_addw{dc}}
|
dt := promote{u64, fold_addw{dc}}
|
||||||
nc := uT~~(mt - jt) # Number of counts to perform: last is implicit
|
nc := uT~~(mt - jt) # Number of counts to perform: last is implicit
|
||||||
if (dt < b * (vec/2) and dt*8 < b * promote{u64,nc}) {
|
if (dt < b * (vec/2) and dt*8 < b * promote{u64,nc}) {
|
||||||
r0 = count_with_runs{V, vec, x, tab, r}
|
r0 = count_with_runs{x, tab, r}
|
||||||
} else if (nc <= 24*vbits/128) {
|
} else if (nc <= 24*vbits/128) {
|
||||||
r0 = rv
|
r0 = rv
|
||||||
count_by_sum{T, V, [vec]uT, xv, b, tab, r0,
|
count_by_sum{T, V, [vec]uT, xv, b, tab, r0,
|
||||||
@ -92,49 +92,55 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = {
|
|||||||
|
|
||||||
# Count adjacent equal elements at once, breaking at w-element groups
|
# Count adjacent equal elements at once, breaking at w-element groups
|
||||||
# May read up to index r from x, hitting one element that's not counted
|
# May read up to index r from x, hitting one element that's not counted
|
||||||
def count_with_runs{V, vec, x, tab:*T, r} = {
|
def count_with_runs{x, tab, r} = {
|
||||||
def w = width{ux}
|
def w = width{ux}
|
||||||
m0:ux = 1 << (w-1) # Last element in each chunk ends a run
|
m0:ux = 1 << (w-1) # Last element in each chunk ends a run
|
||||||
bw := r / w
|
bw := r / w
|
||||||
@for (i to bw) {
|
@for (i to bw) {
|
||||||
xo := x + i*w
|
xo := x + i*w
|
||||||
m := m0
|
m := m0; mark_run_ends{xo, m}
|
||||||
# Mark the end of each run
|
inc_marked_runs{xo, tab, m, m0}
|
||||||
@unroll (j to w / vec) {
|
|
||||||
def jv = j*vec
|
|
||||||
def lv{k} = load{*V~~(xo + k)}
|
|
||||||
m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv
|
|
||||||
}
|
|
||||||
# Iterate over runs
|
|
||||||
jp:T = - T~~1
|
|
||||||
while (m > m0) @unroll (2) {
|
|
||||||
j := trunc{T, ctz{m}}
|
|
||||||
inc{tab, load{xo, j}, cast_i{T, j - jp}}
|
|
||||||
jp = j; m &= m-1
|
|
||||||
}
|
|
||||||
# One step if popc{m} was odd, reducing branch mispredictions above
|
|
||||||
inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}}
|
|
||||||
}
|
}
|
||||||
bw * w
|
bw * w
|
||||||
}
|
}
|
||||||
|
def mark_run_ends{x:*T, m:(ux)} = {
|
||||||
# Condensed version without count_by_sum
|
def vec = arch_defvw/width{T}
|
||||||
fn count_i32_i32(tab:*i32, x:*i32, n:u64) : void = {
|
|
||||||
def T = i32
|
|
||||||
def vbits = arch_defvw
|
|
||||||
def vec = vbits/width{T}
|
|
||||||
def V = [vec]T
|
def V = [vec]T
|
||||||
block_loop{V, n, {r} => {
|
@unroll (j to width{ux} / vec) {
|
||||||
b := r / vec
|
def jv = j*vec
|
||||||
xv := *V~~x
|
def lv{k} = load{*V~~(x + k)}
|
||||||
dc := -(load{xv} != load{*V~~(x+1)})
|
m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv
|
||||||
@for (xv, xp in *V~~(x-1) over _ from 1 to b) dc -= xp != xv
|
}
|
||||||
dt := promote{u64, fold_addw{dc}}
|
}
|
||||||
r0:u64 = 0
|
def inc_marked_runs{x, tab:*T, m, m0} = {
|
||||||
if (dt < b * (vec/2)) r0 = count_with_runs{V, vec, x, tab, r}
|
def w = width{ux}
|
||||||
@for (x over _ from r0 to r) inc{tab, x}
|
# Iterate over runs marked in m
|
||||||
x += r
|
jp:T = - T~~1
|
||||||
}}
|
while (m > m0) @unroll (2) {
|
||||||
|
j := trunc{T, ctz{m}}
|
||||||
|
inc{tab, load{x, j}, cast_i{T, j - jp}}
|
||||||
|
jp = j; m &= m-1
|
||||||
|
}
|
||||||
|
# One step if popc{m} was odd, reducing branch mispredictions above
|
||||||
|
inc{tab, load{x, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}}
|
||||||
|
}
|
||||||
|
|
||||||
|
# No count_by_sum: build each run mask then decide whether to use it
|
||||||
|
fn count_i32_i32(tab:*i32, x:*i32, n:usz) : void = {
|
||||||
|
def w = width{ux}
|
||||||
|
m0:ux = 1 << (w-1)
|
||||||
|
while (n > 0) {
|
||||||
|
b:usz = w
|
||||||
|
if (rare{b > n}) { b = n; goto{'skip_runs'} }
|
||||||
|
m := m0; mark_run_ends{x, m}
|
||||||
|
if (popc{m} < w/2) {
|
||||||
|
inc_marked_runs{x, tab, m, m0}
|
||||||
|
} else {
|
||||||
|
setlabel{'skip_runs'}
|
||||||
|
@for (x over b) inc{tab, x}
|
||||||
|
}
|
||||||
|
x += b; n -= b
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export{'simd_count_i8', count{i8}}
|
export{'simd_count_i8', count{i8}}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user