Run-based i32 /⁼ (leaving out sum-based)
This commit is contained in:
parent
4aea668a6c
commit
3b103aadd0
@ -826,7 +826,7 @@ B slash_im(B t, B x) {
|
|||||||
rp[sum>0] = sum; rp[0] = xia - sum;
|
rp[sum>0] = sum; rp[0] = xia - sum;
|
||||||
r = num_squeeze(r); break;
|
r = num_squeeze(r); break;
|
||||||
}
|
}
|
||||||
#define IIND_INT(N, CHECK_RIA) \
|
#define TRY_SMALL_OUT(N) \
|
||||||
if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \
|
if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \
|
||||||
usz a=1; while (a<xia && xp[a]>xp[a-1]) a++; \
|
usz a=1; while (a<xia && xp[a]>xp[a-1]) a++; \
|
||||||
u##N max=xp[a-1]; \
|
u##N max=xp[a-1]; \
|
||||||
@ -841,11 +841,7 @@ B slash_im(B t, B x) {
|
|||||||
for (usz i=0; i<xia; i++) maxcount|=++tab[xp[i]]; \
|
for (usz i=0; i<xia; i++) maxcount|=++tab[xp[i]]; \
|
||||||
TFREE(tab); \
|
TFREE(tab); \
|
||||||
if (maxcount<=1) a=xia; \
|
if (maxcount<=1) a=xia; \
|
||||||
else if (N>=16 && maxcount<128) { \
|
else if (N>=16 && maxcount<128) { INIT_RES(8) FILL_RES break; } \
|
||||||
i8* rp; r = m_i8arrv(&rp, ria); for (usz i=0; i<ria; i++) rp[i]=0; \
|
|
||||||
for (usz i = 0; i < xia; i++) rp[xp[i]]++; \
|
|
||||||
break; \
|
|
||||||
} \
|
|
||||||
} \
|
} \
|
||||||
} \
|
} \
|
||||||
if (a==xia) { /* Unique argument */ \
|
if (a==xia) { /* Unique argument */ \
|
||||||
@ -855,9 +851,11 @@ B slash_im(B t, B x) {
|
|||||||
for (usz i=0; i<xia; i++) bitp_set(rp, xp[i], 1); \
|
for (usz i=0; i<xia; i++) bitp_set(rp, xp[i], 1); \
|
||||||
break; \
|
break; \
|
||||||
} \
|
} \
|
||||||
usz ria = (usz)max + 1; \
|
usz ria = (usz)max + 1;
|
||||||
CHECK_RIA \
|
#define INIT_RES(N) \
|
||||||
i##N* rp; r = m_i##N##arrv(&rp, ria); for (usz i=0; i<ria; i++) rp[i]=0; \
|
i##N* rp; r = m_i##N##arrv(&rp, ria); \
|
||||||
|
for (usz i=0; i<ria; i++) rp[i]=0;
|
||||||
|
#define FILL_RES \
|
||||||
for (usz i = 0; i < xia; i++) rp[xp[i]]++;
|
for (usz i = 0; i < xia; i++) rp[xp[i]]++;
|
||||||
#define CASE_SMALL(N) \
|
#define CASE_SMALL(N) \
|
||||||
case el_i##N: { \
|
case el_i##N: { \
|
||||||
@ -865,7 +863,9 @@ B slash_im(B t, B x) {
|
|||||||
usz m=1<<N; \
|
usz m=1<<N; \
|
||||||
usz mh = m/2, sa = SINGELI_COUNT_ALLOC; \
|
usz mh = m/2, sa = SINGELI_COUNT_ALLOC; \
|
||||||
if (xia < mh) { \
|
if (xia < mh) { \
|
||||||
IIND_INT(N, if (RIA_SMALL(N)) { sa=mh=ria; goto small_range##N; }) \
|
TRY_SMALL_OUT(N) \
|
||||||
|
if (RIA_SMALL(N)) { sa=mh=ria; goto small_range##N; } \
|
||||||
|
INIT_RES(N) FILL_RES \
|
||||||
} else { \
|
} else { \
|
||||||
small_range##N: TALLOC(usz, t, sa); \
|
small_range##N: TALLOC(usz, t, sa); \
|
||||||
for (usz j=0; j<mh; j++) t[j]=0; \
|
for (usz j=0; j<mh; j++) t[j]=0; \
|
||||||
@ -896,8 +896,21 @@ B slash_im(B t, B x) {
|
|||||||
#undef RIA_SMALL
|
#undef RIA_SMALL
|
||||||
#undef SINGELI_COUNT_ALLOC
|
#undef SINGELI_COUNT_ALLOC
|
||||||
#undef SINGELI_COUNT
|
#undef SINGELI_COUNT
|
||||||
case el_i32: { i32* xp = i32any_ptr(x); IIND_INT(32,) r = num_squeeze(r); break; }
|
case el_i32: {
|
||||||
#undef IIND_INT
|
i32* xp = i32any_ptr(x);
|
||||||
|
TRY_SMALL_OUT(32)
|
||||||
|
INIT_RES(32)
|
||||||
|
#if SINGELI_SIMD
|
||||||
|
simd_count_i32_i32(rp, xp, xia);
|
||||||
|
#else
|
||||||
|
FILL_RES
|
||||||
|
#endif
|
||||||
|
r = num_squeeze(r);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
#undef TRY_SMALL_OUT
|
||||||
|
#undef INIT_RES
|
||||||
|
#undef FILL_RES
|
||||||
case el_f64: {
|
case el_f64: {
|
||||||
f64* xp = f64any_ptr(x);
|
f64* xp = f64any_ptr(x);
|
||||||
usz i,j; f64 max=-1;
|
usz i,j; f64 max=-1;
|
||||||
|
|||||||
@ -3,27 +3,34 @@ include './vecfold'
|
|||||||
|
|
||||||
if_inline (hasarch{'SSE2'}) {
|
if_inline (hasarch{'SSE2'}) {
|
||||||
fn sum_vec{T}(v:T) = vfold{+, fold{+, mzip128{v, T**0}}}
|
fn sum_vec{T}(v:T) = vfold{+, fold{+, mzip128{v, T**0}}}
|
||||||
def fold_addw{v:T=[_]E if E<=u16} = sum_vec{T}(v)
|
def fold_addw{v:T=[_]E if E<=u32} = sum_vec{T}(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}
|
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}
|
||||||
def inc{ptr, ind} = inc{ptr, ind, 1}
|
def inc{ptr, ind} = inc{ptr, ind, 1}
|
||||||
|
|
||||||
|
def block_loop{V=[vec]T, n, iter} = {
|
||||||
|
def block = (2048*8) / width{V} # Target vectors per block
|
||||||
|
def b_max = block + block/4 # Last block max length
|
||||||
|
assert{b_max < 1<<width{T}} # Don't overflow count in vector section
|
||||||
|
i:u64 = 0
|
||||||
|
while (i < n) {
|
||||||
|
# Number of elements to handle in this iteration
|
||||||
|
r:u64 = n - i; if (r > vec*b_max) r = vec*block
|
||||||
|
iter{r}
|
||||||
|
i += r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Write counts /⁼x to tab and return ⌈´x
|
# Write counts /⁼x to tab and return ⌈´x
|
||||||
fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = {
|
fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = {
|
||||||
def vbits = arch_defvw
|
def vbits = arch_defvw
|
||||||
def vec = vbits/width{T}
|
def vec = vbits/width{T}
|
||||||
def uT = ty_u{T}
|
def uT = ty_u{T}
|
||||||
def V = [vec]T
|
def V = [vec]T
|
||||||
def block = (2048*8) / vbits # Target vectors per block
|
|
||||||
def b_max = block + block/4 # Last block max length
|
|
||||||
assert{b_max < 1<<width{T}} # Don't overflow count in vector section
|
|
||||||
x := *T~~xp
|
x := *T~~xp
|
||||||
mx:T = min_allowed # Maximum of x
|
mx:T = min_allowed # Maximum of x
|
||||||
i:u64 = 0
|
block_loop{V, n, {r} => { # Handle r elements
|
||||||
while (i < n) {
|
|
||||||
# Number of elements to handle in this iteration
|
|
||||||
r:u64 = n - i; if (r > vec*b_max) r = vec*block
|
|
||||||
b := r / vec # Vector case does b full vectors if it runs
|
b := r / vec # Vector case does b full vectors if it runs
|
||||||
rv:= b * vec
|
rv:= b * vec
|
||||||
r0:u64 = 0 # Elements actually handled by vector case
|
r0:u64 = 0 # Elements actually handled by vector case
|
||||||
@ -58,9 +65,8 @@ fn count{T}(tab:*usz, xp:*void, n:u64, min_allowed:T) : T = {
|
|||||||
|
|
||||||
# Scalar fallback and cleanup
|
# Scalar fallback and cleanup
|
||||||
@for (x over _ from r0 to r) inc{tab, x}
|
@for (x over _ from r0 to r) inc{tab, x}
|
||||||
i += r
|
|
||||||
x += r
|
x += r
|
||||||
}
|
}}
|
||||||
mx
|
mx
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,7 +92,7 @@ def count_by_sum{T, V, U, xv, b, tab, r0, j0, m} = {
|
|||||||
|
|
||||||
# Count adjacent equal elements at once, breaking at w-element groups
|
# Count adjacent equal elements at once, breaking at w-element groups
|
||||||
# May read up to index r from x, hitting one element that's not counted
|
# May read up to index r from x, hitting one element that's not counted
|
||||||
def count_with_runs{V, vec, x, tab, r} = {
|
def count_with_runs{V, vec, x, tab:*T, r} = {
|
||||||
def w = width{ux}
|
def w = width{ux}
|
||||||
m0:ux = 1 << (w-1) # Last element in each chunk ends a run
|
m0:ux = 1 << (w-1) # Last element in each chunk ends a run
|
||||||
bw := r / w
|
bw := r / w
|
||||||
@ -100,17 +106,37 @@ def count_with_runs{V, vec, x, tab, r} = {
|
|||||||
m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv
|
m |= promote{ux, homMask{lv{jv} != lv{jv+1}}} << jv
|
||||||
}
|
}
|
||||||
# Iterate over runs
|
# Iterate over runs
|
||||||
jp:usz = - usz~~1
|
jp:T = - T~~1
|
||||||
while (m > m0) @unroll (2) {
|
while (m > m0) @unroll (2) {
|
||||||
j := trunc{usz, ctz{m}}
|
j := trunc{T, ctz{m}}
|
||||||
inc{tab, load{xo, j}, j - jp}
|
inc{tab, load{xo, j}, cast_i{T, j - jp}}
|
||||||
jp = j; m &= m-1
|
jp = j; m &= m-1
|
||||||
}
|
}
|
||||||
# One step if popc{m} was odd, reducing branch mispredictions above
|
# One step if popc{m} was odd, reducing branch mispredictions above
|
||||||
inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{usz, m>>(w-1)}}
|
inc{tab, load{xo, w-1}, ((w-1) - jp) & -trunc{T, m>>(w-1)}}
|
||||||
}
|
}
|
||||||
bw * w
|
bw * w
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Condensed version without count_by_sum
|
||||||
|
fn count_i32_i32(tab:*i32, x:*i32, n:u64) : void = {
|
||||||
|
def T = i32
|
||||||
|
def vbits = arch_defvw
|
||||||
|
def vec = vbits/width{T}
|
||||||
|
def V = [vec]T
|
||||||
|
block_loop{V, n, {r} => {
|
||||||
|
b := r / vec
|
||||||
|
xv := *V~~x
|
||||||
|
dc := -(load{xv} != load{*V~~(x+1)})
|
||||||
|
@for (xv, xp in *V~~(x-1) over _ from 1 to b) dc -= xp != xv
|
||||||
|
dt := promote{u64, fold_addw{dc}}
|
||||||
|
r0:u64 = 0
|
||||||
|
if (dt < b * (vec/2)) r0 = count_with_runs{V, vec, x, tab, r}
|
||||||
|
@for (x over _ from r0 to r) inc{tab, x}
|
||||||
|
x += r
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
export{'simd_count_i8', count{i8}}
|
export{'simd_count_i8', count{i8}}
|
||||||
export{'simd_count_i16', count{i16}}
|
export{'simd_count_i16', count{i16}}
|
||||||
|
export{'simd_count_i32_i32', count_i32_i32}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user