Merge pull request #73 from mlochbaum/count
Count by vector compare+add
This commit is contained in:
commit
65f25f735d
@ -604,6 +604,7 @@ cachedBin‿linkerCache ← {
|
||||
"2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan",
|
||||
"2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash",
|
||||
"2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq",
|
||||
"2.."‿"src/builtins/slash.c"‿"count"
|
||||
⟩
|
||||
objs ← ⟨⟩
|
||||
|
||||
|
||||
2
makefile
2
makefile
@ -352,7 +352,7 @@ endif
|
||||
@"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c
|
||||
|
||||
|
||||
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c}
|
||||
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c count.c bits.c transpose.c}
|
||||
@echo $(postmsg)
|
||||
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
|
||||
@echo $< | cut -c 17- | sed 's/^/ /'
|
||||
|
||||
@ -81,6 +81,9 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
if (e==n) {break;} k=e; \
|
||||
}
|
||||
#define WRITE_SPARSE(T) WRITE_SPARSE_##T
|
||||
extern i8 (*const avx2_count_i8)(usz*, i8*, u64, i8);
|
||||
#define SINGELI_COUNT_OR(T) \
|
||||
if (1==sizeof(T)) avx2_count_i8(c0o, (i8*)xp, n, -128); else
|
||||
#else
|
||||
#define COUNT_THRESHOLD 16
|
||||
#define WRITE_SPARSE(T) \
|
||||
@ -88,13 +91,14 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
|
||||
usz js = j; \
|
||||
while (ij<n) { rp[ij]GRADE_UD(++,--); ij+=c0o[GRADE_UD(++j,--j)]; } \
|
||||
for (usz i=0; i<n; i++) js=rp[i]+=js;
|
||||
#define SINGELI_COUNT_OR(T)
|
||||
#endif
|
||||
|
||||
#define COUNTING_SORT(T) \
|
||||
usz C=1<<(8*sizeof(T)); \
|
||||
TALLOC(usz, c0, C); usz *c0o=c0+C/2; \
|
||||
for (usz j=0; j<C; j++) c0[j]=0; \
|
||||
for (usz i=0; i<n; i++) c0o[xp[i]]++; \
|
||||
SINGELI_COUNT_OR(T) for (usz i=0; i<n; i++) c0o[xp[i]]++; \
|
||||
if (n/(COUNT_THRESHOLD*sizeof(T)) <= C) { /* Scan-based */ \
|
||||
T j=GRADE_UD(-C/2,C/2-1); \
|
||||
usz ij; while ((ij=c0o[j])==0) GRADE_UD(j++,j--); \
|
||||
@ -227,6 +231,7 @@ B SORT_C1(B t, B x) {
|
||||
#undef SORT_C1
|
||||
#undef INSERTION_SORT
|
||||
#undef COUNTING_SORT
|
||||
#undef SINGELI_COUNT_OR
|
||||
#if SINGELI_AVX2
|
||||
#undef WRITE_SPARSE_i8
|
||||
#undef WRITE_SPARSE_i16
|
||||
|
||||
@ -85,6 +85,9 @@
|
||||
#define SINGELI_FILE constrep
|
||||
#include "../utils/includeSingeli.h"
|
||||
|
||||
#define SINGELI_FILE count
|
||||
#include "../utils/includeSingeli.h"
|
||||
|
||||
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
|
||||
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
|
||||
@ -850,7 +853,6 @@ B slash_c2(B t, B w, B x) {
|
||||
return c2rt(slash, w, x);
|
||||
}
|
||||
|
||||
|
||||
B slash_im(B t, B x) {
|
||||
if (!isArr(x) || RNK(x)!=1) thrM("/⁼: Argument must be an array");
|
||||
u8 xe = TI(x,elType);
|
||||
@ -904,7 +906,7 @@ B slash_im(B t, B x) {
|
||||
usz m=1<<N; \
|
||||
if (xia < m/2) { \
|
||||
IIND_INT(N) \
|
||||
} else { \
|
||||
} else SINGELI_COUNT_OR(N) { \
|
||||
TALLOC(usz, t, m); \
|
||||
for (usz j=0; j<m/2; j++) t[j]=0; \
|
||||
for (usz i=0; i<xia; i++) t[(u##N)xp[i]]++; \
|
||||
@ -916,8 +918,23 @@ B slash_im(B t, B x) {
|
||||
} \
|
||||
break; \
|
||||
}
|
||||
#if SINGELI_AVX2
|
||||
#define SINGELI_COUNT_OR(N) if (N==8) { \
|
||||
TALLOC(usz, t, m/2); \
|
||||
for (usz j=0; j<m/2; j++) t[j]=0; \
|
||||
i8 max = avx2_count_i8(t, (i8*)xp, xia, 0); \
|
||||
if (max < 0) thrM("/⁼: Argument cannot contain negative numbers"); \
|
||||
usz ria=max+1; \
|
||||
i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i<ria; i++) rp[i]=t[i]; \
|
||||
TFREE(t); \
|
||||
r = num_squeeze(r); \
|
||||
} else
|
||||
#else
|
||||
#define SINGELI_COUNT_OR(N)
|
||||
#endif
|
||||
CASE_SMALL(8) CASE_SMALL(16)
|
||||
#undef CASE_SMALL
|
||||
#undef SINGELI_COUNT_OR
|
||||
case el_i32: { i32* xp = i32any_ptr(x); IIND_INT(32) r = num_squeeze(r); break; }
|
||||
#undef IIND_INT
|
||||
case el_f64: {
|
||||
|
||||
@ -184,8 +184,8 @@ def undefPromote{T, x:X & T==X} = T~~x
|
||||
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
|
||||
def cvt{T, x:X & T==eltype{X}} = x
|
||||
|
||||
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
|
||||
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
|
||||
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}
|
||||
|
||||
|
||||
|
||||
|
||||
72
src/singeli/src/count.singeli
Normal file
72
src/singeli/src/count.singeli
Normal file
@ -0,0 +1,72 @@
|
||||
include './base'
|
||||
include './sse'
|
||||
include './avx'
|
||||
include './avx2'
|
||||
include 'util/tup'
|
||||
include './vecfold'
|
||||
|
||||
fn sum_vec{T==[32]i8}(v:T) = vfold{+, [16]i16~~fold{+, unpackQ{v, T**0}}}
|
||||
|
||||
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}
|
||||
def inc{ptr, ind} = inc{ptr, ind, 1}
|
||||
|
||||
# Write counts /⁼x to tab and return ⌈´x
|
||||
fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = {
|
||||
def vbits = 256
|
||||
def vec = vbits/width{T}
|
||||
def uT = ty_u{T}
|
||||
def V = [vec]T
|
||||
def block = (2048*8) / vbits # Target vectors per block
|
||||
def b_max = block + block/4 # Last block max length
|
||||
assert{b_max < 1<<width{T}} # Don't overflow count in vector section
|
||||
mx:T = min_allowed # Maximum of x
|
||||
i:u64 = 0
|
||||
while (i < n) {
|
||||
# Number of elements to handle in this iteration
|
||||
r:u64 = n - i; if (r > vec*b_max) r = vec*block
|
||||
b := r / vec # Vector case does b full vectors if it runs
|
||||
rv:= b * vec
|
||||
r0:u64 = 0 # Elements actually handled by vector case
|
||||
|
||||
# Find range to check for suitability; return a negative if found
|
||||
xv := *V~~x
|
||||
jv := load{xv}; mv := jv
|
||||
@for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} }
|
||||
@for (x over _ from rv to r) { if (x<min_allowed) return{x}; if (x>mx) mx=x }
|
||||
jt := vfold{min, jv}
|
||||
mt := vfold{max, mv}
|
||||
if (jt < min_allowed) return{jt}
|
||||
if (mt > mx) mx = mt
|
||||
|
||||
nc := uT~~(mt - jt) # Number of counts to perform: last is implicit
|
||||
if (nc <= 48) {
|
||||
r0 = rv
|
||||
j0 := promote{u64, uT~~jt} # Starting count
|
||||
m := promote{u64, nc} # Number of iterations
|
||||
total := trunc{usz, r0} # To compute last count
|
||||
def count_each{js, num} = {
|
||||
j := @collect (k to num) trunc{T, js+k}
|
||||
c := copy{tuplen{j}, [vec]uT ** 0}
|
||||
e := each{{j}=>V**j, j}
|
||||
@for (xv over b) each{{c,e} => c -= xv == e, c, e}
|
||||
def add_sum{c, j} = {
|
||||
s := promote{usz, sum_vec{V}(V~~c)}
|
||||
total -= s; inc{tab, j, s}
|
||||
}
|
||||
each{add_sum, c, j}
|
||||
}
|
||||
m4 := m / 4
|
||||
@for (j4 to m4) count_each{j0 + 4*j4, 4}
|
||||
@for (j from 4*m4 to m) count_each{j0 + j, 1}
|
||||
inc{tab, trunc{T, j0 + m}, trunc{usz,total}}
|
||||
}
|
||||
|
||||
# Scalar fallback and cleanup
|
||||
@for (x over _ from r0 to r) inc{tab, x}
|
||||
i += r
|
||||
x += r
|
||||
}
|
||||
mx
|
||||
}
|
||||
|
||||
export{'avx2_count_i8', count{i8}}
|
||||
@ -71,6 +71,9 @@ def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
|
||||
def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a}
|
||||
def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a}
|
||||
def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
|
||||
def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x}
|
||||
def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x}
|
||||
def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x}
|
||||
|
||||
# TODO don't rely on regular stores being unaligned
|
||||
local def storeu{ptr:P, e:T} = store{ptr, 0, e}
|
||||
|
||||
@ -11,6 +11,7 @@ if (hasarch{'AVX2'}) {
|
||||
include './mask'
|
||||
include './cbqnDefs'
|
||||
include 'util/tup'
|
||||
include './vecfold'
|
||||
|
||||
def preserve_negative_zero = 0
|
||||
|
||||
@ -42,30 +43,6 @@ def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = {
|
||||
|
||||
|
||||
|
||||
def fold{F, x:T} = {
|
||||
show{'WARNING: using fallback fold for ', F, T}
|
||||
def E = eltype{T}
|
||||
r:E = 0
|
||||
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
|
||||
r
|
||||
}
|
||||
def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
|
||||
c:= x
|
||||
def EW = elwidth{T}
|
||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
||||
if (hasarch{'SSSE3'} and 0) {
|
||||
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
|
||||
if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
|
||||
extract{c, 0}
|
||||
} else {
|
||||
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
|
||||
if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} }
|
||||
else extract{c, 0}
|
||||
}
|
||||
}
|
||||
def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
|
||||
|
||||
def makeOptBranch{enable, F} = {
|
||||
if (enable) {
|
||||
def skip = makelabel{}; goto{skip}
|
||||
@ -92,7 +69,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
|
||||
def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E}
|
||||
|
||||
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
|
||||
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
|
||||
def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}}
|
||||
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
|
||||
if (elwidth{T}==64) {
|
||||
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
|
||||
@ -209,4 +186,4 @@ export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}}
|
||||
|
||||
export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}}
|
||||
export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}}
|
||||
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}
|
||||
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}
|
||||
|
||||
21
src/singeli/src/vecfold.singeli
Normal file
21
src/singeli/src/vecfold.singeli
Normal file
@ -0,0 +1,21 @@
|
||||
# Fold associative/commutative operation across a register
|
||||
# Used by squeeze.singeli, count.singeli
|
||||
# Has to be included after util/tup because of name conflict
|
||||
|
||||
def vfold{F, x:T} = {
|
||||
show{'WARNING: using fallback fold for ', F, T}
|
||||
def E = eltype{T}
|
||||
r:E = 0
|
||||
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
|
||||
r
|
||||
}
|
||||
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
|
||||
c:= x
|
||||
def EW = elwidth{T}
|
||||
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
|
||||
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
|
||||
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
|
||||
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} }
|
||||
else extract{c, 0}
|
||||
}
|
||||
def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}
|
||||
Loading…
Reference in New Issue
Block a user