Merge pull request #73 from mlochbaum/count

Count by vector compare+add
This commit is contained in:
dzaima 2023-04-06 19:19:25 +03:00 committed by GitHub
commit 65f25f735d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 128 additions and 32 deletions

View File

@ -604,6 +604,7 @@ cachedBin‿linkerCache ← {
"2..""src/builtins/select.c""select", "2..""src/builtins/scan.c""scan",
"2..""src/builtins/fold.c""fold", "2..""src/builtins/slash.c""slash",
"2..""src/builtins/slash.c""constrep", "2..""src/builtins/scan.c""neq",
"2..""src/builtins/slash.c""count"
objs

View File

@ -352,7 +352,7 @@ endif
@"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c}
build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c count.c bits.c transpose.c}
@echo $(postmsg)
src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin
@echo $< | cut -c 17- | sed 's/^/ /'

View File

@ -81,6 +81,9 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
if (e==n) {break;} k=e; \
}
#define WRITE_SPARSE(T) WRITE_SPARSE_##T
extern i8 (*const avx2_count_i8)(usz*, i8*, u64, i8);
#define SINGELI_COUNT_OR(T) \
if (1==sizeof(T)) avx2_count_i8(c0o, (i8*)xp, n, -128); else
#else
#define COUNT_THRESHOLD 16
#define WRITE_SPARSE(T) \
@ -88,13 +91,14 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2);
usz js = j; \
while (ij<n) { rp[ij]GRADE_UD(++,--); ij+=c0o[GRADE_UD(++j,--j)]; } \
for (usz i=0; i<n; i++) js=rp[i]+=js;
#define SINGELI_COUNT_OR(T)
#endif
#define COUNTING_SORT(T) \
usz C=1<<(8*sizeof(T)); \
TALLOC(usz, c0, C); usz *c0o=c0+C/2; \
for (usz j=0; j<C; j++) c0[j]=0; \
for (usz i=0; i<n; i++) c0o[xp[i]]++; \
SINGELI_COUNT_OR(T) for (usz i=0; i<n; i++) c0o[xp[i]]++; \
if (n/(COUNT_THRESHOLD*sizeof(T)) <= C) { /* Scan-based */ \
T j=GRADE_UD(-C/2,C/2-1); \
usz ij; while ((ij=c0o[j])==0) GRADE_UD(j++,j--); \
@ -227,6 +231,7 @@ B SORT_C1(B t, B x) {
#undef SORT_C1
#undef INSERTION_SORT
#undef COUNTING_SORT
#undef SINGELI_COUNT_OR
#if SINGELI_AVX2
#undef WRITE_SPARSE_i8
#undef WRITE_SPARSE_i16

View File

@ -85,6 +85,9 @@
#define SINGELI_FILE constrep
#include "../utils/includeSingeli.h"
#define SINGELI_FILE count
#include "../utils/includeSingeli.h"
extern void (*const avx2_scan_pluswrap_u8)(uint8_t* v0,uint8_t* v1,uint64_t v2,uint8_t v3);
extern void (*const avx2_scan_pluswrap_u16)(uint16_t* v0,uint16_t* v1,uint64_t v2,uint16_t v3);
extern void (*const avx2_scan_pluswrap_u32)(uint32_t* v0,uint32_t* v1,uint64_t v2,uint32_t v3);
@ -850,7 +853,6 @@ B slash_c2(B t, B w, B x) {
return c2rt(slash, w, x);
}
B slash_im(B t, B x) {
if (!isArr(x) || RNK(x)!=1) thrM("/⁼: Argument must be an array");
u8 xe = TI(x,elType);
@ -904,7 +906,7 @@ B slash_im(B t, B x) {
usz m=1<<N; \
if (xia < m/2) { \
IIND_INT(N) \
} else { \
} else SINGELI_COUNT_OR(N) { \
TALLOC(usz, t, m); \
for (usz j=0; j<m/2; j++) t[j]=0; \
for (usz i=0; i<xia; i++) t[(u##N)xp[i]]++; \
@ -916,8 +918,23 @@ B slash_im(B t, B x) {
} \
break; \
}
#if SINGELI_AVX2
#define SINGELI_COUNT_OR(N) if (N==8) { \
TALLOC(usz, t, m/2); \
for (usz j=0; j<m/2; j++) t[j]=0; \
i8 max = avx2_count_i8(t, (i8*)xp, xia, 0); \
if (max < 0) thrM("/⁼: Argument cannot contain negative numbers"); \
usz ria=max+1; \
i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i<ria; i++) rp[i]=t[i]; \
TFREE(t); \
r = num_squeeze(r); \
} else
#else
#define SINGELI_COUNT_OR(N)
#endif
CASE_SMALL(8) CASE_SMALL(16)
#undef CASE_SMALL
#undef SINGELI_COUNT_OR
case el_i32: { i32* xp = i32any_ptr(x); IIND_INT(32) r = num_squeeze(r); break; }
#undef IIND_INT
case el_f64: {

View File

@ -184,8 +184,8 @@ def undefPromote{T, x:X & T==X} = T~~x
def packQ{a} = packQ{tupsel{0,a}, tupsel{1,a}}
def cvt{T, x:X & T==eltype{X}} = x
def min{a, b & knum{a} & knum{b}} = tern{a<b, a, b}
def max{a, b & knum{a} & knum{b}} = tern{a>b, a, b}
def min{a, b & anyNum{a} & anyNum{b}} = tern{a<b, a, b}
def max{a, b & anyNum{a} & anyNum{b}} = tern{a>b, a, b}

View File

@ -0,0 +1,72 @@
include './base'
include './sse'
include './avx'
include './avx2'
include 'util/tup'
include './vecfold'
fn sum_vec{T==[32]i8}(v:T) = vfold{+, [16]i16~~fold{+, unpackQ{v, T**0}}}
def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}}
def inc{ptr, ind} = inc{ptr, ind, 1}
# Write counts /⁼x to tab and return ⌈´x
fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = {
def vbits = 256
def vec = vbits/width{T}
def uT = ty_u{T}
def V = [vec]T
def block = (2048*8) / vbits # Target vectors per block
def b_max = block + block/4 # Last block max length
assert{b_max < 1<<width{T}} # Don't overflow count in vector section
mx:T = min_allowed # Maximum of x
i:u64 = 0
while (i < n) {
# Number of elements to handle in this iteration
r:u64 = n - i; if (r > vec*b_max) r = vec*block
b := r / vec # Vector case does b full vectors if it runs
rv:= b * vec
r0:u64 = 0 # Elements actually handled by vector case
# Find range to check for suitability; return a negative if found
xv := *V~~x
jv := load{xv}; mv := jv
@for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} }
@for (x over _ from rv to r) { if (x<min_allowed) return{x}; if (x>mx) mx=x }
jt := vfold{min, jv}
mt := vfold{max, mv}
if (jt < min_allowed) return{jt}
if (mt > mx) mx = mt
nc := uT~~(mt - jt) # Number of counts to perform: last is implicit
if (nc <= 48) {
r0 = rv
j0 := promote{u64, uT~~jt} # Starting count
m := promote{u64, nc} # Number of iterations
total := trunc{usz, r0} # To compute last count
def count_each{js, num} = {
j := @collect (k to num) trunc{T, js+k}
c := copy{tuplen{j}, [vec]uT ** 0}
e := each{{j}=>V**j, j}
@for (xv over b) each{{c,e} => c -= xv == e, c, e}
def add_sum{c, j} = {
s := promote{usz, sum_vec{V}(V~~c)}
total -= s; inc{tab, j, s}
}
each{add_sum, c, j}
}
m4 := m / 4
@for (j4 to m4) count_each{j0 + 4*j4, 4}
@for (j from 4*m4 to m) count_each{j0 + j, 1}
inc{tab, trunc{T, j0 + m}, trunc{usz,total}}
}
# Scalar fallback and cleanup
@for (x over _ from r0 to r) inc{tab, x}
i += r
x += r
}
mx
}
export{'avx2_count_i8', count{i8}}

View File

@ -71,6 +71,9 @@ def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a}
def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a}
def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a}
def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a}
def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x}
def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x}
def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x}
# TODO don't rely on regular stores being unaligned
local def storeu{ptr:P, e:T} = store{ptr, 0, e}

View File

@ -11,6 +11,7 @@ if (hasarch{'AVX2'}) {
include './mask'
include './cbqnDefs'
include 'util/tup'
include './vecfold'
def preserve_negative_zero = 0
@ -42,30 +43,6 @@ def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = {
def fold{F, x:T} = {
show{'WARNING: using fallback fold for ', F, T}
def E = eltype{T}
r:E = 0
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
r
}
def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
if (hasarch{'SSSE3'} and 0) {
if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}}
if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}}
extract{c, 0}
} else {
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} }
else extract{c, 0}
}
}
def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}}
def makeOptBranch{enable, F} = {
if (enable) {
def skip = makelabel{}; goto{skip}
@ -92,7 +69,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = {
def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E}
# fold with either Max or Bitwise Or, truncating/zero-extending to TE
def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}}
def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}}
def foldTotal{TE, x:T & hasarch{'AARCH64'}} = {
if (elwidth{T}==64) {
if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}}
@ -209,4 +186,4 @@ export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}}
export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}}
export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}}
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}
export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}}

View File

@ -0,0 +1,21 @@
# Fold associative/commutative operation across a register
# Used by squeeze.singeli, count.singeli
# Has to be included after util/tup because of name conflict
def vfold{F, x:T} = {
show{'WARNING: using fallback fold for ', F, T}
def E = eltype{T}
r:E = 0
each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}}
r
}
def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = {
c:= x
def EW = elwidth{T}
if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}}
if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}}
if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}}
if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} }
else extract{c, 0}
}
def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}