diff --git a/build/src/build.bqn b/build/src/build.bqn index 9ade08f4..b354f478 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -604,6 +604,7 @@ cachedBin‿linkerCache ← { "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", + "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/makefile b/makefile index 9b992474..3d3927c5 100644 --- a/makefile +++ b/makefile @@ -352,7 +352,7 @@ endif @"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c count.c bits.c transpose.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/grade.h b/src/builtins/grade.h index bf270303..828a8e69 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -81,6 +81,9 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); if (e==n) {break;} k=e; \ } #define WRITE_SPARSE(T) WRITE_SPARSE_##T +extern i8 (*const avx2_count_i8)(usz*, i8*, u64, i8); +#define SINGELI_COUNT_OR(T) \ + if (1==sizeof(T)) avx2_count_i8(c0o, (i8*)xp, n, -128); else #else #define COUNT_THRESHOLD 16 #define WRITE_SPARSE(T) \ @@ -88,13 +91,14 @@ extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); usz js = j; \ while (ijb, a, b} +def min{a, b & anyNum{a} & anyNum{b}} = tern{ab, a, b} diff --git a/src/singeli/src/count.singeli b/src/singeli/src/count.singeli new file mode 100644 index 00000000..d7ae54c3 --- /dev/null +++ b/src/singeli/src/count.singeli @@ -0,0 +1,72 @@ +include './base' +include './sse' +include './avx' +include './avx2' +include 'util/tup' +include './vecfold' + +fn sum_vec{T==[32]i8}(v:T) = vfold{+, [16]i16~~fold{+, unpackQ{v, T**0}}} + +def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} +def inc{ptr, ind} = inc{ptr, ind, 1} + +# Write counts /⁼x to tab and return ⌈´x +fn count{T}(tab:*usz, x:*T, n:u64, min_allowed:T) : T = { + def vbits = 256 + def vec = vbits/width{T} + def uT = ty_u{T} + def V = [vec]T + def block = (2048*8) / vbits # Target vectors per block + def b_max = block + block/4 # Last block max length + assert{b_max < 1< vec*b_max) r = vec*block + b := r / vec # Vector case does b full vectors if it runs + rv:= b * vec + r0:u64 = 0 # Elements actually handled by vector case + + # Find range to check for suitability; return a negative if found + xv := *V~~x + jv := load{xv}; mv := jv + @for (xv over _ from 1 to b) { jv = min{jv, xv}; mv = max{mv, xv} } + @for (x over _ from rv to r) { if (xmx) mx=x } + jt := vfold{min, jv} + mt := vfold{max, mv} + if (jt < min_allowed) return{jt} + if (mt > mx) mx = mt + + nc := uT~~(mt - jt) # Number of counts to perform: last is implicit + if (nc <= 48) { + r0 = rv + j0 := promote{u64, uT~~jt} # Starting count + m := promote{u64, nc} # Number of iterations + total := trunc{usz, r0} # To compute last count + def count_each{js, num} = { + j := @collect (k to num) trunc{T, js+k} + c := copy{tuplen{j}, [vec]uT ** 0} + e := each{{j}=>V**j, j} + @for (xv over b) each{{c,e} => c -= xv == e, c, e} + def add_sum{c, j} = { + s := promote{usz, sum_vec{V}(V~~c)} + total -= s; inc{tab, j, s} + } + each{add_sum, c, j} + } + m4 := m / 4 + @for (j4 to m4) count_each{j0 + 4*j4, 4} + @for (j from 4*m4 to m) count_each{j0 + j, 1} + inc{tab, trunc{T, j0 + m}, trunc{usz,total}} + } + + # Scalar fallback and cleanup + @for (x over _ from r0 to r) inc{tab, x} + i += r + x += r + } + mx +} + +export{'avx2_count_i8', count{i8}} diff --git a/src/singeli/src/neon.singeli b/src/singeli/src/neon.singeli index 810a84d4..53575347 100644 --- a/src/singeli/src/neon.singeli +++ b/src/singeli/src/neon.singeli @@ -71,6 +71,9 @@ def fold_add {a:T & nvec{T}} = emit{eltype{T}, ntyp{'vaddv', T}, a} def fold_addw{a:T & nvec{T}} = emit{ty_dbl{eltype{T}}, ntyp{'vaddlv', T}, a} def fold_min {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vminv', T}, a} def fold_max {a:T & nvec{T} & elwidth{T}<=32} = emit{eltype{T}, ntyp{'vmaxv', T}, a} +def vfold{F, x:T & nvec{T} & match{F, min} & elwidth{T}<=32} = fold_min{x} +def vfold{F, x:T & nvec{T} & match{F, max} & elwidth{T}<=32} = fold_max{x} +def vfold{F, x:T & nvec{T} & match{F, +}} = fold_add{x} # TODO don't rely on regular stores being unaligned local def storeu{ptr:P, e:T} = store{ptr, 0, e} diff --git a/src/singeli/src/squeeze.singeli b/src/singeli/src/squeeze.singeli index a282dfd7..35348be3 100644 --- a/src/singeli/src/squeeze.singeli +++ b/src/singeli/src/squeeze.singeli @@ -11,6 +11,7 @@ if (hasarch{'AVX2'}) { include './mask' include './cbqnDefs' include 'util/tup' +include './vecfold' def preserve_negative_zero = 0 @@ -42,30 +43,6 @@ def anyNonChar{M, x:T & isvec{T} & hasarch{'X86_64'}} = { -def fold{F, x:T} = { - show{'WARNING: using fallback fold for ', F, T} - def E = eltype{T} - r:E = 0 - each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} - r -} -def fold{F, x:T & w128{T} & hasarch{'X86_64'}} = { - c:= x - def EW = elwidth{T} - if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} - if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} - if (hasarch{'SSSE3'} and 0) { - if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} - if (EW==8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} - extract{c, 0} - } else { - if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} - if (EW==8) { v:=extract{[8]i16~~c, 0}; cast_i{u8, F{v, v>>8}} } - else extract{c, 0} - } -} -def fold{F, x:T & w256{T} & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} - def makeOptBranch{enable, F} = { if (enable) { def skip = makelabel{}; goto{skip} @@ -92,7 +69,7 @@ fn squeeze{vw, X, CHR, B}(x0:*void, len:Size) : u32 = { def EV = tern{(width{E}*bulk == 64) & hasarch{'X86_64'}, EV2, [bulk]E} # fold with either Max or Bitwise Or, truncating/zero-extending to TE - def foldTotal{TE, x:T} = cast_i{TE, fold{|, x}} + def foldTotal{TE, x:T} = cast_i{TE, vfold{|, x}} def foldTotal{TE, x:T & hasarch{'AARCH64'}} = { if (elwidth{T}==64) { if (width{TE}==64 and bulk==2) cast_i{TE, half{x,0} | half{x,1}} @@ -209,4 +186,4 @@ export{'avx2_squeeze_numB', squeeze{arch_defvw, f64, 0, 1}} export{'avx2_squeeze_c16', squeeze{arch_defvw, u16, 1, 0}} export{'avx2_squeeze_c32', squeeze{arch_defvw, u32, 1, 0}} -export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} \ No newline at end of file +export{'avx2_squeeze_chrB', squeeze{arch_defvw, u64, 1, 1}} diff --git a/src/singeli/src/vecfold.singeli b/src/singeli/src/vecfold.singeli new file mode 100644 index 00000000..330122eb --- /dev/null +++ b/src/singeli/src/vecfold.singeli @@ -0,0 +1,21 @@ +# Fold associative/commutative operation across a register +# Used by squeeze.singeli, count.singeli +# Has to be included after util/tup because of name conflict + +def vfold{F, x:T} = { + show{'WARNING: using fallback fold for ', F, T} + def E = eltype{T} + r:E = 0 + each{{i} => { r = F{r, extract{x, i}} }, iota{vcount{T}}} + r +} +def vfold{F, x:T & w128{T} & hasarch{'X86_64'}} = { + c:= x + def EW = elwidth{T} + if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} + if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} + if (EW<=16) c = F{c, shuf16Lo{c, 4b2301}} + if (EW==8) { v:=extract{[8]i16~~c, 0}; F{cast_i{eltype{T}, v}, cast_i{eltype{T}, v>>8}} } + else extract{c, 0} +} +def vfold{F, x:T & w256{T} & hasarch{'X86_64'}} = vfold{F, F{half{x, 0}, half{x, 1}}}