From b655dd47710199b2fdc65f64839f612a9b6b5600 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 1 Mar 2023 22:21:11 -0500 Subject: [PATCH] =?UTF-8?q?AVX2=20counting=20function=20for=20small-range?= =?UTF-8?q?=201-byte=20/=E2=81=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/src/build.bqn | 1 + makefile | 2 +- src/builtins/slash.c | 10 ++++- src/singeli/src/count.singeli | 71 +++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 src/singeli/src/count.singeli diff --git a/build/src/build.bqn b/build/src/build.bqn index 9ade08f4..b354f478 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -604,6 +604,7 @@ cachedBin‿linkerCache ← { "2.."‿"src/builtins/select.c"‿"select", "2.."‿"src/builtins/scan.c"‿"scan", "2.."‿"src/builtins/fold.c"‿"fold", "2.."‿"src/builtins/slash.c"‿"slash", "2.."‿"src/builtins/slash.c"‿"constrep", "2.."‿"src/builtins/scan.c"‿"neq", + "2.."‿"src/builtins/slash.c"‿"count" ⟩ objs ← ⟨⟩ diff --git a/makefile b/makefile index 9b992474..3d3927c5 100644 --- a/makefile +++ b/makefile @@ -352,7 +352,7 @@ endif @"${MAKE}" i_singeli=0 singeli=0 force_build_dir=build/obj/presingeli REPLXX=0 f= lf= postmsg="singeli sources:" i_t=presingeli i_f='-O1 -DPRE_SINGELI' FFI=0 OUTPUT=build/obj/presingeli/BQN c -build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c bits.c transpose.c} +build_singeli: ${addprefix src/singeli/gen/, cmp.c dyarith.c monarith.c copy.c equal.c squeeze.c select.c fold.c scan.c neq.c slash.c constrep.c count.c bits.c transpose.c} @echo $(postmsg) src/singeli/gen/%.c: src/singeli/src/%.singeli preSingeliBin @echo $< | cut -c 17- | sed 's/^/ /' diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 9d127312..ed82ed13 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -850,6 +850,14 @@ B slash_c2(B t, B w, B x) { return c2rt(slash, w, x); } +#if SINGELI_AVX2 + #define SINGELI_FILE count + #include "../utils/includeSingeli.h" + #define SINGELI_COUNT_OR(N) \ + if (N==8) avx2_count_i8(t, (u8*)xp, xia); else +#else + #define SINGELI_COUNT_OR(N) +#endif B slash_im(B t, B x) { if (!isArr(x) || RNK(x)!=1) thrM("/⁼: Argument must be an array"); @@ -907,7 +915,7 @@ B slash_im(B t, B x) { } else { \ TALLOC(usz, t, m); \ for (usz j=0; jm/2) thrM("/⁼: Argument cannot contain negative numbers"); \ i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i { r = F{r, extract{x, i}} }, iota{vcount{T}}} + r +} +def fold{F, x:T & width{T}==128 & hasarch{'X86_64'}} = { + c:= x + def EW = elwidth{T} + if (EW<=64) c = F{c, shuf{[4]u32, c, 4b1032}} + if (EW<=32) c = F{c, shuf{[4]u32, c, 4b2301}} + if (EW<=16) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^2}}} + if (EW<=8) c = F{c, sel{[16]u8, c, make{[16]i8, iota{16}^1}}} + extract{c, 0} +} +def fold{F, x:T & width{T}==256 & hasarch{'X86_64'}} = fold{F, F{half{x, 0}, half{x, 1}}} + +def inc{ptr, ind, v} = store{ptr, ind, v + load{ptr, ind}} +def inc{ptr, ind} = inc{ptr, ind, 1} + +fn count{T}(tab:*usz, x:*ty_u{T}, n:u64) : u1 = { + def vbits = 256 + def vec = vbits/width{T} + def uT = ty_u{T} + def V = [vec]uT + def iV = [vec]T + def block = (1024*8) / vbits # Desired vectors per block + i:u64 = 0 + while (i < n) { + r:u64 = n - i + b := r / vec + xv := *V~~x + used_eq:u1 = 0 + if (r >= 256) { + b = block; if (r < vec*b) b = r / vec + mv := V**0 + @for (xv over b) mv = max{mv, xv} + mi := iV~~mv + if (homAny{mi < iV**0}) return{1} + if (homAll{mi <= iV**32}) { + used_eq = 1 + r = b * vec + m := fold{max, mv} + total := b*vec + @for (j to promote{u64,m}) { + c := V**0 + e := V**trunc{uT, j} + @for (xv over b) c -= xv == e + s := fold{+, [vec/2]i16~~fold{+, unpackQ{iV~~c, iV**0}}} + total -= promote{u64, s} + inc{tab, j, promote{usz, s}} + } + inc{tab, m, trunc{usz,total}} + } + } + if (not used_eq) @for (x over r) inc{tab, x} + i += r + x += r + } + 0 +} + +export{'avx2_count_i8', count{i8}}