From 7acd9d5688c675f61eec5ca512d5ec9bcfa949f0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 10 Nov 2022 22:21:47 -0500 Subject: [PATCH 1/7] Indices/Replicate implementation comments --- src/builtins/slash.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index fa0ddc8b..b19c0d21 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -1,3 +1,43 @@ +// Indices and Replicate (/) +// In the notes 𝕨 might indicate 𝕩 for Indices too + +// Boolean 𝕨 (Where/Compress) general case based on result type width +// COULD use AVX-512 +// Size 1: pext, or bit-at-a-time +// SHOULD emulate pext if unavailable +// COULD return boolean result from Where +// Size 8, 16: pdep/pext, or branchless +// SHOULD try vector lookup-shuffle if unavailable or old AMD +// Size 32, 64: 16-bit indices from where_block_u16 +// Other sizes: always used grouped code +// Adaptivity based on 𝕨 statistics +// None for 8-bit Where, too short +// COULD try per-block adaptivity for 16-bit Compress +// Sparse if +´𝕨 is small, branchless or branching if very small +// Chosen per-argument for 8, 16 and per-block for larger +// Careful when benchmarking, branch predictor has a long memory +// Grouped if +Β΄Β»βŠΈβ‰ π•¨ is small, always branching +// Chosen per-argument with a threshold that gives up early +// SHOULD implement grouped Where + +// Arbitrary 𝕨 is slower, squeezes if (+Β΄<β‰ )𝕨 to avoid this +// Dense cases (large +´𝕨) use obvious loop, expand 𝕨 to i32 +// Boolean Replicate overwrites to avoid trailing boundary +// Sparse Indices uses ⌈` with Singeli and +` otherwise, i32 output only +// COULD specialize on result type +// Sparse Replicate +// β‰ ` for booleans, +` for CPU types +// TRIED β‰ ` generally, slightly worse +// COULD consolidate refcount updates for nested 𝕩 + +// Replicate by constant +// Boolean uses pdep, β‰ `, or overwriting +// SHOULD make a shift/mask replacement for pdep +// Others use +`, or lots of Singeli +// Fixed shuffles, factorization, partial shuffles, self-overlapping + +// SHOULD do something for odd cell widths in Replicate + #include "../core.h" #include "../utils/mut.h" #include "../utils/calls.h" From 6bea10de219963377974ad7d86a5d9353ab39a95 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 11 Nov 2022 12:14:28 -0500 Subject: [PATCH 2/7] =?UTF-8?q?Implementation=20comments=20and=20minor=20r?= =?UTF-8?q?efactoring=20for=20/=E2=81=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 56 ++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index b19c0d21..5755aef1 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -13,7 +13,7 @@ // Adaptivity based on 𝕨 statistics // None for 8-bit Where, too short // COULD try per-block adaptivity for 16-bit Compress -// Sparse if +´𝕨 is small, branchless or branching if very small +// Sparse if +´𝕨 is small, branchless unless it's very small // Chosen per-argument for 8, 16 and per-block for larger // Careful when benchmarking, branch predictor has a long memory // Grouped if +Β΄Β»βŠΈβ‰ π•¨ is small, always branching @@ -27,7 +27,7 @@ // COULD specialize on result type // Sparse Replicate // β‰ ` for booleans, +` for CPU types -// TRIED β‰ ` generally, slightly worse +// TRIED β‰ ` for CPU types; no better, and clmul would be worse // COULD consolidate refcount updates for nested 𝕩 // Replicate by constant @@ -38,6 +38,18 @@ // SHOULD do something for odd cell widths in Replicate +// Indices inverse (/⁼), a lot like Group +// COULD always give a squeezed result, sometimes expensive +// SHOULD sort large-range 𝕩 to find minimum result type +// Boolean 𝕩: just count 1s +// Long i8 and i16 𝕩: count into zeroed buffer before anything else +// Only zero positive part; if total is too small there were negatives +// Cutoff is set so short 𝕩 gives a result of the same type +// Scan for strictly ascending 𝕩 +// COULD vectorize with find-compare +// COULD find descending too +// Unsigned maximum for integers to avoid a separate negative check + #include "../core.h" #include "../utils/mut.h" #include "../utils/calls.h" @@ -834,32 +846,29 @@ B slash_im(B t, B x) { u8 xe = TI(x,elType); usz xia = IA(x); if (xia==0) { decG(x); return emptyIVec(); } + B r; switch(xe) { default: UD; case el_bit: { usz sum = bit_sum(bitarr_ptr(x), xia); usz ria = 1 + (sum>0); - f64* rp; B r = m_f64arrv(&rp, ria); + f64* rp; r = m_f64arrv(&rp, ria); rp[sum>0] = sum; rp[0] = xia - sum; - decG(x); return num_squeezeChk(r); + r = num_squeeze(r); break; } #define CASE_SMALL(N) \ case el_i##N: { \ i##N* xp = i##N##any_ptr(x); \ usz m=1<xp[a-1]) a++; \ - max=xp[a-1]; \ - if (a==xia) { /* Sorted unique argument */ \ - usz ria = max + 1; \ - u64* rp; r = m_bitarrv(&rp, ria); \ - for (usz i=0; ixp[a-1]) a++; \ + u##N max=xp[a-1]; \ + if (a==xia) { /* Sorted unique argument */ \ + usz ria = max + 1; \ + u64* rp; r = m_bitarrv(&rp, ria); \ + for (usz i=0; imax) max=c; } \ if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ @@ -875,13 +884,13 @@ B slash_im(B t, B x) { i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; imax?c:max; if (c<0) thrM("/⁼: Argument cannot contain negative numbers"); } usz ria = max+1; @@ -892,11 +901,11 @@ B slash_im(B t, B x) { i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; imax?c:max; if (c<0) thrM("/⁼: Argument cannot contain negative numbers"); } usz ria = max+1; if (ria==0) thrOOM(); @@ -907,13 +916,13 @@ B slash_im(B t, B x) { i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; ia; } - usz i,j; B r; i64 max=-1; + usz i,j; i64 max=-1; for (i = 0; i < xia; i++) { i64 c=o2i64(xp[i]); if (c<=max) break; max=c; } for (j = i; j < xia; j++) { i64 c=o2i64(xp[j]); max=c>max?c:max; if (c<0) thrM("/⁼: Argument cannot contain negative numbers"); } if (max > USZ_MAX-1) thrOOM(); @@ -925,9 +934,10 @@ B slash_im(B t, B x) { i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; i Date: Sun, 13 Nov 2022 09:54:13 -0500 Subject: [PATCH 3/7] =?UTF-8?q?Use=20Singeli=20code=20for=20=E2=8D=8Bbool?= =?UTF-8?q?=20and=20=E2=8D=92bool=20with=201-=20and=202-byte=20result?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/grade.h | 23 ++++++++--------------- src/builtins/slash.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index 2ab47965..f59f73ec 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -202,16 +202,17 @@ B SORT_C1(B t, B x) { #endif -#define GRADE_CHR GRADE_UD("⍋","⍒") extern Arr* bitUD[3]; // from fns.c extern B bit2x[2]; +extern B grade_bool(B x, usz ia, bool up); // slash.c + +#define GRADE_CHR GRADE_UD("⍋","⍒") B GRADE_CAT(c1)(B t, B x) { if (isAtm(x) || RNK(x)==0) thrM(GRADE_CHR": Argument cannot be a unit"); if (RNK(x)>1) x = toCells(x); usz ia = IA(x); - if (ia>I32_MAX) thrM(GRADE_CHR": Argument too large"); + B r; if (ia<=2) { - B r; if (ia==2) { SGetU(x); r = incG(bit2x[!(compare(GetU(x,0), GetU(x,1)) GRADE_UD(<=,>=) 0)]); } else if (ia==1) r = taga(ptr_inc(bitUD[1])); else r = emptyIVec(); @@ -220,18 +221,10 @@ B GRADE_CAT(c1)(B t, B x) { } u8 xe = TI(x,elType); - i32* rp; B r = m_i32arrv(&rp, ia); - if (xe==el_bit) { - u64* xp = bitarr_ptr(x); - u64 sum = bit_sum(xp, ia); - u64 r0 = 0; - u64 r1 = GRADE_UD(ia-sum, sum); - for (usz i = 0; i < ia; i++) { - if (bitp_get(xp,i)^GRADE_UD(0,1)) rp[r1++] = i; - else rp[r0++] = i; - } - goto decG_sq; - } else if (xe==el_i8 && ia>8) { + if (xe==el_bit) return grade_bool(x, ia, GRADE_UD(1,0)); + if (ia>I32_MAX) thrM(GRADE_CHR": Argument too large"); + i32* rp; r = m_i32arrv(&rp, ia); + if (xe==el_i8 && ia>8) { i8* xp = i8any_ptr(x); usz n=ia; RADIX_SORT_i8(usz, GRADE); goto decG_sq; diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 5755aef1..1687a102 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -413,6 +413,49 @@ static NOINLINE B zeroCells(B x) { // doesn't consume return r; } +B not_c1(B t, B x); +B grade_bool(B x, usz xia, bool up) { + #define BRANCHLESS_GRADE(T) \ + T* rp; r = m_##T##arrv(&rp, xia); \ + u64 r0 = 0; \ + u64 r1 = l0; \ + if (up) BG_LOOP(!) else BG_LOOP() + #define BG_LOOP(OP) \ + for (usz i = 0; i < xia; i++) { \ + bool b = OP bitp_get(xp,i); \ + rp[b?r0:r1-r0] = i; \ + r0+=b; r1++; \ + } + B r; + u64* xp = bitarr_ptr(x); + u64 sum = bit_sum(xp, xia); + u64 l0 = up? xia-sum : sum; // Length of first set of indices + #if SINGELI && defined(__BMI2__) + if (xia < 16) { BRANCHLESS_GRADE(i8) } + else if (xia <= 1<<15) { + B notx = not_c1(m_f64(0), inc(x)); + u64* xp0 = bitarr_ptr(notx); + u64* xp1 = xp; + if (!up) { u64* t=xp1; xp1=xp0; xp0=t; } + #define BMI_GRADE(W) \ + i##W* rp = m_tyarrvO(&r, W/8, xia, t_i##W##arr, W); \ + bmipopc_1slash##W(xp0, rp , xia); \ + bmipopc_1slash##W(xp1, rp+l0, xia); + if (xia <= 128) { BMI_GRADE(8) } else { BMI_GRADE(16) } + #undef BMI_GRADE + decG(notx); + } + #else + if (xia <= 128) { BRANCHLESS_GRADE(i8) } + else if (xia <= 1<<15) { BRANCHLESS_GRADE(i16) } + #endif + else if (xia <= 1ull<<31) { BRANCHLESS_GRADE(i32) } + else { BRANCHLESS_GRADE(f64) } + #undef BRANCHLESS_GRADE + #undef BG_LOOP + decG(x); return r; +} + void filter_ne_i32(i32* rp, i32* xp, usz len, usz sum, i32 val) { usz b = bsp_max; TALLOC(i16, buf, b + b/16); u64* wp = (u64*)(buf + b); From cd16c8a1c70c4743de295878787434aad2065141 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 13 Nov 2022 11:03:16 -0500 Subject: [PATCH 4/7] =?UTF-8?q?Blocked=204-byte=20=E2=8D=8Bbool=20and=20?= =?UTF-8?q?=E2=8D=92bool=20with=20Singeli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 1687a102..434dd803 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -444,12 +444,27 @@ B grade_bool(B x, usz xia, bool up) { if (xia <= 128) { BMI_GRADE(8) } else { BMI_GRADE(16) } #undef BMI_GRADE decG(notx); + } else if (xia <= 1ull<<31) { + i32* rp0; r = m_i32arrv(&rp0, xia); + i32* rp1 = rp0 + l0; + if (!up) { i32* t=rp1; rp1=rp0; rp0=t; } + usz b = 256; TALLOC(u8, buf, b+b/8); + u64* xp1 = xp; + u64* xp0 = (u64*)(buf + b); + for (usz i=0; ixia-i) b=xia-i; + for (usz j=0; j Date: Sun, 13 Nov 2022 17:59:50 -0500 Subject: [PATCH 5/7] =?UTF-8?q?Check=20for=20a=20small=20return=20type=20i?= =?UTF-8?q?n=20large-range=20/=E2=81=BC=20with=20a=20sparse=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/slash.c | 75 ++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 434dd803..f0c01a10 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -39,16 +39,16 @@ // SHOULD do something for odd cell widths in Replicate // Indices inverse (/⁼), a lot like Group -// COULD always give a squeezed result, sometimes expensive -// SHOULD sort large-range 𝕩 to find minimum result type +// Always gives a squeezed result for integer 𝕩 // Boolean 𝕩: just count 1s // Long i8 and i16 𝕩: count into zeroed buffer before anything else // Only zero positive part; if total is too small there were negatives // Cutoff is set so short 𝕩 gives a result of the same type // Scan for strictly ascending 𝕩 // COULD vectorize with find-compare -// COULD find descending too // Unsigned maximum for integers to avoid a separate negative check +// If (β‰ Γ·βŒˆΒ΄)𝕩 is small, find result type with a sparse u8 table +// COULD use a u16 table for i32 𝕩 to detect i16 result #include "../core.h" #include "../utils/mut.h" @@ -913,26 +913,45 @@ B slash_im(B t, B x) { rp[sum>0] = sum; rp[0] = xia - sum; r = num_squeeze(r); break; } +#define IIND_INT(N) \ + if (xp[0]<0) thrM("/⁼: Argument cannot contain negative numbers"); \ + usz a=1; while (axp[a-1]) a++; \ + u##N max=xp[a-1]; \ + if (amax) max=c; } \ + if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ + usz ria = max + 1; \ + if (xia < ria/8) { \ + u8 maxcount = 0; \ + TALLOC(u8, tab, ria); \ + for (usz i=0; i=16 && maxcount<127) { \ + i8* rp; r = m_i8arrv(&rp, ria); for (usz i=0; ixp[a-1]) a++; \ - u##N max=xp[a-1]; \ - if (a==xia) { /* Sorted unique argument */ \ - usz ria = max + 1; \ - u64* rp; r = m_bitarrv(&rp, ria); \ - for (usz i=0; imax) max=c; } \ - if ((i##N)max<0) thrM("/⁼: Argument cannot contain negative numbers"); \ - usz ria = max+1; \ - i##N* rp; r = m_i##N##arrv(&rp, ria); for (usz i=0; im/2) thrM("/⁼: Argument cannot contain negative numbers"); \ i32* rp; r = m_i32arrv(&rp, ria); for (usz i=0; imax?c:max; if (c<0) thrM("/⁼: Argument cannot contain negative numbers"); } - usz ria = max+1; - if (i==xia) { - u64* rp; r = m_bitarrv(&rp, ria); for (usz i=0; i Date: Mon, 14 Nov 2022 18:00:00 +0200 Subject: [PATCH 6/7] add some NOUNROLLs --- src/builtins/slash.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index f0c01a10..c08e8956 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -453,9 +453,9 @@ B grade_bool(B x, usz xia, bool up) { u64* xp0 = (u64*)(buf + b); for (usz i=0; ixia-i) b=xia-i; - for (usz j=0; j Date: Mon, 14 Nov 2022 19:02:44 +0200 Subject: [PATCH 7/7] better negation buffer handling --- src/builtins/slash.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/builtins/slash.c b/src/builtins/slash.c index c08e8956..019bbffc 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -448,15 +448,15 @@ B grade_bool(B x, usz xia, bool up) { i32* rp0; r = m_i32arrv(&rp0, xia); i32* rp1 = rp0 + l0; if (!up) { i32* t=rp1; rp1=rp0; rp0=t; } - usz b = 256; TALLOC(u8, buf, b+b/8); + usz b = 256; TALLOC(u8, buf, b); + u64 xp0[4]; // 4 ≑ b/64 u64* xp1 = xp; - u64* xp0 = (u64*)(buf + b); for (usz i=0; ixia-i) b=xia-i; - NOUNROLL for (usz j=0; jxia-i? xia-i : b; + usz s0=bit_sum(xp0,b2); bmipopc_1slash8(xp0, (i8*)buf, b2); for (usz j=0; j