From 856eabd493ab73644c74d674a34cea38d0075e59 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 14 Nov 2022 21:05:45 -0500 Subject: [PATCH 01/15] Self-search implementation comments --- src/builtins/selfsearch.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index bc5aa5ef..29fab99d 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -1,3 +1,31 @@ +// Self-search: Mark Firsts (∊), Occurrence Count (⊒), Classify (⊐), Deduplicate (⍷) + +// Except for trivial cases, ⍷ is implemented as ∊⊸/ +// Other functions use adaptations of the same set of methods + +// Boolean ∊: 1 at first element and first ¬⊑𝕩 +// COULD do a branchless thing for boolean ⊒ instead of converting to i8 +// Boolean ⊐: ⥊¬⍟⊑𝕩 +// SHOULD implement boolean ⍷ directly +// Brute force or all-pairs comparison for small lengths +// Branchless, not vectorized (+´∧` structure for ⊐) +// Full-size table lookups for 1- and 2-byte 𝕩 +// 2-byte table can be "sparse" initialized with an extra pass over 𝕩 +// 4-byte ⊐ can use a small-range lookup table +// COULD add small-range 4-byte tables for ∊ and ⊒ +// Radix-assisted lookups are fallbacks for 4-byte ∊ and ⊒ +// COULD do radix-assisted ⊐ as ⍷⊸⊐ or similar +// Specializes on constant top 1/2 bytes, but hashes make this rare +// SHOULD check for sorted flags to use »⊸≠ + +// Specialized 4-byte and 8-byte hash tables +// In-place resizing by factor of 4 based on measured collisions +// Max collisions ensures bounded time spent here before giving up +// First element used as sentinel (not good for ⊒) +// COULD prefetch when table gets larger +// Generic hash table for other cases +// Resizing is pretty expensive here + #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" @@ -6,6 +34,7 @@ B not_c1(B t, B x); B shape_c1(B t, B x); B slash_c2(B t, B w, B x); +// These hashes are stored in tables and must be invertible! #if defined(__SSE4_2__) static inline u32 hash32(u32 x) { return _mm_crc32_u32(0x973afb51, x); } #else From 4506caf3ac2e534fb6f982a4383014097b3d6ee4 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 15 Nov 2022 10:32:01 -0500 Subject: [PATCH 02/15] =?UTF-8?q?Faster=20self-search=20on=20sorted=20?= =?UTF-8?q?=F0=9D=95=A9=20using=20=C2=BB=E2=8A=B8=E2=89=A0=F0=9D=95=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/selfsearch.c | 47 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 29fab99d..c0346d75 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -7,6 +7,9 @@ // COULD do a branchless thing for boolean ⊒ instead of converting to i8 // Boolean ⊐: ⥊¬⍟⊑𝕩 // SHOULD implement boolean ⍷ directly +// Sorted flags: start with r0⌾⊑»⊸≠𝕩 (r0 is 0 for ⊐ and 1 otherwise) +// ∊: ⊢; ⊐: +`; ⊒: ↕∘≠⊸(⊣-⌈`∘×) +// COULD determine ⊒ result type by direct comparisons on 𝕩 // Brute force or all-pairs comparison for small lengths // Branchless, not vectorized (+´∧` structure for ⊐) // Full-size table lookups for 1- and 2-byte 𝕩 @@ -16,7 +19,6 @@ // Radix-assisted lookups are fallbacks for 4-byte ∊ and ⊒ // COULD do radix-assisted ⊐ as ⍷⊸⊐ or similar // Specializes on constant top 1/2 bytes, but hashes make this rare -// SHOULD check for sorted flags to use »⊸≠ // Specialized 4-byte and 8-byte hash tables // In-place resizing by factor of 4 based on measured collisions @@ -29,10 +31,16 @@ #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" +#include "../utils/calls.h" +#include "../builtins.h" -B not_c1(B t, B x); -B shape_c1(B t, B x); -B slash_c2(B t, B w, B x); +extern B not_c1(B, B); +extern B shape_c1(B, B); +extern B slash_c2(B, B, B); +extern B scan_c1(Md1D*, B); +extern B ud_c1(B, B); +extern B sub_c2(B, B, B); +extern B mul_c2(B, B, B); // These hashes are stored in tables and must be invertible! #if defined(__SSE4_2__) @@ -53,6 +61,20 @@ static inline u64 hash64(u64 x) { return x; } +static inline bool use_sorted(B x, u8 logw) { + if (!FL_HAS(x, fl_asc|fl_dsc)) return 0; + if (logw==6) return TI(x, elType) == el_f64; + return 3<=logw & logw<=5; +} +static inline B shift_ne(B x, usz n, u8 lw, bool r0) { // consumes x + u64* rp; B r = m_bitarrv(&rp, n); + u8* xp = tyany_ptr(x); + u8 lb = lw - 3; + CMP_AA_IMM(ne, el_i8+lb, rp, xp-(1<16 && (lw>4 || n<1<<16)) { // ↕∘≠(⊣-⌈`∘×)∊ + B c = shift_ne(x, n, lw, 1); + B i = ud_c1(m_f64(0), m_f64(n)); + Md1D d; d.f = bi_ceil; + B m = scan_c1(&d, mul_c2(m_f64(0), c, inc(i))); + return sub_c2(m_f64(0), i, m); + } void* xv = tyany_ptr(x); #define BRUTE(T) \ i##T* xp = xv; \ @@ -309,7 +341,7 @@ B count_c1(B t, B x) { /*THRESHMUL*/1, THRESH, \ /*INIT*/u32 ctr0 = 1;) if (lw==5) { - if (n<12) { BRUTE(32); } + if (n<20) { BRUTE(32); } i32* rp; B r = m_i32arrv(&rp, n); HASHTAB(u32, 32, 1, n/2, sz==msz? 1 : sz>=(1<<14)? 3 : 5) // Radix-assisted lookup @@ -366,6 +398,11 @@ B indexOf_c1(B t, B x) { B r = 1&*(u64*)xv ? not_c1(m_f64(0), x) : x; return shape_c1(m_f64(0), r); } + if (use_sorted(x, lw) && n>8) { + B r = shift_ne(x, n, lw, 0); + Md1D d; d.f = bi_add; + return scan_c1(&d, r); + } #define BRUTE(T) \ i##T* xp = xv; \ i8* rp; B r = m_i8arrv(&rp, n); rp[0]=0; \ From d6b09adb5792cb7de116898da5c2dd715a909574 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 15 Nov 2022 16:08:34 -0500 Subject: [PATCH 03/15] Add primitive and primitive-1modifier call macros to calls.h --- src/builtins/group.c | 23 +++++++++-------------- src/builtins/search.c | 2 +- src/builtins/selfsearch.c | 17 +++++++---------- src/utils/calls.h | 9 ++++++++- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/builtins/group.c b/src/builtins/group.c index 6d945136..9f2cff82 100644 --- a/src/builtins/group.c +++ b/src/builtins/group.c @@ -100,9 +100,9 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { } if (we==el_bit) { assert(ria == 2); - if (wia>xn) w = take_c2(m_f64(0), m_f64(xn), w); - rp[1] = slash_c2(m_f64(0), inc(w), inc(x)); - rp[0] = slash_c2(m_f64(0), not_c1(m_f64(0), w), x); + if (wia>xn) w = C2(take, m_f64(xn), w); + rp[1] = C2(slash, inc(w), inc(x)); + rp[0] = C2(slash, C1(not, w), x); return taga(r); } // Needed to make sure wia>0 for ip[wia-1] below @@ -128,9 +128,6 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { // Few changes in 𝕨: move in chunks if (xn>64 && notB && change<(xn*width)/32) { - #define C1(F,X ) F##_c1(m_f64(0),X ) - #define C2(F,X,W) F##_c2(m_f64(0),X,W) - u64* mp; B m = m_bitarrv(&mp, xn); u8* wp0 = tyany_ptr(w); we = TI(w,elType); @@ -139,8 +136,6 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { B ind = C1(slash, m); w = C2(select, inc(ind), w); - #undef C1 - #undef C2 if (TI(ind,elType)!=el_i32) ind = taga(cpyI32Arr(ind)); if (TI(w ,elType)!=el_i32) w = taga(cpyI32Arr(w )); wia = IA(ind); @@ -179,10 +174,10 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { // Many ¯1s: filter out, then continue if (xn>32 && neg>(bits?0:xn/4)+xn/8) { - if (wia>xn) w = take_c2(m_f64(0), m_f64(xn), w); - B m = ne_c2(m_f64(0), m_f64(-1), inc(w)); - w = slash_c2(m_f64(0), inc(m), w); - x = slash_c2(m_f64(0), m, x); xn = *SH(x); + if (wia>xn) w = C2(take, m_f64(xn), w); + B m = C2(ne, m_f64(-1), inc(w)); + w = C2(slash, inc(m), w); + x = C2(slash, m, x); xn = *SH(x); neg = 0; } if (TI(w,elType)!=el_i32) w = taga(cpyI32Arr(w)); @@ -331,8 +326,8 @@ B group_c2(B t, B w, B x) { B group_c1(B t, B x) { if (isArr(x) && RNK(x)==1 && TI(x,arrD1)) { usz ia = IA(x); - B range = ud_c1(t, m_f64(ia)); - return group_c2(m_f64(0), x, range); + B range = C1(ud, m_f64(ia)); + return C2(group, x, range); } return c1(rt_group, x); } diff --git a/src/builtins/search.c b/src/builtins/search.c index 330f9285..ff1eeae3 100644 --- a/src/builtins/search.c +++ b/src/builtins/search.c @@ -1,8 +1,8 @@ #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" +#include "../utils/calls.h" -#define C2(F,X,W) F##_c2(m_f64(0),X,W) #define C2i(F, W, X) C2(F, m_i32(W), X) extern B eq_c2(B,B,B); extern B ne_c2(B,B,B); diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index c0346d75..d7d046ea 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -304,10 +304,9 @@ B count_c1(B t, B x) { if (lw==0) { x = toI8Any(x); lw = cellWidthLog(x); } if (use_sorted(x, lw) && n>16 && (lw>4 || n<1<<16)) { // ↕∘≠(⊣-⌈`∘×)∊ B c = shift_ne(x, n, lw, 1); - B i = ud_c1(m_f64(0), m_f64(n)); - Md1D d; d.f = bi_ceil; - B m = scan_c1(&d, mul_c2(m_f64(0), c, inc(i))); - return sub_c2(m_f64(0), i, m); + B i = C1(ud, m_f64(n)); + B m = M1C1(scan, ceil, C2(mul, c, inc(i))); + return C2(sub, i, m); } void* xv = tyany_ptr(x); #define BRUTE(T) \ @@ -395,13 +394,11 @@ B indexOf_c1(B t, B x) { u8 lw = cellWidthLog(x); void* xv = tyany_ptr(x); if (lw == 0) { - B r = 1&*(u64*)xv ? not_c1(m_f64(0), x) : x; - return shape_c1(m_f64(0), r); + B r = 1&*(u64*)xv ? C1(not, x) : x; + return C1(shape, r); } if (use_sorted(x, lw) && n>8) { - B r = shift_ne(x, n, lw, 0); - Md1D d; d.f = bi_add; - return scan_c1(&d, r); + return M1C1(scan, add, shift_ne(x, n, lw, 0)); } #define BRUTE(T) \ i##T* xp = xv; \ @@ -492,5 +489,5 @@ B find_c1(B t, B x) { if (isAtm(x) || RNK(x)==0) thrM("⍷: Argument cannot have rank 0"); usz n = *SH(x); if (n<=1) return x; - return slash_c2(m_f64(0), memberOf_c1(m_f64(0), inc(x)), x); + return C2(slash, C1(memberOf, inc(x)), x); } diff --git a/src/utils/calls.h b/src/utils/calls.h index c9cc4d1b..5fb4bc20 100644 --- a/src/utils/calls.h +++ b/src/utils/calls.h @@ -1,5 +1,12 @@ #pragma once +#define C1(F,X ) F##_c1(m_f64(0),X ) +#define C2(F,X,W) F##_c2(m_f64(0),X,W) +#define M1C1(M,F,X ) m1c1_unsafe(M##_c1, bi_##F, X ) +#define M1C2(M,F,X,W) m1c2_unsafe(M##_c1, bi_##F, X, W) +static inline B m1c1_unsafe(D1C1 m, B f, B x ) { Md1D d; d.f=f; return m(&d, x ); } +static inline B m1c2_unsafe(D1C2 m, B f, B x, B w) { Md1D d; d.f=f; return m(&d, x, w); } + typedef void (*M_CopyF)(void*, usz, B, usz, usz); typedef void (*M_FillF)(void*, usz, B, usz); extern M_CopyF copyFns[el_MAX]; @@ -28,4 +35,4 @@ CMP_DEF(le, AS); #define CMP_AA_IMM(FN, ELT, WHERE, WP, XP, LEN) CMP_AA_CALL(CMP_AA_FN(FN, ELT), WHERE, WP, XP, LEN) #define CMP_AS_IMM(FN, ELT, WHERE, WP, X, LEN) CMP_AS_CALL(CMP_AS_FN(FN, ELT), WHERE, WP, X, LEN) -void bit_negatePtr(u64* rp, u64* xp, usz count); // count is number of u64-s \ No newline at end of file +void bit_negatePtr(u64* rp, u64* xp, usz count); // count is number of u64-s From f1142b4184fdde062a70522d18eb17c89af53e5d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 15 Nov 2022 18:27:54 -0500 Subject: [PATCH 04/15] =?UTF-8?q?Branchless=20=E2=8A=92bool,=20moderately?= =?UTF-8?q?=20faster?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/selfsearch.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index d7d046ea..cd10d5bd 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -4,7 +4,8 @@ // Other functions use adaptations of the same set of methods // Boolean ∊: 1 at first element and first ¬⊑𝕩 -// COULD do a branchless thing for boolean ⊒ instead of converting to i8 +// Boolean ⊒: Branchless sum thing +// COULD vectorize boolean ⊒ with +` // Boolean ⊐: ⥊¬⍟⊑𝕩 // SHOULD implement boolean ⍷ directly // Sorted flags: start with r0⌾⊑»⊸≠𝕩 (r0 is 0 for ⊐ and 1 otherwise) @@ -301,7 +302,27 @@ B count_c1(B t, B x) { if (n>(usz)I32_MAX+1) thrM("⊒: Argument length >2⋆31 not supported"); u8 lw = cellWidthLog(x); - if (lw==0) { x = toI8Any(x); lw = cellWidthLog(x); } + if (lw==0) { + u64* xp = bitarr_ptr(x); + B r; + #define COUNT_BOOL(T) \ + T* rp; r = m_##T##arrv(&rp, n); \ + usz n1 = 0; \ + for (usz i=0; i>=1; \ + rp[i] = b? n1 : i-n1; \ + n1 += b; \ + } \ + } + if (n <= 128) { COUNT_BOOL(i8) } + else if (n <= 1<<15) { COUNT_BOOL(i16) } + else { COUNT_BOOL(i32) } + decG(x); return r; + #undef COUNT_LOOP + #undef COUNT_BOOL + } if (use_sorted(x, lw) && n>16 && (lw>4 || n<1<<16)) { // ↕∘≠(⊣-⌈`∘×)∊ B c = shift_ne(x, n, lw, 1); B i = C1(ud, m_f64(n)); From 93ef4bb2a990c211d43557562689247e3b97fa3e Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 15 Nov 2022 18:37:54 -0500 Subject: [PATCH 05/15] =?UTF-8?q?Direct=20=E2=8D=B7bool=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/selfsearch.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index cd10d5bd..c9e4051b 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -3,16 +3,14 @@ // Except for trivial cases, ⍷ is implemented as ∊⊸/ // Other functions use adaptations of the same set of methods -// Boolean ∊: 1 at first element and first ¬⊑𝕩 -// Boolean ⊒: Branchless sum thing +// Boolean cases all use special code, including ⍷ // COULD vectorize boolean ⊒ with +` -// Boolean ⊐: ⥊¬⍟⊑𝕩 -// SHOULD implement boolean ⍷ directly -// Sorted flags: start with r0⌾⊑»⊸≠𝕩 (r0 is 0 for ⊐ and 1 otherwise) +// Sorted flags: start with r0⌾⊑»⊸≠𝕩 (r0←0 for ⊐, 1 otherwise) // ∊: ⊢; ⊐: +`; ⊒: ↕∘≠⊸(⊣-⌈`∘×) // COULD determine ⊒ result type by direct comparisons on 𝕩 // Brute force or all-pairs comparison for small lengths // Branchless, not vectorized (+´∧` structure for ⊐) +// COULD use direct all-pairs filter, not ∊⊸/, for short ⍷ // Full-size table lookups for 1- and 2-byte 𝕩 // 2-byte table can be "sparse" initialized with an extra pass over 𝕩 // 4-byte ⊐ can use a small-range lookup table @@ -29,6 +27,8 @@ // Generic hash table for other cases // Resizing is pretty expensive here +// SHOULD widen small odd sizes + #include "../core.h" #include "../utils/hash.h" #include "../utils/talloc.h" @@ -510,5 +510,13 @@ B find_c1(B t, B x) { if (isAtm(x) || RNK(x)==0) thrM("⍷: Argument cannot have rank 0"); usz n = *SH(x); if (n<=1) return x; + if (TI(x,elType)==el_bit && RNK(x)==1) { + u64* xp = bitarr_ptr(x); + u64 x0 = 1 & *xp; + usz i = bit_find(xp, n, !x0); decG(x); + u64* rp; B r = m_bitarrv(&rp, 1 + (i Date: Wed, 16 Nov 2022 19:46:51 -0500 Subject: [PATCH 06/15] Move scan to its own file --- makefile | 2 +- src/builtins/md1.c | 221 ------------------------------------------- src/builtins/scan.c | 224 ++++++++++++++++++++++++++++++++++++++++++++ src/opt/single.c | 1 + 4 files changed, 226 insertions(+), 222 deletions(-) create mode 100644 src/builtins/scan.c diff --git a/makefile b/makefile index 34d354ab..d83de3b5 100644 --- a/makefile +++ b/makefile @@ -212,7 +212,7 @@ ${bd}/%.o: src/jit/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< -builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} +builtins: ${addprefix ${bd}/, arithm.o arithd.o cmp.o sfns.o squeeze.o select.o slash.o group.o sort.o search.o selfsearch.o scan.o md1.o md2.o fns.o sysfn.o internal.o inverse.o} ${bd}/%.o: src/builtins/%.c @echo $< | cut -c 5- @$(CC_INC) $@.d -o $@ -c $< diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 4b3c4d9b..01cb30de 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -107,29 +107,6 @@ B each_c2(Md1D* d, B w, B x) { B f = d->f; return homFil2(f, eachd(f, w, x), wf, xf); } -#if SINGELI && __PCLMUL__ - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-variable" - #include "../singeli/gen/neq.c" - #pragma GCC diagnostic pop -#endif -B scan_ne(u64 p, u64* xp, u64 ia) { - u64* rp; B r=m_bitarrv(&rp,ia); -#if SINGELI && __PCLMUL__ - clmul_scan_ne(p, xp, rp, BIT_N(ia)); -#else - for (usz i = 0; i < BIT_N(ia); i++) { - u64 c = xp[i]; - u64 r = c ^ (c<<1); - r^= r<< 2; r^= r<< 4; r^= r<<8; - r^= r<<16; r^= r<<32; r^= p; - rp[i] = r; - p = -(r>>63); // repeat sign bit - } -#endif - return r; -} - static bool fold_ne(u64* x, u64 am) { u64 r = 0; for (u64 i = 0; i < (am>>6); i++) r^= x[i]; @@ -143,204 +120,6 @@ static i64 bit_diff(u64* x, u64 am) { if (am&63) r+= POPC((x[am>>6]^a)<<(64-am & 63)); return r - (i64)(am/2); } - - -#if SINGELI - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wunused-variable" - #include "../singeli/gen/scan.c" - #pragma GCC diagnostic pop -#endif -B slash_c1(B f, B x); -B scan_bit_sum(B x, u64* xp, u64 ia, u64 xs) { // consumes x - u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : el_i32; - if (xs < ia/128) { - B ones = slash_c1(m_f64(0), x); - MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); - SGetU(ones) - usz ri = 0; - for (usz i = 0; i < xs; i++) { - usz e = o2s(GetU(ones, i)); - mut_fillG(r0, ri, m_i32(i), e-ri); - ri = e; - } - if (rif; - if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); - ur xr = RNK(x); - usz ia = IA(x); - if (ia==0) return x; - B xf = getFillQ(x); - u8 xe = TI(x,elType); - if (xr==1 && xe<=el_f64 && isFun(f) && v(f)->flags) { - u8 rtid = v(f)->flags-1; - if (xe==el_bit) { - u64* xp=bitarr_ptr(x); - if (rtid==n_add) { - u64 xs = bit_sum(xp, ia); - if (xs>I32_MAX) goto base; - if (xs<=1) { if (xs==0) return x; goto bit_or; } - return FL_SET(scan_bit_sum(x, xp, ia, xs), fl_asc|fl_squoze); - } - if (rtid==n_or | rtid==n_ceil ) { bit_or:; u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(i>63); - rp[i] = p = x & (m10 ^ (x + c)); - } - decG(x); return r; - } - goto base; - } - if (rtid==n_add) { // + - if (xe==el_i8 ) { i8* xp=i8any_ptr (x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=0; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i16) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=I16_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i32) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=I32_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - #endif - } - if (rtid==n_ne) { // ≠ - f64 x0 = IGetU(x,0).f; if (x0!=0 && x0!=1) goto base; - if (xe==el_i8 ) { i8* xp=i8any_ptr (x); u64* rp; B r=m_bitarrv(&rp,ia); bool c=x0; rp[0]=c; for (usz i=1; if; - if (isAtm(x) || RNK(x)==0) thrM("`: 𝕩 cannot have rank 0"); - ur xr = RNK(x); usz* xsh = SH(x); usz ia = IA(x); - B wf = getFillQ(w); - u8 xe = TI(x,elType); - if (xr==1 && q_i32(w) && elInt(xe) && isFun(f) && v(f)->flags) { - u8 rtid = v(f)->flags-1; - i32 wv = o2iG(w); - if (xe==el_bit) { - u64* xp=bitarr_ptr(x); - if (rtid==n_add) { i32* rp; B r=m_i32arrv(&rp, ia); i64 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i16 && wv==(i16)wv) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i32 && wv==(i32)wv) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - } - if (rtid==n_ne) { // ≠ - if (!q_ibit(wv)) { goto base; } bool c=wv; - if (xe==el_i8 ) { i8* xp=i8any_ptr (x); u64* rp; B r=m_bitarrv(&rp, ia); for (usz i=0; if; if (isAtm(x) || RNK(x)!=1) thrF("´: Argument must be a list (%H ≡ ≢𝕩)", x); usz ia = IA(x); diff --git a/src/builtins/scan.c b/src/builtins/scan.c new file mode 100644 index 00000000..4db03f0f --- /dev/null +++ b/src/builtins/scan.c @@ -0,0 +1,224 @@ +#include "../core.h" +#include "../utils/mut.h" +#include "../builtins.h" + +#if !USE_VALGRIND +static u64 vg_rand(u64 x) { return x; } +#endif + +#if SINGELI + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/scan.c" + #pragma GCC diagnostic pop +#endif + +#if SINGELI && __PCLMUL__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wunused-variable" + #include "../singeli/gen/neq.c" + #pragma GCC diagnostic pop +#endif +B scan_ne(u64 p, u64* xp, u64 ia) { + u64* rp; B r=m_bitarrv(&rp,ia); +#if SINGELI && __PCLMUL__ + clmul_scan_ne(p, xp, rp, BIT_N(ia)); +#else + for (usz i = 0; i < BIT_N(ia); i++) { + u64 c = xp[i]; + u64 r = c ^ (c<<1); + r^= r<< 2; r^= r<< 4; r^= r<<8; + r^= r<<16; r^= r<<32; r^= p; + rp[i] = r; + p = -(r>>63); // repeat sign bit + } +#endif + return r; +} + +B slash_c1(B f, B x); +B scan_bit_sum(B x, u64* xp, u64 ia, u64 xs) { // consumes x + u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : el_i32; + if (xs < ia/128) { + B ones = slash_c1(m_f64(0), x); + MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); + SGetU(ones) + usz ri = 0; + for (usz i = 0; i < xs; i++) { + usz e = o2s(GetU(ones, i)); + mut_fillG(r0, ri, m_i32(i), e-ri); + ri = e; + } + if (rif; + if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); + ur xr = RNK(x); + usz ia = IA(x); + if (ia==0) return x; + B xf = getFillQ(x); + u8 xe = TI(x,elType); + if (xr==1 && xe<=el_f64 && isFun(f) && v(f)->flags) { + u8 rtid = v(f)->flags-1; + if (xe==el_bit) { + u64* xp=bitarr_ptr(x); + if (rtid==n_add) { + u64 xs = bit_sum(xp, ia); + if (xs>I32_MAX) goto base; + if (xs<=1) { if (xs==0) return x; goto bit_or; } + return FL_SET(scan_bit_sum(x, xp, ia, xs), fl_asc|fl_squoze); + } + if (rtid==n_or | rtid==n_ceil ) { bit_or:; u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(i>63); + rp[i] = p = x & (m10 ^ (x + c)); + } + decG(x); return r; + } + goto base; + } + if (rtid==n_add) { // + + if (xe==el_i8 ) { i8* xp=i8any_ptr (x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=0; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + if (xe==el_i16) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=I16_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + if (xe==el_i32) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=I32_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + #endif + } + if (rtid==n_ne) { // ≠ + f64 x0 = IGetU(x,0).f; if (x0!=0 && x0!=1) goto base; + if (xe==el_i8 ) { i8* xp=i8any_ptr (x); u64* rp; B r=m_bitarrv(&rp,ia); bool c=x0; rp[0]=c; for (usz i=1; if; + if (isAtm(x) || RNK(x)==0) thrM("`: 𝕩 cannot have rank 0"); + ur xr = RNK(x); usz* xsh = SH(x); usz ia = IA(x); + B wf = getFillQ(w); + u8 xe = TI(x,elType); + if (xr==1 && q_i32(w) && elInt(xe) && isFun(f) && v(f)->flags) { + u8 rtid = v(f)->flags-1; + i32 wv = o2iG(w); + if (xe==el_bit) { + u64* xp=bitarr_ptr(x); + if (rtid==n_add) { i32* rp; B r=m_i32arrv(&rp, ia); i64 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + if (xe==el_i16 && wv==(i16)wv) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + if (xe==el_i32 && wv==(i32)wv) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } + } + if (rtid==n_ne) { // ≠ + if (!q_ibit(wv)) { goto base; } bool c=wv; + if (xe==el_i8 ) { i8* xp=i8any_ptr (x); u64* rp; B r=m_bitarrv(&rp, ia); for (usz i=0; i Date: Wed, 16 Nov 2022 20:16:33 -0500 Subject: [PATCH 07/15] Pull out scan_add_bool() utility function from scan_c1() --- src/builtins/scan.c | 60 +++++++++++++++++++++++---------------- src/builtins/selfsearch.c | 3 +- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 4db03f0f..68ea2a30 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -36,9 +36,21 @@ B scan_ne(u64 p, u64* xp, u64 ia) { return r; } +static B scan_or(B x, u64 ia) { + u64* xp = bitarr_ptr(x); + u64* rp; B r=m_bitarrv(&rp,ia); + usz n=BIT_N(ia); u64 xi; usz i=0; + while (if; @@ -78,13 +95,8 @@ B scan_c1(Md1D* d, B x) { B f = d->f; u8 rtid = v(f)->flags-1; if (xe==el_bit) { u64* xp=bitarr_ptr(x); - if (rtid==n_add) { - u64 xs = bit_sum(xp, ia); - if (xs>I32_MAX) goto base; - if (xs<=1) { if (xs==0) return x; goto bit_or; } - return FL_SET(scan_bit_sum(x, xp, ia, xs), fl_asc|fl_squoze); - } - if (rtid==n_or | rtid==n_ceil ) { bit_or:; u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(i8) { - return M1C1(scan, add, shift_ne(x, n, lw, 0)); + return scan_add_bool(shift_ne(x, n, lw, 0), n); } #define BRUTE(T) \ i##T* xp = xv; \ From 136c1afacc97596e3179efa5d5edf63baed89eae Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 16 Nov 2022 20:49:57 -0500 Subject: [PATCH 08/15] Macro-ize integer min/max scan and add sorted flag --- src/builtins/grade.h | 12 +++---- src/builtins/scan.c | 66 ++++++++++++++++++++---------------- src/builtins/slash.c | 4 +-- src/singeli/src/scan.singeli | 12 +++---- 4 files changed, 51 insertions(+), 43 deletions(-) diff --git a/src/builtins/grade.h b/src/builtins/grade.h index f59f73ec..66d9e66f 100644 --- a/src/builtins/grade.h +++ b/src/builtins/grade.h @@ -31,22 +31,22 @@ } #if SINGELI -extern void (*const avx2_scan_max8)(int8_t* v0,int8_t* v1,uint64_t v2); -extern void (*const avx2_scan_min8)(int8_t* v0,int8_t* v1,uint64_t v2); -extern void (*const avx2_scan_max16)(int16_t* v0,int16_t* v1,uint64_t v2); -extern void (*const avx2_scan_min16)(int16_t* v0,int16_t* v1,uint64_t v2); +extern void (*const avx2_scan_max_i8)(int8_t* v0,int8_t* v1,uint64_t v2); +extern void (*const avx2_scan_min_i8)(int8_t* v0,int8_t* v1,uint64_t v2); +extern void (*const avx2_scan_max_i16)(int16_t* v0,int16_t* v1,uint64_t v2); +extern void (*const avx2_scan_min_i16)(int16_t* v0,int16_t* v1,uint64_t v2); #define COUNT_THRESHOLD 32 #define WRITE_SPARSE_i8 \ for (usz i=0; i>63); // repeat sign bit } #endif - return r; + decG(x); return r; } -static B scan_or(B x, u64 ia) { +static B scan_or(B x, u64 ia) { // consumes x u64* xp = bitarr_ptr(x); u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; - while (i,MIN,asc) } +#undef MM_CASE +#undef MINMAX +#undef MINMAX_SCAN + B scan_c1(Md1D* d, B x) { B f = d->f; if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); ur xr = RNK(x); @@ -97,8 +125,8 @@ B scan_c1(Md1D* d, B x) { B f = d->f; u64* xp=bitarr_ptr(x); if (rtid==n_add ) return scan_add_bool(x, ia); if (rtid==n_or | rtid==n_ceil ) return scan_or(x, ia); - if (rtid==n_and | rtid==n_mul | rtid==n_floor) { u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(if; if (xe==el_i16) { i16* xp=i16any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=0; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i16) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=I16_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i32) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=I32_MIN; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - #endif - } + if (rtid==n_floor && xef; if (xe==el_bit) { u64* xp=bitarr_ptr(x); if (rtid==n_add) { i32* rp; B r=m_i32arrv(&rp, ia); i64 c=wv; for (usz i=0; i Date: Wed, 16 Nov 2022 21:10:17 -0500 Subject: [PATCH 09/15] f64 min/max scan --- src/builtins/scan.c | 12 ++++++++---- src/singeli/src/scan.singeli | 9 ++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 74871035..617c3457 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -1,6 +1,9 @@ #include "../core.h" #include "../utils/mut.h" #include "../builtins.h" +#include +#define F64_MIN -INFINITY +#define F64_MAX INFINITY #if !USE_VALGRIND static u64 vg_rand(u64 x) { return x; } @@ -104,10 +107,11 @@ B scan_add_bool(B x, u64 ia) { // consumes x MM_CASE(i8 ,NAME,C,I8_##INIT ) \ MM_CASE(i16,NAME,C,I16_##INIT) \ MM_CASE(i32,NAME,C,I32_##INIT) \ + MM_CASE(f64,NAME,C,F64_##INIT) \ } \ decG(x); return FL_SET(r, fl_##ORD); -static B scan_min_int(B x, u8 xe, usz ia) { MINMAX(min,<,MAX,dsc) } -static B scan_max_int(B x, u8 xe, usz ia) { MINMAX(max,>,MIN,asc) } +static B scan_min_num(B x, u8 xe, usz ia) { MINMAX(min,<,MAX,dsc) } +static B scan_max_num(B x, u8 xe, usz ia) { MINMAX(max,>,MIN,asc) } #undef MM_CASE #undef MINMAX #undef MINMAX_SCAN @@ -145,8 +149,8 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (xe==el_i16) { i16* xp=i16any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=0; for (usz i=0; ish{4b1110,sh{4b2200,a}}} +} def avx2_scan_idem{T, op} = { def m = 1 << (width{T}-1) avx2_scan_idem{T, op, (if (match{op,min}) m-1; else -m)} @@ -65,6 +70,8 @@ def avx2_scan_idem{T, op} = { 'avx2_scan_max_i16' = avx2_scan_idem{i16, max} 'avx2_scan_min_i32' = avx2_scan_idem{i32, min} 'avx2_scan_max_i32' = avx2_scan_idem{i32, max} +'avx2_scan_min_f64' = avx2_scan_idem{f64, min, 'F64_MAX'} +'avx2_scan_max_f64' = avx2_scan_idem{f64, max, 'F64_MIN'} # Associative scan avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = { From 98d066c343e10d859c054a19e22a0aeb51d2c499 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 16 Nov 2022 21:20:54 -0500 Subject: [PATCH 10/15] =?UTF-8?q?Use=20scan=5Fmax=5Fnum=20for=20sorted=20?= =?UTF-8?q?=E2=8A=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 19 ++++++++++--------- src/builtins/selfsearch.c | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 617c3457..c036254b 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -102,16 +102,17 @@ B scan_add_bool(B x, u64 ia) { // consumes x #endif #define MM_CASE(T,N,C,I) \ case el_##T : { T* xp=T##any_ptr(x); T* rp; r=m_##T##arrv(&rp, ia); MINMAX_SCAN(T,N,C,I); break; } -#define MINMAX(NAME,C,INIT,ORD) \ - B r; switch (xe) { default:UD; \ - MM_CASE(i8 ,NAME,C,I8_##INIT ) \ - MM_CASE(i16,NAME,C,I16_##INIT) \ - MM_CASE(i32,NAME,C,I32_##INIT) \ - MM_CASE(f64,NAME,C,F64_##INIT) \ - } \ +#define MINMAX(NAME,C,INIT,BIT,ORD) \ + B r; switch (xe) { default:UD; \ + case el_bit: return scan_##BIT(x, ia); \ + MM_CASE(i8 ,NAME,C,I8_##INIT ) \ + MM_CASE(i16,NAME,C,I16_##INIT) \ + MM_CASE(i32,NAME,C,I32_##INIT) \ + MM_CASE(f64,NAME,C,F64_##INIT) \ + } \ decG(x); return FL_SET(r, fl_##ORD); -static B scan_min_num(B x, u8 xe, usz ia) { MINMAX(min,<,MAX,dsc) } -static B scan_max_num(B x, u8 xe, usz ia) { MINMAX(max,>,MIN,asc) } +B scan_min_num(B x, u8 xe, usz ia) { MINMAX(min,<,MAX,and,dsc) } +B scan_max_num(B x, u8 xe, usz ia) { MINMAX(max,>,MIN,or ,asc) } #undef MM_CASE #undef MINMAX #undef MINMAX_SCAN diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index ba26fd9f..b1733a21 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -38,11 +38,11 @@ extern B not_c1(B, B); extern B shape_c1(B, B); extern B slash_c2(B, B, B); -extern B scan_c1(Md1D*, B); extern B ud_c1(B, B); extern B sub_c2(B, B, B); extern B mul_c2(B, B, B); extern B scan_add_bool(B x, u64 ia); +extern B scan_max_num(B x, u8 xe, u64 ia); // These hashes are stored in tables and must be invertible! #if defined(__SSE4_2__) @@ -327,8 +327,8 @@ B count_c1(B t, B x) { if (use_sorted(x, lw) && n>16 && (lw>4 || n<1<<16)) { // ↕∘≠(⊣-⌈`∘×)∊ B c = shift_ne(x, n, lw, 1); B i = C1(ud, m_f64(n)); - B m = M1C1(scan, ceil, C2(mul, c, inc(i))); - return C2(sub, i, m); + B m = C2(mul, c, inc(i)); + return C2(sub, i, scan_max_num(m, TI(m,elType), n)); } void* xv = tyany_ptr(x); #define BRUTE(T) \ From 5985e9283465519b64d77e9f16a346a387d940f8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 17 Nov 2022 07:46:26 -0500 Subject: [PATCH 11/15] Initialized min/max-scan --- src/builtins/scan.c | 42 +++++++++++++++++++++++++----------- src/singeli/src/scan.singeli | 29 +++++++++++++------------ 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index c036254b..aad96763 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -95,8 +95,9 @@ B scan_add_bool(B x, u64 ia) { // consumes x return FL_SET(r, fl_asc|fl_squoze); } +// min/max-scan #if SINGELI - #define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_##T(xp, rp, ia); + #define MINMAX_SCAN(T,NAME,C,I) avx2_scan_##NAME##_init_##T(xp, rp, ia, I); #else #define MINMAX_SCAN(T,NAME,C,I) T c=I; for (usz i=0; i,MIN,or ,asc) } -#undef MM_CASE #undef MINMAX +// Initialized: try to convert 𝕨 to type of 𝕩 +// (could do better for out-of-range floats) +B shape_c2(B, B, B); +#define MM2_ICASE(T,N,C,I) \ + case el_##T : { \ + if (wv!=(T)wv) { if (wv C 0) { r=C2(shape,m_f64(ia),w); break; } else wv=I; } \ + T* xp=T##any_ptr(x); T* rp; r=m_##T##arrv(&rp, ia); MINMAX_SCAN(T,N,C,wv); \ + break; } +#define MINMAX2(NAME,C,INIT,BIT,BI,ORD) \ + i32 wv=0; if (q_i32(w)) { wv=w.f; } else { x=taga(cpyF64Arr(x)); xe=el_f64; } \ + B r; switch (xe) { default:UD; \ + case el_bit: if (wv C BI) r=C2(shape,m_f64(ia),w); else return scan_##BIT(x, ia); break; \ + MM2_ICASE(i8 ,NAME,C,I8_##INIT ) \ + MM2_ICASE(i16,NAME,C,I16_##INIT) \ + MM_CASE(i32,NAME,C,wv) \ + MM_CASE(f64,NAME,C,w.f) \ + } \ + decG(x); return FL_SET(r, fl_##ORD); +static B scan2_min_num(B w, B x, u8 xe, usz ia) { MINMAX2(min,<,MAX,and,1,dsc) } +static B scan2_max_num(B w, B x, u8 xe, usz ia) { MINMAX2(max,>,MIN,or ,0,asc) } +#undef MINMAX2 +#undef MM_CASE +#undef MM_CASE2 #undef MINMAX_SCAN B scan_c1(Md1D* d, B x) { B f = d->f; @@ -190,8 +213,11 @@ B scan_c2(Md1D* d, B w, B x) { B f = d->f; ur xr = RNK(x); usz* xsh = SH(x); usz ia = IA(x); B wf = getFillQ(w); u8 xe = TI(x,elType); - if (xr==1 && q_i32(w) && elInt(xe) && isFun(f) && v(f)->flags) { + if (xr==1 && xe<=el_f64 && isFun(f) && v(f)->flags) { u8 rtid = v(f)->flags-1; + if (rtid==n_floor) return scan2_min_num(w, x, xe, ia); // ⌊ + if (rtid==n_ceil ) return scan2_max_num(w, x, xe, ia); // ⌈ + if (!q_i32(w)) goto base; i32 wv = o2iG(w); if (xe==el_bit) { u64* xp=bitarr_ptr(x); @@ -204,16 +230,6 @@ B scan_c2(Md1D* d, B w, B x) { B f = d->f; if (xe==el_i16) { i16* xp=i16any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i16 && wv==(i16)wv) { i16* xp=i16any_ptr(x); i16* rp; B r=m_i16arrv(&rp, ia); i16 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - if (xe==el_i32 && wv==(i32)wv) { i32* xp=i32any_ptr(x); i32* rp; B r=m_i32arrv(&rp, ia); i32 c=wv; for (usz i=0; ic)c=xp[i]; rp[i]=c; } decG(x); return r; } - } if (rtid==n_ne) { // ≠ if (!q_ibit(wv)) { goto base; } bool c=wv; if (xe==el_i8 ) { i8* xp=i8any_ptr (x); u64* rp; B r=m_bitarrv(&rp, ia); for (usz i=0; ish{4b1110,sh{4b2200,a}}} + scan_post{T, init, x, r, len, op, {a}=>sh{4b1110,sh{4b2200,a}}} } -def avx2_scan_idem{T, op} = { +'avx2_scan_min_init_i8' = avx2_scan_idem{i8 , min}; 'avx2_scan_max_init_i8' = avx2_scan_idem{i8 , max} +'avx2_scan_min_init_i16' = avx2_scan_idem{i16, min}; 'avx2_scan_max_init_i16' = avx2_scan_idem{i16, max} +'avx2_scan_min_init_i32' = avx2_scan_idem{i32, min}; 'avx2_scan_max_init_i32' = avx2_scan_idem{i32, max} +'avx2_scan_min_init_f64' = avx2_scan_idem{f64, min}; 'avx2_scan_max_init_f64' = avx2_scan_idem{f64, max} + +avx2_scan_idem_id{T, op}(x:*T, r:*T, len:u64) : void = { def m = 1 << (width{T}-1) - avx2_scan_idem{T, op, (if (match{op,min}) m-1; else -m)} + def id = (if (match{op,min}) m-1; else -m) + avx2_scan_idem{T, op}(x, r, len, id) } -'avx2_scan_min_i8' = avx2_scan_idem{i8 , min} -'avx2_scan_max_i8' = avx2_scan_idem{i8 , max} -'avx2_scan_min_i16' = avx2_scan_idem{i16, min} -'avx2_scan_max_i16' = avx2_scan_idem{i16, max} -'avx2_scan_min_i32' = avx2_scan_idem{i32, min} -'avx2_scan_max_i32' = avx2_scan_idem{i32, max} -'avx2_scan_min_f64' = avx2_scan_idem{f64, min, 'F64_MAX'} -'avx2_scan_max_f64' = avx2_scan_idem{f64, max, 'F64_MIN'} +'avx2_scan_min_i8' = avx2_scan_idem_id{i8 , min}; 'avx2_scan_max_i8' = avx2_scan_idem_id{i8 , max} +'avx2_scan_min_i16' = avx2_scan_idem_id{i16, min}; 'avx2_scan_max_i16' = avx2_scan_idem_id{i16, max} +'avx2_scan_min_i32' = avx2_scan_idem_id{i32, min}; 'avx2_scan_max_i32' = avx2_scan_idem_id{i32, max} # Associative scan avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = { From be9037cb62f775b63584a68a102909aed7059e34 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 17 Nov 2022 08:32:29 -0500 Subject: [PATCH 12/15] =?UTF-8?q?Squeeze=20for=20=E2=88=A8`=20instead=20of?= =?UTF-8?q?=20scalar=20checking=20for=20bool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index aad96763..e0e05166 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -181,11 +181,7 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (xe==el_i16) { i16* xp=i16any_ptr(x); u64* rp; B r=m_bitarrv(&rp,ia); bool c=x0; rp[0]=c; for (usz i=1; i Date: Thu, 17 Nov 2022 17:50:52 +0200 Subject: [PATCH 13/15] fix function signatures & M1C2; use C1 & bit_negate more --- src/builtins/group.c | 3 +-- src/builtins/scan.c | 6 +++--- src/builtins/selfsearch.c | 3 +-- src/builtins/slash.c | 3 +-- src/utils/calls.h | 8 ++++---- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/builtins/group.c b/src/builtins/group.c index 9f2cff82..c0b486d1 100644 --- a/src/builtins/group.c +++ b/src/builtins/group.c @@ -25,7 +25,6 @@ #include "../utils/mut.h" extern B ud_c1(B, B); -extern B not_c1(B, B); extern B ne_c2(B, B, B); extern B slash_c1(B, B); extern B slash_c2(B, B, B); @@ -102,7 +101,7 @@ static B group_simple(B w, B x, ur xr, usz wia, usz xn, usz* xsh, u8 we) { assert(ria == 2); if (wia>xn) w = C2(take, m_f64(xn), w); rp[1] = C2(slash, inc(w), inc(x)); - rp[0] = C2(slash, C1(not, w), x); + rp[0] = C2(slash, bit_negate(w), x); return taga(r); } // Needed to make sure wia>0 for ip[wia-1] below diff --git a/src/builtins/scan.c b/src/builtins/scan.c index e0e05166..af61f243 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -63,7 +63,7 @@ B scan_add_bool(B x, u64 ia) { // consumes x B r; u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : xs<=I32_MAX? el_i32 : el_f64; if (xs < ia/128) { - B ones = slash_c1(m_f64(0), x); + B ones = C1(slash, x); MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); SGetU(ones) usz ri = 0; @@ -112,8 +112,8 @@ B scan_add_bool(B x, u64 ia) { // consumes x MM_CASE(f64,NAME,C,F64_##INIT) \ } \ decG(x); return FL_SET(r, fl_##ORD); -B scan_min_num(B x, u8 xe, usz ia) { MINMAX(min,<,MAX,and,dsc) } -B scan_max_num(B x, u8 xe, usz ia) { MINMAX(max,>,MIN,or ,asc) } +B scan_min_num(B x, u8 xe, u64 ia) { MINMAX(min,<,MAX,and,dsc) } +B scan_max_num(B x, u8 xe, u64 ia) { MINMAX(max,>,MIN,or ,asc) } #undef MINMAX // Initialized: try to convert 𝕨 to type of 𝕩 // (could do better for out-of-range floats) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index b1733a21..74012a4e 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -35,7 +35,6 @@ #include "../utils/calls.h" #include "../builtins.h" -extern B not_c1(B, B); extern B shape_c1(B, B); extern B slash_c2(B, B, B); extern B ud_c1(B, B); @@ -416,7 +415,7 @@ B indexOf_c1(B t, B x) { u8 lw = cellWidthLog(x); void* xv = tyany_ptr(x); if (lw == 0) { - B r = 1&*(u64*)xv ? C1(not, x) : x; + B r = 1&*(u64*)xv ? bit_negate(x) : x; return C1(shape, r); } if (use_sorted(x, lw) && n>8) { diff --git a/src/builtins/slash.c b/src/builtins/slash.c index 36e12274..be7d9d3e 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -413,7 +413,6 @@ static NOINLINE B zeroCells(B x) { // doesn't consume return r; } -B not_c1(B t, B x); B grade_bool(B x, usz xia, bool up) { #define BRANCHLESS_GRADE(T) \ T* rp; r = m_##T##arrv(&rp, xia); \ @@ -433,7 +432,7 @@ B grade_bool(B x, usz xia, bool up) { #if SINGELI && defined(__BMI2__) if (xia < 16) { BRANCHLESS_GRADE(i8) } else if (xia <= 1<<15) { - B notx = not_c1(m_f64(0), inc(x)); + B notx = bit_negate(inc(x)); u64* xp0 = bitarr_ptr(notx); u64* xp1 = xp; if (!up) { u64* t=xp1; xp1=xp0; xp0=t; } diff --git a/src/utils/calls.h b/src/utils/calls.h index 5fb4bc20..7986303b 100644 --- a/src/utils/calls.h +++ b/src/utils/calls.h @@ -2,10 +2,10 @@ #define C1(F,X ) F##_c1(m_f64(0),X ) #define C2(F,X,W) F##_c2(m_f64(0),X,W) -#define M1C1(M,F,X ) m1c1_unsafe(M##_c1, bi_##F, X ) -#define M1C2(M,F,X,W) m1c2_unsafe(M##_c1, bi_##F, X, W) -static inline B m1c1_unsafe(D1C1 m, B f, B x ) { Md1D d; d.f=f; return m(&d, x ); } -static inline B m1c2_unsafe(D1C2 m, B f, B x, B w) { Md1D d; d.f=f; return m(&d, x, w); } +#define M1C1(M,F, X) m1c1_unsafe(M##_c1, bi_##F, X) +#define M1C2(M,F,W,X) m1c2_unsafe(M##_c2, bi_##F, W, X) +static inline B m1c1_unsafe(D1C1 m, B f, B x) { Md1D d; d.f=f; return m(&d, x); } +static inline B m1c2_unsafe(D1C2 m, B f, B w, B x) { Md1D d; d.f=f; return m(&d, w, x); } typedef void (*M_CopyF)(void*, usz, B, usz, usz); typedef void (*M_FillF)(void*, usz, B, usz); From aee745a7d61e4f53f09302f42becc156bc8d9591 Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 17 Nov 2022 18:13:21 +0200 Subject: [PATCH 14/15] fix argument names of C2 --- src/utils/calls.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/calls.h b/src/utils/calls.h index 7986303b..b0bd741d 100644 --- a/src/utils/calls.h +++ b/src/utils/calls.h @@ -1,7 +1,7 @@ #pragma once -#define C1(F,X ) F##_c1(m_f64(0),X ) -#define C2(F,X,W) F##_c2(m_f64(0),X,W) +#define C1(F, X) F##_c1(m_f64(0), X) +#define C2(F,W,X) F##_c2(m_f64(0),W,X) #define M1C1(M,F, X) m1c1_unsafe(M##_c1, bi_##F, X) #define M1C2(M,F,W,X) m1c2_unsafe(M##_c2, bi_##F, W, X) static inline B m1c1_unsafe(D1C1 m, B f, B x) { Md1D d; d.f=f; return m(&d, x); } From da1062c81cfadf91611fef2fc550483c5f69cc6d Mon Sep 17 00:00:00 2001 From: dzaima Date: Thu, 17 Nov 2022 18:55:59 +0200 Subject: [PATCH 15/15] remove unused #undef --- src/builtins/selfsearch.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/builtins/selfsearch.c b/src/builtins/selfsearch.c index 74012a4e..2750c52c 100644 --- a/src/builtins/selfsearch.c +++ b/src/builtins/selfsearch.c @@ -320,7 +320,6 @@ B count_c1(B t, B x) { else if (n <= 1<<15) { COUNT_BOOL(i16) } else { COUNT_BOOL(i32) } decG(x); return r; - #undef COUNT_LOOP #undef COUNT_BOOL } if (use_sorted(x, lw) && n>16 && (lw>4 || n<1<<16)) { // ↕∘≠(⊣-⌈`∘×)∊