From ab4e5543a0a089f283affbd9f7483ac4715464d1 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 23 May 2024 22:26:05 -0400 Subject: [PATCH 01/38] =?UTF-8?q?Implement=20matrix=20=E2=8A=A3=CB=9D?= =?UTF-8?q?=CB=98=20and=20=E2=8A=A2=CB=9D=CB=98=20with=20select=5Fcells?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index c27bc9b3..47e8da8e 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -450,10 +450,12 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr Md1D* fd = c(Md1D,f); u8 rtid = fd->m1->flags-1; if (rtid==n_const) { f=fd->f; goto const_f; } - if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B && k==1 && xr==2 && isFun(fd->f) && isPervasiveDyExt(fd->f)) { // TODO extend to any rank x with cr==1 + if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B && k==1 && xr==2 && isFun(fd->f)) { // TODO extend to any rank x with cr==1 usz *sh = SH(x); usz m = sh[1]; - if (m == 1) return select_cells(0, x, cam, k, false); - if (m <= 64 && m < sh[0]) return fold_rows(fd, x); + u8 frtid = v(fd->f)->flags-1; + if (m==1 || frtid==n_ltack) return select_cells(0 , x, cam, k, false); + if ( frtid==n_rtack) return select_cells(m-1, x, cam, k, false); + if (isPervasiveDyExt(fd->f) && m <= 64 && m < sh[0]) return fold_rows(fd, x); } } else if (TY(f) == t_md2D) { Md2D* fd = c(Md2D,f); From c76e175719fde4b5e42bd4d39fc51092b747565c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 23 May 2024 22:27:12 -0400 Subject: [PATCH 02/38] =?UTF-8?q?Faster=20boolean=20+=CB=9D=CB=98,=20and?= =?UTF-8?q?=20=E2=89=A0=CB=9D=CB=98=20on=20row=20length=20>64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 8 ++- src/builtins/fold.c | 120 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 47e8da8e..00e55267 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -7,7 +7,8 @@ B fne_c1(B, B); B shape_c2(B, B, B); B transp_c2(B, B, B); -B fold_rows(Md1D* d, B x); // from fold.c +B fold_rows(Md1D* d, B x); // from fold.c +B fold_rows_bit(Md1D* d, B x); // from fold.c B takedrop_highrank(bool take, B w, B x); // from sfns.c B try_interleave_cells(B w, B x, ur xr, ur xk, usz* xsh); // from transpose.c @@ -455,7 +456,10 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr u8 frtid = v(fd->f)->flags-1; if (m==1 || frtid==n_ltack) return select_cells(0 , x, cam, k, false); if ( frtid==n_rtack) return select_cells(m-1, x, cam, k, false); - if (isPervasiveDyExt(fd->f) && m <= 64 && m < sh[0]) return fold_rows(fd, x); + if (isPervasiveDyExt(fd->f)) { + if (TI(x,elType)==el_bit) { B r = fold_rows_bit(fd, x); if (!q_N(r)) return r; } + if (m <= 64 && m < sh[0]) return fold_rows(fd, x); + } } } else if (TY(f) == t_md2D) { Md2D* fd = c(Md2D,f); diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 1a94a6ae..272772b6 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -20,9 +20,13 @@ #include "../utils/includeSingeli.h" #endif -static bool fold_ne(u64* x, u64 am) { +static u64 xor_words(u64* x, u64 l) { u64 r = 0; - for (u64 i = 0; i < (am>>6); i++) r^= x[i]; + for (u64 i = 0; i < l; i++) r^= x[i]; + return r; +} +static bool fold_ne(u64* x, u64 am) { + u64 r = xor_words(x, am>>6); if (am&63) r^= x[am>>6]<<(64-am & 63); return POPC(r) & 1; } @@ -432,3 +436,115 @@ B fold_rows(Md1D* fd, B x) { return mut_fv(r); } } + +B sum_rows_bit(B x) { + usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + u64* xp = bitarr_ptr(x); + if (m < 128) { + if (m == 2) return bi_N; // Transpose is faster + i8* rp; B r = m_i8arrv(&rp, n); + if (m <= 64) { + if (m%8 == 0) { + usz k = m/8; u64 b = (m==64? 0 : 1ull<> (j%8); + rp[i] = POPC(b & xw); + } + } else { + // Row may not fit in an aligned word + // Read a word containing the last bit, combine with saved bits + u64 b = ~(~(u64)0 >> m); + u64 prev = 0; + for (usz i=0, j=m; i> (m-sh); + } + } + } + } else { // 64> (in%64)); + rp[i] = s - o; + j = jn+1; + } + } + decG(x); return r; + } else if (m < 1<<15) { + i16* rp; B r = m_i16arrv(&rp, n); + usz l = m/64; + if (m%64==0) { + for (usz i=0; i= j+l)); + o = POPC(e >> (in%64)); + rp[i] = s - o; + j = jn+1; + } + } + decG(x); return r; + } else { + return bi_N; + } +} + +B xor_rows_bit(B x) { + usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + if (m <= 64) return bi_N; + u64* xp = bitarr_ptr(x); + u64* rp; B r = m_bitarrv(&rp, n); + u64 rw = 0; // Buffer for result bits + #define ADDBIT(I, BIT) \ + rw = rw>>1 | (u64)(BIT)<<63; \ + if ((I+1)%64==0) *rp++ = rw; + #define XOR_LOOP(LEN, MASK) \ + u64 o = 0; /* Carry */ \ + for (usz i=0, j=0; i> (in%64); \ + ADDBIT(i, POPC(s ^ o)); \ + j = jn+1; \ + } + usz l = m/64; + if (m < 128) { + XOR_LOOP(1, (u64)j - (u64)jn) + } else if (m%64==0) { + for (usz i=0; i= j+l)) + } + usz q=(-n)%64; if (q) *rp = rw >> q; + #undef XOR_LOOP + #undef ADDBIT + decG(x); return r; +} + +B fold_rows_bit(Md1D* fd, B x) { + assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); + if (!v(fd->f)->flags) return bi_N; + u8 rtid = v(fd->f)->flags-1; + if (rtid==n_add) return sum_rows_bit(x); + if (rtid==n_ne ) return xor_rows_bit(x); + return bi_N; +} From d29b4df50c7b8d2498ef3af1d140f2207155eb4a Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 25 May 2024 07:04:46 -0400 Subject: [PATCH 03/38] Boolean and, or, eq folds for row length >64 --- src/builtins/fold.c | 63 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 272772b6..bce40bb7 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -505,15 +505,12 @@ B sum_rows_bit(B x) { } } -B xor_rows_bit(B x) { - usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; - if (m <= 64) return bi_N; - u64* xp = bitarr_ptr(x); - u64* rp; B r = m_bitarrv(&rp, n); +void xor_rows_bit(u64* xp, u64* rp, usz n, usz m, bool eq) { u64 rw = 0; // Buffer for result bits + u64 rx = -(u64)(eq &~ m); // ne to eq conversion if needed #define ADDBIT(I, BIT) \ rw = rw>>1 | (u64)(BIT)<<63; \ - if ((I+1)%64==0) *rp++ = rw; + if ((I+1)%64==0) *rp++ = rw ^ rx; #define XOR_LOOP(LEN, MASK) \ u64 o = 0; /* Carry */ \ for (usz i=0, j=0; i= j+l)) } - usz q=(-n)%64; if (q) *rp = rw >> q; + usz q=(-n)%64; if (q) *rp = (rw^rx) >> q; #undef XOR_LOOP #undef ADDBIT - decG(x); return r; +} + +void or_rows_bit(u64* xp, u64* rp, usz n, usz m, u64 and) { + u64 rw = 0; // Buffer for result bits + #define ADDBIT(I, BIT, XOR) \ + rw = rw>>1 | (u64)(BIT)<<63; \ + if ((I+1)%64==0) *rp++ = XOR ^ rw; + if (m < 128) { + usz c = and? m-1 : 0; + u64 o = 0; + for (usz i=0, j=0; i> (in%64)); + ADDBIT(i, s > c+o, 0); + j = jn+1; + } + } else { + u64 rx = -and; u64 id = ~rx; + u64 o = 0; // Saved bits + for (usz i=0, j=0; i> q; + #undef ADDBIT } B fold_rows_bit(Md1D* fd, B x) { @@ -545,6 +582,14 @@ B fold_rows_bit(Md1D* fd, B x) { if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; if (rtid==n_add) return sum_rows_bit(x); - if (rtid==n_ne ) return xor_rows_bit(x); + if (rtid==n_ne|rtid==n_eq|rtid==n_or|rtid==n_and) { + usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + if (m <= 64) return bi_N; + u64* xp = bitarr_ptr(x); + u64* rp; B r = m_bitarrv(&rp, n); + if (rtid==n_ne|rtid==n_eq) xor_rows_bit(xp, rp, n, m, rtid==n_eq); + else or_rows_bit(xp, rp, n, m, rtid==n_and); + decG(x); return r; + } return bi_N; } From 25902229882d10fd0b8e1d5ed4642163bb833fc6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 25 May 2024 15:43:24 -0400 Subject: [PATCH 04/38] Generic-architecture implementations of fold.singeli --- build/src/build.bqn | 2 +- src/builtins/fold.c | 12 ++++++------ src/singeli/src/fold.singeli | 21 ++++++++++++++++----- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/build/src/build.bqn b/build/src/build.bqn index 7ad34a16..6d915d0d 100755 --- a/build/src/build.bqn +++ b/build/src/build.bqn @@ -696,7 +696,7 @@ cachedBin‿linkerCache ← { "xa."‿"src/builtins/squeeze.c"‿"squeeze", "xa."‿"src/utils/mut.c"‿"copy", "xa."‿"src/utils/bits.c"‿"bits", "xag"‿"src/builtins/transpose.c"‿"transpose", "xag"‿"src/builtins/search.c"‿"search", "xag"‿"src/builtins/selfsearch.c"‿"selfsearch" - "xag"‿"src/builtins/scan.c"‿"scan", "xa."‿"src/builtins/fold.c"‿"fold", + "xag"‿"src/builtins/scan.c"‿"scan", "xag"‿"src/builtins/fold.c"‿"fold", "xag"‿"src/builtins/slash.c"‿"slash", "xag"‿"src/builtins/slash.c"‿"replicate", "xag"‿"src/builtins/sort.c"‿"bins", "xa."‿"src/builtins/slash.c"‿"count" diff --git a/src/builtins/fold.c b/src/builtins/fold.c index bce40bb7..262fcb14 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -15,7 +15,7 @@ #include "../builtins.h" #include "../utils/mut.h" -#if SINGELI_SIMD +#if SINGELI #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif @@ -96,8 +96,8 @@ B sum_c1(B t, B x) { } r += s; } else { - #if SINGELI_SIMD - r = simd_sum_f64(xv, ia); + #if SINGELI + r = si_sum_f64(xv, ia); #else r=0; for (usz i=0; i) } DEF_MIN_MAX(i8) DEF_MIN_MAX(i16) DEF_MIN_MAX(i32) -#if SINGELI_SIMD - static f64 min_f64(void* xv, usz ia) { return simd_fold_min_f64(xv,ia); } - static f64 max_f64(void* xv, usz ia) { return simd_fold_max_f64(xv,ia); } +#if SINGELI + static f64 min_f64(void* xv, usz ia) { return si_fold_min_f64(xv,ia); } + static f64 max_f64(void* xv, usz ia) { return si_fold_max_f64(xv,ia); } #else DEF_MIN_MAX(f64) #endif diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 1f262be5..9eec3045 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -20,7 +20,13 @@ def reduce_pairwise{op, plog, x:*T, len, init:T} = { r } -fn fold_idem{T==f64, op}(x:*T, len:u64) : T = { +fn fold_idem{T, op}(x:*T, len:u64) : T = { + assert{len > 0} + a := load{x, 0} + @for (x over _ from 1 to len) a = op{a, x} + a +} +fn fold_idem{T==f64, op if has_simd}(x:*T, len:u64) : T = { def bulk = arch_defvw/width{T} def V = [bulk]T xv:= *V ~~ x @@ -45,10 +51,15 @@ fn fold_idem{T==f64, op}(x:*T, len:u64) : T = { extract{r, 0} } -export{'simd_fold_min_f64', fold_idem{f64,min}} -export{'simd_fold_max_f64', fold_idem{f64,max}} +export{'si_fold_min_f64', fold_idem{f64,min}} +export{'si_fold_max_f64', fold_idem{f64,max}} -fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = { +fn fold_assoc_0{T, op}(x:*T, len:u64) : T = { + a:T = 0 + @for (x over len) a = op{a, x} + a +} +fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { def bulk = arch_defvw/width{T} def V = [bulk]T xv:= *V ~~ x @@ -58,4 +69,4 @@ fn fold_assoc_0{T==f64, op}(x:*T, len:u64) : T = { if (hasarch{'AARCH64'}) vfold{op, r} else extract{mix{op, r}, 0} } -export{'simd_sum_f64', fold_assoc_0{f64,+}} +export{'si_sum_f64', fold_assoc_0{f64,+}} From 7b4468c3941669baa712868c24d9170a9d178028 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 25 May 2024 15:46:19 -0400 Subject: [PATCH 05/38] Move logical fold-rows functions to Singeli --- src/builtins/fold.c | 78 ++---------------------------- src/singeli/src/base.singeli | 2 + src/singeli/src/fold.singeli | 88 ++++++++++++++++++++++++++++++++++ src/singeli/src/search.singeli | 2 - 4 files changed, 94 insertions(+), 76 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 262fcb14..2547d828 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -505,91 +505,21 @@ B sum_rows_bit(B x) { } } -void xor_rows_bit(u64* xp, u64* rp, usz n, usz m, bool eq) { - u64 rw = 0; // Buffer for result bits - u64 rx = -(u64)(eq &~ m); // ne to eq conversion if needed - #define ADDBIT(I, BIT) \ - rw = rw>>1 | (u64)(BIT)<<63; \ - if ((I+1)%64==0) *rp++ = rw ^ rx; - #define XOR_LOOP(LEN, MASK) \ - u64 o = 0; /* Carry */ \ - for (usz i=0, j=0; i> (in%64); \ - ADDBIT(i, POPC(s ^ o)); \ - j = jn+1; \ - } - usz l = m/64; - if (m < 128) { - XOR_LOOP(1, (u64)j - (u64)jn) - } else if (m%64==0) { - for (usz i=0; i= j+l)) - } - usz q=(-n)%64; if (q) *rp = (rw^rx) >> q; - #undef XOR_LOOP - #undef ADDBIT -} - -void or_rows_bit(u64* xp, u64* rp, usz n, usz m, u64 and) { - u64 rw = 0; // Buffer for result bits - #define ADDBIT(I, BIT, XOR) \ - rw = rw>>1 | (u64)(BIT)<<63; \ - if ((I+1)%64==0) *rp++ = XOR ^ rw; - if (m < 128) { - usz c = and? m-1 : 0; - u64 o = 0; - for (usz i=0, j=0; i> (in%64)); - ADDBIT(i, s > c+o, 0); - j = jn+1; - } - } else { - u64 rx = -and; u64 id = ~rx; - u64 o = 0; // Saved bits - for (usz i=0, j=0; i> q; - #undef ADDBIT -} - B fold_rows_bit(Md1D* fd, B x) { assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; if (rtid==n_add) return sum_rows_bit(x); + #if SINGELI if (rtid==n_ne|rtid==n_eq|rtid==n_or|rtid==n_and) { usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; if (m <= 64) return bi_N; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrv(&rp, n); - if (rtid==n_ne|rtid==n_eq) xor_rows_bit(xp, rp, n, m, rtid==n_eq); - else or_rows_bit(xp, rp, n, m, rtid==n_and); + if (rtid==n_ne|rtid==n_eq) si_xor_rows_bit(xp, rp, n, m, rtid==n_eq); + else si_or_rows_bit(xp, rp, n, m, rtid==n_and); decG(x); return r; } + #endif return bi_N; } diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index e63ad62c..7c50724c 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -26,6 +26,8 @@ def elwidth{T} = width{eltype{T}} oper &~ andnot infix none 35 def andnot{a, b if anyNum{a} and anyNum{b}} = a & ~b +oper &- ({v:T,m:(u1)} => v & -promote{T,m}) infix left 35 + def load {p:*[_]_, n } = assert{0} diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 9eec3045..c083a6d7 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -70,3 +70,91 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { else extract{mix{op, r}, 0} } export{'si_sum_f64', fold_assoc_0{f64,+}} + + +fn xor_words(init:u64, x:*u64, l:usz):u64 = { + @for (x over l) init ^= x + init +} +def bit_output{rp:*T} = { + buf:u64 = 0 # Buffer for result bits + def output{i, bit, mod} = { + buf = buf>>1 | promote{u64, bit}<<63 + if ((i+1)%64==0) { store{rp, 0, mod{buf}}; ++rp } + } + def fixbuf{mod} = { buf = mod{buf} } + def flush_bits{n, mod} = { + q:=(-n)%64; if (q!=0) store{rp, 0, mod{buf} >> q} + } + def flush_bits{n} = flush_bits{n, {b}=>b} + tup{output, fixbuf, flush_bits} +} +# word and alignment of start of next row +def next_start{i, m} = { + bn := promote{u64, i+1} * promote{u64, m} + tup{bn/64, bn%64} +} +fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, eq:u1) : void = { + def p64 = promote{u64, .} + def fixout = ^{-(p64{eq} &~ p64{m}), .} # ne to eq conversion + def {add_bit, _, flush_bits} = bit_output{rp} + def add_bit{i, bit} = add_bit{i, bit, fixout} + def xor_loop{len} = { + o:u64 = 0 # Carry + j:u64 = 0; @for (i to n) { + def {jn, sh} = next_start{i, m} + s := xor_words(o, xp + j, len) + e := load{xp, jn} + s ^= e & (if (not same{l,1}) -p64{jn >= j + p64{l}} else j - jn) + o = e >> sh + add_bit{i, popc{s ^ o}} + j = jn+1 + } + } + l := m/64 + if (m < 128) xor_loop{1} + else if (m%64==0) { + @for (i to n) add_bit{i, popc{xor_words(0, xp+l*i, l)}} + } + else xor_loop{l} + flush_bits{n, fixout} +} + +fn or_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, op_and:u1) : void = { + def p64 = promote{u64, .} + def {add_bit, set_out, flush_bits} = bit_output{rp} + if (m < 128) { + c:u64 = (p64{m}-1) &- op_and # a row gives 1 if its sum is >c + o:u64 = 0 + j:u64 = 0; @for (i to n) { + def {jn, sh} = next_start{i, m} + s := o + popc{load{xp,j}} + e := load{xp,jn} + s += popc{e & (j - jn)} # mask is 0 if j==jn, or -1 + o = popc{e >> sh} + add_bit{i, s > c+o, {rw}=>rw} + j = jn+1 + } + } else { + rx := -promote{u64, op_and}; id := ~rx + def fixout = ^{rx, .} + o:u64 = 0 # Saved bits + j:u64 = 0; @for (i to n) { + def {jn, sh} = next_start{i, m} + e := load{xp,jn} ^ rx + m := ~(u64~~0) << sh + rb:u64 = 1 + if ((o | (e &~ m)) == 0) { # Search for shortcut + @for (i from j to jn-1) if (load{xp,i} != id) goto{'found'} + rb = 0; setlabel{'found'} + } + o = e & m + add_bit{i, rb, fixout} + j = jn+1 + } + set_out{fixout} + } + flush_bits{n} +} +export{'si_xor_rows_bit', xor_rows_bit} +export{'si_or_rows_bit', or_rows_bit} diff --git a/src/singeli/src/search.singeli b/src/singeli/src/search.singeli index 8bc5c67c..2716a63b 100644 --- a/src/singeli/src/search.singeli +++ b/src/singeli/src/search.singeli @@ -335,8 +335,6 @@ exportT{'simd_getRangeRaw', each{getRange, tup{i8,i16,i32,f64}}} # Hash tables -oper &- ({v:T,m} => v & -promote{T,m}) infix left 35 - def rty{name} = if (to_prim{name}=='∊') i8 else i32 def ity{name} = (to_prim{name}=='⊒')**(*u32) fn hashtab{T, name}(rpi:*rty{name}, iv:*void, mi:usz, fv:*void, ni:usz, links:ity{name}) = { From 5591d0f4f04e8fb01a3de5d22805f7a2478cc832 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Fri, 31 May 2024 18:16:56 -0400 Subject: [PATCH 06/38] =?UTF-8?q?Boolean=20row-wise=20scans=20=E2=88=A7`?= =?UTF-8?q?=CB=98=20and=20=E2=88=A8`=CB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 12 ++++++ src/builtins/scan.c | 17 ++++++++ src/singeli/src/scan.singeli | 83 ++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 00e55267..e4632510 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -9,6 +9,7 @@ B shape_c2(B, B, B); B transp_c2(B, B, B); B fold_rows(Md1D* d, B x); // from fold.c B fold_rows_bit(Md1D* d, B x); // from fold.c +B scan_rows_bit(Md1D* d, B x); // from scan.c B takedrop_highrank(bool take, B w, B x); // from sfns.c B try_interleave_cells(B w, B x, ur xr, ur xk, usz* xsh); // from transpose.c @@ -461,6 +462,17 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr if (m <= 64 && m < sh[0]) return fold_rows(fd, x); } } + if (rtid==n_scan) { + if (cr==0) goto base; + usz *sh = SH(x); usz m = sh[k]; + if (m<=1 || IA(x)==0) return x; + if (!isFun(fd->f)) goto base; + u8 frtid = v(fd->f)->flags-1; + if (frtid==n_rtack) return x; + if (k==1 && xr==2 && isPervasiveDyExt(fd->f) && TI(x,elType)==el_bit) { + B r = scan_rows_bit(fd, x); if (!q_N(r)) return r; + } + } } else if (TY(f) == t_md2D) { Md2D* fd = c(Md2D,f); u8 rtid = fd->m2->flags-1; diff --git a/src/builtins/scan.c b/src/builtins/scan.c index f798881d..a059c387 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -329,3 +329,20 @@ B scan_c2(Md1D* d, B w, B x) { B f = d->f; decG(x); return withFill(r.b, wf); } + +B scan_rows_bit(Md1D* fd, B x) { + assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); + #if SINGELI + if (!v(fd->f)->flags) return bi_N; + u8 rtid = v(fd->f)->flags-1; + if (rtid==n_and|rtid==n_or) { + usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + u64* xp = bitarr_ptr(x); + u64* rp; B r = m_bitarrc(&rp, x); + if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); + else si_scan_rows_or (xp, rp, n, m); + decG(x); return r; + } + #endif + return bi_N; +} diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index aec9976a..3858c748 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -295,3 +295,86 @@ export{'si_scan_plus_i32_i32', plus_scanC{i32, i32}} export{'si_scan_plus_i16_f64', plus_scanG{i16, f64}} export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} + + + +# Row-wise boolean scan +fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { + def qand = not id + assert{l > 0} + nw := cdiv{n*l, 64} + def res_m1{x,c,m} = { # result word with carry c, popc{m}<=1 + if (qand) x &~ ((x+c) & (x+m)) + else x | ((-x-c) &~ (x-m)) + } + if (l < 64) { + if ((l & (l-1)) == 0) { + if (l == 2) { + @for (r in dst, x in src over nw) { + r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10)) + } + } else { + m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + t := m << (l-1) + @for (r in dst, x in src over nw) { + r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) + else x | ~((t&~x) ^ ((x|t) - m))) + } + } + # could use for l>=8; not much faster and takes up space + # def rowwise{T} = @for (r in *T~~dst, x in *T~~src over (64/width{T})*nw) r = x &~ (x+1) + } else { + d:usz = 64 % l + m:u64 = (~u64~~0 >> d) / ((u64~~1 << l)-1) + m = m<> 1 else ~(x | m) >> 1 ) + a:= (if (qand) s + (x&(m|c)) else s + ((m|c) &~ x)) + r = (if (qand) s ^ a else ~(s ^ a) ) + c = a >> 63 + m = m>>d | m<<(l-d) + } + } + } else if (l < 160) { + q:usz = 0 # distance to next row boundary + c:u64 = id # carry + @for (r in dst, x in src over nw) { + b:= q<64 # whether there's a boundary + p:= q%64 # its position + q-= 64 - (l &- b) + r = res_m1{x, c, promote{u64, b} << p} + c = r >> 63 + } + } else { + ib:usz = 0 # row bit index + wn:usz = 0 # starting word of next row + c:u64 = id # carry + def word{bit} = bit * ((1<<64) - 1) + @for (i to n) { + iw:= wn + r := res_m1{load{src, iw}, c, u64~~1 << (ib%64)} + store{dst, iw, r}; ++iw + ib+= l; wn = ib/64 + c = r>>63 + if (c != id) while (iw < wn) { + x:= load{src, iw} + if (x != word{not id}) { + c = id + store{dst, iw, if (qand) x &~ (x+1) else x | -x}; ++iw + goto{'shortcut'} + } + store{dst, iw, x}; ++iw + } + setlabel{'shortcut'} + @for (r in dst over _ from iw to wn) r = word{id} + } + if (ib%64 != 0) { + x:= load{src, wn} + store{dst, wn, if (qand) x &~ (x+c) else x | (-x-c)} + } + } +} + +export{'si_scan_rows_and', scan_rows_andor{0}} +export{'si_scan_rows_or', scan_rows_andor{1}} From bd64e8bcd29ea759c8037ec1f2c3324da8a8b236 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 2 Jun 2024 16:01:46 -0400 Subject: [PATCH 07/38] =?UTF-8?q?Faster=20boolean=20+`=CB=98=20for=20short?= =?UTF-8?q?=20rows=20based=20on=20flattened=20prefix=20sum?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index a059c387..81d9cafe 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -343,6 +343,25 @@ B scan_rows_bit(Md1D* fd, B x) { else si_scan_rows_or (xp, rp, n, m); decG(x); return r; } + if (rtid==n_add && SH(x)[1]<128) { + usz ia = IA(x); usz m = SH(x)[1]; + usz bl = 128; // block size + i8 buf[bl]; i8 c = 0; + u64* xp = bitarr_ptr(x); + i8* rp; B r = m_i8arrc(&rp, x); + for (usz i = 0, j = m; i < ia; i += bl) { + usz len = ia - i; if (len > bl) len = bl; + usz e = i + len; + si_bcs8(xp + i/64, buf, len); + memset(rp+i, -c, len); + i8* bi = buf-i; + assert(j > i); for (; j < e; j += m) rp[j] = bi[j-1]; + si_scan_max_init_i8(rp+i, rp+i, len, I8_MIN); + for (usz k = i; k < e; k++) rp[k] = bi[k] - rp[k]; + if (j == e) { j += m; c = 0; } else c = rp[e-1]; + } + decG(x); return r; + } #endif return bi_N; } From 8331a05547a611298a398425f13e54fb5c9c76e6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 2 Jun 2024 18:06:55 -0400 Subject: [PATCH 08/38] =?UTF-8?q?Boolean=20=E2=89=A0`=CB=98=20implementati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 7 +++--- src/singeli/src/scan.singeli | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 81d9cafe..c0e78ff4 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -335,12 +335,13 @@ B scan_rows_bit(Md1D* fd, B x) { #if SINGELI if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; - if (rtid==n_and|rtid==n_or) { + if (rtid==n_and|rtid==n_or|rtid==n_ne) { usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); - if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); - else si_scan_rows_or (xp, rp, n, m); + if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); + else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); + else si_scan_rows_ne (xp, rp, n, m); decG(x); return r; } if (rtid==n_add && SH(x)[1]<128) { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 3858c748..910d5e80 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -376,5 +376,50 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { } } +fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { + def scan_word = prefix_byshift{^, <<} + assert{l > 0} + nl := n*l + nw := cdiv{nl, 64} + if (l < 64) { + if ((l & (l-1)) == 0) { + m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + @for (r, x over nw) { + s:= scan_word{x} + b:= s<<1 & m # last bit of previous row + r = s ^ (b<> d) / ((u64~~1 << l)-1) + m = m<>63) + m = m>>d | m<<(l-d) + } + } + } else { + i :usz = 0 # row bit index + iw:usz = 0 # starting word + c:u64 = 0 # carry + while (1) { + i+= l; ii := iw; iw = cdiv{i, 64} + scan_neq{}(c, x+ii, r+ii, promote{u64,iw-ii}) + if (i == nl) return{} + s:= load{r, iw-1} + q := i%64 + s^= -(s<<1 & u64~~1<0)) >> 63) + } + } +} + export{'si_scan_rows_and', scan_rows_andor{0}} export{'si_scan_rows_or', scan_rows_andor{1}} +export{'si_scan_rows_ne', scan_rows_neq} From 8e47221c9f417643e1ab689f4f8ae1a2eef673c8 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 2 Jun 2024 20:28:58 -0400 Subject: [PATCH 09/38] =?UTF-8?q?Implement=20monadic=20boolean=20=3D`=20wi?= =?UTF-8?q?th=20=E2=89=A0`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index c0e78ff4..aa64ea93 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -35,6 +35,12 @@ B scan_ne(B x, u64 p, u64 ia) { // consumes x #endif decG(x); return r; } +B scan_eq(B x, u64 ia) { // consumes x + B r = scan_ne(x, 0, ia); + u64* rp = bitarr_ptr(r); + for (usz i = 0; i < BIT_N(ia); i++) rp[i] ^= 0xAAAAAAAAAAAAAAAA; + return r; +} static B scan_or(B x, u64 ia) { // consumes x u64* xp = bitarr_ptr(x); @@ -209,6 +215,7 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (rtid==n_or | rtid==n_ceil ) return scan_or(x, ia); // ∨⌈ if (rtid==n_and | rtid==n_mul | rtid==n_floor) return scan_and(x, ia); // ∧×⌊ if (rtid==n_ne ) return scan_ne(x, 0, ia); // ≠ + if (rtid==n_eq ) return scan_eq(x, ia); // = if (rtid==n_lt) return scan_lt(x, 0, ia); // < goto base; } @@ -335,14 +342,15 @@ B scan_rows_bit(Md1D* fd, B x) { #if SINGELI if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; - if (rtid==n_and|rtid==n_or|rtid==n_ne) { + if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq) { + if (rtid==n_eq) x = bit_negate(x); usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); else si_scan_rows_ne (xp, rp, n, m); - decG(x); return r; + decG(x); return rtid==n_eq ? bit_negate(r) : r; } if (rtid==n_add && SH(x)[1]<128) { usz ia = IA(x); usz m = SH(x)[1]; From fde29684d749f0bb8925a07a21e1c78e02ca390c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 2 Jun 2024 21:50:49 -0400 Subject: [PATCH 10/38] =?UTF-8?q?Tweak=20for=20faster=20+`=CB=98=20on=20ve?= =?UTF-8?q?ry=20short=20boolean=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index aa64ea93..6d4baa6d 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -358,13 +358,17 @@ B scan_rows_bit(Md1D* fd, B x) { i8 buf[bl]; i8 c = 0; u64* xp = bitarr_ptr(x); i8* rp; B r = m_i8arrc(&rp, x); + u64 ms[7] = { 0x00ff00ff00ff00ff, 0x00ff0000ff0000ff, 0x000000ff000000ff, 0x0000ff00000000ff, 0x00ff0000000000ff, 0xff000000000000ff, 0 }; + u64 mm = ms[m-2>6? 6 : m-2]; usz mk = m*(POPC(mm)/8); for (usz i = 0, j = m; i < ia; i += bl) { usz len = ia - i; if (len > bl) len = bl; usz e = i + len; si_bcs8(xp + i/64, buf, len); memset(rp+i, -c, len); i8* bi = buf-i; - assert(j > i); for (; j < e; j += m) rp[j] = bi[j-1]; + assert(j > i); + if (mk) while (j+mk <= e) { *(u64*)(rp+j) = *(u64*)(bi+j-1) & mm; j+=mk; } + for (; j < e; j += m) rp[j] = bi[j-1]; si_scan_max_init_i8(rp+i, rp+i, len, I8_MIN); for (usz k = i; k < e; k++) rp[k] = bi[k] - rp[k]; if (j == e) { j += m; c = 0; } else c = rp[e-1]; From 3b61568fda5df0089bbe6dffb1678d254a29795d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 3 Jun 2024 16:55:39 -0400 Subject: [PATCH 11/38] =?UTF-8?q?Implement=20boolean=20list=20scans=20->?= =?UTF-8?q?=E2=89=A4=E2=89=A5=20using=20more=20common=20scans?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/scan.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 6d4baa6d..dcfed95a 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -9,6 +9,12 @@ static u64 vg_rand(u64 x) { return x; } #endif +B slash_c1(B, B); +B shape_c2(B, B, B); +B fne_c1(B, B); +B sub_c2(B, B, B); +B add_c2(B, B, B); + #if SINGELI #define SINGELI_FILE scan #include "../utils/includeSingeli.h" @@ -57,7 +63,6 @@ static B scan_and(B x, u64 ia) { // consumes x decG(x); return FL_SET(r, fl_dsc|fl_squoze); } -B slash_c1(B t, B x); B scan_add_bool(B x, u64 ia) { // consumes x u64* xp = bitarr_ptr(x); u64 xs = bit_sum(xp, ia); @@ -119,7 +124,6 @@ B scan_max_num(B x, u8 xe, u64 ia) { MINMAX(max,>,MIN,or ,asc) } #undef MINMAX // Initialized: try to convert 𝕨 to type of 𝕩 // (could do better for out-of-range floats) -B shape_c2(B, B, B); #define MM2_ICASE(T,N,C,I) \ case el_##T : { \ if (wv!=(T)wv) { if (wv C 0) { r=C2(shape,m_f64(ia),w); break; } else wv=I; } \ @@ -180,9 +184,6 @@ static B scan_plus(f64 r0, B x, u8 xe, usz ia) { #endif } -B fne_c1(B, B); -B shape_c2(B, B, B); - extern B scan_arith(B f, B w, B x, usz* xsh); // from cells.c B scan_c1(Md1D* d, B x) { B f = d->f; if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); @@ -210,14 +211,17 @@ B scan_c1(Md1D* d, B x) { B f = d->f; } if (!(xr==1 && xe<=el_f64)) goto base; - if (xe==el_bit) { - if (rtid==n_add ) return scan_add_bool(x, ia); // + - if (rtid==n_or | rtid==n_ceil ) return scan_or(x, ia); // ∨⌈ - if (rtid==n_and | rtid==n_mul | rtid==n_floor) return scan_and(x, ia); // ∧×⌊ - if (rtid==n_ne ) return scan_ne(x, 0, ia); // ≠ - if (rtid==n_eq ) return scan_eq(x, ia); // = - if (rtid==n_lt) return scan_lt(x, 0, ia); // < - goto base; + if (xe==el_bit) switch (rtid) { default: goto base; + case n_add: return scan_add_bool(x, ia); // + + case n_or: case n_ceil: return scan_or(x, ia); // ∨⌈ + case n_and: case n_mul: case n_floor: return scan_and(x, ia); // ∧×⌊ + case n_ne: return scan_ne(x, 0, ia); // ≠ + case n_eq: return scan_eq(x, ia); // = + case n_lt: return scan_lt(x, 0, ia); // < + case n_le: return bit_negate(scan_lt(bit_negate(x), 0, ia)); // ≤ + case n_gt: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_and(x, ia); // > + case n_ge: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_or (x, ia); // ≥ + case n_sub: return C2(sub, m_f64(2 * (*bitarr_ptr(x) & 1)), scan_add_bool(x, ia)); // - } if (rtid==n_add) return scan_plus(0, x, xe, ia); // + if (rtid==n_floor) return scan_min_num(x, xe, ia); // ⌊ @@ -258,7 +262,6 @@ B scan_c1(Md1D* d, B x) { B f = d->f; return withFill(r.b, xf); } -B add_c2(B, B, B); B scan_c2(Md1D* d, B w, B x) { B f = d->f; if (isAtm(x) || RNK(x)==0) thrM("`: 𝕩 cannot have rank 0"); ur xr = RNK(x); usz* xsh = SH(x); usz ia = IA(x); From efde979fe945cdc03bb4ec1873debfda6319e4cf Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 3 Jun 2024 18:28:44 -0400 Subject: [PATCH 12/38] =?UTF-8?q?Fast=20boolean=20=E2=8A=A3`=CB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 2 +- src/builtins/scan.c | 9 ++-- src/singeli/src/scan.singeli | 93 +++++++++++++++++++++++++----------- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index e4632510..b812d579 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -469,7 +469,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr if (!isFun(fd->f)) goto base; u8 frtid = v(fd->f)->flags-1; if (frtid==n_rtack) return x; - if (k==1 && xr==2 && isPervasiveDyExt(fd->f) && TI(x,elType)==el_bit) { + if (k==1 && xr==2 && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && TI(x,elType)==el_bit) { B r = scan_rows_bit(fd, x); if (!q_N(r)) return r; } } diff --git a/src/builtins/scan.c b/src/builtins/scan.c index dcfed95a..c219d8e1 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -345,14 +345,15 @@ B scan_rows_bit(Md1D* fd, B x) { #if SINGELI if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; - if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq) { + if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq|rtid==n_ltack) { if (rtid==n_eq) x = bit_negate(x); usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); - if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); - else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); - else si_scan_rows_ne (xp, rp, n, m); + if (rtid==n_and ) si_scan_rows_and (xp, rp, n, m); + else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); + else if (rtid==n_ltack) si_scan_rows_ltack(xp, rp, n, m); + else si_scan_rows_ne (xp, rp, n, m); decG(x); return rtid==n_eq ? bit_negate(r) : r; } if (rtid==n_add && SH(x)[1]<128) { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 910d5e80..02d3c90f 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -299,6 +299,20 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} # Row-wise boolean scan +def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1) +def loop_with_unaligned_mask{x, r, nw, l, step} = { + d:usz = 64 % l + m:u64 = (~u64~~0 >> d) / ((u64~~1 << l)-1) + m = m< { r=nr; c=nc } + {nr} => { r=nr; c = -(r>>63) } + } + m = m>>d | m<<(l-d) + } +} fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { def qand = not id assert{l > 0} @@ -314,7 +328,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10)) } } else { - m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + m:u64 = aligned_mask{l} t := m << (l-1) @for (r in dst, x in src over nw) { r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) @@ -324,17 +338,12 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { # could use for l>=8; not much faster and takes up space # def rowwise{T} = @for (r in *T~~dst, x in *T~~src over (64/width{T})*nw) r = x &~ (x+1) } else { - d:usz = 64 % l - m:u64 = (~u64~~0 >> d) / ((u64~~1 << l)-1) - m = m< { s:= (if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 ) a:= (if (qand) s + (x&(m|c)) else s + ((m|c) &~ x)) - r = (if (qand) s ^ a else ~(s ^ a) ) - c = a >> 63 - m = m>>d | m<<(l-d) - } + tup{(if (qand) s ^ a else ~(s ^ a) ), + a >> 63} # new c + }} } } else if (l < 160) { q:usz = 0 # distance to next row boundary @@ -347,15 +356,15 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { c = r >> 63 } } else { - ib:usz = 0 # row bit index + i :usz = 0 # row bit index wn:usz = 0 # starting word of next row c:u64 = id # carry def word{bit} = bit * ((1<<64) - 1) - @for (i to n) { + @for (n) { iw:= wn - r := res_m1{load{src, iw}, c, u64~~1 << (ib%64)} + r := res_m1{load{src, iw}, c, u64~~1 << (i%64)} store{dst, iw, r}; ++iw - ib+= l; wn = ib/64 + i+= l; wn = i/64 c = r>>63 if (c != id) while (iw < wn) { x:= load{src, iw} @@ -369,7 +378,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { setlabel{'shortcut'} @for (r in dst over _ from iw to wn) r = word{id} } - if (ib%64 != 0) { + if (i%64 != 0) { x:= load{src, wn} store{dst, wn, if (qand) x &~ (x+c) else x | (-x-c)} } @@ -383,25 +392,19 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + m:u64 = aligned_mask{l} @for (r, x over nw) { s:= scan_word{x} b:= s<<1 & m # last bit of previous row r = s ^ (b<> d) / ((u64~~1 << l)-1) - m = m< { s:= scan_word{x} f:= (m-1)&~m # bits before first full row b:= s<<1 & m # last bit of previous row - r = s ^ ((c & f) | (b<>63) - m = m>>d | m<<(l-d) - } + s ^ ((c & f) | (b< 0} + nl := n*l + nw := cdiv{nl, 64} + if (l < 64) { + if ((l & (l-1)) == 0) { + m:u64 = aligned_mask{l} + @for (r, x over nw) { b:= x & m; r = b< { + f:= (m-1)&~m # bits before first full row + b:= x & m + (c & f) | (b<>63) + i+= l; wn = i/64 + @for (r in r over _ from iw+1 to wn) r = c + } + if (i%64 != 0) store{r, wn, c} + } +} + +export{'si_scan_rows_and', scan_rows_andor{0}} +export{'si_scan_rows_or', scan_rows_andor{1}} +export{'si_scan_rows_ne', scan_rows_neq} +export{'si_scan_rows_ltack', scan_rows_left} From 6d27dd391b70325eacc25bba50d9471ad2571ade Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 3 Jun 2024 18:49:32 -0400 Subject: [PATCH 13/38] =?UTF-8?q?Clean=20up=20scan=5Frows=5Fbit=20logic=20?= =?UTF-8?q?and=20implement=20-`=CB=98=20as=20(2=C3=97=E2=8A=A3`=CB=98)-+`?= =?UTF-8?q?=CB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 4 +-- src/builtins/scan.c | 80 ++++++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index b812d579..0fb83ed6 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -9,7 +9,7 @@ B shape_c2(B, B, B); B transp_c2(B, B, B); B fold_rows(Md1D* d, B x); // from fold.c B fold_rows_bit(Md1D* d, B x); // from fold.c -B scan_rows_bit(Md1D* d, B x); // from scan.c +B scan_rows_bit(u8, B x); // from scan.c B takedrop_highrank(bool take, B w, B x); // from sfns.c B try_interleave_cells(B w, B x, ur xr, ur xk, usz* xsh); // from transpose.c @@ -470,7 +470,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr u8 frtid = v(fd->f)->flags-1; if (frtid==n_rtack) return x; if (k==1 && xr==2 && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && TI(x,elType)==el_bit) { - B r = scan_rows_bit(fd, x); if (!q_N(r)) return r; + B r = scan_rows_bit(frtid, x); if (!q_N(r)) return r; } } } else if (TY(f) == t_md2D) { diff --git a/src/builtins/scan.c b/src/builtins/scan.c index c219d8e1..fc733332 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -12,8 +12,9 @@ static u64 vg_rand(u64 x) { return x; } B slash_c1(B, B); B shape_c2(B, B, B); B fne_c1(B, B); -B sub_c2(B, B, B); B add_c2(B, B, B); +B sub_c2(B, B, B); +B mul_c2(B, B, B); #if SINGELI #define SINGELI_FILE scan @@ -340,45 +341,50 @@ B scan_c2(Md1D* d, B w, B x) { B f = d->f; return withFill(r.b, wf); } -B scan_rows_bit(Md1D* fd, B x) { +B scan_rows_bit(u8 rtid, B x) { assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); #if SINGELI - if (!v(fd->f)->flags) return bi_N; - u8 rtid = v(fd->f)->flags-1; - if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq|rtid==n_ltack) { - if (rtid==n_eq) x = bit_negate(x); - usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; - u64* xp = bitarr_ptr(x); - u64* rp; B r = m_bitarrc(&rp, x); - if (rtid==n_and ) si_scan_rows_and (xp, rp, n, m); - else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); - else if (rtid==n_ltack) si_scan_rows_ltack(xp, rp, n, m); - else si_scan_rows_ne (xp, rp, n, m); - decG(x); return rtid==n_eq ? bit_negate(r) : r; - } - if (rtid==n_add && SH(x)[1]<128) { - usz ia = IA(x); usz m = SH(x)[1]; - usz bl = 128; // block size - i8 buf[bl]; i8 c = 0; - u64* xp = bitarr_ptr(x); - i8* rp; B r = m_i8arrc(&rp, x); - u64 ms[7] = { 0x00ff00ff00ff00ff, 0x00ff0000ff0000ff, 0x000000ff000000ff, 0x0000ff00000000ff, 0x00ff0000000000ff, 0xff000000000000ff, 0 }; - u64 mm = ms[m-2>6? 6 : m-2]; usz mk = m*(POPC(mm)/8); - for (usz i = 0, j = m; i < ia; i += bl) { - usz len = ia - i; if (len > bl) len = bl; - usz e = i + len; - si_bcs8(xp + i/64, buf, len); - memset(rp+i, -c, len); - i8* bi = buf-i; - assert(j > i); - if (mk) while (j+mk <= e) { *(u64*)(rp+j) = *(u64*)(bi+j-1) & mm; j+=mk; } - for (; j < e; j += m) rp[j] = bi[j-1]; - si_scan_max_init_i8(rp+i, rp+i, len, I8_MIN); - for (usz k = i; k < e; k++) rp[k] = bi[k] - rp[k]; - if (j == e) { j += m; c = 0; } else c = rp[e-1]; + switch (rtid) { default: return bi_N; + case n_eq: return bit_negate(scan_rows_bit(n_ne, bit_negate(x))); + case n_and: case n_or: case n_ne: case n_ltack: { + usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + u64* xp = bitarr_ptr(x); + u64* rp; B r = m_bitarrc(&rp, x); + switch (rtid) { default:UD; + case n_and: si_scan_rows_and (xp, rp, n, m); break; + case n_or: si_scan_rows_or (xp, rp, n, m); break; + case n_ne: si_scan_rows_ne (xp, rp, n, m); break; + case n_ltack: si_scan_rows_ltack(xp, rp, n, m); break; + } + decG(x); return r; + } + case n_add: case n_sub: { + usz ia = IA(x); usz m = SH(x)[1]; + if (m >= 128) return bi_N; + usz bl = 128; // block size + i8 buf[bl]; i8 c = 0; + u64* xp = bitarr_ptr(x); + i8* rp; B r = m_i8arrc(&rp, x); + u64 ms[7] = { 0x00ff00ff00ff00ff, 0x00ff0000ff0000ff, 0x000000ff000000ff, 0x0000ff00000000ff, 0x00ff0000000000ff, 0xff000000000000ff, 0 }; + u64 mm = ms[m-2>6? 6 : m-2]; usz mk = m*(POPC(mm)/8); + for (usz i = 0, j = m; i < ia; i += bl) { + usz len = ia - i; if (len > bl) len = bl; + usz e = i + len; + si_bcs8(xp + i/64, buf, len); + memset(rp+i, -c, len); + i8* bi = buf-i; + assert(j > i); + if (mk) while (j+mk <= e) { *(u64*)(rp+j) = *(u64*)(bi+j-1) & mm; j+=mk; } + for (; j < e; j += m) rp[j] = bi[j-1]; + si_scan_max_init_i8(rp+i, rp+i, len, I8_MIN); + for (usz k = i; k < e; k++) rp[k] = bi[k] - rp[k]; + if (j == e) { j += m; c = 0; } else c = rp[e-1]; + } + if (rtid!=n_sub) { decG(x); return r; } + return C2(sub, C2(mul, m_f64(2), scan_rows_bit(n_ltack, x)), r); } - decG(x); return r; } - #endif + #else return bi_N; + #endif } From 40bf3bfd1ceca442d1e7f363ba340802a8bbd690 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 4 Jun 2024 20:35:46 -0400 Subject: [PATCH 14/38] AVX2-based boolean short-row scans --- src/singeli/src/base.singeli | 2 +- src/singeli/src/scan.singeli | 52 +++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/base.singeli b/src/singeli/src/base.singeli index 7c50724c..b9b6ec8c 100644 --- a/src/singeli/src/base.singeli +++ b/src/singeli/src/base.singeli @@ -158,7 +158,7 @@ def lvec = match { {[n]T, n, (width{T})} => 1; {T, n, w} => 0 } # base cases def { - absu,andAllZero,andnz,b_getBatch,clmul,cvt,extract,fold_addw,half, + absu,andAllZero,andnz,b_getBatch,blend,clmul,cvt,extract,fold_addw,half, homAll,homAny,homBlend,homMask,homMaskStore,homMaskStoreF,loadBatchBit, loadLow,make,maskStore,maskToHom,mulw,mulh,narrow,narrowPair,packHi,packLo,packQ,pair,pdep, pext,popcRand,sel,shl,shr,shuf,shuf16Hi,shuf16Lo,shufHalves,storeLow, diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 02d3c90f..4b65c5ae 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -5,6 +5,7 @@ if_inline (hasarch{'X86_64'}) { } include './mask' include './f64' +include 'util/tup' include './scan_common' # Initialized scan, generic implementation @@ -300,10 +301,13 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} # Row-wise boolean scan def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1) +def unaligned_mask{l} = { + def d = 64 % l + def m = (~u64~~0 >> d) / ((u64~~1 << l)-1) + tup{m<> d) / ((u64~~1 << l)-1) - m = m<>d | m<<(l-d) } } +def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { + {ms, d} := unaligned_mask{l} + def V = [4]u64 + d4:usz = width{V} % l + m:= make{V, scan{{a,_} => a>>d | a<<(l-d), tup{ms, ...iota{3}}}} + c:= V**0 + @maskedLoop{4} (x in tup{V, xp}, + r in tup{V, rp} over promote{u64,nw}) { + s := scan_words{x, m} + pc:= c; c = shuf{V, -(s>>63), 4b2103} + r = apply_carry{s, blend{V, c, pc, 2b0001}, (m-V**1)&~m} + m = m>>d4 | m<<(l-d4) + } +} + fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { def qand = not id assert{l > 0} @@ -337,6 +356,17 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { } # could use for l>=8; not much faster and takes up space # def rowwise{T} = @for (r in *T~~dst, x in *T~~src over (64/width{T})*nw) r = x &~ (x+1) + } else if (hasarch{'AVX2'}) { + def scan_words{x, m:V} = { + mb:= m | V**1 + p:= if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 + a:= if (qand) p + (mb & x) else p + (mb &~ x) + if (qand) p ^ a else ~(p ^ a) + } + def apply_carry{s, c, f} = { + if (qand) s & (~f | c) else s | (f & c) + } + avx2_loop_with_unaligned_mask{src, dst, nw, l, scan_words, apply_carry} } else { loop_with_unaligned_mask{src, dst, nw, l, {x, c, m} => { s:= (if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 ) @@ -398,6 +428,18 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { b:= s<<1 & m # last bit of previous row r = s ^ (b< pre{v, 1} + } + s:= vec_prefix_byshift{^, <<}{x} + b:= s<<1 & m # last bit of previous row + s ^ (b< { s:= scan_word{x} @@ -432,6 +474,10 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = { if ((l & (l-1)) == 0) { m:u64 = aligned_mask{l} @for (r, x over nw) { b:= x & m; r = b< { f:= (m-1)&~m # bits before first full row From e6f1e04de21d23336c9326ff37a0dfe6f9380547 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 13 Jun 2024 16:52:52 -0400 Subject: [PATCH 15/38] =?UTF-8?q?Fast=20generic=20and=20pext-based=20?= =?UTF-8?q?=E2=88=A7=CB=9D=CB=98=20and=20=E2=88=A8=CB=9D=CB=98=20on=20widt?= =?UTF-8?q?h<64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/fold.c | 18 ++++- src/singeli/src/fold.singeli | 142 ++++++++++++++++++++++++++++++++--- 2 files changed, 148 insertions(+), 12 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 2547d828..70587410 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -14,6 +14,7 @@ #include "../core.h" #include "../builtins.h" #include "../utils/mut.h" +#include "../utils/calls.h" #if SINGELI #define SINGELI_FILE fold @@ -512,12 +513,23 @@ B fold_rows_bit(Md1D* fd, B x) { if (rtid==n_add) return sum_rows_bit(x); #if SINGELI if (rtid==n_ne|rtid==n_eq|rtid==n_or|rtid==n_and) { + bool andor = rtid==n_or|rtid==n_and; usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; - if (m <= 64) return bi_N; + if (andor && m < 256) while (m%8 == 0) { + usz f = CTZ(m|32); + m >>= f; usz c = m*n; + u64* yp; B y = m_bitarrv(&yp, c); + u8 e = el_i8 + f-3; + CmpASFn cmp = rtid==n_or ? CMP_AS_FN(ne, e) : CMP_AS_FN(eq, e); + CMP_AS_CALL(cmp, yp, bitarr_ptr(x), m_f64((rtid==n_or)-1), c); + decG(x); if (m==1) return y; + x = y; + } + if (!andor && m <= 64) return bi_N; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrv(&rp, n); - if (rtid==n_ne|rtid==n_eq) si_xor_rows_bit(xp, rp, n, m, rtid==n_eq); - else si_or_rows_bit(xp, rp, n, m, rtid==n_and); + if (andor) si_or_rows_bit(xp, rp, n, m, rtid==n_and); + else si_xor_rows_bit(xp, rp, n, m, rtid==n_eq); decG(x); return r; } #endif diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index c083a6d7..ff257bf4 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -1,5 +1,8 @@ include './base' include './mask' +if_inline (hasarch{'BMI2'}) include './bmi2' + +include 'util/tup' def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}} def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}} @@ -120,14 +123,135 @@ fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, eq:u1) : void = { flush_bits{n, fixout} } -fn or_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, op_and:u1) : void = { - def p64 = promote{u64, .} +def unaligned_mask{l} = { + def d = 64 % l + def m = (~u64~~0 >> d) / ((u64~~1 << l)-1) + tup{m< loop_pext{u32, {x}=>op{x, x<<1}, t} + } else { + def all = 1<<64-1 + ms:5**u64 = reverse{slice{scan{{x,s}=>x^((x< { + def T = u32 + @for (x in xp, r in *T~~rp over nw) { + a := op{x, x>>1} + each{{m, sh} => { a &= m; a |= a>>sh }, ms, 1< { + def T = u16 + @for (x in xp, r in *T~~rp over nw) { + a := get{x} & t + a = (a * 4r2b001) & (64w0xf000) + a = (a * fold{+, 1<<(12*iota{4})}) >> (3*16) + r = cast_i{T, a} + } + } + } + } + if (op_and) loop{{x} => x & ((x&~t) + m)} + else loop{{x} => x | ((x| t) - m)} + } else { # odd row length + d:usz = 64 % l + e := ((~u64~~0 >> d) / ((u64~~1 << l)-1)) << (l-1) + r:u64 = 0 + rh := *u32~~rp + ri:ux = 0 + if (fast_BMI2{}) { + @for (xo in xp over i to nw) { + m := e<<1 | 1 + t := e | 1<<63 + x := xo^xx + r |= pext{x | ((x|t) - m), t} << ri + ri += popc{e} + e = e>>d | e<<(l-d) + if (ri >= 32) { + store{rh, 0, cast_i{u32,r^xx}}; ++rh; + r >>= 32; ri -= 32 + } + } + } else { + dm:= 64/l + e0:= e<<1 | 1 + c:u64 = 0 + def loop{...par} = { + @for (xo in xp over nw) { + m := e<<1 | 1 + t := e | (1<<63) + x := xo^xx + s := x | ((x|t) - m) # Fold results + cs:= c; c = (s&~e) >> 63 + nb:= dm + promote{usz, e>=e0} # = popc{e} + def extract = match { + {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q + { {q}, {}} => (s & e) * (q << clz{e}) + } + rb:= extract{...par} >> (64 - nb) + r |= (rb|cs) << ri + ri += promote{ux, nb} + if (ri >= 32) { + store{rh, 0, cast_i{u32,r^xx}}; ++rh; + r >>= 32; ri -= 32 + } + e = e>>d | e<<(l-d) + } + } + if (l == 3) { + mult0:u64 = base{1<< 2, 3**1}; top3:u64 = base{1<<9, 8**(1<<3-1)}>>2 + mult1:u64 = base{1<< 6, 3**1}; top9:u64 = base{1<<27, 3**(1<<9-1)}<<1 + mult2:u64 = base{1<<18, 3**1} + loop{tup{mult0,mult1,mult2}, tup{top3, top9}} + } else if (l < 8) { + assert{l > 4} + ld:= l-1; lld:= l*ld + {mult0, _} := unaligned_mask{ld} + mult0 &= u64~~1<>l; topk|= topk<>ll + loop{tup{mult0,mult1}, tup{topk}} + } else { + {mult, _} := unaligned_mask{l-1} + loop{tup{mult}, tup{}} + } + } + if (ri > 0) store{rh, 0, cast_i{u32,r^xx}} + } +} + +fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { def {add_bit, set_out, flush_bits} = bit_output{rp} - if (m < 128) { - c:u64 = (p64{m}-1) &- op_and # a row gives 1 if its sum is >c + if (l < 64) { + or_rows_bit_lt64{xp, rp, n, l, op_and} + return{} + } else if (l < 128) { + c:u64 = (promote{u64, l}-1) &- op_and # a row gives 1 if its sum is >c o:u64 = 0 j:u64 = 0; @for (i to n) { - def {jn, sh} = next_start{i, m} + def {jn, sh} = next_start{i, l} s := o + popc{load{xp,j}} e := load{xp,jn} s += popc{e & (j - jn)} # mask is 0 if j==jn, or -1 @@ -140,15 +264,15 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, op_and:u1) : void = { def fixout = ^{rx, .} o:u64 = 0 # Saved bits j:u64 = 0; @for (i to n) { - def {jn, sh} = next_start{i, m} + def {jn, sh} = next_start{i, l} e := load{xp,jn} ^ rx - m := ~(u64~~0) << sh + l := ~(u64~~0) << sh rb:u64 = 1 - if ((o | (e &~ m)) == 0) { # Search for shortcut + if ((o | (e &~ l)) == 0) { # Search for shortcut @for (i from j to jn-1) if (load{xp,i} != id) goto{'found'} rb = 0; setlabel{'found'} } - o = e & m + o = e & l add_bit{i, rb, fixout} j = jn+1 } From f0f130c42e4cbc11cc96bd24137c31d0c7bffe18 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 15 Jun 2024 18:30:46 -0400 Subject: [PATCH 16/38] Use a lookup table for evenly-spaced masks, getting division results from those --- src/builtins/fold.c | 2 ++ src/builtins/scan.c | 2 ++ src/builtins/slash.c | 6 +++++- src/singeli/src/fold.singeli | 24 +++++++++--------------- src/singeli/src/scan.singeli | 17 ++++++----------- src/singeli/src/slash.singeli | 8 ++++++++ src/singeli/src/spaced.singeli | 14 ++++++++++++++ 7 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 src/singeli/src/spaced.singeli diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 70587410..9d1617f3 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -17,6 +17,8 @@ #include "../utils/calls.h" #if SINGELI + extern uint64_t* const si_spaced_masks; + #define get_spaced_mask(i) si_spaced_masks[i-1] #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif diff --git a/src/builtins/scan.c b/src/builtins/scan.c index fc733332..c4fb2885 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -17,6 +17,8 @@ B sub_c2(B, B, B); B mul_c2(B, B, B); #if SINGELI + extern uint64_t* const si_spaced_masks; + #define get_spaced_mask(i) si_spaced_masks[i-1] #define SINGELI_FILE scan #include "../utils/includeSingeli.h" #endif diff --git a/src/builtins/slash.c b/src/builtins/slash.c index ce4a7deb..27357e79 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -777,7 +777,11 @@ B slash_c2(B t, B w, B x) { u64* rp; r = m_bitarrv(&rp, s); #if FAST_PDEP if (wv <= 52) { - u64 m = (u64)-1 / (((u64)1<> d) / ((u64~~1 << l)-1) - tup{m< x & ((x&~t) + m)} else loop{{x} => x | ((x| t) - m)} - } else { # odd row length - d:usz = 64 % l - e := ((~u64~~0 >> d) / ((u64~~1 << l)-1)) << (l-1) + } else { + {e0, d} := unaligned_spaced_mask_mod{l} + e := e0 << (l-1) r:u64 = 0 rh := *u32~~rp ri:ux = 0 @@ -193,8 +188,7 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { } } } else { - dm:= 64/l - e0:= e<<1 | 1 + dm:= cast_i{usz, popc{e}} c:u64 = 0 def loop{...par} = { @for (xo in xp over nw) { @@ -226,15 +220,15 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { } else if (l < 8) { assert{l > 4} ld:= l-1; lld:= l*ld - {mult0, _} := unaligned_mask{ld} + {mult0, _} := unaligned_spaced_mask_mod{ld} mult0 &= u64~~1<>l; topk|= topk<>ll loop{tup{mult0,mult1}, tup{topk}} } else { - {mult, _} := unaligned_mask{l-1} + {mult, _} := unaligned_spaced_mask_mod{l-1} loop{tup{mult}, tup{}} } } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 4b65c5ae..0f8417f5 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -5,6 +5,7 @@ if_inline (hasarch{'X86_64'}) { } include './mask' include './f64' +include './spaced' include 'util/tup' include './scan_common' @@ -300,14 +301,8 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} # Row-wise boolean scan -def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1) -def unaligned_mask{l} = { - def d = 64 % l - def m = (~u64~~0 >> d) / ((u64~~1 << l)-1) - tup{m< a>>d | a<<(l-d), tup{ms, ...iota{3}}}} @@ -347,7 +342,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10)) } } else { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} t := m << (l-1) @for (r in dst, x in src over nw) { r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) @@ -422,7 +417,7 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} @for (r, x over nw) { s:= scan_word{x} b:= s<<1 & m # last bit of previous row @@ -472,7 +467,7 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} @for (r, x over nw) { b:= x & m; r = b<>d | m<<(l-d), d} +} From a8b036ad080d93241962ae29715d51b66d45b53c Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jun 2024 09:18:06 -0400 Subject: [PATCH 17/38] =?UTF-8?q?Implement=20=E2=89=A0=CB=9D=CB=98=20and?= =?UTF-8?q?=20=3D=CB=9D=CB=98=20like=20=E2=88=A7=E2=88=A8=20on=20width<64?= =?UTF-8?q?=20boolean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/fold.c | 1 - src/singeli/src/fold.singeli | 196 ++++++++++++++++++++++------------- 2 files changed, 122 insertions(+), 75 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 9d1617f3..8a41e8d1 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -527,7 +527,6 @@ B fold_rows_bit(Md1D* fd, B x) { decG(x); if (m==1) return y; x = y; } - if (!andor && m <= 64) return bi_N; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrv(&rp, n); if (andor) si_or_rows_bit(xp, rp, n, m, rtid==n_and); diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 259bcb11..110e1b0f 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -2,8 +2,8 @@ include './base' include './mask' if_inline (hasarch{'BMI2'}) include './bmi2' include './spaced' - include 'util/tup' +include './scan_common' def opsh64{op}{v:([4]f64), perm} = op{v, shuf{[4]u64, v, perm}} def opsh32{op}{v:([2]f64), perm} = op{v, shuf{[4]u32, v, perm}} @@ -76,63 +76,17 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { export{'si_sum_f64', fold_assoc_0{f64,+}} -fn xor_words(init:u64, x:*u64, l:usz):u64 = { - @for (x over l) init ^= x - init -} -def bit_output{rp:*T} = { - buf:u64 = 0 # Buffer for result bits - def output{i, bit, mod} = { - buf = buf>>1 | promote{u64, bit}<<63 - if ((i+1)%64==0) { store{rp, 0, mod{buf}}; ++rp } - } - def fixbuf{mod} = { buf = mod{buf} } - def flush_bits{n, mod} = { - q:=(-n)%64; if (q!=0) store{rp, 0, mod{buf} >> q} - } - def flush_bits{n} = flush_bits{n, {b}=>b} - tup{output, fixbuf, flush_bits} -} -# word and alignment of start of next row -def next_start{i, m} = { - bn := promote{u64, i+1} * promote{u64, m} - tup{bn/64, bn%64} -} -fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, m:usz, eq:u1) : void = { - def p64 = promote{u64, .} - def fixout = ^{-(p64{eq} &~ p64{m}), .} # ne to eq conversion - def {add_bit, _, flush_bits} = bit_output{rp} - def add_bit{i, bit} = add_bit{i, bit, fixout} - def xor_loop{len} = { - o:u64 = 0 # Carry - j:u64 = 0; @for (i to n) { - def {jn, sh} = next_start{i, m} - s := xor_words(o, xp + j, len) - e := load{xp, jn} - s ^= e & (if (not same{l,1}) -p64{jn >= j + p64{l}} else j - jn) - o = e >> sh - add_bit{i, popc{s ^ o}} - j = jn+1 - } - } - l := m/64 - if (m < 128) xor_loop{1} - else if (m%64==0) { - @for (i to n) add_bit{i, popc{xor_words(0, xp+l*i, l)}} - } - else xor_loop{l} - flush_bits{n, fixout} -} - -def or_rows_bit_lt64{xp, rp, n, l, op_and} = { +def fold_rows_bit_lt64{ + op, run_loop2, run_loop4, pext_res, mult_in, xx, rx, rxs, + xp, rp, n, l +} = { nw := cdiv{n*l,64} - xx := -promote{u64, op_and} def loop_pext{T, get, b} = { - @for (x in xp, r in *T~~rp over nw) r = cast_i{T, pext{get{x}, b}} + @for (x in xp, r in *T~~rp over nw) r = cast_i{T, rxs{pext{get{x}, b}}} } if (l == 2) { t:u64 = 64w2b10 - def loop = { + run_loop2{ if (fast_BMI2{}) { {op} => loop_pext{u32, {x}=>op{x, x<<1}, t} } else { @@ -143,15 +97,14 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { @for (x in xp, r in *T~~rp over nw) { a := op{x, x>>1} each{{m, sh} => { a &= m; a |= a>>sh }, ms, 1<> (3*16) - r = cast_i{T, a} + r = cast_i{T, rxs{a}} } } } } - if (op_and) loop{{x} => x & ((x&~t) + m)} - else loop{{x} => x | ((x| t) - m)} } else { {e0, d} := unaligned_spaced_mask_mod{l} e := e0 << (l-1) @@ -175,38 +126,41 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { rh := *u32~~rp ri:ux = 0 if (fast_BMI2{}) { - @for (xo in xp over i to nw) { - m := e<<1 | 1 - t := e | 1<<63 - x := xo^xx - r |= pext{x | ((x|t) - m), t} << ri + c:u64 = 0 + @for (x in xp over i to nw) { + r |= pext_res{xx{x}, e, c} << ri ri += popc{e} e = e>>d | e<<(l-d) if (ri >= 32) { - store{rh, 0, cast_i{u32,r^xx}}; ++rh; + store{rh, 0, cast_i{u32,rx{r}}}; ++rh; r >>= 32; ri -= 32 } } } else { dm:= cast_i{usz, popc{e}} + e0 |= -promote{u64, l & (l-1) == 0} c:u64 = 0 def loop{...par} = { @for (xo in xp over nw) { - m := e<<1 | 1 - t := e | (1<<63) - x := xo^xx - s := x | ((x|t) - m) # Fold results - cs:= c; c = (s&~e) >> 63 + x:= xx{xo} + def {s, mod_rb} = if (length{select{par,0}} == 3) { + # fast path for l==3 + s:= op{op{x, x<<1}, op{x<<2, c}} + c = op{x>>63, x>>62} + tup{s, {rb}=>rb} + } else { + mult_in{x, e, c} + } nb:= dm + promote{usz, e>=e0} # = popc{e} def extract = match { {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q { {q}, {}} => (s & e) * (q << clz{e}) } rb:= extract{...par} >> (64 - nb) - r |= (rb|cs) << ri + r |= mod_rb{rb} << ri ri += promote{ux, nb} if (ri >= 32) { - store{rh, 0, cast_i{u32,r^xx}}; ++rh; + store{rh, 0, cast_i{u32,rx{r}}}; ++rh; r >>= 32; ri -= 32 } e = e>>d | e<<(l-d) @@ -229,17 +183,111 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { loop{tup{mult0,mult1}, tup{topk}} } else { {mult, _} := unaligned_spaced_mask_mod{l-1} + if (l==8) mult &= 1<<(7*8) - 1 loop{tup{mult}, tup{}} } } - if (ri > 0) store{rh, 0, cast_i{u32,r^xx}} + if (ri > 0) store{rh, 0, cast_i{u32,rx{r}}} + } +} + + +fn xor_words(init:u64, x:*u64, l:usz):u64 = { + @for (x over l) init ^= x + init +} +def bit_output{rp:*T} = { + buf:u64 = 0 # Buffer for result bits + def output{i, bit, mod} = { + buf = buf>>1 | promote{u64, bit}<<63 + if ((i+1)%64==0) { store{rp, 0, mod{buf}}; ++rp } + } + def fixbuf{mod} = { buf = mod{buf} } + def flush_bits{n, mod} = { + q:=(-n)%64; if (q!=0) store{rp, 0, mod{buf} >> q} + } + def flush_bits{n} = flush_bits{n, {b}=>b} + tup{output, fixbuf, flush_bits} +} +# word and alignment of start of next row +def next_start{i, m} = { + bn := promote{u64, i+1} * promote{u64, m} + tup{bn/64, bn%64} +} +fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, eq:u1) : void = { + def p64 = promote{u64, .} + rx:= -(p64{eq} &~ p64{l}) # ne to eq conversion + if (l <= 64 and not (l%8==0 and l>16)) { + def run_loop2{loop} = loop{^} + def run_loop4{m, t, loop} = loop{{x} => { x2:= x^(x<<1); x2^(x2<<2) }} + def xor_word = prefix_byshift{^, <<} + def pext_in{xo, e, c} = { + x := xor_word{xo} + rb:= x ^ (x<>63))>>(64-l) + rb + } + def pext_res{x, e, c} = pext{pext_in{x, e, c}, e} + def mult_in{x, e, c} = tup{pext_in{x, e, c}, {r}=>r} + fold_rows_bit_lt64{ + ^, run_loop2, run_loop4, pext_res, mult_in, {x}=>x, ^{rx,.}, ^{rx,.}, + xp, rp, n, l + } + } else { + def fixout = ^{rx, .} + def {add_bit, _, flush_bits} = bit_output{rp} + def add_bit{i, bit} = add_bit{i, bit, fixout} + def xor_loop{len} = { + o:u64 = 0 # Carry + j:u64 = 0; @for (i to n) { + def {jn, sh} = next_start{i, l} + s := xor_words(o, xp + j, len) + e := load{xp, jn} + s ^= e & (if (not same{len,1}) -p64{jn >= j + p64{len}} else j - jn) + o = e >> sh + add_bit{i, popc{s ^ o}} + j = jn+1 + } + } + ll := l/64 + if (l <= 64) { + bm:= u64~~2<<(l-1) - 1 + k:= l/8 + @for (i to n) add_bit{i, popc{bm & load{*u64~~(*u8~~xp + k*i)}}} + } else if (l < 128) xor_loop{1} + else if (l%64==0) { + @for (i to n) add_bit{i, popc{xor_words(0, xp+ll*i, ll)}} + } + else xor_loop{ll} + flush_bits{n, fixout} } } fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { def {add_bit, set_out, flush_bits} = bit_output{rp} if (l < 64) { - or_rows_bit_lt64{xp, rp, n, l, op_and} + def run_loop2{loop} = if (op_and) loop{&} else loop{|} + def run_loop4{m, t, loop} = { + if (op_and) loop{{x} => x & ((x&~t) + m)} + else loop{{x} => x | ((x| t) - m)} + } + def xor_word = prefix_byshift{^, <<} + def pext_in{x, e, c} = { + m := e<<1 | 1 + t := e | 1<<63 + tup{x | ((x|t) - m), t} + } + def pext_res{x, e, c} = pext{...pext_in{x, e, c}} + def mult_in{x, e, c} = { + {s, t} := pext_in{x, e, c} + cs:= c; c = (s&~e) >> 63 + tup{s, {rb}=>rb|cs} + } + def xx = ^{-promote{u64, op_and}, .} + fold_rows_bit_lt64{ + |, run_loop2, run_loop4, pext_res, mult_in, xx, xx, {r}=>r, + xp, rp, n, l + } return{} } else if (l < 128) { c:u64 = (promote{u64, l}-1) &- op_and # a row gives 1 if its sum is >c From 5e3cc6de814076ba9086da33e368aca62694c4c9 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jun 2024 11:29:40 -0400 Subject: [PATCH 18/38] Refactoring, simplification, comments --- src/singeli/src/fold.singeli | 104 ++++++++++++++++------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 110e1b0f..cadc4307 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -76,70 +76,67 @@ fn fold_assoc_0{T==f64, op if has_simd}(x:*T, len:u64) : T = { export{'si_sum_f64', fold_assoc_0{f64,+}} +# Short-row boolean folds: main challenge is bit packing def fold_rows_bit_lt64{ - op, run_loop2, run_loop4, pext_res, mult_in, xx, rx, rxs, + op, run_loop2, run_loop4, pext_res, mult_in, + xx, rx, # input and output xor for cases not specialized to individual functions + rxs, # output xor only, where and/or are specialized xp, rp, n, l } = { nw := cdiv{n*l,64} - def loop_pext{T, get, b} = { - @for (x in xp, r in *T~~rp over nw) r = cast_i{T, rxs{pext{get{x}, b}}} + def loop_T{T, proc_word} = { + @for (x in xp, r in *T~~rp over nw) r = cast_i{T, rxs{proc_word{x}}} } if (l == 2) { - t:u64 = 64w2b10 - run_loop2{ - if (fast_BMI2{}) { - {op} => loop_pext{u32, {x}=>op{x, x<<1}, t} - } else { - def all = 1<<64-1 - ms:5**u64 = reverse{slice{scan{{x,s}=>x^((x< { - def T = u32 - @for (x in xp, r in *T~~rp over nw) { - a := op{x, x>>1} - each{{m, sh} => { a &= m; a |= a>>sh }, ms, 1<>sh } + def ss = 1< loop_T{u32, {x} => extract{op{x, x>>1}}}} } else if (l == 4) { m:u64 = 64w2b0001; t := m<<3 - run_loop4{m, t, - if (fast_BMI2{}) { - loop_pext{u16, ., t} - } else { - {get} => { - def T = u16 - @for (x in xp, r in *T~~rp over nw) { - a := get{x} & t - a = (a * 4r2b001) & (64w0xf000) - a = (a * fold{+, 1<<(12*iota{4})}) >> (3*16) - r = cast_i{T, rxs{a}} - } - } + def extract = if (fast_BMI2{}) pext{., t} else { + {x} => { + a:= x & t + a = (a * 4r2b001) & 64w0xf000 + a = (a * 4r0x001) >> (64-16) } } - } else { + run_loop4{m, t, {get} => loop_T{u16, {x} => extract{get{x}}}} + } else { # generic width<64 {e0, d} := unaligned_spaced_mask_mod{l} - e := e0 << (l-1) - r:u64 = 0 - rh := *u32~~rp - ri:ux = 0 - if (fast_BMI2{}) { - c:u64 = 0 - @for (x in xp over i to nw) { - r |= pext_res{xx{x}, e, c} << ri - ri += popc{e} - e = e>>d | e<<(l-d) + e := e0 << (l-1) # ending bit of each row + c:u64 = 0 # carry, use depends on algorithm + def {write_bits, flush_bits} = { + r:u64 = 0 + rh := *u32~~rp + ri:ux = 0 + def write{rb, n_bits} = { + r |= rb << ri + ri += promote{ux, n_bits} if (ri >= 32) { store{rh, 0, cast_i{u32,rx{r}}}; ++rh; r >>= 32; ri -= 32 } } + def flush{} = if (ri > 0) store{rh, 0, cast_i{u32,rx{r}}} + tup{write, flush} + } + def finish_step{...a} = { + write_bits{...a} + e = e>>d | e<<(l-d) # update end mask for next iteration + } + if (fast_BMI2{}) { + @for (x in xp over nw) finish_step{pext_res{xx{x}, e, c}, popc{e}} } else { - dm:= cast_i{usz, popc{e}} - e0 |= -promote{u64, l & (l-1) == 0} - c:u64 = 0 + # Emulate pext with 1, 2, or 3 multiply/mask steps. + # To move size-a groups spaced at distance b together, + # the multiplier has up to b/a bits spaced by b-a. + dm:= cast_i{usz, popc{e}} # minimum output bits per word + dm-= promote{usz, l&(l-1) == 0} # for divisors of 64, e0 effectively overflows; subtract 1 to correct def loop{...par} = { @for (xo in xp over nw) { x:= xx{xo} @@ -152,18 +149,13 @@ def fold_rows_bit_lt64{ mult_in{x, e, c} } nb:= dm + promote{usz, e>=e0} # = popc{e} + # the multiply-mask sequence (last "mask" is the shift by (64 - nb)) def extract = match { - {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q { {q}, {}} => (s & e) * (q << clz{e}) + {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q } rb:= extract{...par} >> (64 - nb) - r |= mod_rb{rb} << ri - ri += promote{ux, nb} - if (ri >= 32) { - store{rh, 0, cast_i{u32,rx{r}}}; ++rh; - r >>= 32; ri -= 32 - } - e = e>>d | e<<(l-d) + finish_step{mod_rb{rb}, nb} } } if (l == 3) { @@ -187,7 +179,7 @@ def fold_rows_bit_lt64{ loop{tup{mult}, tup{}} } } - if (ri > 0) store{rh, 0, cast_i{u32,rx{r}}} + flush_bits{} } } From 1e6c7057e8fe2ce1a94bf0ba411792c4817e42b6 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 17 Jun 2024 21:51:32 -0400 Subject: [PATCH 19/38] =?UTF-8?q?Extend=20boolean=20F`=CB=98=20special=20c?= =?UTF-8?q?ode=20to=20any=20frame=20and=20cell=20rank?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 10 +++++----- src/builtins/scan.c | 21 +++++++++++---------- src/singeli/src/scan.singeli | 14 ++++++-------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 0fb83ed6..cff6890e 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -7,9 +7,9 @@ B fne_c1(B, B); B shape_c2(B, B, B); B transp_c2(B, B, B); -B fold_rows(Md1D* d, B x); // from fold.c -B fold_rows_bit(Md1D* d, B x); // from fold.c -B scan_rows_bit(u8, B x); // from scan.c +B fold_rows(Md1D* d, B x); // from fold.c +B fold_rows_bit(Md1D* d, B x); // from fold.c +B scan_rows_bit(u8, B x, usz m); // from scan.c B takedrop_highrank(bool take, B w, B x); // from sfns.c B try_interleave_cells(B w, B x, ur xr, ur xk, usz* xsh); // from transpose.c @@ -469,8 +469,8 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr if (!isFun(fd->f)) goto base; u8 frtid = v(fd->f)->flags-1; if (frtid==n_rtack) return x; - if (k==1 && xr==2 && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && TI(x,elType)==el_bit) { - B r = scan_rows_bit(frtid, x); if (!q_N(r)) return r; + if (TI(x,elType)==el_bit && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && 1==shProd(sh, k+1, xr)) { + B r = scan_rows_bit(frtid, x, m); if (!q_N(r)) return r; } } } else if (TY(f) == t_md2D) { diff --git a/src/builtins/scan.c b/src/builtins/scan.c index c4fb2885..0ebd813e 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -343,25 +343,26 @@ B scan_c2(Md1D* d, B w, B x) { B f = d->f; return withFill(r.b, wf); } -B scan_rows_bit(u8 rtid, B x) { - assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); +// scan cells of size m, stride 1 +B scan_rows_bit(u8 rtid, B x, usz m) { + assert(isArr(x) && TI(x,elType)==el_bit); #if SINGELI switch (rtid) { default: return bi_N; - case n_eq: return bit_negate(scan_rows_bit(n_ne, bit_negate(x))); + case n_eq: return bit_negate(scan_rows_bit(n_ne, bit_negate(x), m)); case n_and: case n_or: case n_ne: case n_ltack: { - usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; + usz ia = IA(x); u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); switch (rtid) { default:UD; - case n_and: si_scan_rows_and (xp, rp, n, m); break; - case n_or: si_scan_rows_or (xp, rp, n, m); break; - case n_ne: si_scan_rows_ne (xp, rp, n, m); break; - case n_ltack: si_scan_rows_ltack(xp, rp, n, m); break; + case n_and: si_scan_rows_and (xp, rp, ia, m); break; + case n_or: si_scan_rows_or (xp, rp, ia, m); break; + case n_ne: si_scan_rows_ne (xp, rp, ia, m); break; + case n_ltack: si_scan_rows_ltack(xp, rp, ia, m); break; } decG(x); return r; } case n_add: case n_sub: { - usz ia = IA(x); usz m = SH(x)[1]; + usz ia = IA(x); if (m >= 128) return bi_N; usz bl = 128; // block size i8 buf[bl]; i8 c = 0; @@ -383,7 +384,7 @@ B scan_rows_bit(u8 rtid, B x) { if (j == e) { j += m; c = 0; } else c = rp[e-1]; } if (rtid!=n_sub) { decG(x); return r; } - return C2(sub, C2(mul, m_f64(2), scan_rows_bit(n_ltack, x)), r); + return C2(sub, C2(mul, m_f64(2), scan_rows_bit(n_ltack, x, m)), r); } } #else diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 0f8417f5..3d15a81b 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -327,10 +327,10 @@ def avx2_loop_with_unaligned_mask{xp, rp, nw, l, scan_words, apply_carry} = { } } -fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { +fn scan_rows_andor{id}(src:*u64, dst:*u64, nl:usz, l:usz) : void = { def qand = not id assert{l > 0} - nw := cdiv{n*l, 64} + nw := cdiv{nl, 64} def res_m1{x,c,m} = { # result word with carry c, popc{m}<=1 if (qand) x &~ ((x+c) & (x+m)) else x | ((-x-c) &~ (x-m)) @@ -385,7 +385,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { wn:usz = 0 # starting word of next row c:u64 = id # carry def word{bit} = bit * ((1<<64) - 1) - @for (n) { + we:= nl/64; while (wn < we) { iw:= wn r := res_m1{load{src, iw}, c, u64~~1 << (i%64)} store{dst, iw, r}; ++iw @@ -410,10 +410,9 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { } } -fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { +fn scan_rows_neq(x:*u64, r:*u64, nl:usz, l:usz) : void = { def scan_word = prefix_byshift{^, <<} assert{l > 0} - nl := n*l nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { @@ -460,10 +459,9 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { } } -fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = { +fn scan_rows_left(x:*u64, r:*u64, nl:usz, l:usz) : void = { def scan_word = prefix_byshift{^, <<} assert{l > 0} - nl := n*l nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { @@ -484,7 +482,7 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = { i :usz = 0 # row bit index wn:usz = 0 # starting word of next row c:u64 = 0 # carry - @for (n) { + we:= nl/64; while (wn < we) { iw:= wn m := u64~~1 << (i%64) xw:= -(load{x, iw} & m) From cb1b72fbb239752962e95cfd00c13c9f6fea53bc Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 18 Jun 2024 07:46:18 -0400 Subject: [PATCH 20/38] =?UTF-8?q?Extend=20boolean=20F=CB=9D=CB=98=20specia?= =?UTF-8?q?l=20code=20to=20any=20ranks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 34 +++++++++++++++++++++++++--------- src/builtins/fold.c | 12 ++++++------ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index cff6890e..244f5e0e 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -7,10 +7,10 @@ B fne_c1(B, B); B shape_c2(B, B, B); B transp_c2(B, B, B); -B fold_rows(Md1D* d, B x); // from fold.c -B fold_rows_bit(Md1D* d, B x); // from fold.c -B scan_rows_bit(u8, B x, usz m); // from scan.c -B takedrop_highrank(bool take, B w, B x); // from sfns.c +B fold_rows(Md1D* d, B x); // from fold.c +B fold_rows_bit(Md1D* d, B x, usz n, usz m); // from fold.c +B scan_rows_bit(u8, B x, usz m); // from scan.c +B takedrop_highrank(bool take, B w, B x); // from sfns.c B try_interleave_cells(B w, B x, ur xr, ur xk, usz* xsh); // from transpose.c // X - variable name; XSH - its shape; K - number of leading axes that get iterated over; SLN - number of slices that will be made; DX - additional refcount count to add to x @@ -452,14 +452,29 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr Md1D* fd = c(Md1D,f); u8 rtid = fd->m1->flags-1; if (rtid==n_const) { f=fd->f; goto const_f; } - if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B && k==1 && xr==2 && isFun(fd->f)) { // TODO extend to any rank x with cr==1 - usz *sh = SH(x); usz m = sh[1]; + usz *sh = SH(x); + if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B + && isFun(fd->f) && 1==shProd(sh, k+1, xr)) { + usz m = sh[k]; u8 frtid = v(fd->f)->flags-1; if (m==1 || frtid==n_ltack) return select_cells(0 , x, cam, k, false); if ( frtid==n_rtack) return select_cells(m-1, x, cam, k, false); if (isPervasiveDyExt(fd->f)) { - if (TI(x,elType)==el_bit) { B r = fold_rows_bit(fd, x); if (!q_N(r)) return r; } - if (m <= 64 && m < sh[0]) return fold_rows(fd, x); + if (TI(x,elType)==el_bit) { + incG(x); // keep shape alive + B r = fold_rows_bit(fd, x, shProd(sh, 0, k), m); + if (!q_N(r)) { + if (xr > 2) { + usz* rsh = arr_shAlloc(a(r), xr-1); + shcpy(rsh, sh, k); + shcpy(rsh+k, sh+k+1, xr-1-k); + } + decG(x); return r; + } + decG(x); + } + // TODO extend to any rank + if (xr==2 && k==1 && m<=64 && mf)) goto base; u8 frtid = v(fd->f)->flags-1; if (frtid==n_rtack) return x; - if (TI(x,elType)==el_bit && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && 1==shProd(sh, k+1, xr)) { + if (TI(x,elType)==el_bit && (isPervasiveDyExt(fd->f)||frtid==n_ltack) + && 1==shProd(sh, k+1, xr)) { B r = scan_rows_bit(frtid, x, m); if (!q_N(r)) return r; } } diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 8a41e8d1..a02d953d 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -440,8 +440,7 @@ B fold_rows(Md1D* fd, B x) { } } -B sum_rows_bit(B x) { - usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; +B sum_rows_bit(B x, usz n, usz m) { u64* xp = bitarr_ptr(x); if (m < 128) { if (m == 2) return bi_N; // Transpose is faster @@ -508,15 +507,16 @@ B sum_rows_bit(B x) { } } -B fold_rows_bit(Md1D* fd, B x) { - assert(isArr(x) && RNK(x)==2 && TI(x,elType)==el_bit); +// Fold n cells of size m, stride 1 +// Return a vector regardless of argument shape, or bi_N if not handled +B fold_rows_bit(Md1D* fd, B x, usz n, usz m) { + assert(isArr(x) && TI(x,elType)==el_bit && IA(x)==n*m); if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; - if (rtid==n_add) return sum_rows_bit(x); + if (rtid==n_add) return sum_rows_bit(x, n, m); #if SINGELI if (rtid==n_ne|rtid==n_eq|rtid==n_or|rtid==n_and) { bool andor = rtid==n_or|rtid==n_and; - usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; if (andor && m < 256) while (m%8 == 0) { usz f = CTZ(m|32); m >>= f; usz c = m*n; From 4b0f105a7f4d82f104c41e5e67320d3bf663c174 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 18 Jun 2024 14:05:16 -0400 Subject: [PATCH 21/38] =?UTF-8?q?Implement=20short-row=20num=E2=8A=8F?= =?UTF-8?q?=CB=98bool=20(including=20=E2=8A=A3=CB=9D=CB=98=20=E2=8A=A2?= =?UTF-8?q?=CB=9D=CB=98)=20with=20fold=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 9 ++++++++- src/singeli/src/fold.singeli | 25 ++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 244f5e0e..316faa8e 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -177,6 +177,7 @@ NOINLINE B leading_axis_arith(FC2 fc2, B w, B x, usz* wsh, usz* xsh, ur mr) { // // fast special-case implementations +extern void (*const si_select_cells_bit_lt64)(uint64_t*,uint64_t*,uint32_t,uint32_t,uint32_t); // from fold.c (fold.singeli) static NOINLINE B select_cells(usz n, B x, usz cam, usz k, bool leaf) { // n {leaf? <∘⊑; ⊏}⎉¯k x; TODO probably can share some parts with takedrop_highrank and/or call ⊏? ur xr = RNK(x); assert(xr>1 && k loop_T{u16, {x} => extract{get{x}}}} } else { # generic width<64 {e0, d} := unaligned_spaced_mask_mod{l} - e := e0 << (l-1) # ending bit of each row - c:u64 = 0 # carry, use depends on algorithm + el:= e0 << (l-1) # ending bit of each row + e := if (same{off,-1}) el else e0< a>>o} + def run_loop4{m, t, loop} = loop{{x} => x<<(l-1-o)} + def pext_res{x, e, c} = pext{x, e} + def mult_in{x, e, c} = tup{x, {r}=>r} + def id{x} = x + fold_rows_bit_lt64{ + {a,b}=>a, run_loop2, run_loop4, pext_res, mult_in, o, id, id, id, + xp, rp, n, l + } +} fn xor_words(init:u64, x:*u64, l:usz):u64 = { @for (x over l) init ^= x @@ -222,7 +236,7 @@ fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, eq:u1) : void = { def pext_res{x, e, c} = pext{pext_in{x, e, c}, e} def mult_in{x, e, c} = tup{pext_in{x, e, c}, {r}=>r} fold_rows_bit_lt64{ - ^, run_loop2, run_loop4, pext_res, mult_in, {x}=>x, ^{rx,.}, ^{rx,.}, + ^, run_loop2, run_loop4, pext_res, mult_in, -1, {x}=>x, ^{rx,.}, ^{rx,.}, xp, rp, n, l } } else { @@ -277,7 +291,7 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { } def xx = ^{-promote{u64, op_and}, .} fold_rows_bit_lt64{ - |, run_loop2, run_loop4, pext_res, mult_in, xx, xx, {r}=>r, + |, run_loop2, run_loop4, pext_res, mult_in, -1, xx, xx, {r}=>r, xp, rp, n, l } return{} @@ -316,3 +330,4 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { } export{'si_xor_rows_bit', xor_rows_bit} export{'si_or_rows_bit', or_rows_bit} +export{'si_select_cells_bit_lt64', select_rows_bit_lt64} From 355efd1eb4ea4e12d0c17d3971d6ba68ab9fd8d0 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Wed, 19 Jun 2024 18:36:53 -0400 Subject: [PATCH 22/38] Insert-cells and scan implementation comments, and minor tweaks --- src/builtins/fold.c | 28 ++++++++++++++++ src/builtins/scan.c | 65 ++++++++++++++++++++++++++++-------- src/singeli/src/fold.singeli | 3 +- 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index a02d953d..ac7100f6 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -11,6 +11,34 @@ // •math.Sum: +´ with faster and more precise SIMD code for i32, f64 +// Insert with rank (˝˘ or ˝⎉k), or fold on flat array +// SHOULD optimize dyadic insert with rank +// Length 1, ⊣⊢: implemented as ⊏˘ +// SHOULD reshape identity for fast ˝˘ on empty rows +// Boolean operand, cell size 1: +// +: popcount +// Rows length ≤64: extract rows, popcount each +// TRIED scan-based version, faster for width 3 only +// COULD have bit-twiddling versions of +˝˘ on very short rows +// >64: popcount, boundary corrections (clang auto-vectorizes) +// ∨∧≠=, rows <64: extract from scan or windowed op +// Dedicated auto-vectorizing code for sizes 2, 4 +// Extract is generic with multiplies or BMI2 (SHOULD add SIMD) +// COULD use CLMUL for faster windowed ≠ +// ∨∧: +// SHOULD handle ∨∧ synonyms for boolean fold on rows +// Multiples of 8 <256: comparison function, recurse if needed +// Rows 6424: read word, mask, last bit of popcount +// Rows l<64: word-based scan, correct with SWAR +// ≥64: xor words (auto-vectorizes), correct boundary, popcount +// COULD implement boolean -˝˘ with xor, +˝˘, offset +// Arithmetic on rank 2, short rows: transpose then insert, blocked +// SHOULD extend transpose-insert code to any frame and cell rank +// SHOULD have dedicated +⌊⌈ insert with rank + #include "../core.h" #include "../builtins.h" #include "../utils/mut.h" diff --git a/src/builtins/scan.c b/src/builtins/scan.c index 0ebd813e..f083314c 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -1,3 +1,39 @@ +// Scan (`) +// Empty 𝕩, and length 1 if no 𝕨: return 𝕩 +// Generic operand: +// Constant: copy +// ⊢ identity, ⊣ reshape 𝕨 or first cell +// Boolean operand, rank 1: +// + AVX2 expansion (SHOULD have better generic, add SSE, NEON) +// ∨⌈ ∧×⌊ search+copy, then memset (COULD vectorize search) +// ≠ SWAR shifts, CLMUL, VPCLMUL (SHOULD add SSE, NEON) +// < SWAR +// =≤≥>- in terms of ≠<∨∧+ with adjustments +// Arithmetic operand, rank 1: +// ⌈⌊ Scalar, SSE, AVX in log(vector width) steps +// + Overflow-checked scalar or AVX2 +// Ad-hoc boolean-valued handling for ≠∨ +// SHOULD extend rank 1 special cases to cell bound 1 +// Higher-rank arithmetic, non-tiny cells: apply operand cell-wise +// SHOULD have dedicated high-rank scan optimizations + +// Scan with rank (`˘ or `⎉k) +// SHOULD optimize dyadic scan with rank +// Empty 𝕩, length 1, ⊢: return 𝕩 +// Boolean operand, cell size 1: +// ≠∨∧⊣ and synonyms, rows <64: SWAR, AVX2 (SHOULD add SSE, NEON) +// Power of two row size: autovectorized +// COULD have dedicated SIMD for CPU widths, little improvement +// ⊣ SWAR for <64, select for ≥ +// ∨⌈ ∧×⌊ SWAR with addition for small rows, search for large +// Rows 64≤l<160: SWAR specialized for ≤1 boundary +// Large rows: word-at-a-time search +// ≠ power-of-two shifts for <64, rank-1 scans and boundary corrections if ≥ +// SHOULD have a better intermediate-size (< ~256) SIMD method +// + scan in blocks, correct with mask, ⌊`, subtract +// = as ≠`⌾¬, - as (2×⊣`)-+` +// SHOULD optimize non-boolean scan with rank + #include "../core.h" #include "../utils/mut.h" #include "../builtins.h" @@ -5,6 +41,9 @@ #define F64_MIN -INFINITY #define F64_MAX INFINITY +#define CASE_N_AND case n_and: case n_mul: case n_floor +#define CASE_N_OR case n_or: case n_ceil + #if !USE_VALGRIND static u64 vg_rand(u64 x) { return x; } #endif @@ -215,16 +254,16 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (!(xr==1 && xe<=el_f64)) goto base; if (xe==el_bit) switch (rtid) { default: goto base; - case n_add: return scan_add_bool(x, ia); // + - case n_or: case n_ceil: return scan_or(x, ia); // ∨⌈ - case n_and: case n_mul: case n_floor: return scan_and(x, ia); // ∧×⌊ - case n_ne: return scan_ne(x, 0, ia); // ≠ - case n_eq: return scan_eq(x, ia); // = - case n_lt: return scan_lt(x, 0, ia); // < - case n_le: return bit_negate(scan_lt(bit_negate(x), 0, ia)); // ≤ - case n_gt: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_and(x, ia); // > - case n_ge: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_or (x, ia); // ≥ - case n_sub: return C2(sub, m_f64(2 * (*bitarr_ptr(x) & 1)), scan_add_bool(x, ia)); // - + case n_add: return scan_add_bool(x, ia); // + + CASE_N_OR: return scan_or(x, ia); // ∨⌈ + CASE_N_AND: return scan_and(x, ia); // ∧×⌊ + case n_ne: return scan_ne(x, 0, ia); // ≠ + case n_eq: return scan_eq(x, ia); // = + case n_lt: return scan_lt(x, 0, ia); // < + case n_le: return bit_negate(scan_lt(bit_negate(x), 0, ia)); // ≤ + case n_gt: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_and(x, ia); // > + case n_ge: x=bit_negate(x); *bitarr_ptr(x)^= 1; return scan_or (x, ia); // ≥ + case n_sub: return C2(sub, m_f64(2 * (*bitarr_ptr(x) & 1)), scan_add_bool(x, ia)); // - } if (rtid==n_add) return scan_plus(0, x, xe, ia); // + if (rtid==n_floor) return scan_min_num(x, xe, ia); // ⌊ @@ -349,13 +388,13 @@ B scan_rows_bit(u8 rtid, B x, usz m) { #if SINGELI switch (rtid) { default: return bi_N; case n_eq: return bit_negate(scan_rows_bit(n_ne, bit_negate(x), m)); - case n_and: case n_or: case n_ne: case n_ltack: { + CASE_N_AND: CASE_N_OR: case n_ne: case n_ltack: { usz ia = IA(x); u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); switch (rtid) { default:UD; - case n_and: si_scan_rows_and (xp, rp, ia, m); break; - case n_or: si_scan_rows_or (xp, rp, ia, m); break; + CASE_N_AND: si_scan_rows_and (xp, rp, ia, m); break; + CASE_N_OR: si_scan_rows_or (xp, rp, ia, m); break; case n_ne: si_scan_rows_ne (xp, rp, ia, m); break; case n_ltack: si_scan_rows_ltack(xp, rp, ia, m); break; } diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 94f88970..8aada08d 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -140,6 +140,7 @@ def fold_rows_bit_lt64{ dm:= cast_i{usz, popc{el}} # minimum output bits per word dm-= promote{usz, l&(l-1) == 0} # for divisors of 64, e0 effectively overflows; subtract 1 to correct def loop{...par} = { + et:= e0 << clz{e0} @for (xo in xp over nw) { x:= xx{xo} def {s, mod_rb} = if (length{select{par,0}} == 3) { @@ -153,7 +154,7 @@ def fold_rows_bit_lt64{ nb:= dm + promote{usz, e>=e0} # = popc{e} # the multiply-mask sequence (last "mask" is the shift by (64 - nb)) def extract = match { - { {q}, {}} => (s & e) * (q << clz{e}) + { {q}, {}} => (et &~ (et - (s & e))) * q {{...qs, q}, {...bs, b}} => (extract{qs, bs} & b) * q } rb:= extract{...par} >> (64 - nb) From 5f1c677d480c9518363325168b53f56bc1536ce4 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Thu, 20 Jun 2024 11:00:57 -0400 Subject: [PATCH 23/38] =?UTF-8?q?Support=20=E2=88=A7=E2=88=A8=20synonyms?= =?UTF-8?q?=20in=20fold=5Frows=5Fbit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/fold.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index ac7100f6..169e2068 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -21,12 +21,11 @@ // TRIED scan-based version, faster for width 3 only // COULD have bit-twiddling versions of +˝˘ on very short rows // >64: popcount, boundary corrections (clang auto-vectorizes) -// ∨∧≠=, rows <64: extract from scan or windowed op +// ∨∧≠= and synonyms, rows <64: extract from scan or windowed op // Dedicated auto-vectorizing code for sizes 2, 4 // Extract is generic with multiplies or BMI2 (SHOULD add SIMD) // COULD use CLMUL for faster windowed ≠ -// ∨∧: -// SHOULD handle ∨∧ synonyms for boolean fold on rows +// ∨∧ and synonyms: // Multiples of 8 <256: comparison function, recurse if needed // Rows 64f)->flags-1; if (rtid==n_add) return sum_rows_bit(x, n, m); #if SINGELI - if (rtid==n_ne|rtid==n_eq|rtid==n_or|rtid==n_and) { - bool andor = rtid==n_or|rtid==n_and; + bool is_or = rtid==n_or |rtid==n_ceil; + bool andor = rtid==n_and|rtid==n_floor|rtid==n_mul|is_or; + if (rtid==n_ne|rtid==n_eq|andor) { if (andor && m < 256) while (m%8 == 0) { usz f = CTZ(m|32); m >>= f; usz c = m*n; u64* yp; B y = m_bitarrv(&yp, c); u8 e = el_i8 + f-3; - CmpASFn cmp = rtid==n_or ? CMP_AS_FN(ne, e) : CMP_AS_FN(eq, e); - CMP_AS_CALL(cmp, yp, bitarr_ptr(x), m_f64((rtid==n_or)-1), c); + CmpASFn cmp = is_or ? CMP_AS_FN(ne, e) : CMP_AS_FN(eq, e); + CMP_AS_CALL(cmp, yp, bitarr_ptr(x), m_f64(is_or-1), c); decG(x); if (m==1) return y; x = y; } u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrv(&rp, n); - if (andor) si_or_rows_bit(xp, rp, n, m, rtid==n_and); + if (andor) si_or_rows_bit(xp, rp, n, m, !is_or); else si_xor_rows_bit(xp, rp, n, m, rtid==n_eq); decG(x); return r; } From 5758853ccbd61dc3d04d787f27d0fbf60391f720 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sat, 22 Jun 2024 21:57:59 -0400 Subject: [PATCH 24/38] Can't call fold-rows special code with reduction length 0 --- src/builtins/cells.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 316faa8e..7d921dca 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -461,7 +461,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr if (rtid==n_const) { f=fd->f; goto const_f; } usz *sh = SH(x); if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B - && isFun(fd->f) && 1==shProd(sh, k+1, xr)) { + && isFun(fd->f) && 1==shProd(sh, k+1, xr) && sh[k] > 0) { usz m = sh[k]; u8 frtid = v(fd->f)->flags-1; if (m==1 || frtid==n_ltack) return select_cells(0 , x, cam, k, false); From e952e4ca3e53bd1efaff07719f8b7645798154e7 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 23 Jun 2024 17:06:37 -0400 Subject: [PATCH 25/38] =?UTF-8?q?Disable=20F=C2=B4=CB=98=20special=20code?= =?UTF-8?q?=20when=20cell=20rank=20isn't=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 7d921dca..069fa802 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -460,7 +460,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr u8 rtid = fd->m1->flags-1; if (rtid==n_const) { f=fd->f; goto const_f; } usz *sh = SH(x); - if ((rtid==n_fold || rtid==n_insert) && TI(x,elType)!=el_B + if (((rtid==n_fold && cr==1) || rtid==n_insert) && TI(x,elType)!=el_B && isFun(fd->f) && 1==shProd(sh, k+1, xr) && sh[k] > 0) { usz m = sh[k]; u8 frtid = v(fd->f)->flags-1; From 7e15dfe1047aa96c6ed4a8365ea33684ff1b7ddb Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:03:57 +0300 Subject: [PATCH 26/38] make loadu_*/storeu_* take void* --- src/core/stuff.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/stuff.h b/src/core/stuff.h index 6221d449..a16324c5 100644 --- a/src/core/stuff.h +++ b/src/core/stuff.h @@ -3,9 +3,9 @@ #define MAP_NORESERVE 0 // apparently needed for freebsd or something #endif -static void storeu_u64(u64* p, u64 v) { memcpy((char*)p, &v, 8); } static u64 loadu_u64(u64* p) { u64 v; memcpy(&v, (char*)p, 8); return v; } -static void storeu_u32(u32* p, u32 v) { memcpy((char*)p, &v, 4); } static u32 loadu_u32(u32* p) { u32 v; memcpy(&v, (char*)p, 4); return v; } -static void storeu_u16(u16* p, u16 v) { memcpy((char*)p, &v, 2); } static u16 loadu_u16(u16* p) { u16 v; memcpy(&v, (char*)p, 2); return v; } +static void storeu_u64(void* p, u64 v) { memcpy(p, &v, 8); } static u64 loadu_u64(void* p) { u64 v; memcpy(&v, p, 8); return v; } +static void storeu_u32(void* p, u32 v) { memcpy(p, &v, 4); } static u32 loadu_u32(void* p) { u32 v; memcpy(&v, p, 4); return v; } +static void storeu_u16(void* p, u16 v) { memcpy(p, &v, 2); } static u16 loadu_u16(void* p) { u16 v; memcpy(&v, p, 2); return v; } #define ptr_roundUp(P, N) ({ AUTO p_ = (P); u64 n_ = (N); TOPTR(typeof(*p_), (ptr2u64(p_)+n_-1) & ~(n_-1)); }) #define ptr_roundUpToEl(P) ({ AUTO p2_ = (P); ptr_roundUp(p2_, _Alignof(typeof(*p2_))); }) From ace06e96ae227bf8dffbb10841a5915cb404be07 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:09:30 +0300 Subject: [PATCH 27/38] document CMP_* non-zero length requirement --- src/utils/calls.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/calls.h b/src/utils/calls.h index 137b8810..a133241e 100644 --- a/src/utils/calls.h +++ b/src/utils/calls.h @@ -16,7 +16,7 @@ CMP_DEF(ge, AS); CMP_DEF(ge, AA); CMP_DEF(lt, AS); CMP_DEF(le, AS); -// will write up to 8×⌈len÷64 bytes to WHERE, i.e. whole u64-s +// will write up to 8×⌈len÷64 bytes to WHERE, i.e. whole u64-s; LEN must not be 0 #define CMP_AA_FN(FN, ELT) cmp_fns_##FN##AA[ELT] #define CMP_AS_FN(FN, ELT) cmp_fns_##FN##AS[ELT] From 6c110a29eec5053f76e40e04884c20280e4f9323 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:07:02 +0300 Subject: [PATCH 28/38] use loadu_* & storeu_* for unaligned loads/stores --- src/builtins/fold.c | 6 +++--- src/builtins/scan.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 169e2068..0b35c113 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -475,12 +475,12 @@ B sum_rows_bit(B x, usz n, usz m) { if (m <= 64) { if (m%8 == 0) { usz k = m/8; u64 b = (m==64? 0 : 1ull<> (j%8); + u64 xw = loadu_u64((u8*)xp+j/8) >> (j%8); rp[i] = POPC(b & xw); } } else { @@ -489,7 +489,7 @@ B sum_rows_bit(B x, usz n, usz m) { u64 b = ~(~(u64)0 >> m); u64 prev = 0; for (usz i=0, j=m; i> (m-sh); diff --git a/src/builtins/scan.c b/src/builtins/scan.c index f083314c..f86f2074 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -416,7 +416,7 @@ B scan_rows_bit(u8 rtid, B x, usz m) { memset(rp+i, -c, len); i8* bi = buf-i; assert(j > i); - if (mk) while (j+mk <= e) { *(u64*)(rp+j) = *(u64*)(bi+j-1) & mm; j+=mk; } + if (mk) while (j+mk <= e) { storeu_u64(rp+j, loadu_u64(bi+j-1) & mm); j+=mk; } for (; j < e; j += m) rp[j] = bi[j-1]; si_scan_max_init_i8(rp+i, rp+i, len, I8_MIN); for (usz k = i; k < e; k++) rp[k] = bi[k] - rp[k]; From 024ab669d57cb6938dcf4837edf9b1f68ff4ada6 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:08:58 +0300 Subject: [PATCH 29/38] =?UTF-8?q?fix=20=E2=88=A7=CB=9D=CB=980=E2=80=BF8?= =?UTF-8?q?=E2=A5=8A0=20running=20comparison=20functions=20on=20length-0?= =?UTF-8?q?=20inputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/fold.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 0b35c113..ccf88581 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -545,6 +545,7 @@ B fold_rows_bit(Md1D* fd, B x, usz n, usz m) { bool is_or = rtid==n_or |rtid==n_ceil; bool andor = rtid==n_and|rtid==n_floor|rtid==n_mul|is_or; if (rtid==n_ne|rtid==n_eq|andor) { + if (n==0) { decG(x); return taga(arr_shVec(allZeroes(0))); } if (andor && m < 256) while (m%8 == 0) { usz f = CTZ(m|32); m >>= f; usz c = m*n; From 8d5efa72f0dae158cb3ba9baa89b8e91e36359cc Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:16:41 +0300 Subject: [PATCH 30/38] =?UTF-8?q?testing=20of=20=CB=9D=CB=98=20&=20`=CB=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/cases/cells.bqn | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/cases/cells.bqn b/test/cases/cells.bqn index 9ca95b06..d0106260 100644 --- a/test/cases/cells.bqn +++ b/test/cases/cells.bqn @@ -27,3 +27,31 @@ %USE tcc ⋄ ⟨⊏⟩ {𝕎 _tcc 𝕩}⌜ ⟨4‿5⥊↕20, 4‿5‿6⥊↕120, 2‿3‿4‿5⥊↕120⟩ %USE tcc ⋄ ⊏_tcc 4‿2⥊↕8 ⋄ 1⊏_tcc 4‿2⥊↕8 ⋄ ⊏_tcc ↕4‿2 ⋄ 1⊏_tcc ↕4‿2 %USE tcc ⋄ ⊑_tcc 4‿2⥊↕8 ⋄ 1⊑_tcc 4‿2⥊↕8 ⋄ ⊑_tcc ↕4‿2 ⋄ 1⊑_tcc ↕4‿2 + +( + # big ˝˘ & `˘ tester + _basicArgs ← { Test _𝕣 ranks: + {𝕊f: {{! f Test 0‿1⥊˜ 𝕩⊏0‿2‿4‿8‿32}¨ ↕𝕩⥊3 ⋄ 1}¨ ranks}¨ +‿∧‿∨‿=‿≠‿⊣‿⊢ + } + + # TODO replace ○⊢ with some pureness-preserving •internal.Keep + # w F _thing x tests F _thing⎉w x + _testFoldCells ← {∨´𝕗=⊢‿⊣? 0=×´𝕨↓≢𝕩? 1; (𝔽˝⎉𝕨 ≡ 𝔽˝○⊢⎉𝕨) 𝕩} + _testScanCells ← { (𝔽`⎉𝕨 ≡ 𝔽`○⊢⎉𝕨) 𝕩} + + {1 𝕨 _testFoldCells 𝕩} _basicArgs 2‿3‿4 + {1 𝕨 _testScanCells 𝕩} _basicArgs 2‿3‿4 + {2 𝕨 _testScanCells 𝕩} _basicArgs 3‿4 + {2 𝕨 _testScanCells 𝕩} _basicArgs 3‿4 + + + ⟨0, 0‿1, 1‿0, 1, 10, 10‿1, 1‿10, 100⟩ { + 𝕨‿𝕩 ⥊¨↩ + sh ← 𝕨∾𝕩 + cr ← ≠𝕩 + +‿∧‿∨‿=‿≠‿⊣‿⊢ { + cr 𝕎 _testFoldCells 𝕩 + cr 𝕎 _testScanCells 𝕩 + }⌜ {0=×´sh? 𝕩; ∾⟨𝕩, ¬⌾⊑¨ 𝕩, ¬⌾(¯1⊑⥊)¨ 𝕩⟩} sh⊸⥊¨ 0‿1 + }⌜ ⟨0, 1, 2, 8, 8‿1, 4‿8, 4‿2, 59, 60, 63, 80, 81, 200⟩ +) From b6bf3f2da62be4f3e3e17c335fcc87c7e2ab3abf Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:35:25 +0300 Subject: [PATCH 31/38] suppress clang ubsan warning it's a true warning of actual UB and could conceivably be taken advantage of by a compiler, but is not currently so keep things pretty for now --- src/builtins/scan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index f86f2074..b0a462e4 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -414,7 +414,7 @@ B scan_rows_bit(u8 rtid, B x, usz m) { usz e = i + len; si_bcs8(xp + i/64, buf, len); memset(rp+i, -c, len); - i8* bi = buf-i; + i8* bi = buf; bi-=i; // yeah this makes the pointer go out of bounds, but whatever assert(j > i); if (mk) while (j+mk <= e) { storeu_u64(rp+j, loadu_u64(bi+j-1) & mm); j+=mk; } for (; j < e; j += m) rp[j] = bi[j-1]; From 8e7f19a883a1f51e81e4a50dfdabf01dd2ffac8b Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 03:43:10 +0300 Subject: [PATCH 32/38] make cells test work under heapverify disables checking whether the result is correct, which is not important during heapverify --- test/cases/cells.bqn | 5 +++-- test/run.bqn | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/cases/cells.bqn b/test/cases/cells.bqn index d0106260..d0f4ea09 100644 --- a/test/cases/cells.bqn +++ b/test/cases/cells.bqn @@ -29,6 +29,7 @@ %USE tcc ⋄ ⊑_tcc 4‿2⥊↕8 ⋄ 1⊑_tcc 4‿2⥊↕8 ⋄ ⊑_tcc ↕4‿2 ⋄ 1⊑_tcc ↕4‿2 ( + %USE IS_HEAPVERIFY # big ˝˘ & `˘ tester _basicArgs ← { Test _𝕣 ranks: {𝕊f: {{! f Test 0‿1⥊˜ 𝕩⊏0‿2‿4‿8‿32}¨ ↕𝕩⥊3 ⋄ 1}¨ ranks}¨ +‿∧‿∨‿=‿≠‿⊣‿⊢ @@ -36,8 +37,8 @@ # TODO replace ○⊢ with some pureness-preserving •internal.Keep # w F _thing x tests F _thing⎉w x - _testFoldCells ← {∨´𝕗=⊢‿⊣? 0=×´𝕨↓≢𝕩? 1; (𝔽˝⎉𝕨 ≡ 𝔽˝○⊢⎉𝕨) 𝕩} - _testScanCells ← { (𝔽`⎉𝕨 ≡ 𝔽`○⊢⎉𝕨) 𝕩} + _testFoldCells ← {∨´𝕗=⊢‿⊣? 0=×´𝕨↓≢𝕩? 1; heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽˝⎉𝕨 𝕩; (𝔽˝⎉𝕨 ≡ 𝔽˝○⊢⎉𝕨) 𝕩} + _testScanCells ← { heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽`⎉𝕨 𝕩; (𝔽`⎉𝕨 ≡ 𝔽`○⊢⎉𝕨) 𝕩} {1 𝕨 _testFoldCells 𝕩} _basicArgs 2‿3‿4 {1 𝕨 _testScanCells 𝕩} _basicArgs 2‿3‿4 diff --git a/test/run.bqn b/test/run.bqn index 783854a7..dc40b1fb 100644 --- a/test/run.bqn +++ b/test/run.bqn @@ -63,7 +63,7 @@ searchRefine ← { exitCode ← 0 Run ← { 𝕊 testname: lns ← •FLines fullfile ← "cases"•file.At filename ← testname∾".bqn" - defk‿defv ← ⟨⟩‿⟨⟩ + defk‿defv ← ⟨"IS_HEAPVERIFY"⟩‿⟨"heapverify←"∾•Repr o.heapverify⟩ i ← 0 Pos ← {∾⟨filename, ":", •Repr 1+𝕩⟩} Next ← {𝕊: (i+↩1) ⊢ i⊑lns} From 445da5a659bc826c005b7526330595b91bfb6acf Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 04:21:46 +0300 Subject: [PATCH 33/38] actually test for correctness --- test/cases/cells.bqn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cases/cells.bqn b/test/cases/cells.bqn index d0f4ea09..9fea925d 100644 --- a/test/cases/cells.bqn +++ b/test/cases/cells.bqn @@ -32,13 +32,13 @@ %USE IS_HEAPVERIFY # big ˝˘ & `˘ tester _basicArgs ← { Test _𝕣 ranks: - {𝕊f: {{! f Test 0‿1⥊˜ 𝕩⊏0‿2‿4‿8‿32}¨ ↕𝕩⥊3 ⋄ 1}¨ ranks}¨ +‿∧‿∨‿=‿≠‿⊣‿⊢ + {𝕊f: {{f Test 0‿1⥊˜ 𝕩⊏0‿2‿4‿8‿32}¨ ↕𝕩⥊3 ⋄ 1}¨ ranks}¨ +‿∧‿∨‿=‿≠‿⊣‿⊢ } # TODO replace ○⊢ with some pureness-preserving •internal.Keep # w F _thing x tests F _thing⎉w x - _testFoldCells ← {∨´𝕗=⊢‿⊣? 0=×´𝕨↓≢𝕩? 1; heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽˝⎉𝕨 𝕩; (𝔽˝⎉𝕨 ≡ 𝔽˝○⊢⎉𝕨) 𝕩} - _testScanCells ← { heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽`⎉𝕨 𝕩; (𝔽`⎉𝕨 ≡ 𝔽`○⊢⎉𝕨) 𝕩} + _testFoldCells ← {∨´𝕗=⊢‿⊣? 0=×´(-𝕨)↑≢𝕩? 1; heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽˝⎉𝕨 𝕩; ! (𝔽˝⎉𝕨 ≡ 𝔽˝○⊢⎉𝕨) 𝕩} + _testScanCells ← { heapverify? 0=≠⥊𝕩? 1 ⊣ 𝔽`⎉𝕨 𝕩; ! (𝔽`⎉𝕨 ≡ 𝔽`○⊢⎉𝕨) 𝕩} {1 𝕨 _testFoldCells 𝕩} _basicArgs 2‿3‿4 {1 𝕨 _testScanCells 𝕩} _basicArgs 2‿3‿4 @@ -54,5 +54,5 @@ cr 𝕎 _testFoldCells 𝕩 cr 𝕎 _testScanCells 𝕩 }⌜ {0=×´sh? 𝕩; ∾⟨𝕩, ¬⌾⊑¨ 𝕩, ¬⌾(¯1⊑⥊)¨ 𝕩⟩} sh⊸⥊¨ 0‿1 - }⌜ ⟨0, 1, 2, 8, 8‿1, 4‿8, 4‿2, 59, 60, 63, 80, 81, 200⟩ + }⌜ ⟨0, 1, 2, 8, 8‿1, 4‿8, 4‿2, 59, 60, 63, 80, 81, 200, 640, 641⟩ ) From ace6105bf645a57901ab11b2685001eaa898153d Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 25 Jun 2024 22:30:56 -0400 Subject: [PATCH 34/38] =?UTF-8?q?Fixes=20for=20=E2=88=A7=CB=9D=CB=98=20and?= =?UTF-8?q?=20=E2=88=A8=CB=9D=CB=98=20on=20long=20rows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/singeli/src/fold.singeli | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 8aada08d..3b68016f 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -309,7 +309,7 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { j = jn+1 } } else { - rx := -promote{u64, op_and}; id := ~rx + rx := -promote{u64, op_and} def fixout = ^{rx, .} o:u64 = 0 # Saved bits j:u64 = 0; @for (i to n) { @@ -318,7 +318,7 @@ fn or_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, op_and:u1) : void = { l := ~(u64~~0) << sh rb:u64 = 1 if ((o | (e &~ l)) == 0) { # Search for shortcut - @for (i from j to jn-1) if (load{xp,i} != id) goto{'found'} + @for (i from j to jn) if (load{xp,i} != rx) goto{'found'} rb = 0; setlabel{'found'} } o = e & l From e365ecf7dc85124c51412d8d7a986c30b56cc18f Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Tue, 25 Jun 2024 22:31:47 -0400 Subject: [PATCH 35/38] =?UTF-8?q?Fix=20F`=CB=98=20on=20rank=200?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/builtins/cells.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/cells.c b/src/builtins/cells.c index 069fa802..68d36f30 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -485,7 +485,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr } } if (rtid==n_scan) { - if (cr==0) goto base; + if (cr==0) goto noSpecial; usz *sh = SH(x); usz m = sh[k]; if (m<=1 || IA(x)==0) return x; if (!isFun(fd->f)) goto base; From 452065adaf4b1875bbf0752d6230ba920b452f5c Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 05:00:11 +0300 Subject: [PATCH 36/38] add magic incantation to make optimizer happy --- src/builtins/scan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/scan.c b/src/builtins/scan.c index b0a462e4..5e582904 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -407,7 +407,7 @@ B scan_rows_bit(u8 rtid, B x, usz m) { i8 buf[bl]; i8 c = 0; u64* xp = bitarr_ptr(x); i8* rp; B r = m_i8arrc(&rp, x); - u64 ms[7] = { 0x00ff00ff00ff00ff, 0x00ff0000ff0000ff, 0x000000ff000000ff, 0x0000ff00000000ff, 0x00ff0000000000ff, 0xff000000000000ff, 0 }; + static const u64 ms[7] = { 0x00ff00ff00ff00ff, 0x00ff0000ff0000ff, 0x000000ff000000ff, 0x0000ff00000000ff, 0x00ff0000000000ff, 0xff000000000000ff, 0 }; u64 mm = ms[m-2>6? 6 : m-2]; usz mk = m*(POPC(mm)/8); for (usz i = 0, j = m; i < ia; i += bl) { usz len = ia - i; if (len > bl) len = bl; From 7474e51642e2f3693cc378461350df51750def32 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 05:23:41 +0300 Subject: [PATCH 37/38] rename mask_of in spaced.singeli otherwise confusable with maskOf in mask.singeli --- src/singeli/src/spaced.singeli | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/singeli/src/spaced.singeli b/src/singeli/src/spaced.singeli index 7b89d6f9..ff477cb3 100644 --- a/src/singeli/src/spaced.singeli +++ b/src/singeli/src/spaced.singeli @@ -1,14 +1,14 @@ -def mask_of{l} = emit{u64, 'get_spaced_mask', l} # see slash.singeli +def spaced_mask_of{l} = emit{u64, 'get_spaced_mask', l} # see slash.singeli def aligned_spaced_mask{l} = { assert{l <= 64} assert{l&(l-1) == 0} - mask_of{l} + spaced_mask_of{l} } def unaligned_spaced_mask_mod{l:T} = { assert{l < 64} - def m = mask_of{l} + def m = spaced_mask_of{l} def d = cast_i{T, ctz{m}} # = 64%l tup{m>>d | m<<(l-d), d} } From 6824b9a747850bee486a769459f7a0c35b6558a5 Mon Sep 17 00:00:00 2001 From: dzaima Date: Wed, 26 Jun 2024 06:09:30 +0300 Subject: [PATCH 38/38] fix another unaligned load --- src/singeli/src/fold.singeli | 2 +- test/cases/cells.bqn | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/singeli/src/fold.singeli b/src/singeli/src/fold.singeli index 3b68016f..f6019c71 100644 --- a/src/singeli/src/fold.singeli +++ b/src/singeli/src/fold.singeli @@ -260,7 +260,7 @@ fn xor_rows_bit(xp:*u64, rp:*u64, n:usz, l:usz, eq:u1) : void = { if (l <= 64) { bm:= u64~~2<<(l-1) - 1 k:= l/8 - @for (i to n) add_bit{i, popc{bm & load{*u64~~(*u8~~xp + k*i)}}} + @for (i to n) add_bit{i, popc{bm & loadu{*u64~~(*u8~~xp + k*i)}}} } else if (l < 128) xor_loop{1} else if (l%64==0) { @for (i to n) add_bit{i, popc{xor_words(0, xp+ll*i, ll)}} diff --git a/test/cases/cells.bqn b/test/cases/cells.bqn index 9fea925d..260e9c42 100644 --- a/test/cases/cells.bqn +++ b/test/cases/cells.bqn @@ -45,8 +45,7 @@ {2 𝕨 _testScanCells 𝕩} _basicArgs 3‿4 {2 𝕨 _testScanCells 𝕩} _basicArgs 3‿4 - - ⟨0, 0‿1, 1‿0, 1, 10, 10‿1, 1‿10, 100⟩ { + ⟨0, 0‿1, 1‿0, 2‿2, 1, 10, 10‿1, 1‿10, 30⟩ { 𝕨‿𝕩 ⥊¨↩ sh ← 𝕨∾𝕩 cr ← ≠𝕩 @@ -54,5 +53,5 @@ cr 𝕎 _testFoldCells 𝕩 cr 𝕎 _testScanCells 𝕩 }⌜ {0=×´sh? 𝕩; ∾⟨𝕩, ¬⌾⊑¨ 𝕩, ¬⌾(¯1⊑⥊)¨ 𝕩⟩} sh⊸⥊¨ 0‿1 - }⌜ ⟨0, 1, 2, 8, 8‿1, 4‿8, 4‿2, 59, 60, 63, 80, 81, 200, 640, 641⟩ + }⌜ ⟨0, 1, 2, 8, 32, 8‿1, 4‿8, 4‿2, 59, 60, 63, 80, 81, 200, 640, 641⟩ )