diff --git a/src/builtins/cells.c b/src/builtins/cells.c index e4632510..b812d579 100644 --- a/src/builtins/cells.c +++ b/src/builtins/cells.c @@ -469,7 +469,7 @@ B for_cells_c1(B f, u32 xr, u32 cr, u32 k, B x, u32 chr) { // F⎉cr x, with arr if (!isFun(fd->f)) goto base; u8 frtid = v(fd->f)->flags-1; if (frtid==n_rtack) return x; - if (k==1 && xr==2 && isPervasiveDyExt(fd->f) && TI(x,elType)==el_bit) { + if (k==1 && xr==2 && (isPervasiveDyExt(fd->f)||frtid==n_ltack) && TI(x,elType)==el_bit) { B r = scan_rows_bit(fd, x); if (!q_N(r)) return r; } } diff --git a/src/builtins/scan.c b/src/builtins/scan.c index dcfed95a..c219d8e1 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -345,14 +345,15 @@ B scan_rows_bit(Md1D* fd, B x) { #if SINGELI if (!v(fd->f)->flags) return bi_N; u8 rtid = v(fd->f)->flags-1; - if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq) { + if (rtid==n_and|rtid==n_or|rtid==n_ne|rtid==n_eq|rtid==n_ltack) { if (rtid==n_eq) x = bit_negate(x); usz *sh = SH(x); usz n = sh[0]; usz m = sh[1]; u64* xp = bitarr_ptr(x); u64* rp; B r = m_bitarrc(&rp, x); - if (rtid==n_and) si_scan_rows_and(xp, rp, n, m); - else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); - else si_scan_rows_ne (xp, rp, n, m); + if (rtid==n_and ) si_scan_rows_and (xp, rp, n, m); + else if (rtid==n_or ) si_scan_rows_or (xp, rp, n, m); + else if (rtid==n_ltack) si_scan_rows_ltack(xp, rp, n, m); + else si_scan_rows_ne (xp, rp, n, m); decG(x); return rtid==n_eq ? bit_negate(r) : r; } if (rtid==n_add && SH(x)[1]<128) { diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 910d5e80..02d3c90f 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -299,6 +299,20 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} # Row-wise boolean scan +def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1) +def loop_with_unaligned_mask{x, r, nw, l, step} = { + d:usz = 64 % l + m:u64 = (~u64~~0 >> d) / ((u64~~1 << l)-1) + m = m< { r=nr; c=nc } + {nr} => { r=nr; c = -(r>>63) } + } + m = m>>d | m<<(l-d) + } +} fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { def qand = not id assert{l > 0} @@ -314,7 +328,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10)) } } else { - m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + m:u64 = aligned_mask{l} t := m << (l-1) @for (r in dst, x in src over nw) { r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) @@ -324,17 +338,12 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { # could use for l>=8; not much faster and takes up space # def rowwise{T} = @for (r in *T~~dst, x in *T~~src over (64/width{T})*nw) r = x &~ (x+1) } else { - d:usz = 64 % l - m:u64 = (~u64~~0 >> d) / ((u64~~1 << l)-1) - m = m< { s:= (if (qand) (x &~ m) >> 1 else ~(x | m) >> 1 ) a:= (if (qand) s + (x&(m|c)) else s + ((m|c) &~ x)) - r = (if (qand) s ^ a else ~(s ^ a) ) - c = a >> 63 - m = m>>d | m<<(l-d) - } + tup{(if (qand) s ^ a else ~(s ^ a) ), + a >> 63} # new c + }} } } else if (l < 160) { q:usz = 0 # distance to next row boundary @@ -347,15 +356,15 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { c = r >> 63 } } else { - ib:usz = 0 # row bit index + i :usz = 0 # row bit index wn:usz = 0 # starting word of next row c:u64 = id # carry def word{bit} = bit * ((1<<64) - 1) - @for (i to n) { + @for (n) { iw:= wn - r := res_m1{load{src, iw}, c, u64~~1 << (ib%64)} + r := res_m1{load{src, iw}, c, u64~~1 << (i%64)} store{dst, iw, r}; ++iw - ib+= l; wn = ib/64 + i+= l; wn = i/64 c = r>>63 if (c != id) while (iw < wn) { x:= load{src, iw} @@ -369,7 +378,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { setlabel{'shortcut'} @for (r in dst over _ from iw to wn) r = word{id} } - if (ib%64 != 0) { + if (i%64 != 0) { x:= load{src, wn} store{dst, wn, if (qand) x &~ (x+c) else x | (-x-c)} } @@ -383,25 +392,19 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = (~u64~~0) / ((u64~~1 << l)-1) + m:u64 = aligned_mask{l} @for (r, x over nw) { s:= scan_word{x} b:= s<<1 & m # last bit of previous row r = s ^ (b<> d) / ((u64~~1 << l)-1) - m = m< { s:= scan_word{x} f:= (m-1)&~m # bits before first full row b:= s<<1 & m # last bit of previous row - r = s ^ ((c & f) | (b<>63) - m = m>>d | m<<(l-d) - } + s ^ ((c & f) | (b< 0} + nl := n*l + nw := cdiv{nl, 64} + if (l < 64) { + if ((l & (l-1)) == 0) { + m:u64 = aligned_mask{l} + @for (r, x over nw) { b:= x & m; r = b< { + f:= (m-1)&~m # bits before first full row + b:= x & m + (c & f) | (b<>63) + i+= l; wn = i/64 + @for (r in r over _ from iw+1 to wn) r = c + } + if (i%64 != 0) store{r, wn, c} + } +} + +export{'si_scan_rows_and', scan_rows_andor{0}} +export{'si_scan_rows_or', scan_rows_andor{1}} +export{'si_scan_rows_ne', scan_rows_neq} +export{'si_scan_rows_ltack', scan_rows_left}