diff --git a/src/builtins/fold.c b/src/builtins/fold.c index 70587410..9d1617f3 100644 --- a/src/builtins/fold.c +++ b/src/builtins/fold.c @@ -17,6 +17,8 @@ #include "../utils/calls.h" #if SINGELI + extern uint64_t* const si_spaced_masks; + #define get_spaced_mask(i) si_spaced_masks[i-1] #define SINGELI_FILE fold #include "../utils/includeSingeli.h" #endif diff --git a/src/builtins/scan.c b/src/builtins/scan.c index fc733332..c4fb2885 100644 --- a/src/builtins/scan.c +++ b/src/builtins/scan.c @@ -17,6 +17,8 @@ B sub_c2(B, B, B); B mul_c2(B, B, B); #if SINGELI + extern uint64_t* const si_spaced_masks; + #define get_spaced_mask(i) si_spaced_masks[i-1] #define SINGELI_FILE scan #include "../utils/includeSingeli.h" #endif diff --git a/src/builtins/slash.c b/src/builtins/slash.c index ce4a7deb..27357e79 100644 --- a/src/builtins/slash.c +++ b/src/builtins/slash.c @@ -777,7 +777,11 @@ B slash_c2(B t, B w, B x) { u64* rp; r = m_bitarrv(&rp, s); #if FAST_PDEP if (wv <= 52) { - u64 m = (u64)-1 / (((u64)1<> d) / ((u64~~1 << l)-1) - tup{m< x & ((x&~t) + m)} else loop{{x} => x | ((x| t) - m)} - } else { # odd row length - d:usz = 64 % l - e := ((~u64~~0 >> d) / ((u64~~1 << l)-1)) << (l-1) + } else { + {e0, d} := unaligned_spaced_mask_mod{l} + e := e0 << (l-1) r:u64 = 0 rh := *u32~~rp ri:ux = 0 @@ -193,8 +188,7 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { } } } else { - dm:= 64/l - e0:= e<<1 | 1 + dm:= cast_i{usz, popc{e}} c:u64 = 0 def loop{...par} = { @for (xo in xp over nw) { @@ -226,15 +220,15 @@ def or_rows_bit_lt64{xp, rp, n, l, op_and} = { } else if (l < 8) { assert{l > 4} ld:= l-1; lld:= l*ld - {mult0, _} := unaligned_mask{ld} + {mult0, _} := unaligned_spaced_mask_mod{ld} mult0 &= u64~~1<>l; topk|= topk<>ll loop{tup{mult0,mult1}, tup{topk}} } else { - {mult, _} := unaligned_mask{l-1} + {mult, _} := unaligned_spaced_mask_mod{l-1} loop{tup{mult}, tup{}} } } diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 4b65c5ae..0f8417f5 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -5,6 +5,7 @@ if_inline (hasarch{'X86_64'}) { } include './mask' include './f64' +include './spaced' include 'util/tup' include './scan_common' @@ -300,14 +301,8 @@ export{'si_scan_plus_i32_f64', plus_scanG{i32, f64}} # Row-wise boolean scan -def aligned_mask{l} = (~u64~~0) / ((u64~~1 << l)-1) -def unaligned_mask{l} = { - def d = 64 % l - def m = (~u64~~0 >> d) / ((u64~~1 << l)-1) - tup{m< a>>d | a<<(l-d), tup{ms, ...iota{3}}}} @@ -347,7 +342,7 @@ fn scan_rows_andor{id}(src:*u64, dst:*u64, n:usz, l:usz) : void = { r = (if (qand) x & (x<<1 | 64w2b01) else x | (x<<1 & 64w2b10)) } } else { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} t := m << (l-1) @for (r in dst, x in src over nw) { r = (if (qand) x &~ ((t&x) ^ ((x&~t) + m)) @@ -422,7 +417,7 @@ fn scan_rows_neq(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} @for (r, x over nw) { s:= scan_word{x} b:= s<<1 & m # last bit of previous row @@ -472,7 +467,7 @@ fn scan_rows_left(x:*u64, r:*u64, n:usz, l:usz) : void = { nw := cdiv{nl, 64} if (l < 64) { if ((l & (l-1)) == 0) { - m:u64 = aligned_mask{l} + m:u64 = aligned_spaced_mask{l} @for (r, x over nw) { b:= x & m; r = b<>d | m<<(l-d), d} +}