From 7b3c69f9dad650eacde490dbeb913051a675d803 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Nov 2022 15:35:53 -0500 Subject: [PATCH 1/6] Take sum first in boolean plus-scan; special code for sparse or boolean sum --- src/builtins/md1.c | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index a87b339f..5fc42dec 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -156,6 +156,8 @@ static i64 bit_diff(u64* x, u64 am) { static u64 vg_rand(u64 x) { return x; } #endif +B slash_c1(B f, B x); + B scan_c1(Md1D* d, B x) { B f = d->f; if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); ur xr = RNK(x); @@ -167,15 +169,35 @@ B scan_c1(Md1D* d, B x) { B f = d->f; u8 rtid = v(f)->flags-1; if (xe==el_bit) { u64* xp=bitarr_ptr(x); - if (rtid==n_add && iaI32_MAX) goto base; + if (xs<=1) { if (xs==0) return x; goto bit_or; } + if (xs < ia/128) { + B ones = slash_c1(m_f64(0), x); + u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : el_i32; + MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); + SGetU(ones) + usz ri = 0; + for (usz i = 0; i < xs; i++) { + usz e = o2s(GetU(ones, i)); + mut_fillG(r0, ri, m_i32(i), e-ri); + ri = e; + } + if (ri Date: Mon, 7 Nov 2022 16:06:06 -0500 Subject: [PATCH 2/6] Implement multiple result types for non-Singeli +`bool --- src/builtins/md1.c | 54 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 5fc42dec..5ef00382 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -151,13 +151,40 @@ static i64 bit_diff(u64* x, u64 am) { #include "../singeli/gen/scan.c" #pragma GCC diagnostic pop #endif +B slash_c1(B f, B x); +B scan_bit_sum(B x, u64* xp, u64 ia, u64 xs) { // consumes x + u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : el_i32; + if (xs < ia/128) { + B ones = slash_c1(m_f64(0), x); + MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); + SGetU(ones) + usz ri = 0; + for (usz i = 0; i < xs; i++) { + usz e = o2s(GetU(ones, i)); + mut_fillG(r0, ri, m_i32(i), e-ri); + ri = e; + } + if (rif; if (isAtm(x) || RNK(x)==0) thrM("`: Argument cannot have rank 0"); ur xr = RNK(x); @@ -173,28 +200,7 @@ B scan_c1(Md1D* d, B x) { B f = d->f; u64 xs = bit_sum(xp, ia); if (xs>I32_MAX) goto base; if (xs<=1) { if (xs==0) return x; goto bit_or; } - if (xs < ia/128) { - B ones = slash_c1(m_f64(0), x); - u8 re = xs<=I8_MAX? el_i8 : xs<=I16_MAX? el_i16 : el_i32; - MAKE_MUT(r0, ia) mut_init(r0, re); MUTG_INIT(r0); - SGetU(ones) - usz ri = 0; - for (usz i = 0; i < xs; i++) { - usz e = o2s(GetU(ones, i)); - mut_fillG(r0, ri, m_i32(i), e-ri); - ri = e; - } - if (ri Date: Mon, 7 Nov 2022 17:53:38 -0500 Subject: [PATCH 3/6] 1- and 2-byte Singeli boolean +` --- src/builtins/md1.c | 13 ++++++------ src/singeli/src/scan.singeli | 40 +++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 5ef00382..a8d4397b 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -169,15 +169,16 @@ B scan_bit_sum(B x, u64* xp, u64 ia, u64 xs) { // consumes x return mut_fv(r0); } B r; + void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); #if SINGELI - i32* rp; r=m_i32arrv(&rp, ia); - avx2_bcs32(xp, rp, ia); + #define SUM(W,T) avx2_bcs##W(xp, rp, ia); #else - void* rp = m_tyarrv(&r, elWidth(re), ia, el2t(re)); - #define CASE(T) case el_##T: { T c=0; for (usz i=0; i> make{[8]u32, 4*tail{1, iota{8}}} s:= sel8{[32]u8~~b, ii32>>3 + bit{2}} p:= s & make{[32]u8, (1<<(1+tail{2})) - 1} # Prefixes d:= sel{[16]u8, make{[32]u8, merge{sums{4},sums{4}}}, [32]i8~~p} d+= sel8{d, bit{2}*(1+bit{3}>>2)-1} - d+= sel8{d, bit{3}-1} - #d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100} - j:= 4*i + d + sel8{d, bit{3}-1} + } + def step{x:u32, i, store1} = { + d:= sumlanes{x} + if (w==8) d+= [32]u8~~shuf{[4]u64, [8]i32~~sel8{d, bit{3}<<4-1}, 4b1100} + j:= (w/8)*i def out{v, k} = each{out, widen{v}, 2*k+iota{2}} - def out{v0:[8]i32, k} = { - v := [8]u32~~v0 + c - if (tail{1,k}) c = sel{[8]u32, v, make{[8]i32, broadcast{8, 7}}} + def out{v0:[vl]T, k} = { + v := V~~v0 + c + # Update carry at the lane boundary + if (w!=32 or tail{1,k}) { + c = sel{[8]u32, spread{v}, make{[8]i32, broadcast{8, 7}}} + } store1{rv, j+k, v} } out{[32]i8~~d, 0} @@ -120,15 +130,17 @@ avx2_bcs32(x:*u64, r:*i32, l:u64) : void = { if (e*32 != l) { def st{p, j, v} = { - j8 := 8*j - if (j8+8 <= l) { + jv := vl*j + if (jv+vl <= l) { store{p, j, v} } else { - if (j8 < l) maskstoreF{rv, maskOf{[8]i32, l - j8}, j, v} + if (jv < l) maskstoreF{rv, maskOf{V, l - jv}, j, v} return{} } } step{load{xv, e}, e, st} } } -'avx2_bcs32' = avx2_bcs32 +'avx2_bcs8' = avx2_bcs{i8} +'avx2_bcs16' = avx2_bcs{i16} +'avx2_bcs32' = avx2_bcs{i32} From 812b97dc51407eab62d68e817762deae8835c277 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Mon, 7 Nov 2022 20:23:15 -0500 Subject: [PATCH 4/6] Mark boolean +` result sorted and squoze --- src/builtins/md1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index a8d4397b..2d911cc7 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -201,7 +201,7 @@ B scan_c1(Md1D* d, B x) { B f = d->f; u64 xs = bit_sum(xp, ia); if (xs>I32_MAX) goto base; if (xs<=1) { if (xs==0) return x; goto bit_or; } - return scan_bit_sum(x, xp, ia, xs); + return FL_SET(scan_bit_sum(x, xp, ia, xs), fl_asc|fl_squoze); } if (rtid==n_or | rtid==n_ceil ) { bit_or: u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(i Date: Tue, 8 Nov 2022 16:56:43 +0200 Subject: [PATCH 5/6] use ty_u --- src/singeli/src/scan.singeli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/singeli/src/scan.singeli b/src/singeli/src/scan.singeli index 5a7989ff..a62671eb 100644 --- a/src/singeli/src/scan.singeli +++ b/src/singeli/src/scan.singeli @@ -87,7 +87,7 @@ avx2_scan_assoc_0{T, op}(x:*T, r:*T, len:u64, init:T) : void = { # Boolean cumulative sum avx2_bcs{T}(x:*u64, r:*T, l:u64) : void = { - def U = primtype{'u',width{T}} + def U = ty_u{T} def w = width{T} def vl= 256 / w def V = [vl]U From 87d0fa5dc794883a8fb757fa61b102773391c652 Mon Sep 17 00:00:00 2001 From: dzaima Date: Tue, 8 Nov 2022 16:59:17 +0200 Subject: [PATCH 6/6] fix gcc 9 build --- src/builtins/md1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/builtins/md1.c b/src/builtins/md1.c index 2d911cc7..b2e57601 100644 --- a/src/builtins/md1.c +++ b/src/builtins/md1.c @@ -203,8 +203,8 @@ B scan_c1(Md1D* d, B x) { B f = d->f; if (xs<=1) { if (xs==0) return x; goto bit_or; } return FL_SET(scan_bit_sum(x, xp, ia, xs), fl_asc|fl_squoze); } - if (rtid==n_or | rtid==n_ceil ) { bit_or: u64* rp; B r=m_bitarrv(&rp,ia); usz n=BIT_N(ia); u64 xi; usz i=0; while(i